commit d69f5709119eff379ff2e676fa8d0eb61c18cd44
parent a826ba0c86cac9bc046e8e4c418fc655ad715807
Author: I Mean I'm Just Saying <imeanimjustsaying@kiwifarms.st>
Date: Tue, 29 Sep 2026 23:37:34 -0400
Merge main (2ad584c0: release 14 HS, S1 and CF, release 15 UT) into r15/drive-stall
Two conflicts, both adjacent additions, both sides kept:
- editor/CHANGELOG.md: [Unreleased] carries UT's bullet (merged first) and
then DS's; both stay under [Unreleased].
- plans/release-15.md: main's file, with DS's row in the slices table (in
place of "per its prompt") and DS's "as shipped" section after UT's, before
the Rollout. Every line of either side is in the result.
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Diffstat:
53 files changed, 2840 insertions(+), 216 deletions(-)
diff --git a/SITE.md b/SITE.md
@@ -23,6 +23,7 @@ Regenerate this file with `pnpm --filter yt-dlp-transcript-common exec tsx bin/f
| [`cloudflareProject`](#cloudflareproject) | absent |
| [`accent`](#accent) | absent |
| [`siteUrl`](#siteurl) | absent |
+| [`listed`](#listed) | `true` |
| [`relatedSites`](#relatedsites) | `[]` |
| [`pwa`](#pwa) | `false` |
| [`archives`](#archives) | `true` |
@@ -157,6 +158,12 @@ Absolute public URL of this site's deployment, e.g. `https://jeralyzer.pages.dev
Default: absent
+## `listed`
+
+Whether the family lists this site. Opt-OUT: absent/true = listed, only an explicit `false` is written. An unlisted site still builds and deploys as before, and its own pages are unchanged; it is left out of the homepage (cards, chart, `/stats`), the hub (members, federated search, `/corpus.json`, `/llms.txt`), every other site's footer, and the published `channel-sites.json` and pooled `stats/`. A channel only unlisted sites expose is in none of the family's public totals; a channel a listed site also exposes is credited to the listed one.
+
+Default: `true`
+
## `relatedSites`
Pulls specific siblings to the front of the footer's cross-site list, in named groups. Siblings not named here fall into a trailing "Other sites" group. Absent/empty = one flat list of every sibling.
diff --git a/common/bin/compose-homepage.ts b/common/bin/compose-homepage.ts
@@ -8,6 +8,9 @@
// public/channel-sites.json <- channel slug -> [siteId, ...]
// public/homepage-summary.json <- small cross-site landing summary
//
+// None of the three names an unlisted site (site.json `listed: false`) or holds
+// a channel only unlisted sites expose (lib/siteSchema.ts isListedSite).
+//
// Requires build:index to have populated the cues LMDB first (the homepage
// prebuild chains it), same as the export pipeline.
@@ -15,6 +18,7 @@ import path from "node:path";
import { getPaths, type Paths } from "../lib/paths";
import { writeJsonAtomic as writeJsonAtomicShared } from "../lib/jsonFile-server";
import { buildPoolSummary } from "../controller/poolSummary";
+import { isListedSite } from "../lib/site";
import { runIfEntryPoint } from "./_cli";
// Where the homepage Next.js app serves static assets from. Overridable for e2e
@@ -45,7 +49,7 @@ export async function main(opts: { paths?: Paths } = {}): Promise<void> {
statsDir,
});
- // channel slug -> the ids of the content sites that expose it. Drives
+ // channel slug -> the ids of the listed content sites that expose it. Drives
// `groupBy: "site"` in the hub's dashboard (see channelSites.tsx).
await writeJsonAtomic(
path.join(publicDir, "channel-sites.json"),
@@ -60,8 +64,11 @@ export async function main(opts: { paths?: Paths } = {}): Promise<void> {
summary,
);
+ const unlisted = sites.filter((s) => !isListedSite(s)).length;
console.log(
- `compose-homepage: ${Object.keys(channelSites).length} channel(s) mapped across ${sites.length} site(s); ` +
+ `compose-homepage: ${Object.keys(channelSites).length} channel(s) mapped across ${sites.length - unlisted} listed site(s)` +
+ (unlisted > 0 ? ` (${unlisted} unlisted left out)` : "") +
+ "; " +
`summary covers ${summary.totals.transcripts} transcription(s) / ${summary.totals.downloads} download(s) ` +
`across ${summary.sites.length} public site(s) into ${publicDir}.`,
);
diff --git a/common/bin/compose-hub.test.ts b/common/bin/compose-hub.test.ts
@@ -130,3 +130,43 @@ test("in a worktree, compose-hub writes its own files and never through the link
rmSync(root, { recursive: true, force: true });
}
});
+
+// Release 14 slice HS: an unlisted site builds and deploys, and the hub does
+// not list it — not a member, so not in federated search, corpus.json or
+// llms.txt.
+test("an unlisted site is in none of the hub's files; a listed one is in each", async () => {
+ const root = mkdtempSync(path.join(tmpdir(), "compose-hub-"));
+ const log = console.log;
+ try {
+ const paths = fixturePaths(root);
+ const write = (id: string, site: Record<string, unknown>) => {
+ mkdirSync(path.join(paths.sitesDir, id), { recursive: true });
+ writeFileSync(path.join(paths.sitesDir, id, "site.json"), JSON.stringify(site));
+ };
+ write("fixture-shown", { siteTitle: "Shown Fixture", siteUrl: "https://fixture-shown.example" });
+ write("fixture-unlisted", {
+ siteTitle: "Unlisted Fixture",
+ siteUrl: "https://fixture-unlisted.example",
+ listed: false,
+ });
+ const lines: string[] = [];
+ console.log = (...a: unknown[]) => void lines.push(a.join(" "));
+ await main({ paths });
+ console.log = log;
+ const pool = JSON.parse(
+ readFileSync(path.join(paths.exportPublicDir, "hub-sites.json"), "utf8"),
+ ) as Array<{ siteId: string }>;
+ assert.deepEqual(pool.map((s) => s.siteId), ["fixture-shown"]);
+ for (const f of ["hub-sites.json", "corpus.json", "llms.txt"]) {
+ const text = readFileSync(path.join(paths.exportPublicDir, f), "utf8");
+ assert.ok(text.includes("fixture-shown.example"), `${f} lists the listed site`);
+ for (const needle of ["fixture-unlisted", "Unlisted Fixture"]) {
+ assert.ok(!text.includes(needle), `${f} names ${needle}`);
+ }
+ }
+ assert.match(lines.join("\n"), /compose-hub: 1 built-in pool site\(s\)/);
+ } finally {
+ console.log = log;
+ rmSync(root, { recursive: true, force: true });
+ }
+});
diff --git a/common/bin/compose-hub.ts b/common/bin/compose-hub.ts
@@ -4,7 +4,8 @@
// there is no SITE_ID and no per-site data: the hub is a federating shell that
// reads every archive cross-origin at runtime. It emits:
//
-// public/hub-sites.json <- the built-in trusted pool (listSites with a siteUrl)
+// public/hub-sites.json <- the built-in trusted pool (listSites with a siteUrl,
+// listed — site.json `listed`, isListedSite)
// public/hub-summary.json <- the official instances' numbers, the homepage's
// own (lib/hubSummary.ts) — OPTIONAL: skipped when
// there is no index to walk
@@ -19,7 +20,7 @@ import { existsSync } from "node:fs";
import { cp, rm, access } from "node:fs/promises";
import { getPaths, type Paths } from "../lib/paths";
import { accentHex } from "../lib/accent";
-import { listSites, resolveHubUrl } from "../lib/site";
+import { isListedSite, listSites, resolveHubUrl } from "../lib/site";
import { getHomepageConfig } from "../lib/homepage";
import { SITE_DESCRIPTOR_VERSION } from "../lib/siteDescriptor";
import {
@@ -79,7 +80,10 @@ export async function main(opts: { paths?: Paths } = {}): Promise<void> {
const paths = opts.paths ?? getPaths();
const publicDir = paths.exportPublicDir;
- // Built-in pool: every configured site that publishes a public URL. The entry
+ // Built-in pool: every configured site that publishes a public URL and is
+ // listed. An unlisted site (`listed: false`) still builds and deploys, but the
+ // hub does not list it: not a member, not in federated search, not in the
+ // hub's corpus.json or llms.txt (both are built from this list). The entry
// the hub registry loads at boot to seed its trusted built-in pool, and the
// input buildHubCorpus maps — ONE type for both (lib/archive/contract.ts
// HubMemberInput), where this file used to restate it as a local
@@ -88,7 +92,7 @@ export async function main(opts: { paths?: Paths } = {}): Promise<void> {
// a siteUrl can't be federated and is dropped — by both consumers.
const builtins: HubMemberInput[] = [];
for (const site of listSites(paths)) {
- if (!site.siteUrl) continue;
+ if (!site.siteUrl || !isListedSite(site)) continue;
builtins.push({
siteId: site.siteId,
siteTitle: site.siteTitle,
diff --git a/common/components/SearchBar.tsx b/common/components/SearchBar.tsx
@@ -30,6 +30,7 @@ export default function SearchBar({ nav }: { nav?: ReactNode }) {
groupStates,
queryDirty,
filtersDirty,
+ searchedThisPageLife,
handleResetLayers,
layersResetDisabled,
hasSubs,
@@ -79,6 +80,11 @@ export default function SearchBar({ nav }: { nav?: ReactNode }) {
draftTags,
]);
+ // The line under the bar says what to do: after an edit that is not applied
+ // yet, and before the first Search of the page life, when the results area
+ // is empty and waits for it.
+ const promptApply = queryDirty || filtersDirty || !searchedThisPageLife;
+
const submit = (
<Button
type="submit"
@@ -220,12 +226,12 @@ export default function SearchBar({ nav }: { nav?: ReactNode }) {
{/* One status line under the bar instead of two hints competing for room
inside it — and a SIBLING of the form, so the pinned block is never
more than the input row plus the chips. */}
- {(queryDirty || filtersDirty || hasSubs) && (
+ {(promptApply || hasSubs) && (
<p className="flex flex-wrap items-center gap-x-3 gap-y-0.5 text-xs">
{/* The site's accent, not the warning hue: an unapplied edit is
the next step, not a fault. `--brand` clears 4.5:1 on every
base's page ground (lib/brand.ts MIN_ACCENT_CONTRAST). */}
- {(queryDirty || filtersDirty) && (
+ {promptApply && (
<span className="text-brand">
Press Enter or click Search to apply
</span>
diff --git a/common/components/SearchResults.tsx b/common/components/SearchResults.tsx
@@ -55,6 +55,7 @@ const CARD_HIT_CAP = 8;
export default function SearchResults() {
const {
+ searchedThisPageLife,
hasActiveQuery,
resultGroups,
totalHits,
@@ -112,6 +113,13 @@ export default function SearchResults() {
// results" and the footer sit underneath it and cannot be reached.
const selectionBar = view !== "chart" && resultGroups.length > 0 && selectedCount > 0;
+ // A clear screen until the visitor asks: no count, no controls, no chart and
+ // no listing — the bar, the page's intro and the footer, with the bar's
+ // "Press Enter or click Search" line saying what to do. An empty Search shows
+ // every video; a link with a query or a filter shows its results on load.
+ // `resultGroups` is still computed underneath.
+ if (!searchedThisPageLife) return null;
+
return (
<section
data-testid="results-section"
diff --git a/common/components/SearchSessionContext.tsx b/common/components/SearchSessionContext.tsx
@@ -52,6 +52,7 @@ import {
} from "../lib/availability";
import { useUrlParams, writeUrlParams } from "./urlState";
import {
+ FILTER_URL_KEYS_V1,
buildShareSearchParams,
hasShareV1,
parseShareV1,
@@ -197,13 +198,23 @@ export type GroundingMode = "search" | "selection";
// export-filter parser.
const SELECTION_KEY = "ytdlp-tb:selection";
-// Whether a search has run in this page life: a commit, a profile load, or a
-// query arriving on the URL. Module state, so it survives client-side
-// navigation and resets on a reload or a new visit. A stored query is held
-// (see `runHeld`) only on the page life's FIRST load — the hub mounts a fresh
-// provider per route, and `/` → `/ask` must keep grounding in the search the
-// visitor just ran there.
+// Two facts about this page life, each with one job. Module state, so both
+// survive client-side navigation and reset on a reload or a new visit.
+//
+// `ranThisPageLife`: a search has run — a commit (Search, Enter, a Filters
+// Apply), a profile load, or an active query arriving on the URL. A stored
+// query is held (see `runHeld`) while it is unset: on the page life's first
+// load, and on any later mount while nothing has run. Once it is set a later
+// mount runs the stored query — the hub mounts a fresh provider per route, and
+// `/` → `/ask` must keep grounding in the search the visitor just ran there. A
+// filter-only link does not set it: it shows its results, but it ran no query.
+//
+// `askedThisPageLife`: the visitor has asked — a search ran, or a link carries
+// a query or a filter (`urlAsks`). The results area shows nothing until it is
+// set (`searchedThisPageLife`, its reactive copy): until the visitor asks, the
+// page is the bar, the intro and the footer.
let ranThisPageLife = false;
+let askedThisPageLife = false;
function loadSelection(): string[] {
if (typeof window === "undefined") return [];
@@ -257,6 +268,22 @@ function urlHasAnyFilterParam(): boolean {
return FILTER_URL_KEYS.some((k) => p.has(k));
}
+// A link that carries a query (`qt`, legacy `q` / `m`) or a filter (`tg`, the
+// share-link keys, the legacy filter keys) is the visitor asking: it shows its
+// results on load, as a Search would (`askedThisPageLife`). A video (`v`, `t`)
+// or a chart's shape (`view`, `cs`) alone is not a search.
+const ASKING_URL_KEYS: readonly string[] = [
+ "qt",
+ "q",
+ "tg",
+ ...FILTER_URL_KEYS,
+ ...FILTER_URL_KEYS_V1,
+];
+
+function urlAsks(params: URLSearchParams): boolean {
+ return ASKING_URL_KEYS.some((k) => params.has(k));
+}
+
// Resolve a stored snapshot's per-channel deltas (plus group defaults) into
// the concrete set of EXCLUDED channel names.
function snapshotToExcluded(
@@ -457,18 +484,14 @@ function useSearchSessionState() {
const [view, setView] = useState<"results" | "chart">("results");
const [chartShape, setChartShape] = useState<ChartShape | null>(null);
- // Tracks whether the user has committed at least once this session. Used
- // by the placeholder copy below — until the user has searched, we show a
- // "tip" hint rather than the "no matching videos" empty state.
- const [, setSearchExecuted] = useState<boolean>(() => {
- if (typeof window === "undefined") return true;
- const params = new URLSearchParams(window.location.search);
- return !(
- params.has("v") &&
- !params.has("qt") &&
- (params.get("q") ?? "") === ""
- );
- });
+ // The reactive copy of `askedThisPageLife`, which stays the source of truth
+ // across remounts: a mount later in the page life starts from it, and every
+ // place that sets it sets this too. False until the visitor asks, and the
+ // results area renders nothing until then. False on the server, where the
+ // module variable is never set, so the first client render matches.
+ const [searchedThisPageLife, setSearchedThisPageLife] = useState(
+ () => askedThisPageLife,
+ );
// Defer rendering of the QueryBuilder to the client. The builder's leaf
// IDs come from a module-scoped counter that's necessarily out of sync
@@ -809,8 +832,9 @@ function useSearchSessionState() {
[committedTags, postScopeSlugs],
);
- // False while a restored query is held (see `runHeld`): the results area
- // shows the browse listing under the restored filters, and nothing fetches.
+ // False while a restored query is held (see `runHeld`): nothing fetches, and
+ // the results area is empty until the visitor asks — or, when a filter-only
+ // link asked, it lists every video under the link's filters.
const hasActiveQuery = useMemo(
() => !runHeld && isNodeActive(committedRoot),
[committedRoot, runHeld],
@@ -1273,9 +1297,13 @@ function useSearchSessionState() {
setDraftRoot(resolvedRoot);
setCommittedRoot(resolvedRoot);
if (holdRestored) setRunHeld(true);
- else setSearchExecuted(true);
- if (rootFromUrl && isNodeActive(rootFromUrl)) ranThisPageLife = true;
}
+ // After `holdRestored` is decided. Only an active URL query counts as a
+ // run; a filter-only link asks (its results show) without releasing a
+ // stored query on this mount or a later one.
+ if (rootFromUrl && isNodeActive(rootFromUrl)) ranThisPageLife = true;
+ if (ranThisPageLife || urlAsks(params)) askedThisPageLife = true;
+ setSearchedThisPageLife(askedThisPageLife);
setDraftExcludedChannels(initialExcluded);
setDraftNov(initialNov);
@@ -1388,9 +1416,10 @@ function useSearchSessionState() {
}, [persistUiCollapse]);
const commitSearch = () => {
- setSearchExecuted(true);
setRunHeld(false);
ranThisPageLife = true;
+ askedThisPageLife = true;
+ setSearchedThisPageLife(true);
setCommittedRoot(draftRoot);
const nextSnapshot = buildDraftSnapshot();
const current = loadStoredState() ?? emptyStoredState();
@@ -1455,6 +1484,8 @@ function useSearchSessionState() {
// Loading a profile commits it, which runs it — as it always has.
setRunHeld(false);
ranThisPageLife = true;
+ askedThisPageLife = true;
+ setSearchedThisPageLife(true);
applyDraftSnapshot(snapshot);
const excluded = snapshotToExcluded(
snapshot,
@@ -2040,6 +2071,9 @@ function useSearchSessionState() {
hitBatchSize,
setHitLimit,
// ── Results ──
+ // False until the visitor asks in this page life; nothing in the results
+ // area renders until then (see `askedThisPageLife`).
+ searchedThisPageLife,
hasActiveQuery,
resultGroups,
totalHits,
diff --git a/common/controller/buildStats.test.ts b/common/controller/buildStats.test.ts
@@ -65,6 +65,7 @@ const { buildIndex } = await import("./buildIndex");
const { buildStats, STATS_DOWNGRADE_ENV } = await import("./buildStats");
const { normalizeTranscript } = await import("./normalizeTranscript");
const { readStatsPages } = await import("./poolSummary");
+const { siteStatsDir } = await import("../lib/site");
const { STATS_SCHEMA_VERSION } = await import("../lib/stats");
const { open } = await import("lmdb");
@@ -594,6 +595,62 @@ test("(j) the schema guard: an older cache is cleared, a newer one is refused un
}
});
+// Release 14 slice HS: the whole-pool bundle is published as the homepage's
+// `stats/`, and an unlisted site's content is in no public total.
+test("(k) the whole-pool bundle leaves out a channel only an unlisted site exposes; the site's own bundle keeps it", async () => {
+ resetCorpus();
+ const seedChannel = (slug: string, ids: string[]) => {
+ writeJson(path.join(paths.channelsDir, slug, "config.json"), {
+ handling: "youtube",
+ name: slug,
+ url: `https://www.youtube.com/@${slug}/videos`,
+ });
+ for (const id of ids) {
+ seedVideo(id, "2026-07-11T11:00:00Z", {}, slug);
+ addCaptions(id, "2026-07-11T12:00:00Z", slug);
+ }
+ };
+ seedVideo("listed-1");
+ seedChannel("unlisted-channel", ["u1", "u2"]);
+ seedChannel("shared-channel", ["s1"]);
+ seedChannel("pool-channel", ["p1"]);
+ // The listed site also exposes the shared channel; the unlisted site exposes
+ // its own channel and the shared one. The pool channel is on no site.
+ const siteFile = path.join(paths.sitesDir, SITE, "site.json");
+ const listed = JSON.parse(readFileSync(siteFile, "utf8"));
+ listed.channels.push({ slug: "shared-channel", groupId: "default" });
+ writeJson(siteFile, listed);
+ writeJson(path.join(paths.sitesDir, "fixture-unlisted", "site.json"), {
+ ...listed,
+ siteId: "fixture-unlisted",
+ siteTitle: "Unlisted",
+ headerTitle: "Unlisted",
+ siteUrl: "https://unlisted.example",
+ listed: false,
+ channels: [
+ { slug: "unlisted-channel", groupId: "default" },
+ { slug: "shared-channel", groupId: "default" },
+ ],
+ });
+ await runIndex();
+ const log: string[] = [];
+ const { byId } = await runStats(log);
+ assert.deepEqual([...byId.keys()].sort(), ["listed-1", "p1", "s1"]);
+ const manifest = JSON.parse(readFileSync(path.join(POOL, "manifest.json"), "utf8"));
+ assert.equal(manifest.totalCount, 3);
+ assert.deepEqual(
+ manifest.channels.map((c: { slug: string }) => c.slug),
+ ["pool-channel", "shared-channel", CHANNEL],
+ );
+ assert.ok(
+ log.includes("Stats whole-pool: 3 videos, 1 page(s); 1 channel(s) only unlisted sites expose left out."),
+ log.join("\n"),
+ );
+ // The unlisted site still builds as before: its own bundle has its videos.
+ const own = await readStatsPages(siteStatsDir(paths, "fixture-unlisted"));
+ assert.deepEqual(own.map((s) => s.id).sort(), ["s1", "u1", "u2"]);
+});
+
test("(z) no write this file caused landed outside its temp root", () => {
// LMDB writes natively, past the spy: its file must be under the root too.
assert.ok(paths.lmdbPath.startsWith(ROOT + path.sep), paths.lmdbPath);
diff --git a/common/controller/buildStats.ts b/common/controller/buildStats.ts
@@ -72,7 +72,11 @@ import {
} from "../lib/channelMediaHold";
import { getSettings } from "../lib/settings";
import type { Paths } from "../lib/paths";
-import { listSites, siteStatsDir } from "../lib/site";
+import {
+ channelsOnlyOnUnlistedSites,
+ listSites,
+ siteStatsDir,
+} from "../lib/site";
import {
INDEX_SCANNED_AT_KEY,
STATS_SCHEMA_VERSION,
@@ -171,11 +175,12 @@ export type BuildStatsOptions = {
paths: Paths;
onLog?: (msg: string) => void;
signal?: AbortSignal;
- // When set, also write an UNFILTERED whole-pool stats bundle (every non-
- // excluded channel) into this dir as {manifest,page-NNNN}.json. Used by the
- // Archilyzer hub (compose-homepage), whose cross-site charts need the full
- // dataset rather than any one site's filtered slice. Forces collection of the
- // full dataset even when no per-site bundle needs a rebuild.
+ // When set, also write a whole-pool stats bundle (every non-excluded
+ // channel, but for those only unlisted sites expose) into this dir as
+ // {manifest,page-NNNN}.json. Used by the Archilyzer hub (compose-homepage),
+ // whose cross-site charts need the full dataset rather than any one site's
+ // filtered slice. Forces collection of the full dataset even when no per-site
+ // bundle needs a rebuild.
wholePoolStatsDir?: string;
};
@@ -709,16 +714,25 @@ export async function buildStats({
log(`Stats site ${plan.site.siteId}: ${filtered.length} videos, ${pageCount} page(s).`);
}
- // Whole-pool bundle for the hub: every non-excluded channel, unfiltered. Built
+ // Whole-pool bundle for the hub: every non-excluded channel, except a channel
+ // only unlisted sites expose (site.json `listed: false`): the bundle is
+ // published as the homepage's `stats/`, and an unlisted site's content is in
+ // no public total. Its own per-site bundle above is built as before. Built
// from the same in-memory dataset so it stays consistent with the per-site
// bundles. Always rewritten when requested (stats records are small).
if (wholePoolStatsDir) {
+ const unlistedOnly = channelsOnlyOnUnlistedSites(sites);
+ const pooled =
+ unlistedOnly.size > 0
+ ? all.filter((s) => !unlistedOnly.has(s.channelSlug))
+ : all;
const { pageCount } = await writePages(
wholePoolStatsDir,
- all,
+ pooled,
STATS_MAX_PAGE_BYTES,
);
const channelEntries: StatsChannelEntry[] = [...channels.keys()]
+ .filter((slug) => !unlistedOnly.has(slug))
.map((slug) => ({
slug,
name: channels.get(slug)?.name ?? slug,
@@ -728,13 +742,18 @@ export async function buildStats({
const manifest: StatsManifest = {
version: STATS_MANIFEST_VERSION,
generatedAt: new Date().toISOString(),
- totalCount: all.length,
+ totalCount: pooled.length,
pageCount,
maxPageBytes: STATS_MAX_PAGE_BYTES,
channels: channelEntries,
};
await writeJsonAtomic(path.join(wholePoolStatsDir, "manifest.json"), manifest);
- log(`Stats whole-pool: ${all.length} videos, ${pageCount} page(s).`);
+ log(
+ `Stats whole-pool: ${pooled.length} videos, ${pageCount} page(s)` +
+ (unlistedOnly.size > 0
+ ? `; ${unlistedOnly.size} channel(s) only unlisted sites expose left out.`
+ : "."),
+ );
}
// Prune fingerprints for sites that no longer exist (their staging dirs are
diff --git a/common/controller/poolSummary.test.ts b/common/controller/poolSummary.test.ts
@@ -0,0 +1,28 @@
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import { parseSite } from "../lib/siteSchema";
+import { channelSitesOf } from "./poolSummary";
+
+// Run with:
+// pnpm --filter yt-dlp-transcript-common test
+//
+// channelSitesOf is the published `channel-sites.json` (compose-homepage) and
+// the map the summary attributes channels by. Release 14 slice HS: an unlisted
+// site (site.json `listed: false`) is in neither.
+
+test("channel-sites.json names listed sites only; a channel only an unlisted site exposes is absent", () => {
+ const sites = [
+ parseSite("fixture-a", { channels: [{ slug: "a1" }, { slug: "shared" }] }),
+ parseSite("fixture-b", { channels: [{ slug: "b1" }, { slug: "shared" }] }),
+ parseSite("fixture-unlisted", {
+ listed: false,
+ channels: [{ slug: "shared" }, { slug: "own" }],
+ }),
+ ];
+ assert.deepEqual(channelSitesOf(sites), {
+ a1: ["fixture-a"],
+ shared: ["fixture-a", "fixture-b"],
+ b1: ["fixture-b"],
+ });
+ assert.ok(!JSON.stringify(channelSitesOf(sites)).includes("fixture-unlisted"));
+});
diff --git a/common/controller/poolSummary.ts b/common/controller/poolSummary.ts
@@ -11,7 +11,7 @@ import path from "node:path";
import { mkdir, readFile } from "node:fs/promises";
import type { Paths } from "../lib/paths";
import { buildStats } from "./buildStats";
-import { listSites, type Site } from "../lib/site";
+import { isListedSite, listSites, type Site } from "../lib/site";
import {
statsPageFileName,
type StatsManifest,
@@ -45,11 +45,14 @@ export async function readStatsPages(statsDir: string): Promise<VideoStat[]> {
return out;
}
-// channel slug -> the ids of the content sites that expose it. A channel on
-// multiple sites maps to all of them; a pool-only channel is simply absent.
-export function channelSitesOf(sites: Site[]): ChannelSitesMap {
+// channel slug -> the ids of the LISTED content sites that expose it: the
+// published `channel-sites.json`. A channel on multiple sites maps to all of
+// them; a pool-only channel is simply absent, and so is an unlisted site
+// (site.json `listed: false`) and a channel only unlisted sites expose.
+export function channelSitesOf(sites: readonly Site[]): ChannelSitesMap {
const channelSites: ChannelSitesMap = {};
for (const site of sites) {
+ if (!isListedSite(site)) continue;
for (const c of site.channels) {
(channelSites[c.slug] ??= []).push(site.siteId);
}
@@ -71,8 +74,9 @@ export async function buildPoolSummary(opts: {
}): Promise<PoolSummary> {
const { paths, statsDir } = opts;
await mkdir(statsDir, { recursive: true });
- // Whole-pool stats dataset (every non-excluded channel). buildStats also
- // refreshes the per-site bundles as a side effect, which is harmless.
+ // Whole-pool stats dataset (every non-excluded channel but those only
+ // unlisted sites expose). buildStats also refreshes the per-site bundles as a
+ // side effect, which is harmless.
await buildStats({ paths, wholePoolStatsDir: statsDir });
const sites = listSites(paths);
const channelSites = channelSitesOf(sites);
diff --git a/common/lib/homepageSummary.test.ts b/common/lib/homepageSummary.test.ts
@@ -277,3 +277,70 @@ test("a site's wordmark lead travels when it is a proper prefix of the title; ot
assert.ok(plain.sites.every((x) => !("wordmarkLead" in x)));
assert.equal(s.version, HOMEPAGE_SUMMARY_VERSION);
});
+
+// Release 14 slice HS: site.json `listed: false`. The site still builds and
+// deploys; the family's public pages do not list it, and no public total counts
+// the channels only it exposes.
+test("an unlisted site is in no array and no total; a channel it shares is the listed site's", () => {
+ const unlisted = { ...site("zeta", ["q1", "a1"], "https://zeta.example"), listed: false } as Site;
+ const sites = [...SITES, unlisted];
+ const channelSites = { ...CHANNEL_SITES, a1: ["alpha", "zeta"], q1: ["zeta"] };
+ const own = [
+ stat({ channelSlug: "q1", id: "q-1", uploadDate: "20251101", duration: 7200 }),
+ stat({ channelSlug: "q1", id: "q-2", uploadDate: "20260201", status: "deleted" }),
+ stat({ channelSlug: "q1", id: "q-3", hasTranscript: false, transcribedDate: null }),
+ ];
+ const s = buildHomepageSummary([...STATS, ...own], channelSites, sites, NOW);
+ const base = buildHomepageSummary(STATS, CHANNEL_SITES, SITES, NOW);
+ // Byte for byte the summary without the unlisted site: every array (sites,
+ // channels, series, recent, monthly) and every total (totals, official,
+ // availability, monthlyUnplaced).
+ assert.deepEqual(s, base);
+ const text = JSON.stringify(s);
+ for (const needle of ["zeta", "ZETA", "q1", "Q1", "q-1"]) {
+ assert.ok(!text.includes(needle), needle);
+ }
+ // The shared channel stays credited to alpha.
+ assert.equal(s.sites.find((x) => x.siteId === "alpha")!.channels, 2);
+ assert.equal(s.version, 6);
+});
+
+test("an unlisted site with no siteUrl, or with every channel shared, changes nothing either", () => {
+ const base = buildHomepageSummary(STATS, CHANNEL_SITES, SITES, NOW);
+ // No siteUrl: never public, and its own channel is still in no total (unlike
+ // a pool-only channel, which `totals` counts).
+ const urlless = { ...site("zeta", ["r1"]), listed: false } as Site;
+ assert.deepEqual(
+ buildHomepageSummary(
+ [...STATS, stat({ channelSlug: "r1", id: "r-1" })],
+ { ...CHANNEL_SITES, r1: ["zeta"] },
+ [...SITES, urlless],
+ NOW,
+ ),
+ base,
+ );
+ const sharedOnly = { ...site("zeta", ["a1", "b1"], "https://zeta.example"), listed: false } as Site;
+ assert.deepEqual(
+ buildHomepageSummary(STATS, { ...CHANNEL_SITES, a1: ["alpha", "zeta"], b1: ["beta", "zeta"] }, [...SITES, sharedOnly], NOW),
+ base,
+ );
+ // Listed explicitly is the default: the same summary as no key at all.
+ const listedTrue = SITES.map((x) => ({ ...x, listed: true })) as Site[];
+ assert.deepEqual(buildHomepageSummary(STATS, CHANNEL_SITES, listedTrue, NOW), base);
+});
+
+test("a shared channel is the listed site's even when the unlisted site's id sorts first", () => {
+ const base = buildHomepageSummary(STATS, CHANNEL_SITES, SITES, NOW);
+ // "aaa-hidden" < "beta": the primary pick sorts ids, so only the listed
+ // filter keeps b1 with beta.
+ const unlisted = { ...site("aaa-hidden", ["b1", "q1"], "https://aaa-hidden.example"), listed: false } as Site;
+ const s = buildHomepageSummary(
+ [...STATS, stat({ channelSlug: "q1", id: "q-1", uploadDate: "20251101" })],
+ { ...CHANNEL_SITES, b1: ["aaa-hidden", "beta"], q1: ["aaa-hidden"] },
+ [unlisted, ...SITES],
+ NOW,
+ );
+ assert.deepEqual(s, base);
+ assert.ok(s.recent.filter((r) => r.slug.startsWith("b1/")).every((r) => r.siteId === "beta"));
+ assert.ok(!JSON.stringify(s).includes("aaa-hidden"));
+});
diff --git a/common/lib/homepageSummary.ts b/common/lib/homepageSummary.ts
@@ -1,6 +1,7 @@
import type { Platform } from "./platform";
import type { VideoStat } from "./stats";
import type { Site } from "./site";
+import { channelsOnlyOnUnlistedSites, isListedSite } from "./siteSchema";
import { accentHex, accentIdOf } from "./accent";
import { wordmarkLeadFor, type AccentId } from "./brand";
import { VIDEO_STATES, type VideoState } from "./availability";
@@ -10,11 +11,15 @@ import { VIDEO_STATES, type VideoState } from "./availability";
// HTML, so the landing renders instantly without the browser fetching the
// multi-MB whole-pool stats dataset.
//
-// Scope: the chart "universe" is PUBLIC sites only (those with a siteUrl), and
-// every video is attributed to a single PRIMARY public site (the first, by
-// sorted id, exposing its channel) so the Site and Channel breakdowns partition
-// the same set and combined totals stay honest. The KPI `totals` are instance-
-// wide (count pool-only / URL-less content too) — a deliberate scope difference.
+// Scope: the chart "universe" is PUBLIC sites only (those with a siteUrl that
+// are listed — site.json `listed`, lib/siteSchema.ts isListedSite), and every
+// video is attributed to a single PRIMARY public site (the first, by sorted id,
+// exposing its channel) so the Site and Channel breakdowns partition the same
+// set and combined totals stay honest. The KPI `totals` (and `availability`)
+// are instance-wide (count pool-only / URL-less content too) — a deliberate
+// scope difference — EXCEPT a channel only unlisted sites expose, which no
+// part of the summary counts (channelsOnlyOnUnlistedSites). An unlisted site is
+// in no array here; a channel it shares with a listed site is the listed one's.
//
// Two metrics are pre-binned at two granularities; Cumulative and Share (100%)
// are derived client-side from these, so no extra precompute is needed.
@@ -33,7 +38,12 @@ import { VIDEO_STATES, type VideoState } from "./availability";
// the same way — a summary without it paints its sites' hex, as before. So is
// the per-site `wordmarkLead` (release 14): a summary without it shows each
// card's title plain, as before. Nothing reads this number to accept a file.
-export const HOMEPAGE_SUMMARY_VERSION = 5;
+//
+// v6 (release 14): an unlisted site (site.json `listed: false`) is in no array,
+// and a channel only unlisted sites expose is in no total — `totals` and
+// `availability` included. No field was added or removed; the number says the
+// totals' scope moved.
+export const HOMEPAGE_SUMMARY_VERSION = 6;
// Day buckets are capped to this many trailing days so the embedded summary stays
// small regardless of archive age (daily detail is only useful recently).
@@ -166,7 +176,8 @@ export type HomepageOfficialTotals = {
export type HomepageSummary = {
version: number;
generatedAt: string; // ISO timestamp
- // Instance-wide headline numbers (count everything, not just public sites).
+ // Instance-wide headline numbers (count everything, not just public sites —
+ // but never a channel only unlisted sites expose).
totals: {
transcripts: number;
downloads: number;
@@ -351,12 +362,21 @@ export function buildHomepageSummary(
const nowWeek = weekOf(nowDay.replace(/-/g, ""))!;
const dayFloor = isoDate(new Date(now.getTime() - (DAY_WINDOW - 1) * 86400000));
- // Public sites only (need a link target + a stable place on the chart).
- const publicSites = sites.filter((s): s is Site & { siteUrl: string } => !!s.siteUrl);
+ // Public sites only (need a link target + a stable place on the chart), and
+ // only the listed ones.
+ const publicSites = sites.filter(
+ (s): s is Site & { siteUrl: string } => !!s.siteUrl && isListedSite(s),
+ );
const siteById = new Map(publicSites.map((s) => [s.siteId, s]));
+ // An unlisted site's own channels: counted nowhere below, totals included.
+ const unlistedOnly = channelsOnlyOnUnlistedSites(sites);
+ const inScope = unlistedOnly.size > 0
+ ? stats.filter((s) => !unlistedOnly.has(s.channelSlug))
+ : stats;
// channel slug -> primary public site (first by sorted id). Channels with no
- // public site are out of the chart universe entirely.
+ // public site are out of the chart universe entirely. Over listed sites only,
+ // so a channel an unlisted site shares with a listed one is the listed one's.
const primarySiteOf = new Map<string, string>();
for (const slug of Object.keys(channelSites)) {
const primary = [...channelSites[slug]]
@@ -369,7 +389,7 @@ export function buildHomepageSummary(
const channelName = new Map<string, string>();
const transcribedItems: Attributed[] = [];
const downloadedItems: Attributed[] = [];
- // Instance-wide KPI accumulators (count everything, not just public).
+ // Instance-wide KPI accumulators (count everything in scope, not just public).
let transcripts = 0;
let downloads = 0;
let hoursSeconds = 0;
@@ -405,7 +425,7 @@ export function buildHomepageSummary(
const uploadItems: { month: string; siteId: string }[] = [];
let monthlyUnplaced = 0;
- for (const s of stats) {
+ for (const s of inScope) {
// A transcript COUNTS whether or not it carries a date: transcripts,
// channels, hours, upload-month placement. Only the transcribed time series
// (and "this month", and the recent rail) need `transcribedDate`. buildStats
@@ -496,7 +516,7 @@ export function buildHomepageSummary(
.sort((a, b) => a.name.localeCompare(b.name));
// Recent feed (public universe), attributed to the primary site for its link.
- const recent: HomepageRecentItem[] = stats
+ const recent: HomepageRecentItem[] = inScope
.filter((s): s is VideoStat & { transcribedDate: string } =>
Boolean(s.hasTranscript && s.transcribedDate && primarySiteOf.get(s.channelSlug)),
)
@@ -522,16 +542,17 @@ export function buildHomepageSummary(
};
});
- // State census over every record, matching `totals`' instance-wide scope
- // (pool-only channels included) rather than the charts' public-site universe.
+ // State census over every in-scope record, matching `totals`' instance-wide
+ // scope (pool-only channels included, a channel only unlisted sites expose
+ // not) rather than the charts' public-site universe.
// Seeded with every state at zero so the shape is stable across corpora.
const byState = Object.fromEntries(
VIDEO_STATES.map((s) => [s, 0]),
) as Record<VideoState, number>;
- for (const s of stats) byState[s.status] = (byState[s.status] ?? 0) + 1;
+ for (const s of inScope) byState[s.status] = (byState[s.status] ?? 0) + 1;
const availability: HomepageAvailability = {
byState,
- counted: stats.length,
+ counted: inScope.length,
};
// Monthly back-catalogue series, keyed by the sites that survived the
diff --git a/common/lib/paths.ts b/common/lib/paths.ts
@@ -178,9 +178,10 @@ let cached: Paths | null = null;
// reference — to every file under it when the path is a directory
// (plans/FACTS.md, "A path joined from `process.cwd()` …"). So every join on
// such a value goes through this one opted-out call. Nothing changes at run
-// time.
-function under(...parts: string[]): string {
- return path.join(/* turbopackIgnore: true */ ...parts);
+// time. The comment sits before a named first argument, not a spread: that is
+// the form Turbopack's own advice shows.
+function under(first: string, ...rest: string[]): string {
+ return path.join(/* turbopackIgnore: true */ first, ...rest);
}
export function getPaths(): Paths {
diff --git a/common/lib/site.ts b/common/lib/site.ts
@@ -13,6 +13,7 @@ import {
import { socialLinksForSave } from "./socialLinks";
import { readJsonFileSync, writeJsonAtomic } from "./jsonFile-server";
import {
+ isListedSite,
isValidSiteId,
parseSite,
parseSiteUrl,
@@ -138,7 +139,9 @@ export type CrossSiteLink = { siteId: string; title: string; url: string };
export type CrossSiteGroup = { label?: string; sites: CrossSiteLink[] };
// Resolve the footer's cross-site list for `current` against the full pool.
-// Siblings that lack a siteUrl (or are `current`) are not linkable and dropped.
+// Siblings that lack a siteUrl (or are `current`) are not linkable and dropped,
+// and so is an unlisted sibling (`listed: false`, isListedSite) — even one a
+// featured group names. An unlisted `current` still lists its siblings.
// `current.relatedSites` groups render first, in order, each filtered to known
// linkable ids (unknown/used/self skipped, empty groups dropped). Every still-
// unused sibling lands in a trailing remainder group — unlabeled when there
@@ -149,7 +152,7 @@ export function resolveRelatedSites(
): CrossSiteGroup[] {
const byId = new Map<string, CrossSiteLink>();
for (const s of all) {
- if (s.siteId === current.siteId || !s.siteUrl) continue;
+ if (s.siteId === current.siteId || !s.siteUrl || !isListedSite(s)) continue;
byId.set(s.siteId, { siteId: s.siteId, title: s.siteTitle, url: s.siteUrl });
}
const used = new Set<string>();
diff --git a/common/lib/siteSchema.test.ts b/common/lib/siteSchema.test.ts
@@ -9,12 +9,14 @@ import type { z } from "zod";
import {
SITE_FIELD_DOCS,
SITE_KEYS,
+ channelsOnlyOnUnlistedSites,
+ isListedSite,
parseSite,
siteFieldsSchema,
siteToDisk,
type Site,
} from "./siteSchema";
-import { getSite, siteConfigFile, writeSite } from "./site";
+import { getSite, resolveRelatedSites, siteConfigFile, writeSite } from "./site";
import type { Paths } from "./paths";
const HERE = path.dirname(fileURLToPath(import.meta.url));
@@ -60,6 +62,7 @@ test("empty, null, [] and a number all read as the defaults, every key emitted",
assert.equal(want.duplicates, true);
assert.equal(want.transcriptDownloads, true);
assert.equal(want.pwa, false);
+ assert.equal(want.listed, true);
assert.deepEqual(want.relatedSites, []);
assert.equal(want.socialLinks, undefined);
assert.ok("socialLinks" in want);
@@ -231,6 +234,7 @@ function fixtures(): Array<[string, unknown]> {
channels: [{ slug: "c1", groupId: "a", order: 3 }, { slug: "c2", groupId: "q" }],
cloudflareProject: "p",
siteUrl: "https://s.example//",
+ listed: false,
relatedSites: [{ siteIds: ["x", "x", "BAD"] }, { label: " ", siteIds: [] }],
pwa: true,
archives: false,
@@ -285,6 +289,59 @@ test("writeSite throws on no groups, a default outside the groups, and an unsafe
assert.equal(fs.existsSync(siteConfigFile(paths, "s")), false);
});
+test("listed: absent reads listed, only an explicit false unlists, and only false is written", async () => {
+ for (const v of [undefined, true, 0, "false", null]) {
+ assert.equal(parseSite("s", { listed: v }).listed, true, String(v));
+ assert.equal("listed" in siteToDisk(parseSite("s", { listed: v })), false, String(v));
+ }
+ assert.equal(parseSite("s", { listed: false }).listed, false);
+ // `true` in a caller's Site is the default, so it is not written either.
+ assert.equal("listed" in siteToDisk({ ...parseSite("s", {}), listed: true }), false);
+
+ // Through the real writer and reader: false survives, absent reads listed.
+ const paths = scratchPaths(await mkdtemp(path.join(os.tmpdir(), "site-")));
+ const hidden = parseSite("s", { siteTitle: "Hidden", siteUrl: "https://h.example", listed: false });
+ await writeSite(hidden, paths);
+ assert.equal(JSON.parse(await readFile(siteConfigFile(paths, "s"), "utf8")).listed, false);
+ assert.deepEqual(getSite("s", paths), hidden);
+ await writeSite({ ...hidden, listed: true }, paths);
+ const disk = JSON.parse(await readFile(siteConfigFile(paths, "s"), "utf8"));
+ assert.equal("listed" in disk, false);
+ assert.equal(getSite("s", paths).listed, true);
+});
+
+test("isListedSite is the key's default; channelsOnlyOnUnlistedSites keeps a shared channel with the listed site", () => {
+ assert.equal(isListedSite({}), true);
+ assert.equal(isListedSite({ listed: true }), true);
+ assert.equal(isListedSite({ listed: false }), false);
+ const sites = [
+ parseSite("shown", { channels: [{ slug: "shared" }, { slug: "mine" }] }),
+ parseSite("hidden", { listed: false, channels: [{ slug: "shared" }, { slug: "secret" }] }),
+ parseSite("hidden2", { listed: false, channels: [{ slug: "secret" }, { slug: "secret2" }] }),
+ ];
+ assert.deepEqual([...channelsOnlyOnUnlistedSites(sites)].sort(), ["secret", "secret2"]);
+ assert.deepEqual([...channelsOnlyOnUnlistedSites([sites[0]])], []);
+});
+
+test("the footer never links an unlisted sibling, and an unlisted site's own footer still lists the rest", () => {
+ const current = parseSite("cur", {
+ siteUrl: "https://cur.example",
+ relatedSites: [{ label: "Friends", siteIds: ["hidden", "shown"] }],
+ });
+ const shown = parseSite("shown", { siteTitle: "Shown", siteUrl: "https://shown.example" });
+ const hidden = parseSite("hidden", { siteTitle: "Hidden", siteUrl: "https://hidden.example", listed: false });
+ const other = parseSite("other", { siteTitle: "Other", siteUrl: "https://other.example" });
+ const ids = (groups: ReturnType<typeof resolveRelatedSites>) =>
+ groups.flatMap((g) => g.sites.map((s) => s.siteId));
+ // Named in a featured group or not, the unlisted site is not linked.
+ assert.deepEqual(ids(resolveRelatedSites(current, [current, shown, hidden, other])), ["shown", "other"]);
+ // The unlisted site is still built as before: its footer lists its siblings.
+ assert.deepEqual(
+ ids(resolveRelatedSites({ ...hidden, relatedSites: [] }, [current, shown, hidden, other])),
+ ["cur", "shown", "other"],
+ );
+});
+
test("writeSite → getSite round-trips, and the file holds only non-defaults", async () => {
const paths = scratchPaths(await mkdtemp(path.join(os.tmpdir(), "site-")));
const site = parseSite("s", { siteTitle: "Mine", archives: false });
@@ -295,4 +352,5 @@ test("writeSite → getSite round-trips, and the file holds only non-defaults",
assert.equal("duplicates" in disk, false);
assert.equal("transcriptDownloads" in disk, false);
assert.equal("pwa" in disk, false);
+ assert.equal("listed" in disk, false);
});
diff --git a/common/lib/siteSchema.ts b/common/lib/siteSchema.ts
@@ -84,6 +84,7 @@ export type Site = {
cloudflareProject?: string;
accent?: string;
siteUrl?: string;
+ listed?: boolean;
relatedSites?: RelatedSiteGroup[];
pwa?: boolean;
archives?: boolean;
@@ -116,6 +117,8 @@ export const SITE_FIELD_DOCS: FieldDocs<Site> = {
'Per-site brand accent: a named accent id (`signal`, `brass`, `vermilion`, `violet`, `sakura`, `blue`, `green`) or a custom `"#rrggbb"`. It is the site\'s accent on every page; a reader does not pick one. Absent = `signal`, the family default. A custom hex is darkened or lightened per base until it reaches 4.5:1. The public `/site.json` always carries a hex: an id is published as its on-dark value. Any other spelling is dropped.',
siteUrl:
"Absolute public URL of this site's deployment, e.g. `https://jeralyzer.pages.dev` (trimmed, trailing slashes removed; anything not absolute http(s) is dropped). Drives the cross-site footer: a site with no siteUrl is omitted from every other site's list.",
+ listed:
+ "Whether the family lists this site. Opt-OUT: absent/true = listed, only an explicit `false` is written. An unlisted site still builds and deploys as before, and its own pages are unchanged; it is left out of the homepage (cards, chart, `/stats`), the hub (members, federated search, `/corpus.json`, `/llms.txt`), every other site's footer, and the published `channel-sites.json` and pooled `stats/`. A channel only unlisted sites expose is in none of the family's public totals; a channel a listed site also exposes is credited to the listed one.",
relatedSites:
"Pulls specific siblings to the front of the footer's cross-site list, in named groups. Siblings not named here fall into a trailing \"Other sites\" group. Absent/empty = one flat list of every sibling.",
pwa:
@@ -139,6 +142,36 @@ export function isValidSiteId(id: unknown): id is string {
return typeof id === "string" && SITE_ID_RE.test(id);
}
+// THE ONE PREDICATE for `listed` (site.json's opt-out; absent = listed). Every
+// public output that enumerates the family's sites filters through it: the
+// homepage summary (lib/homepageSummary.ts), channel-sites.json and the pooled
+// stats (controller/poolSummary.ts, controller/buildStats.ts), the hub's
+// member list (bin/compose-hub.ts) and the footer's siblings
+// (lib/site.ts resolveRelatedSites). The editor's own pages list every site.
+// Here, beside the key, and exported from lib/site like isValidSiteId, so the
+// pure summary builder can use it without importing file I/O.
+export function isListedSite(site: Pick<Site, "listed">): boolean {
+ return site.listed !== false;
+}
+
+// The channels whose content belongs to unlisted sites alone: exposed by at
+// least one site, and by no listed one. No public total counts them. A channel
+// a listed site also exposes is not here (it is credited to the listed site),
+// and a channel no site exposes (pool-only) is not here either — the family's
+// instance-wide totals have always counted it.
+export function channelsOnlyOnUnlistedSites(
+ sites: readonly Pick<Site, "listed" | "channels">[],
+): Set<string> {
+ const onListed = new Set<string>();
+ const onUnlisted = new Set<string>();
+ for (const site of sites) {
+ const into = isListedSite(site) ? onListed : onUnlisted;
+ for (const c of site.channels) into.add(c.slug);
+ }
+ for (const slug of onListed) onUnlisted.delete(slug);
+ return onUnlisted;
+}
+
export const SITE_DEFAULT_TITLE = "Transcript Browser";
export const SITE_DEFAULT_DESCRIPTION = "Browse and search video transcripts";
@@ -246,6 +279,8 @@ export const siteFieldsSchema = z.object({
).describe(d.cloudflareProject),
accent: settingsField(parseAccentSetting).describe(d.accent),
siteUrl: settingsField(parseSiteUrl).describe(d.siteUrl),
+ // Opt-out: only an explicit false unlists. Absent/true stays listed.
+ listed: settingsField((v): boolean => v !== false).describe(d.listed),
relatedSites: settingsField(parseRelatedSites).describe(d.relatedSites),
pwa: settingsField((v): boolean => v === true).describe(d.pwa),
// Opt-out: only an explicit false disables. Absent/true stays on.
@@ -335,6 +370,8 @@ export function siteToDisk(site: Site): Site {
: {}),
...(accent ? { accent } : {}),
...(siteUrl ? { siteUrl } : {}),
+ // Listed is the default: only the opt-out is persisted.
+ ...(site.listed === false ? { listed: false } : {}),
...(relatedSites.length > 0 ? { relatedSites } : {}),
...(site.pwa ? { pwa: true } : {}),
// Persist only the non-default: archives is on unless explicitly disabled.
diff --git a/common/styles/tokens.css b/common/styles/tokens.css
@@ -255,6 +255,15 @@ html[data-base="light"] {
--chart-surface: #ffffff;
--chart-grid: rgba(22, 28, 33, 0.09);
--chart-axis: #55646e;
+ /* OTHER: the homepage growth chart's band for the sites it groups
+ (homepage/app/lib/growthGaps.ts), a near-neutral slate at the foot of
+ the palette's lightness band (OKLCH L 0.43, C 0.03). 7.36:1 on the
+ ground, 7.99:1 on the chart surface. Against each slot it can sit on
+ (the dataviz validator, normal / worst CVD): blue 21.9 / 21.4, green
+ 17.4 / 15.2, violet 16.2 / 13.4, amber 22.1 / 18.2, magenta 24.5 /
+ 9.1, rust 14.1 / 10.5 — every CVD pair clears 8; rust is the one
+ normal pair under 15. */
+ --chart-other: #3e545c;
--chart-tooltip-bg: #ffffff;
}
@@ -338,6 +347,14 @@ html[data-base="dark"] {
--chart-surface: #16110a;
--chart-grid: rgba(233, 220, 197, 0.08);
--chart-axis: #8a8170;
+ /* OTHER, on this base: a near-neutral grey (OKLCH L 0.49, C 0.01),
+ the dimmest chart mark. 3.22:1 on the ground, 3.06:1 on the chart
+ surface (the slots: 3.37–6.46:1 on the ground). Against each slot (normal / worst CVD): blue 17.8 / 17.9,
+ green 19.2 / 15.7, violet 23.4 / 21.5, amber 20.3 / 18.1, magenta
+ 19.5 / 7.3, rust 13.5 / 9.3 — magenta in the CVD 6–8 floor band
+ (legal with the legend, the table and Other's place on top), rust the
+ one normal pair under 15. */
+ --chart-other: #62625c;
--chart-tooltip-bg: #1b150d;
}
diff --git a/editor/CHANGELOG.md b/editor/CHANGELOG.md
@@ -4,6 +4,7 @@
- **Transcripts that arrived after a video was first seen are counted.** The stats behind the homepage, the hub and every site's charts were cached per video and refreshed only when the video's metadata changed, so a transcript that came later — a Whisper run days after the download, or a video downloaded after the last index build — never reached them, and a video with YouTube captions alone had no transcription date. Counts and charts were low; the homepage could show a site with 0 transcripts, 0 channels and 0 hours while it served its videos. A stat is now also redone whenever the index re-reads the video, every transcript has a date, and a captioned video is dated by when its captions arrived rather than by a later Normalize run, so its place on "Transcribed over time" can move. **After updating, rebuild and restart the editor before anything else:** until then, **Build stats dataset** runs the old code and would undo the new stats, while a site, hub or homepage build already runs the new code — and the first stats build of any kind re-reads every video once (about 10–30 minutes on a large archive; it can be stopped and picks up where it stopped). Then build the index, the stats, the homepage, the hub, and the sites.
- **A stats build keeps the stats of a channel whose drive is not mounted, and will not undo a newer version's stats.** A channel whose media is on a drive that is not mounted (or is being moved) is left as it was instead of being read as a channel with no videos; a stats rebuild that has to start over refuses until the drive is back. A stats build refuses to clear stats written by a newer version of the editor; set `ARCHILYZER_STATS_ALLOW_DOWNGRADE=1` to roll back on purpose. Its log also says apart how many videos were downloaded since the last index build (they catch up after the next one) and how many the index skipped (no upload date, or it failed on them).
- **An index build keeps a channel whose drive is not mounted, instead of dropping it from the sites.** **Build index**, a site build's data phase and `archilyzer index` read a channel whose media is on a drive that is not mounted (or is being moved, or whose link and config disagree) as a channel with no videos: they removed its videos from the index, and the next site build published the channel as gone. Such a channel is now left as the last build had it — its videos stay in the index, its pages stay as they were, and the sites built next still list it — and the log names it, with its storage location: one line per channel, ` Held: N channel(s), K video(s) kept.` at the end of the `Diff:` line, and the channels again on the last line. A data folder that fails to read is held the same way, and a channel with no data folder at all is said in the log instead of passed over. An index rebuild that has to start over (after an update that changes the index's format, or with no index yet) refuses while any channel is held and says which; mount the drive first, or set `ARCHILYZER_INDEX_ALLOW_HELD=1` to rebuild without that channel until its drive is back and the index is built again — on the command for a command-line build (`ARCHILYZER_INDEX_ALLOW_HELD=1 pnpm archilyzer index`), or in the editor's own environment, with a restart, for **Build index** and the site builds started from the editor.
+- **umtool's build no longer lists its e2e test data, the e2e server's build folder or `.env.local` among a route's files.** The clip-audio route named its cache files in a way the bundler read as a pattern reaching into umtool's hidden folders, so its list of files took in the e2e fixture (where the tests link the song data), the e2e dev server's build folder and the env file: 1,704 of its 2,167 entries. It now lists what the other routes list (463). Those folders and env files are also excluded from every route's list, and `pnpm test:scripts` reads the last umtool build's lists back and fails on any such entry. A checkout whose umtool build predates its code (this change included) skips that check, saying so, until umtool is rebuilt (`pnpm --filter umtool exec next build`). Nothing changes when umtool runs.
- **A drive that stops answering no longer stops the editor answering.** When a storage location's drive is mounted but not answering (an SMR disk in a USB enclosure resetting under a long write), every page and poll that touched it waited on it, and a few such waits froze the whole editor until the drive came back. Every 15 seconds the editor now reads each location's disk activity counters from the kernel, which never waits on the drive: a disk with requests waiting and none finished since the last look is marked **Not answering**, and the mark comes off after two looks in a row find it working. Where no disk can be named (in a container, say) it asks the drive from a separate process with a 3-second limit instead. Any page or poll that reads the drive also gives up after 3 seconds and marks it the same way, and no more than four such reads wait on one drive at a time. While it is marked, the editor's pages and polls do not read that drive: `/storage` shows the location as **Not answering** with the time it stopped and how it is watched (**Refresh** asks again at once), the `/channels` volume chip reads "not answering since HH:MM" and its channels' badges "not answering", their videos list, video pages and Cleanup stage say so instead of reading the drive, `/saved-videos` names the channels it did not read, and the index and stats builds keep those channels as they do for an unmounted drive, and a channel that is in the middle of a move still shows as moving. Jobs for those channels are refused until the drive answers. Pages and polls also reuse each channel's media check for 5 seconds. The editor's `start` script and the container now give Node 16 threads for file access instead of 4 (`UV_THREADPOOL_SIZE`); that buys time for reads already waiting on a drive, and a read that was already waiting when the drive stalled still waits until the drive answers.
- **Building the homepage now publishes the source: a read-only git mirror, its raw tree and a fresh tarball, behind a gate.** `archilyzer build homepage`, the `/sites` Homepage jobs and `pnpm ops build-homepage` run `archilyzer source publish` between compose and `next build`. It makes a fresh clone of the private `main` (the repository itself is never rewritten), rewrites that copy with git-filter-repo using your scrub rules (file contents and commit messages; your home directory becomes `/home/user` without a rule), and publishes it under `homepage/public` for `git clone https://archilyzer.pages.dev/source/archilyzer.git`, beside `/source/tree/` and the Downloads tarball. Before anything is written, every object of the rewritten history and every file about to be published is searched for every string you have denied; **one hit refuses the build**, and its log names the string only by where you wrote it (`denylist line 3 (len 5)`) and each hit by its object, field and byte offset — never a byte of the object. **A refusal withdraws the source**: the last publish is removed from `homepage/public` and the last build's copy from `homepage/out`, and **Deploy homepage refuses** a build whose source was not audited under today's rules and today's `main` ("run `archilyzer build homepage`, then deploy"). The rules live outside the repo, in `~/.config/archilyzer/source-scrub.txt` and `source-denylist.txt` (`ARCHILYZER_CONFIG_DIR`, `SOURCE_SCRUB_FILE`, `SOURCE_DENYLIST_FILE`); **without them the build refuses**, naming the missing file. **Put everything private in the denylist before any deploy, a preview included**: previews are public, and every deployment stays reachable at its own address until you delete it. Install git-filter-repo once (`pipx install git-filter-repo`; the editor's process needs `~/.local/bin` on its `PATH` to find it) — without it the build fetches it through `pipx run`, which needs the network — and gitleaks if you want its secret scan too. An unchanged `main` with unchanged rules is skipped, so a rebuild costs about 20 seconds only when something moved. A checkout with no git repository (the docker image, a tarball install) builds with the /source page's empty state. `archilyzer source publish --check` audits without writing, `archilyzer source audit <clone>/.git` checks any clone, `archilyzer build homepage --no-source` removes the published source instead, and `archilyzer doctor` reports the tools, the two files (rule counts and permissions, never their contents) and the last publish. `create-archives.sh` is gone. See PUBLISH.md, "The source mirror (homepage)".
- **umtool reads the corpus from its checkout (or `TRANSCRIPTS_DIR`), and the song project's data defaults to `~/.local/share/archilyzer/song`.** If yours is elsewhere, link it there before restarting umtool: `mkdir -p ~/.local/share/archilyzer && ln -s <where the data is> ~/.local/share/archilyzer/song` (the data stays where it is). With no `CHANNELS_DIR`, umtool reads the corpus at `$TRANSCRIPTS_DIR/channels`, else the checkout's own `transcripts/channels`; it used to fall back to an absolute path that existed on one machine only. The song project's videos default to `~/reports/quartering-uh-song/videos`; `SONG_DIR` and `VIDEO_ROOT` still win. The song project's tracked manifests record their paths relative to the song folders, and the twenty one-off `umtool/song/*.sh` run logs, which only ever ran on the machine that wrote them, are gone.
@@ -12,6 +13,7 @@
- **A social icon is checked by what it may contain, when it is saved new or edited and every time it is shown, and a refused one says why.** An icon must be one well-formed `<svg>` of shapes, groups, gradients, clips, masks, filters, text and simple animation, with SVG presentation attributes: no script, `style` block, `foreignObject`, link, embedded image, `title`/`desc` with anything but text (text-only ones are removed), or HTML element; no event handler, however it is written; a `style` attribute of presentation properties only; a reference only to something inside the icon, written plainly; and nothing that could load from elsewhere (a CSS escape or comment, `image-set(`, `image(`, `cross-fade(`, `element(`, `src(`, `paint(`, `@import`). Comments, a leading XML declaration and a plain DOCTYPE are removed. A refused save ends with the reason ("… has an invalid SVG: it has an event handler attribute.", "… it links to something outside the icon.") and never repeats the markup; for a drawing program's file it says to export it with presentation attributes rather than a style block (in Inkscape, save as Plain SVG). **Upgrading:** an icon an older build stored is kept as it is when a save does not change it — a pause, a priority or a title still saves — but a page shows it as its label until its SVG is replaced; `archilyzer doctor`'s new "social icons" line names every stored icon that fails, by file and label, with the reason.
- **Two grounds, Light and Dark, and no accent picker in the header.** The editor's header keeps its theme toggle, which cycles System, Light and Dark; the theme menu (Base and Accent) is gone, and the editor wears its own accent, Signal. A stored choice of the retired third ground loads as Light and is rewritten once; a stored accent is not read and is left in storage. A site's accent is still set in its form; the form's hint no longer says a reader can pick another.
- **The hub URL hints say what the setting does now.** Settings' **Family hub URL** and a site's **Hub URL** no longer promise a Hub link in the header (it was removed): the value is published as `hubUrl` in each site's `/site.json` and `/corpus.json`, so the hub can tell its member sites. `SETTINGS.md` and `SITE.md` say the same.
+- **A site can be left off the homepage and the hub.** A site's settings have a new checkbox, **List on the Archilyzer homepage and hub**, on by default (`listed` in `site.json`; only `false` is written). Turned off, the site still builds and deploys at its own URL as before, but the homepage has no card, chart series, `/stats` entry or recent item for it; the hub does not list it as a member, search it, or name it in its `corpus.json` and `llms.txt`; no other site's footer links it; and `channel-sites.json` and the homepage's `stats/` leave it out. A channel only unlisted sites carry is in none of the published totals, the homepage's headline numbers included; a channel a listed site also carries is counted under the listed site. The editor's own pages still show every site. It takes effect at the next homepage, hub and site builds.
## [0.10.0] - 2026-09-28
- **The homepage can be built and deployed from `/sites`.** Under a new **Homepage** section, after Hub, there is **Build homepage** (tick **Deploy after build** to ship it in the same job, only if the build succeeds) and **Deploy homepage**, which ships the build already in `homepage/out`. A **Preview branch** box beside them sends either deploy to a Cloudflare Pages preview of the `archilyzer` project instead of production, and shows the preview's address as you type; a name Cloudflare would refuse or rewrite, or `main`, greys the deploy buttons out and says why. A line under the buttons says what a deploy would ship: when `homepage/out` was built (or that it holds no build yet), and where it goes, with the live URL. Deploy homepage with nothing built is refused before any job starts. The homepage reads the search index as it stands, so run **Build index** first when its numbers should move. The jobs run the same code as `archilyzer build homepage` / `deploy homepage`, and show on `/jobs` as `build-homepage`, `deploy-homepage` and `build-deploy-homepage`. The Hub section no longer describes the homepage.
diff --git a/editor/app/sites/actions.ts b/editor/app/sites/actions.ts
@@ -92,6 +92,9 @@ export async function saveSiteAction(
};
}
const siteUrl = parseSiteUrl(siteUrlRaw);
+ // Listed on the homepage and hub by default: the same opt-out idiom as
+ // archives below (an unchecked box sends no key → persisted as false).
+ const listed = formData.get("listed") === "on";
// Hub parent (per-site override of the family default) + PWA opt-in.
const hubUrlRaw = String(formData.get("hubUrl") ?? "").trim();
@@ -217,6 +220,8 @@ export async function saveSiteAction(
...(accent ? { accent } : {}),
...(cloudflareProject ? { cloudflareProject } : {}),
...(siteUrl ? { siteUrl } : {}),
+ // The Site is rebuilt from the form: a key missing here is dropped on save.
+ ...(listed ? {} : { listed: false }),
...(hubUrl ? { hubUrl } : {}),
...(pwa ? { pwa: true } : {}),
...(archives ? {} : { archives: false }),
diff --git a/editor/app/sites/components/SiteForm.tsx b/editor/app/sites/components/SiteForm.tsx
@@ -327,6 +327,22 @@ export function SiteForm({ initial, channels, allSites, isNew }: Props) {
defaultValue={initial.siteUrl ?? ""}
hint="Absolute URL this site is served at (e.g. https://jeralyzer.com). Used so other sites can link to it in their footer. Leave blank to omit this site from cross-site lists."
/>
+ <label className="flex items-center gap-2 text-sm">
+ <input
+ type="checkbox"
+ name="listed"
+ defaultChecked={initial.listed !== false}
+ className="accent-brand"
+ />
+ List on the Archilyzer homepage and hub
+ </label>
+ <p className="-mt-2 text-xs text-muted-foreground">
+ On by default. Turn off to leave this site out of the homepage (its
+ card, chart and stats), the hub (its members, search, corpus.json and
+ llms.txt) and every other site's footer, and to count its own
+ channels in none of the published totals. The site still builds and
+ deploys as before, at its own URL.
+ </p>
<Field
label="Hub URL"
name="hubUrl"
diff --git a/editor/e2e/helpers.ts b/editor/e2e/helpers.ts
@@ -152,6 +152,7 @@ export async function writeSite(
? { cloudflareProject: site.cloudflareProject }
: {}),
...(site.siteUrl ? { siteUrl: site.siteUrl } : {}),
+ ...(site.listed === false ? { listed: false } : {}),
...(site.relatedSites ? { relatedSites: site.relatedSites } : {}),
};
await writeFile(
diff --git a/editor/e2e/sites-crud.spec.ts b/editor/e2e/sites-crud.spec.ts
@@ -345,6 +345,62 @@ test("archives + per-video transcript downloads opt-outs round-trip", async ({
await expect(downloads).toBeChecked();
});
+test("the listed opt-out round-trips, and a save of another field keeps it", async ({
+ page,
+}) => {
+ await resetData("empty");
+ // An invented fixture id: no real site is named in a test.
+ await writeSite("fixture-unlisted", {
+ siteTitle: "Unlisted Fixture",
+ siteUrl: "https://fixture-unlisted.example",
+ listed: false,
+ });
+
+ type ListedSiteFile = { siteTitle?: string; listed?: boolean };
+ const file = "test-transcripts/sites/fixture-unlisted/site.json";
+ const listed = page.getByRole("checkbox", {
+ name: "List on the Archilyzer homepage and hub",
+ });
+ const save = async () => {
+ await page.getByRole("button", { name: /save site/i }).click();
+ await expect(
+ page.getByRole("status").filter({ hasText: "Saved" }),
+ ).toBeVisible();
+ };
+
+ // `false` on disk opens unticked, and a save that changes only the title
+ // keeps it: the action rebuilds the site from the form.
+ await page.goto("/sites/fixture-unlisted");
+ await expect(listed).not.toBeChecked();
+ await page.getByLabel(/site title/i).fill("Unlisted Fixture Renamed");
+ await save();
+ await expect(async () => {
+ const site = await readJson<ListedSiteFile>(file);
+ expect(site.siteTitle).toBe("Unlisted Fixture Renamed");
+ expect(site.listed).toBe(false);
+ }).toPass({ timeout: 10_000 });
+
+ // Ticking it removes the key — listed is the default.
+ await page.goto("/sites/fixture-unlisted");
+ await expect(listed).not.toBeChecked();
+ await listed.check();
+ await save();
+ await expect(async () => {
+ const site = await readJson<ListedSiteFile>(file);
+ expect("listed" in site).toBe(false);
+ }).toPass({ timeout: 10_000 });
+
+ // And unticking writes the explicit false again.
+ await page.goto("/sites/fixture-unlisted");
+ await expect(listed).toBeChecked();
+ await listed.uncheck();
+ await save();
+ await expect(async () => {
+ const site = await readJson<ListedSiteFile>(file);
+ expect(site.listed).toBe(false);
+ }).toPass({ timeout: 10_000 });
+});
+
test("the hub form's per-video transcript downloads opt-out round-trips to homepage.json", async ({
page,
}) => {
diff --git a/export/CHANGELOG.md b/export/CHANGELOG.md
@@ -6,6 +6,8 @@
- **A chart's stacked bars are separated by a 2 px gap in the chart card's colour.** A stacked bar's segments were drawn touching; they now have a 2 px gap in the card's colour between them, and in high-contrast mode the system's background colour. Stacked areas keep their line in each series' colour along the top, charts of one series, line charts and side-by-side bars are unchanged. Needs a rebuild and deploy of each site.
- **Two grounds, Light and Dark, and each site in its own accent.** The third ground, the warm paper one, is gone: the header's toggle cycles System, Light and Dark. A reader who had chosen it gets Light, before the page first paints and with no other ground on the way, and the stored choice becomes Light (the old paper theme's `archive` + `light` too). The theme menu's accent picker is gone from the header and the slide-out menu: every page wears the site's own accent (`site.json` `accent`), and a reader's stored pick from before is not read and is left in storage. Needs a rebuild and deploy of each site.
- **The header carries the operator's social links and one theme toggle, keeps the site's name on a small screen, and links to the Archilyzer home in place of the sites menu.** Every site's header and the hub's end with the social icons (the site's `socialLinks`, else `settings.json`'s) followed by the theme toggle, all 36 px keys (44 px on a touch screen) with a focus ring. From 520 px wide the header shows every link, up to four (with more, the ones marked **Keep in header on small screens** first, then the last of the rest); below 520 px it shows only the marked ones (none marked → none) and keeps the site's name beside them. The switch is 32.5rem, so at a larger text size it comes later. With one marked link, every current site's name shows in full from 360 px wide on a touch screen. The footer keeps every link, in the same keys (its icons were 20 px and turned the accent on hover; they now turn the text colour), and wraps them rather than widen the page. The **Sites** dropdown and the **Hub** link are gone from the header and the slide-out menu: in their place a link, **Archilyzer**, goes to the Archilyzer home's Official Instances, in the same tab (not on the hub, which lists them itself). **Changelog** moved from the header and the menu to the footer, after Use with AI. The nav and the Archilyzer link are inline from 1024 px wide; below that they are in the slide-out menu, which now holds only them. Only as last resorts, for a very long name on a phone, does the name drop (its mark stays; the same before and after the page's font has loaded, and never with its last letter cut off) and do the icons scroll sideways in their own box. A site's `hubUrl` still loads and is no longer shown. Needs a rebuild and deploy of each site.
+- **An unlisted site is not in the hub or in another site's footer.** The hub's members (`hub-sites.json`), and so its federated search, `corpus.json` and `llms.txt`, leave out a site whose `listed` is `false`; the hub's instance figures count none of the channels only it carries; and no other site's footer links it, even from a featured group. The unlisted site's own pages are unchanged.
+- **A clear screen until the first Search.** A plain visit to a site's search page, and to the hub's, shows the search bar, the page's intro and the footer: no count, no listing and no results controls, and the line under the bar, "Press Enter or click Search to apply", says what to do. Search with the box empty lists every video, as before. A link that carries a query or a filter (`qt=`, `q=`, `tg=`, a share link, the older filter keys) still shows its results on load. Within one visit the results stay: going to Ask AI or another page and coming back keeps them. A reload starts over, and shows results at once only when the address carries a query or a filter. A query restored from the last visit waits in the box until Search, over the clear screen or under a filter link's results, and it still waits after another page and Back. Needs a rebuild and deploy of each site and the hub.
## [0.10.0] - 2026-09-28
- **A video whose recheck failed shows as possibly missing rather than available.** When a video drops out of its channel's listing it is marked "Missing?" until a recheck says why. A recheck that could not reach the video — a blocked request or a network error — used to clear the mark as if the video had been found. It now leaves "Missing?" in place until a recheck actually reaches the video. Needs a rebuild and deploy of every export site.
diff --git a/export/e2e-hub/federated-search.spec.ts b/export/e2e-hub/federated-search.spec.ts
@@ -4,6 +4,7 @@ import {
LIVE_CHAT_MISSING,
NO_ARCHIVES_IN_SCOPE,
} from "../app/ask/hubScopeCopy";
+import { showAll } from "../e2e/helpers";
// The hub's federated search, per archive: two official members (hub-sites.json)
// served by route mocks WITH CORS, each with one video. Proves the scope chips
@@ -26,6 +27,10 @@ import {
// later; a member whose live chat cannot be read says so on its chip
// (LIVE_CHAT_MISSING), with a Retry for the live chat alone that keeps its
// videos in the search while it runs.
+// Release 14 (S1): the results — the listing and the "N of M archives
+// answered" line with it — show only after the first Search of the page life,
+// so a spec that reads them presses Search first (`showAll`); the chips carry
+// each archive's state before that.
const ORIGIN_A = "http://localhost:4598";
const ORIGIN_B = "http://localhost:4599";
@@ -199,6 +204,14 @@ test.describe("hub federated search — scope, per-archive state, attribution",
"aria-pressed",
"true",
);
+ // Both archives are in, and nothing has been asked: no results area yet,
+ // and the bar says what to do.
+ await expect(page.getByTestId("results-summary")).toHaveCount(0);
+ await expect(resultFrom(page, A)).toHaveCount(0);
+ await expect(resultFrom(page, B)).toHaveCount(0);
+ await expect(page.getByText("Press Enter or click Search to apply")).toBeVisible();
+ await showAll(page);
+ await expect(page.getByTestId("results-summary")).toHaveText("All videos (2)");
// Browse listing: one card per archive, each naming its source in text.
const b = resultFrom(page, B);
await expect(b).toHaveCount(1);
@@ -216,6 +229,7 @@ test.describe("hub federated search — scope, per-archive state, attribution",
}) => {
const mocks = await setup(page);
await page.goto("/");
+ await showAll(page);
await expect(resultFrom(page, B)).toHaveCount(1);
await expect(resultFrom(page, A)).toHaveCount(1);
@@ -234,6 +248,7 @@ test.describe("hub federated search — scope, per-archive state, attribution",
await page.reload();
await expect(chip(page, A)).toHaveAttribute("data-status", "ready");
await expect(chip(page, B)).toHaveAttribute("data-status", "off");
+ await showAll(page);
await expect(resultFrom(page, A)).toHaveCount(1);
await expect(resultFrom(page, B)).toHaveCount(0);
expect(mocks.b.requests).toEqual([]);
@@ -251,6 +266,7 @@ test.describe("hub federated search — scope, per-archive state, attribution",
const mocks = await setup(page);
mocks.b.pages = "abort";
await page.goto("/");
+ await showAll(page);
await expect(chip(page, B)).toHaveAttribute("data-status", "failed");
await expect(chip(page, B)).toContainText("failed");
@@ -314,6 +330,7 @@ test.describe("hub federated search — scope, per-archive state, attribution",
mocks.b.pages = "abort";
mocks.c!.pages = "hold";
await page.goto("/");
+ await showAll(page);
await expect(chip(page, B)).toHaveAttribute("data-status", "failed");
await expect(chip(page, C)).toHaveAttribute("data-status", "loading");
@@ -335,6 +352,7 @@ test.describe("hub federated search — scope, per-archive state, attribution",
const mocks = await setup(page);
mocks.b.subs = "missing";
await page.goto("/");
+ await showAll(page);
// Ready means every feed of the archive is in, subs included. A 404 is an
// empty subs manifest, at once. It used to be an error, retried after ~1 s,
@@ -354,6 +372,7 @@ test.describe("hub federated search — scope, per-archive state, attribution",
// transcripts first) lists B then A. Neither sets an accent.
await setup(page, [A, B], { summary: [B, A] });
await page.goto("/");
+ await showAll(page);
const cards = page.getByTestId("shelf-spine");
await expect(cards).toHaveCount(2);
@@ -521,6 +540,7 @@ test.describe("hub federated search — scope, per-archive state, attribution",
const mocks = await setup(page);
mocks.b.subs = "error";
await page.goto("/");
+ await showAll(page);
// Live chat is the auxiliary layer: after one retry the error counts as
// settled, so Origin B is ready — its videos searched, without live chat —
@@ -544,6 +564,7 @@ test.describe("hub federated search — scope, per-archive state, attribution",
const mocks = await setup(page);
mocks.b.subs = "error";
await page.goto("/");
+ await showAll(page);
await expect(chip(page, B)).toHaveAttribute("data-status", "ready");
await expect(resultFrom(page, B)).toHaveCount(1);
diff --git a/export/e2e/browse-all.spec.ts b/export/e2e/browse-all.spec.ts
@@ -5,11 +5,12 @@ import {
VIDEO_CHAT_SMALL,
VIDEO_TRANSCRIPT_ONLY,
} from "./fixtures/data";
-import { installRoutes } from "./helpers";
+import { installRoutes, showAll } from "./helpers";
// Browse-all e2e: with no search query active the results section lists every
-// video passing the committed filters (no hits). Typing a query narrows it;
-// clearing it returns to the full browse list.
+// video passing the committed filters (no hits) — once the visitor has pressed
+// Search; until then the page shows no results area at all (first-search.spec).
+// Typing a query narrows it; clearing it returns to the full browse list.
const TRANSCRIPT_ONLY_SLUG = `${CHANNEL_SLUG}/${VIDEO_TRANSCRIPT_ONLY}`;
const CHAT_SMALL_SLUG = `${CHANNEL_SLUG}/${VIDEO_CHAT_SMALL}`;
@@ -31,8 +32,20 @@ test.describe("browse all (no query)", () => {
await installRoutes(page);
});
- test("lists every video on load with no query", async ({ page }) => {
+ test("a clear screen on load, then every video on an empty Search", async ({
+ page,
+ }) => {
await page.goto("/");
+ await page.getByTestId("query-builder").waitFor();
+ // Give the session time to hydrate (it waits for the manifest) and to
+ // show a listing if it were going to.
+ await page.waitForTimeout(1_000);
+ await expect(page.getByTestId("results-section")).toHaveCount(0);
+ await expect(page.getByTestId("results-summary")).toHaveCount(0);
+ await expect(page.getByTestId("browse-hint")).toHaveCount(0);
+ await expect(page.locator("[data-card-header]")).toHaveCount(0);
+
+ await page.getByTestId("search-submit").click();
await expect(page.getByTestId("results-section")).toBeVisible();
await expect(page.getByTestId("results-summary")).toHaveText(
"All videos (3)",
@@ -43,6 +56,7 @@ test.describe("browse all (no query)", () => {
test("filters apply to the browse list on Search", async ({ page }) => {
await page.goto("/");
+ await showAll(page);
await expectResultSlugs(page, ALL_SLUGS);
// All fixture videos are non-livestream, so unchecking "Videos" excludes
@@ -62,6 +76,7 @@ test.describe("browse all (no query)", () => {
page,
}) => {
await page.goto("/");
+ await showAll(page);
await expectResultSlugs(page, ALL_SLUGS);
const leafInput = page.locator('input[data-testid^="leaf-query-"]').first();
diff --git a/export/e2e/charts.spec.ts b/export/e2e/charts.spec.ts
@@ -1,5 +1,5 @@
import { expect, test, type Page } from "@playwright/test";
-import { installChartRoutes, urlParams } from "./helpers";
+import { installChartRoutes, showAll, urlParams } from "./helpers";
import { over, painted, rgbOf } from "../../common/testing/chartPixels";
// Charts are now a VIEW MODE of the search page: a "Results | Chart" toggle
@@ -74,6 +74,7 @@ test.describe("charts (search view mode)", () => {
test("Chart toggle with no query plots metadata", async ({ page }) => {
await page.goto("/");
+ await showAll(page);
await expect(page.getByTestId("results-summary")).toContainText("All videos");
await openChart(page);
await expect(page.locator(".recharts-surface")).toBeVisible({
@@ -106,6 +107,7 @@ test.describe("charts (search view mode)", () => {
page,
}) => {
await page.goto("/");
+ await showAll(page);
await expect(page.getByTestId("results-summary")).toContainText("All videos");
await openChart(page);
const opts = page.getByTestId("chart-options");
@@ -122,6 +124,7 @@ test.describe("charts (search view mode)", () => {
page,
}) => {
await page.goto("/");
+ await showAll(page);
await expect(page.getByTestId("results-summary")).toContainText("All videos");
await openChart(page);
const opts = page.getByTestId("chart-options");
@@ -150,6 +153,7 @@ test.describe("charts (search view mode)", () => {
page,
}) => {
await page.goto("/");
+ await showAll(page);
await expect(page.getByTestId("results-summary")).toContainText("All videos");
await openChart(page);
await expect(page.locator(".recharts-surface")).toBeVisible({
@@ -231,6 +235,7 @@ test.describe("charts (search view mode)", () => {
page,
}) => {
await page.goto("/");
+ await showAll(page);
await expect(page.getByTestId("results-summary")).toContainText("All videos");
await openChart(page);
const opts = page.getByTestId("chart-options");
@@ -272,6 +277,7 @@ test.describe("charts (search view mode)", () => {
test(`${width} px: a stacked area paints its true total and every band`, async ({ page }) => {
await page.setViewportSize({ width, height: 1000 });
await page.goto("/");
+ await showAll(page);
await expect(page.getByTestId("results-summary")).toContainText("All videos");
await openChart(page);
const opts = page.getByTestId("chart-options");
diff --git a/export/e2e/first-search.spec.ts b/export/e2e/first-search.spec.ts
@@ -0,0 +1,194 @@
+import { expect, test, type Page } from "@playwright/test";
+import {
+ CHANNEL,
+ CHANNEL_SLUG,
+ TAG_TOPIC,
+ VIDEO_CHAT_LARGE,
+ VIDEO_CHAT_SMALL,
+ VIDEO_TRANSCRIPT_ONLY,
+} from "./fixtures/data";
+import { installRoutes, installTagRoutes, openFilters, showAll } from "./helpers";
+
+// Release 14 (S1): a clear screen until the first Search. A plain visit shows
+// the search bar, the page's intro and the footer — no count, no controls, no
+// listing — and the bar's line says what to do. The first Search of the page
+// life (Enter, the button, a Filters Apply, a profile load) shows the results;
+// an empty one lists every video, exactly as before. A link that carries a
+// query (`qt=`) or a filter (`tg=`, `ch=`, a share link) is the visitor asking
+// and shows its results on load. The gate is once per page life: client-side
+// navigation keeps the listing, a reload clears it.
+
+const STORAGE_KEY = "ytdlp-tb:export-filters";
+const HINT = "Press Enter or click Search to apply";
+
+const ALPHA_TREE = { k: "g", o: "AND", c: [{ k: "l", q: "alpha", s: "transcripts" }] };
+const qt = (tree: unknown) => encodeURIComponent(JSON.stringify(tree));
+
+const leafInput = (page: Page) =>
+ page.locator('input[data-testid^="leaf-query-"]').first();
+
+const card = (page: Page, id: string) =>
+ page.locator(`[data-result-slug="${CHANNEL_SLUG}/${id}"]`);
+
+// The bar has mounted; then give the session time to hydrate (it waits for the
+// manifest) and to show a listing, were it going to.
+async function settle(page: Page) {
+ await page.getByTestId("query-builder").waitFor();
+ await page.waitForTimeout(1_000);
+}
+
+async function expectClearScreen(page: Page) {
+ await expect(page.getByTestId("results-section")).toHaveCount(0);
+ await expect(page.getByTestId("results-summary")).toHaveCount(0);
+ await expect(page.getByTestId("view-toggle")).toHaveCount(0);
+ await expect(page.getByTestId("selection-toolbar")).toHaveCount(0);
+ await expect(page.getByTestId("browse-hint")).toHaveCount(0);
+ await expect(page.locator("[data-card-header]")).toHaveCount(0);
+ await expect(page.getByText(HINT)).toBeVisible();
+}
+
+async function expectAllVideos(page: Page) {
+ await expect(page.getByTestId("results-summary")).toHaveText("All videos (3)");
+ await expect(page.getByTestId("browse-hint")).toBeVisible();
+ await expect(page.locator("[data-card-header]")).toHaveCount(3);
+}
+
+test.describe("a clear screen until the first Search", () => {
+ test.beforeEach(async ({ page }) => {
+ await installRoutes(page);
+ });
+
+ for (const { width, height } of [
+ { width: 1280, height: 800 },
+ { width: 390, height: 844 },
+ ]) {
+ test(`${width}×${height}: on load, the bar, the intro and the footer — no results`, async ({
+ page,
+ }) => {
+ await page.setViewportSize({ width, height });
+ await page.goto("/");
+ await settle(page);
+ await expectClearScreen(page);
+ // The page's intro stays: the transcript count.
+ await expect(page.getByRole("heading", { level: 1 })).toContainText("transcripts");
+ // The footer is on the first screen, whole.
+ await expect(page.getByRole("contentinfo")).toBeInViewport({ ratio: 1 });
+ });
+ }
+
+ test("Enter on an empty box shows every video", async ({ page }) => {
+ await page.goto("/");
+ await settle(page);
+ await expectClearScreen(page);
+ await leafInput(page).press("Enter");
+ await expectAllVideos(page);
+ // The line has done its job.
+ await expect(page.getByText(HINT)).toHaveCount(0);
+ });
+
+ test("the Search button shows every video", async ({ page }) => {
+ await page.goto("/");
+ await settle(page);
+ await page.getByTestId("search-submit").click();
+ await expectAllVideos(page);
+ });
+
+ test("changing a filter before the first Search does not show the listing", async ({
+ page,
+ }) => {
+ await page.goto("/");
+ await settle(page);
+ await openFilters(page);
+ await page.getByRole("checkbox", { name: "Videos" }).uncheck();
+ await expectClearScreen(page);
+ await page.getByTestId("search-submit").click();
+ await expect(page.getByTestId("results-summary")).toHaveText("All videos (0)");
+ });
+
+ test("/ → /ask → / keeps the listing", async ({ page }) => {
+ await page.goto("/");
+ await showAll(page);
+ await expectAllVideos(page);
+ const nav = page.getByTestId("workspace-nav");
+ await nav.getByRole("link", { name: "Chat" }).click();
+ // A dev server compiles /ask on its first visit.
+ await expect(page).toHaveURL(/\/ask\/?$/, { timeout: 20_000 });
+ await expect(page.getByPlaceholder(/Ask about the transcripts/)).toBeVisible();
+ await nav.getByRole("link", { name: "Search" }).click();
+ await expect(page).not.toHaveURL(/\/ask/);
+ await expectAllVideos(page);
+ });
+
+ test("leaving the search page and coming Back keeps the listing", async ({ page }) => {
+ await page.goto("/");
+ await showAll(page);
+ await expectAllVideos(page);
+ // A route outside the workspace: the search session unmounts with it.
+ await page
+ .getByRole("banner")
+ .getByRole("link", { name: "Use with AI", exact: true })
+ .click();
+ await expect(page).toHaveURL(/\/use-with-ai\/?$/);
+ await page.goBack();
+ await expect(page).not.toHaveURL(/use-with-ai/);
+ await expectAllVideos(page);
+ });
+
+ test("a reload clears it", async ({ page }) => {
+ await page.goto("/");
+ await showAll(page);
+ await expectAllVideos(page);
+ await page.reload();
+ await settle(page);
+ await expectClearScreen(page);
+ });
+
+ test("a qt= link shows its results on load", async ({ page }) => {
+ await page.goto(`/?qt=${qt(ALPHA_TREE)}`);
+ await expect(page.getByTestId("results-summary")).toContainText("Matching videos");
+ await expect(page.locator("[data-card-header]")).toHaveCount(3);
+ await expect(page.getByText(HINT)).toHaveCount(0);
+ });
+
+ test("a filter-only link shows its results on load", async ({ page }) => {
+ await installTagRoutes(page);
+ await page.goto(`/?tg=${TAG_TOPIC}`);
+ await expect(page.getByTestId("results-summary")).toHaveText("All videos (1)");
+ await expect(card(page, VIDEO_CHAT_LARGE)).toBeVisible();
+ await expect(card(page, VIDEO_CHAT_SMALL)).toHaveCount(0);
+
+ // A legacy channel link: this one leaves the only channel out.
+ await page.goto(`/?ch=${encodeURIComponent(CHANNEL)}`);
+ await expect(page.getByTestId("results-summary")).toHaveText("All videos (0)");
+ await expect(page.getByText("No videos match the current filters.")).toBeVisible();
+ });
+
+ test("a restored query shows the clear screen and the filled form", async ({ page }) => {
+ await page.goto("/");
+ await page.evaluate(
+ ({ key, value }) => window.localStorage.setItem(key, value),
+ {
+ key: STORAGE_KEY,
+ value: JSON.stringify({
+ v: 1,
+ working: {
+ channels: { included: [], excluded: [] },
+ nol: true,
+ query: JSON.stringify(ALPHA_TREE),
+ },
+ profiles: {},
+ activeProfileName: null,
+ }),
+ },
+ );
+ await page.goto("/");
+ await expect(leafInput(page)).toHaveValue("alpha");
+ await expect(page.getByTestId("search-submit")).toHaveAttribute("data-dirty", "true");
+ await settle(page);
+ await expectClearScreen(page);
+ // The visitor runs it.
+ await leafInput(page).press("Enter");
+ await expect(page.getByTestId("results-summary")).toContainText("Matching videos");
+ await expect(card(page, VIDEO_TRANSCRIPT_ONLY)).toBeVisible();
+ });
+});
diff --git a/export/e2e/helpers.ts b/export/e2e/helpers.ts
@@ -156,6 +156,17 @@ export async function openFilters(page: Page) {
await profiles.waitFor();
}
+// The search page shows nothing under the bar until the first Search of the
+// page life (a link that carries a query or a filter shows its results on
+// load). A spec that wants the listing of every video presses Search, as a
+// visitor does. It waits for the query builder first, which renders only once
+// the bar has mounted, so the click is never lost to the pre-hydration window.
+export async function showAll(page: Page) {
+ await page.getByTestId("query-builder").waitFor();
+ await page.getByTestId("search-submit").click();
+ await page.getByTestId("results-summary").waitFor();
+}
+
export async function urlParams(page: Page): Promise<URLSearchParams> {
const search = await page.evaluate(() => window.location.search);
return new URLSearchParams(search);
diff --git a/export/e2e/responsive.spec.ts b/export/e2e/responsive.spec.ts
@@ -1,6 +1,6 @@
import { expect, test, type Page } from "@playwright/test";
import { CHANNEL_SLUG, VIDEO_TRANSCRIPT_ONLY } from "./fixtures/data";
-import { expectModalOpen, installRoutes } from "./helpers";
+import { expectModalOpen, installRoutes, showAll } from "./helpers";
import { INSTANCES_URL } from "../../common/lib/project";
// The phone. Every other spec in this suite runs at the project's 1440×1200
@@ -109,6 +109,7 @@ test.describe("phone layout", () => {
page,
}) => {
await page.goto("/");
+ await showAll(page);
const summary = page.getByTestId("results-summary");
await expect(summary).toContainText("All videos (3)");
@@ -146,6 +147,7 @@ test.describe("phone layout", () => {
page,
}) => {
await page.goto("/");
+ await showAll(page);
const toolbar = page.getByTestId("selection-toolbar");
// Nothing selected: it is an ordinary row above the cards, as the 1440
// specs see it.
diff --git a/export/e2e/restore-no-refire.spec.ts b/export/e2e/restore-no-refire.spec.ts
@@ -1,5 +1,6 @@
import { expect, test, type Page } from "@playwright/test";
-import { installRoutes } from "./helpers";
+import { TAG_TOPIC } from "./fixtures/data";
+import { installRoutes, installTagRoutes } from "./helpers";
// Release 8 slice E. The search session restored from localStorage loads the
// query and the filters but does NOT run the search on first load: a restored
@@ -7,6 +8,10 @@ import { installRoutes } from "./helpers";
// shards (up to 8 MB each on a cold device) for a search the visitor had not
// asked for this time. The visitor runs it — Search / Enter. A query on the
// URL (`qt=`) is a shared link, i.e. the visitor asking, and still runs.
+// Release 14: until then the page shows no results area at all — the held
+// query no longer sits over a browse listing. A link that carries only a
+// filter (`tg=`, `ch=`) shows its results but runs no query, so a stored one
+// stays held on that load AND on every later mount in the same visit.
const STORAGE_KEY = "ytdlp-tb:export-filters";
const SHARD = /\/transcripts\/[^/]+\/page-\d+\.json$/;
@@ -51,6 +56,24 @@ async function seedStoredSearch(page: Page) {
);
}
+// A route outside the workspace unmounts the search session; coming back
+// mounts it again in the same page life.
+async function leaveForUseWithAi(page: Page) {
+ await page
+ .getByRole("banner")
+ .getByRole("link", { name: "Use with AI", exact: true })
+ .click();
+ await expect(page).toHaveURL(/use-with-ai/, { timeout: 20_000 });
+}
+
+async function expectHeld(page: Page) {
+ await expect(leafInput(page)).toHaveValue("alpha");
+ await expect(page.getByTestId("search-submit")).toHaveAttribute(
+ "data-dirty",
+ "true",
+ );
+}
+
test.describe("restored search waits for the visitor", () => {
test.beforeEach(async ({ page }) => {
await installRoutes(page);
@@ -78,14 +101,14 @@ test.describe("restored search waits for the visitor", () => {
await expect(
page.getByText("Press Enter or click Search to apply"),
).toBeVisible();
- // The results area is the browse listing under the restored filters.
- await expect(page.getByTestId("results-summary")).toHaveText(
- "All videos (3)",
- );
- await expect(page.getByTestId("browse-hint")).toBeVisible();
// Give a would-be pipeline every chance to start.
await page.waitForTimeout(1_500);
expect(shards.n).toBe(0);
+ // Nothing has been asked in this page life, so there is no results area
+ // at all: no count, no listing under the restored filters.
+ await expect(page.getByTestId("results-summary")).toHaveCount(0);
+ await expect(page.getByTestId("browse-hint")).toHaveCount(0);
+ await expect(page.locator("[data-card-header]")).toHaveCount(0);
// The visitor runs it.
const fetched = page.waitForRequest(
@@ -122,4 +145,54 @@ test.describe("restored search waits for the visitor", () => {
"false",
);
});
+
+ test("a filter-only link shows its results and leaves a stored query held, then and after Back", async ({
+ page,
+ }) => {
+ await installTagRoutes(page);
+ const shards = countShards(page);
+ await seedStoredSearch(page);
+
+ shards.n = 0;
+ await page.goto(`/?tg=${TAG_TOPIC}`);
+ // The link asked: its results show, under its filter…
+ await expect(page.getByTestId("results-summary")).toHaveText("All videos (1)");
+ // …and the stored query is in the box, held.
+ await expectHeld(page);
+
+ await leaveForUseWithAi(page);
+ await page.goBack();
+ await expect(page).not.toHaveURL(/use-with-ai/);
+ // A second mount in the same visit: still held, still the listing.
+ await expectHeld(page);
+ await expect(page.getByTestId("results-summary")).toHaveText("All videos (1)");
+ await page.waitForTimeout(1_500);
+ expect(shards.n).toBe(0);
+ });
+
+ test("a filter-only link, then the header's Search link: the stored query stays held", async ({
+ page,
+ }) => {
+ const shards = countShards(page);
+ await seedStoredSearch(page);
+
+ shards.n = 0;
+ // Under a filter link the stored session is not read at all.
+ await page.goto(`/?ch=${encodeURIComponent("Nobody")}`);
+ await expect(page.getByTestId("results-summary")).toContainText("All videos");
+ await expect(leafInput(page)).toHaveValue("");
+
+ await leaveForUseWithAi(page);
+ await page
+ .getByRole("banner")
+ .getByRole("link", { name: "Search", exact: true })
+ .click();
+ await expect(page).not.toHaveURL(/use-with-ai/, { timeout: 20_000 });
+ // The plain search page reads the stored session: the query is held, and
+ // the listing shows because the visitor asked earlier in this visit.
+ await expectHeld(page);
+ await expect(page.getByTestId("results-summary")).toHaveText("All videos (3)");
+ await page.waitForTimeout(1_500);
+ expect(shards.n).toBe(0);
+ });
});
diff --git a/export/e2e/tag-chips.spec.ts b/export/e2e/tag-chips.spec.ts
@@ -1,5 +1,5 @@
import { expect, test, type Page } from "@playwright/test";
-import { installRoutes, installTagRoutes } from "./helpers";
+import { installRoutes, installTagRoutes, showAll } from "./helpers";
import {
CHANNEL_SLUG,
TAG_COLLAB,
@@ -146,6 +146,7 @@ test.describe("curated tag chips", () => {
test("a tagged card shows its tags; an untagged one shows none", async ({
page,
}) => {
+ await showAll(page);
await expect(card(page, VIDEO_CHAT_LARGE).getByTestId("card-tag")).toHaveCount(2);
await expect(
card(page, VIDEO_CHAT_LARGE).locator('[data-testid="card-tag"][data-tag-id="' + TAG_TOPIC + '"]'),
@@ -161,6 +162,7 @@ test.describe("curated tag chips", () => {
page,
}) => {
// All three videos before any tag filter.
+ await showAll(page);
await expect(card(page, VIDEO_TRANSCRIPT_ONLY)).toBeVisible();
await chip(page, TAG_TOPIC).click();
@@ -378,6 +380,7 @@ test.describe("curated tag chips", () => {
await installRoutes(page);
await page.goto("/");
await waitForHydration(page);
+ await showAll(page);
await expect(card(page, VIDEO_TRANSCRIPT_ONLY)).toBeVisible();
await expect(chipRow(page)).toHaveCount(0);
diff --git a/export/e2e/workspace-shell.spec.ts b/export/e2e/workspace-shell.spec.ts
@@ -5,7 +5,7 @@ import {
VIDEO_CHAT_SMALL,
VIDEO_TRANSCRIPT_ONLY,
} from "./fixtures/data";
-import { installRoutes } from "./helpers";
+import { installRoutes, showAll } from "./helpers";
// The unified workspace shell: `/` (results) and `/ask` (chat) share one layout,
// so the persistent search bar and its committed search survive navigation
@@ -61,9 +61,10 @@ test.describe("workspace shell", () => {
"aria-current",
"page",
);
- // Wait for the results pane to hydrate (browse mode lists every video) before
- // clicking — a client Link click lost to a pre-hydration window would leave us
- // stranded on `/`.
+ // Wait for the results pane to hydrate (an empty Search lists every video)
+ // before clicking — a client Link click lost to a pre-hydration window would
+ // leave us stranded on `/`.
+ await showAll(page);
await expect(page.locator("[data-card-header]").first()).toBeVisible();
await nav.getByRole("link", { name: "Chat" }).click();
await expect(page.getByPlaceholder(/Ask about the transcripts/)).toBeVisible();
diff --git a/homepage/CHANGELOG.md b/homepage/CHANGELOG.md
@@ -1,6 +1,8 @@
# Homepage Changelog
## [Unreleased]
+- **The growth chart draws its smallest instances together as Other.** Two or more instances that each hold under 5% of the chart's total are one band, **Other**, on top of the stack, in a near-neutral grey of its own (`--chart-other`: 7.36:1 on the Light ground, 3.22:1 on the Dark one, and apart from every instance colour for colour-blind readers); an instance at exactly 5% keeps its band, and a single one under 5% is not grouped. The other instances keep their bands and their colours. The legend lists them and Other; the caption says what Other is and the chart's description names the instances in it; every month's hover title and the Numbers by year table still name every instance. At this release's numbers Hasanalyzer, Rekietalyzer and Jasolyzer are Other. The instance cards and `/stats` are unchanged. The e2e fixture's fifth site transcribes 4 a day rather than 5, so two of its six sites are grouped.
+- **An unlisted site is not on the homepage.** A site whose settings turn off **List on the Archilyzer homepage and hub** (`listed: false`) has no Official Instances card, chart series, `/stats` entry or recent item, is not in `channel-sites.json` or `stats/`, and the channels only it carries count in none of the numbers, the headline totals included. The summary's version is 6. The e2e fixture has a seventh, unlisted site that no page names.
- **`/#instances` goes straight to Official Instances.** The section carries `id="instances"`, clear of the sticky header, and every archive's header now links there (`INSTANCES_URL` in `common/lib/project.ts`). With no sites the link lands on the top of the page.
- **The social links are in the header beside the theme toggle, and on a small screen the header keeps the name.** The operator's social icons (`homepage.json`'s, else `settings.json`'s) sit in the header's bar as well as in the footer's Elsewhere column, followed by the theme toggle, all spaced alike. From 768 px wide the bar is wordmark, nav, icons, toggle; below 768 px it is wordmark, icons, toggle, and the nav has the rule below to itself, where its four links fit. From 520 px wide the header shows every link, up to four (with more, the ones marked **Keep in header on small screens** first, then the last of the rest); below 520 px it shows only the marked ones (none marked → none; the switch is 32.5rem, so at a larger text size it comes later), and the wordmark keeps its full name: "Archilyzer" shows from 292 px wide on a touch screen with one marked link. The footer always shows every link. Only as last resorts, on a screen narrower still or at a much larger text size, does the wordmark's text drop (its mark stays) and do the icons scroll sideways in their own box, the last one in view first. Each is an icon named by its label, with no text beside it.
diff --git a/homepage/app/components/ArchiveGrowthChart.tsx b/homepage/app/components/ArchiveGrowthChart.tsx
@@ -3,8 +3,16 @@ import type {
HomepageSummarySite,
} from "yt-dlp-transcript-common/lib/homepageSummary";
import { monthLabel } from "yt-dlp-transcript-common/lib/homepageChart";
-import { siteChartColors } from "yt-dlp-transcript-common/lib/siteColor";
-import { PLOT_SIZES, gapSegments, growthStack, runsOf } from "../lib/growthGaps";
+import {
+ FOLD_PERCENT,
+ OTHER_LABEL,
+ PLOT_SIZES,
+ gapSegments,
+ growthLayers,
+ growthStack,
+ layerColors,
+ runsOf,
+} from "../lib/growthGaps";
// The front page's showpiece: every official instance's back catalogue as
// stacked strata, one month per step, from the oldest upload to the last
@@ -36,6 +44,28 @@ import { PLOT_SIZES, gapSegments, growthStack, runsOf } from "../lib/growthGaps"
// The legend above the plot wears the same colours. The layers stack in the
// summary's order (STACK_ORDER, unchanged).
//
+// OTHER (release 14, slice CF; lib/growthGaps.ts). Two or more sites each
+// under 5 % of the chart's total fold into ONE band, "Other", on top of the
+// stack. The kept sites keep their slots — siteChartColors runs over EVERY
+// site, so a fold never repaints one — and Other wears its own near-neutral
+// grey, `--chart-other` (tokens.css: Light #3e545c, Dark #62625c; 7.36 /
+// 3.22:1 on the ground, 7.99 / 3.06:1 on the chart surface), not the axis
+// labels' colour. The legend shows the kept sites and Other; the hover title
+// and the table name every site, and the image's label and the caption say
+// what Other holds. A grey is below the validator's chroma floor by
+// definition (it is the de-emphasis role, not a categorical slot); against
+// each slot it can sit on, Other is (normal ΔE / worst of protan and deutan,
+// light · dark):
+// blue 21.9 / 21.4 · 17.8 / 17.9 — the only neighbour in today's data
+// (Bonnellyzer, the top kept site in every month Other has data);
+// green 17.4 / 15.2 · 19.2 / 15.7; violet 16.2 / 13.4 · 23.4 / 21.5;
+// amber 22.1 / 18.2 · 20.3 / 18.1; magenta 24.5 / 9.1 · 19.5 / 7.3;
+// rust 14.1 / 10.5 · 13.5 / 9.3.
+// Every CVD pair clears the floor (6), and on Light the target (8); Dark's
+// magenta is in the 6–8 band and rust is under the normal-vision 15 on both
+// bases: there the gap between the bands, Other's place on top, the legend
+// and the table carry it.
+//
// VALIDATED (the dataviz skill's validator, each base's values on its
// --chart-surface and on the page ground), with the operator's accents
// (Jeralyzer Brass, Anilyzer Sakura, Bonnellyzer Blue, Hasanalyzer Violet,
@@ -69,23 +99,32 @@ export function ArchiveGrowthChart({
const n = months.length;
if (n === 0 || sites.length === 0) return null;
- const { totals, peak, yMax, ticks, order, stack: stacked } = growthStack(months, sites);
+ // The bands, bottom-up: the kept sites, then Other when two or more fold.
+ const {
+ totals,
+ peak,
+ yMax,
+ ticks,
+ layers: bands,
+ stack: stacked,
+ } = growthStack(months, sites, growthLayers(months, sites));
if (peak === 0) return null;
const x = (i: number) => (n === 1 ? W / 2 : (i / (n - 1)) * W);
const y = (v: number) => H - (v / yMax) * H;
const r = (v: number) => Math.round(v * 10) / 10;
- // One colour per site, in `sites` order (the legend's too).
- const colors = siteChartColors(sites);
+ // One colour per band, in stack order (the legend's too).
+ const colors = layerColors(bands, sites);
+ const folded = bands.find((b) => b.other)?.sites.map((si) => sites[si].siteTitle) ?? [];
const layers = stacked.map(({ lo, hi }, k) => {
- const si = order[k];
const top = hi.map((v, i) => `${r(x(i))},${r(y(v))}`);
const bottom = lo.map((v, i) => `${r(x(i))},${r(y(v))}`).reverse();
return {
- site: sites[si],
- color: colors[si],
+ key: bands[k].key,
+ label: bands[k].other ? OTHER_LABEL : sites[bands[k].sites[0]].siteTitle,
+ color: colors[k],
top,
area: `M${top.join("L")}L${bottom.join("L")}Z`,
};
@@ -95,7 +134,7 @@ export function ArchiveGrowthChart({
const gapSets = PLOT_SIZES.map((size) => {
const segments = gapSegments(stacked, yMax, size);
const paths = layers.map((l, k) => ({
- key: l.site.siteId,
+ key: l.key,
d: runsOf(segments[k])
.map((run) => `M${[...run, run.at(-1)! + 1].map((i) => l.top[i]).join("L")}`)
.join(""),
@@ -113,11 +152,18 @@ export function ArchiveGrowthChart({
const last = monthLabel(months[n - 1].month);
const peakIdx = totals.indexOf(peak);
const all = totals.reduce((a, b) => a + b, 0);
+ // The legend is hidden from assistive tech, so the image's label says what
+ // Other holds; the caption says what it is.
+ const otherNote = folded.length
+ ? `${folded.slice(0, -1).join(", ")} and ${folded.at(-1)}, each under ` +
+ `${FOLD_PERCENT}% of the total, are drawn together as ${OTHER_LABEL}.`
+ : "";
const label =
`Stacked area chart of ${all.toLocaleString()} transcripts by the month each ` +
`video was published, ${first} to ${last}, across ${sites.length} official ` +
`instances. The busiest month is ${monthLabel(months[peakIdx].month)}, ` +
- `with ${peak.toLocaleString()}.`;
+ `with ${peak.toLocaleString()}.` +
+ (otherNote ? ` ${otherNote}` : "");
// Per-year table for anyone who wants the numbers rather than the shape.
const byYear = new Map<string, Record<string, number>>();
@@ -130,10 +176,10 @@ export function ArchiveGrowthChart({
return (
<figure className="flex flex-col gap-4">
<ul className="flex flex-wrap gap-x-5 gap-y-2 list-none" aria-hidden="true">
- {sites.map((s, i) => (
- <li key={s.siteId} className="flex items-center gap-2 text-sm text-[var(--muted-foreground)]">
- <span className="h-2.5 w-2.5 shrink-0 rounded-[2px]" style={{ backgroundColor: colors[i] }} />
- {s.siteTitle}
+ {layers.map((l) => (
+ <li key={l.key} className="flex items-center gap-2 text-sm text-[var(--muted-foreground)]">
+ <span className="h-2.5 w-2.5 shrink-0 rounded-[2px]" style={{ backgroundColor: l.color }} />
+ {l.label}
</li>
))}
</ul>
@@ -163,7 +209,7 @@ export function ArchiveGrowthChart({
/>
))}
{layers.map((l) => (
- <path key={l.site.siteId} d={l.area} fill={l.color} />
+ <path key={l.key} d={l.area} fill={l.color} />
))}
{/* THE SURFACE GAP (the marks spec): touching bands are parted by
a 2 px gap in the colour behind the plot — the page ground, as
@@ -194,6 +240,8 @@ export function ArchiveGrowthChart({
))}
{months.map((m, i) => {
const w = W / n;
+ // Every site with data that month, by name — a folded one too:
+ // the title and the table are where Other's sites are found.
const parts = sites
.map((s) => [s.siteTitle, m.bySite[s.siteId] ?? 0] as const)
.filter(([, v]) => v > 0)
@@ -245,6 +293,8 @@ export function ArchiveGrowthChart({
<figcaption className="text-sm text-[var(--muted-foreground)]">
Transcripts by the month each video was published, all official instances.
+ {folded.length > 0 &&
+ ` Instances under ${FOLD_PERCENT}% of the total are drawn together as ${OTHER_LABEL}.`}
</figcaption>
<details className="text-sm">
<summary className="cursor-pointer text-[var(--muted-foreground)] underline decoration-[var(--border-strong)] underline-offset-4 hover:text-[var(--foreground)]">
diff --git a/homepage/app/lib/growthGaps.test.ts b/homepage/app/lib/growthGaps.test.ts
@@ -1,13 +1,20 @@
import { test } from "node:test";
import assert from "node:assert/strict";
+import { siteChartColors } from "yt-dlp-transcript-common/lib/siteColor";
import { buildFixtureSummary } from "../../e2e/fixture-summary";
import {
GAP_PX,
MIN_KEEP_PX,
+ OTHER_COLOR,
+ OTHER_KEY,
PLOT_SIZES,
bandAbove,
+ foldedSites,
gapSegments,
+ growthLayers,
growthStack,
+ layerColors,
+ ownLayers,
type Band,
type PlotSize,
} from "./growthGaps";
@@ -83,15 +90,30 @@ function slivers(): { stack: Band[]; yMax: number } {
return { stack, yMax };
}
-test("the fixture summary: gaps are drawn, and no band is ever covered, at every height", () => {
+test("the fixture summary, as the chart draws it (Other on top): gaps are drawn, and no band is ever covered, at every height", () => {
const summary = buildFixtureSummary();
- const { stack, yMax } = growthStack(summary.monthly ?? [], summary.sites);
+ const months = summary.monthly ?? [];
+ const layers = growthLayers(months, summary.sites);
+ // The fixture's last two sites are under 5 %: they fold.
+ assert.deepEqual(foldedSites(months, summary.sites), [4, 5]);
+ assert.equal(layers.at(-1)!.key, OTHER_KEY);
+ const { stack, yMax } = growthStack(months, summary.sites, layers);
let drawn = 0;
+ let underOther = 0;
for (const size of PLOT_SIZES) {
- drawn += gapSegments(stack, yMax, size).flat().length;
+ const segs = gapSegments(stack, yMax, size);
+ drawn += segs.flat().length;
+ // The gap under Other parts it from the top kept site; nothing sits on
+ // Other, so its own upper edge has none.
+ underOther += segs[stack.length - 2].filter((i) => bandAbove(stack, stack.length - 2, i) === stack.length - 1).length;
+ assert.deepEqual(segs[stack.length - 1], [], `${size.px}px: a gap along Other's top`);
assert.deepEqual(covered(stack, yMax, size), [], `${size.px}px`);
}
assert.ok(drawn > 0, "no gap drawn at all");
+ assert.ok(underOther > 0, "no gap between the top kept site and Other");
+ // Unfolded, the same summary keeps its promise too.
+ const own = growthStack(months, summary.sites);
+ for (const size of PLOT_SIZES) assert.deepEqual(covered(own.stack, own.yMax, size), [], `unfolded ${size.px}px`);
});
test("slivers beside large bands, with one-month spikes: no band is ever covered", () => {
@@ -126,3 +148,142 @@ test("no gap along the stack's top, and none under the run length", () => {
assert.deepEqual(gapSegments(stack, yMax, size), [[], []], `${size.px}px`);
}
});
+
+// ── The fold ──────────────────────────────────────────────────────────────────
+//
+// A site under 5 % of the placed total (every site's transcripts over the whole
+// plotted range) folds into ONE Other band on top, when two or more do. The
+// range's sums are what count: `monthsOf` spreads each site's sum over three
+// months, so no single month decides.
+
+type Named = { siteId: string; siteTitle: string; accentId?: string };
+
+function sitesOf(ids: readonly string[], accents: readonly (string | undefined)[] = []): Named[] {
+ return ids.map((siteId, i) => ({ siteId, siteTitle: siteId.toUpperCase(), accentId: accents[i] }));
+}
+
+function monthsOf(sites: readonly Named[], sums: readonly number[]) {
+ // Thirds, the remainder in the last month.
+ return [0, 1, 2].map((k) => ({
+ bySite: Object.fromEntries(
+ sites.map((s, i) => [s.siteId, k < 2 ? Math.floor(sums[i] / 3) : sums[i] - 2 * Math.floor(sums[i] / 3)]),
+ ),
+ }));
+}
+
+test("the fold with today's proportions: the three largest keep their bands, the other three are one Other band on top", () => {
+ // The family's shares in the 2026-09-28 summary, in its order and with its
+ // accents: 42.03, 38.40, 9.18, 4.33, 3.79, 2.26 %.
+ const sites = sitesOf(["a", "b", "c", "d", "e", "f"], ["brass", "sakura", "blue", "violet", "green", "vermilion"]);
+ const sums = [4203, 3840, 918, 433, 379, 226];
+ const months = monthsOf(sites, sums);
+ assert.deepEqual(foldedSites(months, sites), [3, 4, 5]);
+ const layers = growthLayers(months, sites);
+ assert.deepEqual(
+ layers.map((l) => [l.key, l.sites, l.other]),
+ [
+ ["a", [0], false],
+ ["b", [1], false],
+ ["c", [2], false],
+ [OTHER_KEY, [3, 4, 5], true],
+ ],
+ );
+ // Other is the sum of its sites, on top: the stack's top is every month's
+ // total, as before the fold.
+ const { stack, totals } = growthStack(months, sites, layers);
+ assert.deepEqual(stack[3].hi.map((v, i) => v - stack[3].lo[i]), months.map((m) => m.bySite.d + m.bySite.e + m.bySite.f));
+ assert.deepEqual(stack[3].hi, totals);
+ // Colours: the kept sites their families' slots, Other the neutral grey.
+ assert.deepEqual(layerColors(layers, sites), ["var(--chart-4)", "var(--chart-5)", "var(--chart-1)", OTHER_COLOR]);
+ assert.equal(OTHER_COLOR, "var(--chart-other)");
+});
+
+test("the threshold's edge: exactly 5 % keeps its band, just under folds", () => {
+ const sites = sitesOf(["a", "b", "c", "d"]);
+ // b and c are 999 of 20,000 (4.995 %), d exactly 1,000 (5 %).
+ assert.deepEqual(foldedSites(monthsOf(sites, [17_002, 999, 999, 1_000]), sites), [1, 2]);
+ // Two sites at exactly 5 %: neither folds.
+ const three = sitesOf(["a", "b", "c"]);
+ assert.deepEqual(foldedSites(monthsOf(three, [18_000, 1_000, 1_000]), three), []);
+ assert.deepEqual(growthLayers(monthsOf(three, [18_000, 1_000, 1_000]), three), ownLayers(three));
+});
+
+test("a fold of one is no fold: a single site under 5 % keeps its band", () => {
+ const sites = sitesOf(["a", "b", "c"]);
+ const months = monthsOf(sites, [60, 36, 4]);
+ assert.deepEqual(foldedSites(months, sites), []);
+ assert.deepEqual(growthLayers(months, sites), ownLayers(sites));
+ assert.ok(growthLayers(months, sites).every((l) => !l.other));
+});
+
+test("no site under 5 %: every site keeps its band, as before the fold", () => {
+ const sites = sitesOf(["a", "b", "c"]);
+ const months = monthsOf(sites, [50, 30, 20]);
+ assert.deepEqual(foldedSites(months, sites), []);
+ const layers = growthLayers(months, sites);
+ assert.deepEqual(layers, ownLayers(sites));
+ // The same stack as growthStack's default.
+ assert.deepEqual(growthStack(months, sites, layers), growthStack(months, sites));
+});
+
+test("all but one under 5 %: one kept band and one Other band", () => {
+ const sites = sitesOf(["a", "b", "c", "d", "e", "f"]);
+ const months = monthsOf(sites, [80, 4, 4, 4, 4, 4]);
+ const layers = growthLayers(months, sites);
+ assert.deepEqual(
+ layers.map((l) => [l.key, l.sites]),
+ [
+ ["a", [0]],
+ [OTHER_KEY, [1, 2, 3, 4, 5]],
+ ],
+ );
+ const { stack, totals } = growthStack(months, sites, layers);
+ assert.equal(stack.length, 2);
+ assert.deepEqual(stack[1].hi, totals);
+});
+
+test("a site with nothing in the plotted range is under 5 %, and folds with another", () => {
+ const sites = sitesOf(["a", "b", "c"]);
+ assert.deepEqual(foldedSites(monthsOf(sites, [96, 4, 0]), sites), [1, 2]);
+ // Nothing plotted at all: nothing folds.
+ assert.deepEqual(foldedSites(monthsOf(sites, [0, 0, 0]), sites), []);
+});
+
+test("every site under 5 % (over twenty sites): every site folds, one Other band", () => {
+ const ids = Array.from({ length: 25 }, (_, i) => `s${i}`);
+ const sites = sitesOf(ids);
+ const layers = growthLayers(monthsOf(sites, ids.map(() => 40)), sites);
+ assert.deepEqual(
+ layers.map((l) => [l.key, l.sites.length]),
+ [[OTHER_KEY, 25]],
+ );
+});
+
+test("a kept site's colour is the one it wears unfolded, whoever folds", () => {
+ // No accents: each site's slot is its index's (siteChartColors), so a list
+ // of the kept sites alone would repaint d — the chart runs it over them all.
+ const sites = sitesOf(["a", "b", "c", "d"]);
+ const months = monthsOf(sites, [500, 20, 20, 460]);
+ const layers = growthLayers(months, sites);
+ assert.deepEqual(
+ layers.map((l) => l.key),
+ ["a", "d", OTHER_KEY],
+ );
+ const unfolded = layerColors(ownLayers(sites), sites);
+ assert.deepEqual(unfolded, siteChartColors(sites));
+ const colours = layerColors(layers, sites);
+ assert.deepEqual(colours, [unfolded[0], unfolded[3], OTHER_COLOR]);
+ assert.notEqual(colours[1], siteChartColors([sites[0], sites[3]])[1], "the kept list alone would repaint d");
+ // With accents, the same: each kept site keeps its family's slot.
+ const named = sitesOf(["a", "b", "c", "d", "e", "f"], ["brass", "sakura", "blue", "violet", "green", "vermilion"]);
+ const all = siteChartColors(named);
+ const nm = monthsOf(named, [4203, 3840, 918, 433, 379, 226]);
+ const nl = growthLayers(nm, named);
+ assert.deepEqual(
+ layerColors(nl, named).slice(0, -1),
+ nl.slice(0, -1).map((l) => all[l.sites[0]]),
+ );
+ // The fold never hands a kept site the grey, nor Other a site's slot.
+ assert.ok(!layerColors(nl, named).slice(0, -1).includes(OTHER_COLOR));
+ assert.ok(!all.includes(OTHER_COLOR));
+});
diff --git a/homepage/app/lib/growthGaps.ts b/homepage/app/lib/growthGaps.ts
@@ -1,16 +1,34 @@
-// THE GROWTH CHART'S SURFACE GAPS, worked out (ArchiveGrowthChart.tsx draws
-// them). Pure: no React, no DOM, so a unit test can check every segment.
+import { siteChartColors } from "yt-dlp-transcript-common/lib/siteColor";
+
+// THE GROWTH CHART'S LAYERS AND SURFACE GAPS, worked out (ArchiveGrowthChart.tsx
+// draws them). Pure: no React, no DOM, so a unit test can check every layer and
+// every segment.
+//
+// THE FOLD. A site under FOLD_PERCENT % of the placed total — every site's
+// transcripts over the whole plotted range, the sum the bands are placed from
+// — is drawn in ONE "Other" band on top of the stack, in the chart's neutral
+// grey (OTHER_COLOR). Exactly FOLD_PERCENT % keeps its band; the comparison is
+// on integers (100 × a site's sum < FOLD_PERCENT × the total), so the edge is
+// exact. A fold of one is no fold: with a single site under the line, every
+// site keeps its band. The kept sites keep their colours: each is its
+// siteChartColors slot over EVERY site, folded or not, so a site's colour never
+// changes because another one folded. The legend shows the kept sites and
+// Other; the hover title and the "Numbers by year" table still name every site.
//
-// The marks spec parts touching bands by a 2 px gap in the colour behind the
-// plot. The chart draws it centred on each band's upper edge, so each of the
-// two bands gives 1 px. A band too thin to give its pixel and keep one of its
-// own colour must touch its neighbour instead, or the gap erases it. "Thin" is
-// measured AT RIGHT ANGLES to the edge, where the stroke's 2 px are measured:
-// on a steep edge a band's perpendicular thickness is its vertical height ×
-// cos θ, and on a one-month dip at a phone's width it is nearly nothing. And it
-// is measured at the NARROWEST plot each of the chart's three heights is drawn
-// at, where the edges are steepest. A gap is drawn along a segment only where
-// both bands keep at least MIN_KEEP_PX there after every gap that touches them.
+// THE GAPS. The marks spec parts touching bands by a 2 px gap in the colour
+// behind the plot. The chart draws it centred on each band's upper edge, so
+// each of the two bands gives 1 px. A band too thin to give its pixel and keep
+// one of its own colour must touch its neighbour instead, or the gap erases
+// it. "Thin" is measured AT RIGHT ANGLES to the edge, where the stroke's 2 px
+// are measured: on a steep edge a band's perpendicular thickness is its
+// vertical height × cos θ, and on a one-month dip at a phone's width it is
+// nearly nothing. And it is measured at the NARROWEST plot each of the chart's
+// three heights is drawn at, where the edges are steepest. A gap is drawn along
+// a segment only where both bands keep at least MIN_KEEP_PX there after every
+// gap that touches them. The gaps work on bands, not sites: Other is one band,
+// the top one, so the gap under it parts it from the kept site beneath (the
+// highest with a height that month), and its own upper edge, where nothing
+// sits, has none.
export const GAP_PX = 2;
export const MIN_KEEP_PX = 1;
@@ -34,6 +52,8 @@ export type PlotSize = (typeof PLOT_SIZES)[number];
// The stack, bottom-up: each band's lower and upper value per month.
export type Band = { lo: readonly number[]; hi: readonly number[] };
+type Month = { bySite: Record<string, number | undefined> };
+
// The layers stack in the summary's order (the first five as they come, any
// more after them).
const STACK_ORDER = [0, 1, 2, 3, 4];
@@ -43,6 +63,68 @@ function stackOrder(n: number): number[] {
return [...head, ...tail];
}
+// ── The fold ──────────────────────────────────────────────────────────────────
+
+export const FOLD_PERCENT = 5;
+export const OTHER_LABEL = "Other";
+// Other's own chart token (tokens.css `--chart-other`), a near-neutral grey:
+// 7.36:1 on the Light ground and 3.22:1 on the Dark one (7.99 / 3.06 on the
+// chart surface). tokens.css has its separation from each chart slot.
+export const OTHER_COLOR = "var(--chart-other)";
+
+// One band of the stack: a site's own (`sites` is its index in the summary's
+// list), or Other (the indexes of every folded site, in the list's order).
+// `key` is the site's id, or OTHER_KEY, which no site id can be (an id is
+// `[a-z0-9][a-z0-9-]*`).
+export type GrowthLayer = { key: string; sites: readonly number[]; other: boolean };
+export const OTHER_KEY = "(other)";
+
+// Each site's transcripts over the whole plotted range.
+function siteSums(months: readonly Month[], sites: readonly { siteId: string }[]): number[] {
+ return sites.map((s) => months.reduce((a, m) => a + (m.bySite[s.siteId] ?? 0), 0));
+}
+
+// The sites folded into Other, as indexes into `sites` in its order: every
+// site under FOLD_PERCENT % of the placed total, when there are two or more of
+// them; none otherwise.
+export function foldedSites(months: readonly Month[], sites: readonly { siteId: string }[]): number[] {
+ const sums = siteSums(months, sites);
+ const all = sums.reduce((a, b) => a + b, 0);
+ if (all <= 0) return [];
+ const small = sums.flatMap((v, i) => (100 * v < FOLD_PERCENT * all ? [i] : []));
+ return small.length >= 2 ? small : [];
+}
+
+// Every site its own band, in stack order: the chart before the fold.
+export function ownLayers(sites: readonly { siteId: string }[]): GrowthLayer[] {
+ return stackOrder(sites.length).map((i) => ({ key: sites[i].siteId, sites: [i], other: false }));
+}
+
+// The chart's bands, bottom-up: the kept sites in stack order, then Other on
+// top when anything folds.
+export function growthLayers(
+ months: readonly Month[],
+ sites: readonly { siteId: string }[],
+): GrowthLayer[] {
+ const folded = foldedSites(months, sites);
+ if (folded.length === 0) return ownLayers(sites);
+ const out = new Set(folded);
+ return [
+ ...ownLayers(sites).filter((l) => !out.has(l.sites[0])),
+ { key: OTHER_KEY, sites: folded, other: true },
+ ];
+}
+
+// Each layer's colour: a kept site its siteChartColors slot over EVERY site
+// (so folding never repaints it), Other the neutral grey.
+export function layerColors(
+ layers: readonly GrowthLayer[],
+ sites: readonly { accentId?: string }[],
+): string[] {
+ const chart = siteChartColors(sites);
+ return layers.map((l) => (l.other ? OTHER_COLOR : chart[l.sites[0]]));
+}
+
// A clean tick step giving three or four gridlines under `max`.
function niceStep(max: number): number {
const raw = max / 3.5;
@@ -53,12 +135,16 @@ function niceStep(max: number): number {
return 10 * pow;
}
-// The chart's numbers: each month's total, the peak, the value scale (yMax
-// and its gridlines), and the bands bottom-up (`order[k]` is band k's index in
-// `sites`).
+// ── The stack ─────────────────────────────────────────────────────────────────
+
+// The chart's numbers: each month's total (every site, folded or not), the
+// peak, the value scale (yMax and its gridlines), and one band per layer,
+// bottom-up (`stack[k]` is `layers[k]`'s; a layer's value in a month is the sum
+// of its sites'). Without `layers`, every site is its own band.
export function growthStack(
- months: readonly { bySite: Record<string, number | undefined> }[],
+ months: readonly Month[],
sites: readonly { siteId: string }[],
+ layers: readonly GrowthLayer[] = ownLayers(sites),
) {
const n = months.length;
const totals = months.map((m) => sites.reduce((a, s) => a + (m.bySite[s.siteId] ?? 0), 0));
@@ -67,16 +153,19 @@ export function growthStack(
const yMax = Math.ceil(peak / step) * step;
const ticks: number[] = [];
if (peak > 0) for (let t = step; t <= yMax; t += step) ticks.push(t);
- const order = stackOrder(sites.length);
const base = new Array<number>(n).fill(0);
- const stack: Band[] = order.map((si) => {
+ const stack: Band[] = layers.map((layer) => {
const lo = base.slice();
- const hi = months.map((m, i) => (base[i] += m.bySite[sites[si].siteId] ?? 0));
+ const hi = months.map(
+ (m, i) => (base[i] += layer.sites.reduce((a, si) => a + (m.bySite[sites[si].siteId] ?? 0), 0)),
+ );
return { lo, hi };
});
- return { totals, peak, yMax, ticks, order, stack };
+ return { totals, peak, yMax, ticks, layers, stack };
}
+// ── The gaps ──────────────────────────────────────────────────────────────────
+
const height = (b: Band, i: number) => b.hi[i] - b.lo[i];
// The band a gap along band k's upper edge would share at month i: the next
diff --git a/homepage/e2e/fixture-summary.ts b/homepage/e2e/fixture-summary.ts
@@ -26,7 +26,17 @@ import type { VideoStat } from "../../common/lib/stats";
// symlog axis reaches 10K while the Recent weeks stay linear and small;
// • every site transcribes every day of the last 200, so every line is full
// and every series has a non-zero start for Indexed;
-// • uploads spread over 2019–2026, for the growth chart's months.
+// • uploads spread over 2019–2026, for the growth chart's months;
+// • the last two sites each under 5 % of the growth chart's total (4.35 and
+// 3.26 %), so the chart folds them into its Other band (release 14 slice
+// CF; fixture-five transcribes 4 a day, not 5, for it);
+// • a seventh site, UNLISTED (site.json `listed: false`, release 14 slice
+// HS): its own two channels transcribe every day like the rest, and it
+// also exposes the first site's first channel. The summary names it nowhere
+// and counts its own channels in no total, so `buildFixtureSummary()` is
+// exactly `buildFixtureSummary(FIXTURE_SITES, [])` — every number the specs
+// read is the six listed sites'. Every site in the second list is unlisted:
+// `buildFixtureInputs` sets `listed: false` on it, whatever it carries.
export const FIXTURE_SUMMARY_NAME = ".e2e-summary.json";
@@ -42,7 +52,8 @@ export const FIXTURE_PALE_HEX = "#f4c2d7";
// site.json one (a named accent's site, the custom hex's and a site with no
// accent have one — the last ends MID-WORD, "Fix" + "ture Three", as
// "Jer" + "alyzer" does; the rest show their title plain); `daily` how many
-// recordings each channel transcribes a day.
+// recordings each channel transcribes a day. Whether a site is listed is the
+// list it is passed in (buildFixtureInputs), not a field.
export type FixtureSite = {
siteId: string;
siteTitle: string;
@@ -57,10 +68,20 @@ export const FIXTURE_SITES: readonly FixtureSite[] = [
{ siteId: "fixture-two", siteTitle: "Fixture Two", accent: FIXTURE_PALE_HEX, wordmarkLead: "Fixture", channels: 3, daily: 9 },
{ siteId: "fixture-three", siteTitle: "Fixture Three", wordmarkLead: "Fix", channels: 3, daily: 7 },
{ siteId: "fixture-four", siteTitle: "Fixture Four", channels: 2, daily: 8 },
- { siteId: "fixture-five", siteTitle: "Fixture Five", channels: 2, daily: 5 },
+ { siteId: "fixture-five", siteTitle: "Fixture Five", channels: 2, daily: 4 },
{ siteId: "fixture-six", siteTitle: "Fixture Six", accent: "vermilion", channels: 2, daily: 3 },
];
+// The unlisted site (an invented id, as every fixture's). Not in FIXTURE_SITES,
+// which is the six the pages show; buildFixtureInputs' second list, which
+// unlists it.
+export const FIXTURE_UNLISTED_SITE: FixtureSite = {
+ siteId: "fixture-unlisted",
+ siteTitle: "Fixture Unlisted",
+ channels: 2,
+ daily: 6,
+};
+
const DAY = 86_400_000;
const SPIKE = { site: 0, start: Date.UTC(2026, 4, 11), days: 7, count: 12_000 };
const HISTORY_DAYS = 200;
@@ -76,7 +97,13 @@ function uploadDate(n: number): string {
return ymd(d.getTime());
}
-export function buildFixtureSummary(fixtureSites: readonly FixtureSite[] = FIXTURE_SITES) {
+// The builder's inputs: the recordings, the channel → sites map and the sites.
+// `fixtureSites` are listed; every site in `unlistedSites` is written with
+// `listed: false`, and each also exposes the first listed site's first channel.
+export function buildFixtureInputs(
+ fixtureSites: readonly FixtureSite[] = FIXTURE_SITES,
+ unlistedSites: readonly FixtureSite[] = [FIXTURE_UNLISTED_SITE],
+): { stats: VideoStat[]; channelSites: Record<string, string[]>; sites: Site[] } {
const stats: VideoStat[] = [];
const channelSites: Record<string, string[]> = {};
const sites: Site[] = [];
@@ -112,17 +139,24 @@ export function buildFixtureSummary(fixtureSites: readonly FixtureSite[] = FIXTU
});
};
const today = Date.UTC(2026, 8, 15);
- fixtureSites.forEach((s, i) => {
+ // A site's own channels, `<siteId>-ch<N>`, each transcribing every day of the
+ // last HISTORY_DAYS; `shared` are other sites' channels it also exposes.
+ const addSite = (
+ s: FixtureSite,
+ i: number,
+ { listed = true, shared = [] }: { listed?: boolean; shared?: string[] } = {},
+ ) => {
const slugs = Array.from({ length: s.channels }, (_, c) => `${s.siteId}-ch${c + 1}`);
- for (const slug of slugs) channelSites[slug] = [s.siteId];
+ for (const slug of [...slugs, ...shared]) (channelSites[slug] ??= []).push(s.siteId);
sites.push({
siteId: s.siteId,
siteTitle: s.siteTitle,
siteDescription: `${s.siteTitle}, an e2e fixture archive.`,
siteUrl: `https://${s.siteId}.example`,
- channels: slugs.map((slug) => ({ slug })),
+ channels: [...slugs, ...shared].map((slug) => ({ slug })),
...(s.accent ? { accent: s.accent } : {}),
...(s.wordmarkLead ? { wordmarkLead: s.wordmarkLead } : {}),
+ ...(listed ? {} : { listed: false }),
} as unknown as Site);
slugs.forEach((slug, c) => {
const name = `${s.siteTitle} Channel ${c + 1}`;
@@ -133,13 +167,31 @@ export function buildFixtureSummary(fixtureSites: readonly FixtureSite[] = FIXTU
for (let k = 0; k < count; k++) record(i, slug, name, day);
}
});
- });
+ };
+ fixtureSites.forEach((s, i) => addSite(s, i));
// The megaspike: one week of bulk transcription on the first site's first
// channel.
const spikeSite = fixtureSites[SPIKE.site];
for (let k = 0; k < SPIKE.count; k++) {
record(SPIKE.site, `${spikeSite.siteId}-ch1`, `${spikeSite.siteTitle} Channel 1`, SPIKE.start + (k % SPIKE.days) * DAY);
}
+ // The unlisted sites LAST, so every listed site's recordings (their numbers,
+ // dates and states) are the same with them or without.
+ unlistedSites.forEach((s, j) =>
+ addSite(s, fixtureSites.length + j, {
+ listed: false,
+ shared: [`${spikeSite.siteId}-ch1`],
+ }),
+ );
+ return { stats, channelSites, sites };
+}
+
+// The summary the e2e dev server reads: the real builder over those inputs.
+export function buildFixtureSummary(
+ fixtureSites: readonly FixtureSite[] = FIXTURE_SITES,
+ unlistedSites: readonly FixtureSite[] = [FIXTURE_UNLISTED_SITE],
+) {
+ const { stats, channelSites, sites } = buildFixtureInputs(fixtureSites, unlistedSites);
return buildHomepageSummary(stats, channelSites, sites, FIXTURE_NOW);
}
diff --git a/homepage/e2e/growth-chart.spec.ts b/homepage/e2e/growth-chart.spec.ts
@@ -1,13 +1,17 @@
import { test, expect, type Page } from "@playwright/test";
import { buildFixtureSummary } from "./fixture-summary";
import { painted, rgbOf } from "../../common/testing/chartPixels";
+import { siteChartColors } from "../../common/lib/siteColor";
+import { OTHER_COLOR, growthLayers } from "../app/lib/growthGaps";
// The growth chart's bands are parted by the marks spec's SURFACE GAP: 2 px
// along each band's upper edge, in the colour behind the plot — the page
// ground, which the chart sits on — never a line in the text colour
// (ArchiveGrowthChart.tsx, globals.css `.growth-gap`), and the reader's Canvas
// in forced colours. The fixture summary (fixture-summary.ts) has six sites
-// with monthly data, so the chart and its gaps render.
+// with monthly data, so the chart and its gaps render; its last two are each
+// under 5 % of the chart's total, so they are drawn as one Other band on top
+// (lib/growthGaps.ts, release 14 slice CF).
const BASE_KEY = "ytdlp-tb:base";
@@ -87,8 +91,9 @@ for (const [width, shown] of [
// THE DATA IS WHAT IS PAINTED. Read back from a screenshot of the plot: at the
// busiest month the stack's topmost painted row is within 1 px of where the
-// month's true total sits on the value scale, and every site with data shows
-// pixels of its own colour — the gaps take no band away.
+// month's true total sits on the value scale, and every band with data — each
+// kept site's and Other's — shows pixels of its own colour: the gaps take no
+// band away.
for (const width of [390, 1280]) {
test(`${width} px: the chart paints its peak at its true height and every band`, async ({ page }) => {
await page.setViewportSize({ width, height: 900 });
@@ -96,6 +101,7 @@ for (const width of [390, 1280]) {
const summary = buildFixtureSummary();
const months = summary.monthly ?? [];
const sites = summary.sites;
+ const layers = growthLayers(months, sites);
const totals = months.map((m) => sites.reduce((a, x) => a + (m.bySite[x.siteId] ?? 0), 0));
const peak = Math.max(...totals);
const peakAt = totals.indexOf(peak);
@@ -111,10 +117,14 @@ for (const width of [390, 1280]) {
const expected = (1 - peak / yMax) * box.height;
const x = (peakAt / (months.length - 1)) * box.width;
const ground = rgbOf(await page.evaluate(() => getComputedStyle(document.body).backgroundColor));
+ // The legend's swatches, one per band, bottom-up (kept sites, then Other).
const swatches = await page
.locator("figure ul li span")
.evaluateAll((els) => els.map((e) => getComputedStyle(e).backgroundColor));
- const withData = sites.map((st) => months.some((m) => (m.bySite[st.siteId] ?? 0) > 0));
+ expect(swatches).toHaveLength(layers.length);
+ const withData = layers.map((l) =>
+ months.some((m) => l.sites.some((si) => (m.bySite[sites[si].siteId] ?? 0) > 0)),
+ );
const shot = await painted(page, plot, {
columns: [x - 1, x, x + 1],
ground,
@@ -124,8 +134,116 @@ for (const width of [390, 1280]) {
});
const top = Math.min(...shot.tops.filter((t): t is number => t !== null));
expect(Math.abs(top - expected), `top ${top} vs ${expected.toFixed(1)}`).toBeLessThanOrEqual(1);
- sites.forEach((st, i) => {
- if (withData[i]) expect(shot.counts[i].columns, `${st.siteTitle}'s colour`).toBeGreaterThan(0);
+ layers.forEach((l, k) => {
+ if (withData[k]) expect(shot.counts[k].columns, `${l.key}'s colour`).toBeGreaterThan(0);
});
});
}
+
+const resolveFill = (page: Page, css: string) =>
+ page.evaluate((c) => {
+ const el = document.createElement("span");
+ el.style.backgroundColor = c;
+ document.body.append(el);
+ const out = getComputedStyle(el).backgroundColor;
+ el.remove();
+ return out;
+ }, css);
+
+// WCAG contrast of two "rgb(…)" colours.
+function contrast(a: string, b: string): number {
+ const lum = (css: string) => {
+ const [r, g, bl] = rgbOf(css).map((v) => {
+ const c = v / 255;
+ return c <= 0.04045 ? c / 12.92 : ((c + 0.055) / 1.055) ** 2.4;
+ });
+ return 0.2126 * r + 0.7152 * g + 0.0722 * bl;
+ };
+ const [hi, lo] = [lum(a), lum(b)].sort((p, q) => q - p);
+ return (hi + 0.05) / (lo + 0.05);
+}
+
+// THE FOLD (lib/growthGaps.ts). The fixture's last two sites are each under
+// 5 % of the chart's total: ONE Other band, drawn last (on top), in the chart's
+// neutral grey. The legend shows the kept sites and Other; the table and
+// every month's title name every site.
+test("two sites under 5 % are one Other band on top: the legend shows the kept sites and Other, the table and the titles name all six", async ({
+ page,
+}) => {
+ const summary = buildFixtureSummary();
+ const months = summary.monthly ?? [];
+ const sites = summary.sites;
+ const layers = growthLayers(months, sites);
+ expect(layers.map((l) => l.key)).toEqual([
+ "fixture-one",
+ "fixture-two",
+ "fixture-three",
+ "fixture-four",
+ "(other)",
+ ]);
+ expect(layers.at(-1)!.sites).toEqual([4, 5]);
+ const titles = sites.map((s) => s.siteTitle);
+ await page.goto("/");
+ const figure = page.locator("figure").filter({ has: page.locator('[role="img"]') });
+
+ // The legend: the four kept sites, then Other.
+ await expect(figure.locator("ul[aria-hidden='true'] li")).toHaveText([...titles.slice(0, 4), "Other"]);
+
+ // The bands, in paint order: Other's is the last, in the neutral grey.
+ const fills = await figure
+ .locator("svg > path:not(.growth-gap)")
+ .evaluateAll((els) => els.map((e) => e.getAttribute("fill")));
+ expect(fills).toEqual([...siteChartColors(sites).slice(0, 4), OTHER_COLOR]);
+
+ // The image's label names the folded sites; the caption says what Other is.
+ await expect(figure.locator('[role="img"]')).toHaveAttribute(
+ "aria-label",
+ /Fixture Five and Fixture Six, each under 5% of the total, are drawn together as Other\.$/,
+ );
+ await expect(figure.locator("figcaption")).toContainText(
+ "Instances under 5% of the total are drawn together as Other.",
+ );
+
+ // Every month's title names every site with data that month, folded or not.
+ const hits = await figure
+ .locator("svg rect.growth-hit title")
+ .evaluateAll((els) => els.map((e) => e.textContent ?? ""));
+ expect(hits).toHaveLength(months.length);
+ let foldedNamed = 0;
+ months.forEach((m, i) => {
+ for (const s of sites) {
+ const v = m.bySite[s.siteId] ?? 0;
+ if (v > 0) expect(hits[i], `${m.month}`).toContain(`\n${s.siteTitle} ${v.toLocaleString("en-US")}`);
+ }
+ if (hits[i].includes("\nFixture Five ") || hits[i].includes("\nFixture Six ")) foldedNamed++;
+ });
+ expect(foldedNamed).toBeGreaterThan(0);
+ expect(hits.join("\n")).not.toContain("\nOther");
+
+ // The table: a column per site, all six, then Total.
+ await figure.locator("summary", { hasText: "Numbers by year" }).click();
+ await expect(figure.locator("table thead th")).toHaveText(["Year", ...titles, "Total"]);
+});
+
+test("Other is its own grey (--chart-other), at least 3:1 on both grounds, and no kept site's colour", async ({
+ page,
+}) => {
+ await page.goto("/");
+ for (const base of ["light", "dark"] as const) {
+ await page.evaluate(([k, v]) => localStorage.setItem(k, v), [BASE_KEY, base]);
+ await page.reload();
+ await expect(page.locator("html")).toHaveAttribute("data-base", base);
+ const swatches = await page
+ .locator("figure ul[aria-hidden='true'] li > span")
+ .evaluateAll((els) => els.map((e) => getComputedStyle(e).backgroundColor));
+ const other = swatches.at(-1)!;
+ expect(other, base).toBe(await resolveFill(page, OTHER_COLOR));
+ // Its own token, set on each base (a Dark block without it would fall
+ // back to the Light value from :root).
+ expect(other, base).toBe(base === "light" ? "rgb(62, 84, 92)" : "rgb(98, 98, 92)");
+ expect(other, base).not.toBe(await resolveFill(page, "var(--chart-axis)"));
+ const ground = await page.evaluate(() => getComputedStyle(document.body).backgroundColor);
+ expect(contrast(other, ground), `${base}: Other on the ground`).toBeGreaterThanOrEqual(3);
+ expect(swatches.slice(0, -1), base).not.toContain(other);
+ }
+});
diff --git a/homepage/e2e/instance-colours.spec.ts b/homepage/e2e/instance-colours.spec.ts
@@ -4,6 +4,7 @@ import { test, expect, type Page } from "@playwright/test";
import { resolveAccent } from "../../common/lib/accent";
import { ACCENTS } from "../../common/lib/brand";
import { ACCENT_CHART_SLOT, siteChartColors } from "../../common/lib/siteColor";
+import { OTHER_COLOR } from "../app/lib/growthGaps";
import { FIXTURE_PALE_HEX, FIXTURE_SITES, FIXTURE_SUMMARY_NAME } from "./fixture-summary";
// The Official Instances cards wear their site's accent (release 10): a named
@@ -16,7 +17,9 @@ import { FIXTURE_PALE_HEX, FIXTURE_SITES, FIXTURE_SUMMARY_NAME } from "./fixture
// The dev server reads the synthetic summary (fixture-summary.ts): site 0
// Brass, site 1 a pale custom hex, 2–4 none, 5 Vermilion — so site 3's own
// slot (chart-4) is Brass's and it takes the lowest free one, and six sites
-// wear all six validated slots.
+// wear all six validated slots. Sites 4 and 5 are each under 5 % of the
+// chart's total, so the chart draws them as one Other band (release 14 slice
+// CF): the legend has the four kept sites, in their own slots, and Other.
const BASE_KEY = "ytdlp-tb:base";
@@ -89,6 +92,8 @@ test("each instance card wears its site's accent on every base, in its chart lay
expect(chart[0]).toBe(`var(--chart-${ACCENT_CHART_SLOT.brass! + 1})`);
expect(chart[5]).toBe("var(--chart-6)");
expect([...chart].sort()).toEqual([1, 2, 3, 4, 5, 6].map((k) => `var(--chart-${k})`));
+ // The chart's bands: the four kept sites, then Other (sites 4 and 5).
+ const kept = 4;
await page.goto("/");
const on = { dark: "onDark", light: "onLight" } as const;
@@ -99,13 +104,16 @@ test("each instance card wears its site's accent on every base, in its chart lay
expect(stripes).toHaveLength(n);
// The summary's monthly series draws the chart, legend and all: the
- // legend (and so each layer) wears each site's chart colour, six apart.
+ // legend (and so each layer) wears each kept site's chart colour — its
+ // slot among all six, unchanged by the fold — then Other's grey, five
+ // apart.
await expect(page.getByRole("img", { name: /transcripts by the month/i })).toHaveCount(1);
- expect(legend, "legend swatches").toHaveLength(n);
- for (let i = 0; i < n; i++) {
+ expect(legend, "legend swatches").toHaveLength(kept + 1);
+ for (let i = 0; i < kept; i++) {
expect(legend[i], `legend ${i}`).toBe(await resolve(page, chart[i]));
}
- expect(new Set(legend).size, "six layers, six colours").toBe(n);
+ expect(legend[kept], "Other").toBe(await resolve(page, OTHER_COLOR));
+ expect(new Set(legend).size, "five layers, five colours").toBe(kept + 1);
// Site 0: the NAMED accent at this base's value — not the published
// on-dark hex on every base — and its legend swatch is the amber slot,
@@ -124,16 +132,42 @@ test("each instance card wears its site's accent on every base, in its chart lay
}
// Sites 2–4 have no accent: their chart colour, the same as their legend
- // swatch.
+ // swatch where they keep a band (2 and 3; 4 is in Other).
for (let i = 2; i <= 4; i++) {
expect(stripes[i], `site ${i}`).toBe(await resolve(page, chart[i]));
- expect(stripes[i], `site ${i} vs legend`).toBe(legend[i]);
+ if (i < kept) expect(stripes[i], `site ${i} vs legend`).toBe(legend[i]);
}
- // Site 5: Vermilion, the sixth slot's family — the rust layer, not the
- // golden angle's hue 328 beside the magenta slot.
+ // Site 5: Vermilion, the sixth slot's family — its chart colour is the
+ // rust (chart[5], asserted above), not the golden angle's hue 328 beside
+ // the magenta slot. It is in Other on the growth chart; the test below
+ // reads the rust off its line on /stats/.
expect(stripes[5]).toBe(await resolve(page, ACCENTS.vermilion[on[base]]));
- expect(legend[5]).toBe(await resolve(page, "var(--chart-6)"));
- expect(hueGap(stripes[5], legend[5]), "vermilion vs its layer").toBeLessThan(25);
+ expect(hueGap(stripes[5], await resolve(page, chart[5])), "vermilion vs its chart colour").toBeLessThan(25);
+ }
+});
+
+// The growth chart folds site 5 into Other, so the rust is checked where it is
+// still drawn: /stats/' site lines (homepageChartData, one line per site in the
+// summary's order, each in its siteChartColors colour).
+test("on /stats/ each site's line wears its chart colour: the Vermilion site's is the rust", async ({
+ page,
+}) => {
+ const sites = fixtureSites();
+ const chart = siteChartColors(sites);
+ await page.goto("/stats/");
+ for (const base of ["dark", "light"] as const) {
+ await useBase(page, base);
+ const lines = page.locator(".recharts-line-curve");
+ await expect(lines).toHaveCount(sites.length);
+ const strokes = await lines.evaluateAll((els) => els.map((el) => getComputedStyle(el).stroke));
+ const want = [];
+ for (const c of chart) want.push(await resolve(page, c));
+ expect(strokes, base).toEqual(want);
+ expect(strokes[5], `${base}: the Vermilion site's line`).toBe(await resolve(page, "var(--chart-6)"));
+ expect(
+ hueGap(strokes[5], await resolve(page, ACCENTS.vermilion[base === "dark" ? "onDark" : "onLight"])),
+ "vermilion vs its line",
+ ).toBeLessThan(25);
}
});
diff --git a/homepage/e2e/marketing.spec.ts b/homepage/e2e/marketing.spec.ts
@@ -113,8 +113,10 @@ test("the growth chart renders from the summary, or not at all", async ({
return;
}
await expect(chart).toBeVisible();
+ // The caption; with sites folded into Other (lib/growthGaps.ts) a second
+ // sentence says what Other is.
await expect(
- page.getByText(/all official instances\.$/i),
+ page.getByText(/all official instances\.( |$)/i),
).toBeVisible();
});
diff --git a/homepage/e2e/unlisted-site.spec.ts b/homepage/e2e/unlisted-site.spec.ts
@@ -0,0 +1,61 @@
+import fs from "node:fs";
+import path from "node:path";
+import { test, expect } from "@playwright/test";
+import {
+ FIXTURE_SITES,
+ FIXTURE_SUMMARY_NAME,
+ FIXTURE_UNLISTED_SITE,
+ buildFixtureInputs,
+ buildFixtureSummary,
+} from "./fixture-summary";
+
+// Release 14 slice HS: a site with site.json `listed: false` builds and
+// deploys, and the homepage lists it nowhere — not a card, not a chart series,
+// not a recent item — and counts the channels only it exposes in no total.
+//
+// The fixture summary (fixture-summary.ts) is built over six listed sites and
+// one unlisted one, which has two channels of its own and shares the first
+// site's first channel.
+
+const NEEDLES = [FIXTURE_UNLISTED_SITE.siteId, FIXTURE_UNLISTED_SITE.siteTitle];
+
+test("the summary the server reads is the one built without the unlisted site", () => {
+ // The site is in the builder's input: unlisted, with recordings of its own
+ // and a channel shared with the first listed site.
+ const { stats, channelSites, sites } = buildFixtureInputs();
+ const unlisted = sites.find((s) => s.siteId === FIXTURE_UNLISTED_SITE.siteId);
+ expect(unlisted?.listed).toBe(false);
+ const own = stats.filter((s) => s.channelSlug.startsWith(`${FIXTURE_UNLISTED_SITE.siteId}-ch`));
+ expect(own.length).toBeGreaterThan(0);
+ expect(channelSites[`${FIXTURE_SITES[0].siteId}-ch1`]).toContain(FIXTURE_UNLISTED_SITE.siteId);
+ // Listed, the same site would change the summary: a card, and its recordings.
+ const withoutIt = buildFixtureSummary(FIXTURE_SITES, []);
+ const asListed = buildFixtureSummary([...FIXTURE_SITES, FIXTURE_UNLISTED_SITE], []);
+ expect(asListed.sites.map((s) => s.siteId)).toContain(FIXTURE_UNLISTED_SITE.siteId);
+ expect(asListed.totals.transcripts).toBe(withoutIt.totals.transcripts + own.length);
+
+ const onDisk = JSON.parse(
+ fs.readFileSync(path.resolve(process.cwd(), "e2e", FIXTURE_SUMMARY_NAME), "utf8"),
+ );
+ // Every array and every total: what the six listed sites alone produce.
+ expect(onDisk).toEqual(JSON.parse(JSON.stringify(withoutIt)));
+ const text = JSON.stringify(onDisk);
+ for (const needle of NEEDLES) expect(text).not.toContain(needle);
+ expect(onDisk.sites.map((s: { siteId: string }) => s.siteId)).toEqual(
+ FIXTURE_SITES.map((s) => s.siteId),
+ );
+});
+
+for (const route of ["/", "/stats/"]) {
+ test(`${route} names no unlisted site, in its HTML or on the page`, async ({ page }) => {
+ const res = await page.goto(route);
+ expect(res?.ok()).toBe(true);
+ const served = (await res?.text()) ?? "";
+ // The listed sites are there, so the check below is not vacuous.
+ for (const s of FIXTURE_SITES) expect(served).toContain(s.siteId);
+ for (const needle of NEEDLES) {
+ expect(served).not.toContain(needle);
+ expect(await page.content()).not.toContain(needle);
+ }
+ });
+}
diff --git a/plans/FACTS.md b/plans/FACTS.md
@@ -153,6 +153,16 @@ THROWS at declaration — module load — on a name `SUB_FILE_RE` matches;
Never name the curated field `tags`. Never assume a `tags.json` is the keyword list:
`transcripts/tags.json` and `sites/<id>/tags.json` are curated tags.
+**`listed` / `unlisted` mean three unrelated things** (release 14, slice HS):
+
+| Which | Where | What it is |
+| --- | --- | --- |
+| A video's visibility | `common/lib/availability.ts` (the `"unlisted"` state, `isUnlisted`), `common/lib/transcripts{,-server}.ts`, `common/components/shareUrl.ts`, `common/controller/buildIndex.ts` | The platform's own "unlisted" (reachable by link, not listed on the channel). |
+| The hub's list has loaded | `export/app/components/hub/useHubSites.ts` — `listed` | `/hub-sites.json` has been answered and `/hub-summary.json` has settled. |
+| **A site the family lists** | `site.json` `listed` (`common/lib/siteSchema.ts` — `isListedSite`, `channelsOnlyOnUnlistedSites`) | Absent = listed; `false` keeps the site off the homepage, the hub and the other sites' footers, and out of the public totals. |
+
+A grep for either word finds all three; read the file before assuming which.
+
**Never publish a path segment named `.git`.** wrangler's Pages upload drops `**/.git` (and
`**/node_modules`) SILENTLY, and Cloudflare's managed rules block `/.git/` requests. The source
mirror is `/source/archilyzer.git/` for that reason (release 12, below). A mirror named `.git` would
@@ -6790,8 +6800,12 @@ S4, as shipped"; `release-10.md` "Slice L2 / L1, as shipped". Every anchor below
release 11), else `seriesColor(i)` when free (the first SIX are `var(--chart-1..6)`), else the
lowest free slot — never two sites in one colour, and any six wear the six validated slots. The growth
chart, its legend and `/stats` (`SiteGrid`, `homepageChartData`, the By-site leaderboard) wear
- it; the accents themselves fail the dataviz validator as a chart palette (no five with Brass
- and Blue pass). `useHubSites` lists the official
+ it (release 14 slice CF: the growth chart draws two or more sites each under 5 % of its total as
+ ONE Other band on top, in its own `--chart-other` (tokens.css, Light `#3e545c`, Dark `#62625c`;
+ not in `REQUIRED_TOKENS`); `homepage/app/lib/growthGaps.ts` `growthLayers` /
+ `layerColors`, which run `siteChartColors` over EVERY site, so a kept site keeps its slot); the
+ accents themselves fail the dataviz validator as a chart palette (no five with Brass and Blue
+ pass). `useHubSites` lists the official
instances only once `/hub-summary.json` has SETTLED (found, missing or unreadable;
`useHubSummary.ts:45`, `networkMode: "always"`), so they never reorder a moment later. The order
is applied in the browser: `hub-sites.json` on disk is unchanged.
@@ -7329,7 +7343,8 @@ Slices Q (`4855f70b`) and R (`ffdeb2cd`): [`release-12.md`](release-12.md), the
`path.resolve` / fs call on the result becomes an asset reference: to a file, or, when the joined
path is a directory, to every file under it (`DirAssetReference`).
- **A worktree build will not show it.** A worktree has no `transcripts/`, so the reference is
- empty. **Test with the corpus visible.**
+ empty. **Test with the corpus visible.** Nor does it have umtool's e2e fixture (`.e2e-song`,
+ `.next-e2e`), which is what slice UT's pattern reached (below).
- **What happened:** slice Q wrote `path.join(REPO_ROOT, "transcripts", "channels")` with
`REPO_ROOT = findRepoRoot(process.cwd())` in `umtool/lib/paths.mjs`. The walk's fallback,
`path.resolve(start, "..")`, evaluates to the project root.
@@ -7340,18 +7355,51 @@ Slices Q (`4855f70b`) and R (`ffdeb2cd`): [`release-12.md`](release-12.md), the
ModuleReference>::resolve_reference failed … Symlink [project]/transcripts/channels/<slug>/archive
is invalid, it points out of the filesystem root`.
- A worktree built it in 30 s at 0.8 GB, which is how the gate passed.
-- **The opt-out is per expression and documented:** `path.join(/* turbopackIgnore: true */
- process.cwd(), bar)`. That exact text is Turbopack's own advice in its "whole project was traced"
- message; the table is in the Next docs, `03-api-reference/08-turbopack.md`, "Magic comments". It
- goes before the FIRST argument of each path or fs call on such a value, and it changes nothing
- at run time. Per call, not per value:
+- **The opt-out is per expression, and it is not documented for path or fs calls** (corrected by
+ release 15, slice UT). `path.join(/* turbopackIgnore: true */ process.cwd(), bar)` is Turbopack's
+ own advice, in the text of its "Encountered unexpected file in NFT list" issue (the "whole project
+ was traced" warning; the 16.2.3 native binary carries it), and Next's own server uses it, on
+ the join and on the fs call around it: `next/dist/server/next-server.js:620` (16.2.3) reads
+ `existsSync(/* turbopackIgnore: true */ (0, _path.join)(/* turbopackIgnore: true */ this.dir,
+ 'static'))`. The Next docs (`08-turbopack.md` and
+ `02-guides/lazy-loading.md`, "Magic Comments") list the comment only for `import()`, `require()`,
+ `require.resolve()` and `new Worker()`. It goes before the FIRST argument of each call, and it
+ changes nothing at run time. Measured in slice UT, it works on a `path.join` and on an fs call.
+ Per call, not per value:
- a nested call needs its own marker (`path.dirname(/* turbopackIgnore: true */
fileURLToPath(import.meta.url))`);
- - an outer fs call on an opted-out `path.join` is covered.
-- **Not followed by the tracer:** `os.homedir()` and `process.env.*`. A build with `HOME` pointed at
- a synthetic home inside the project, full of out-of-root symlinks under `reports/`,
- `.local/share/archilyzer/song` and `.cache/`, succeeded. So `~/reports` and the XDG song path are
- safe as `path.join(os.homedir(), …)`.
+ - **an fs call on an opted-out `path.join` is NOT covered**, nested or through a variable: it
+ traces the join's value. Slice UT, on umtool's clip-audio route. With 4 probe files in its dot
+ directories: the join opted out and the fs calls on its value kept, 4 traced; those calls
+ stubbed, 0. With the primary's fixture: one `existsSync(path.join(/* opt-out */ CACHE_DIR, …))`,
+ 1,704; the same with the `existsSync` opted out as well, 0; the join held in a variable and
+ only `existsSync(/* opt-out */ cached)` reading it, 0. On a cwd-derived value the join is a
+ known path, so the outer call traces the one file it names; the guard lets that through (its
+ comment says so; the release 15 review ruled its four sites safe). Next's own
+ `next-server.js:620` (above) opts out both calls.
+- **A value the tracer cannot know is a dynamic part, not ignored** (corrected by release 15, slice
+ UT). `process.env.*`, `os.homedir()`, a parameter and an imported binding all make patterns over
+ the app's own directory (`umtool/`):
+ - 66 of umtool's 68 routes trace its whole tree outside dot-directories (361 files,
+ `next.config.ts` among them). Opting out every path op in `song/paths.mjs`, `lib/paths.mjs`
+ and `lib/paths.ts` (`path.resolve(process.env.X ?? path.join(os.homedir(), …))` and the like)
+ left 31 of the 68 routes clean; every path op in all 53 modules that have one (319 calls),
+ 49 (`_global-error` and `_not-found` were clean before). The rest come through fs calls
+ (`lib/report/snapshots.mjs` is the first the warning names).
+ - The clip-audio route's `path.join(CACHE_DIR, `${stamp}.${asMp3 ? "mp3" : "wav"}`)` (CACHE_DIR
+ imported, built on the env or home directory) also took in the dot-directories: 1,704 of its
+ 2,167 entries were the e2e fixture (`.e2e-song`), the e2e server's build directory
+ (`.next-e2e`) and `.env.local`. Hoisting the ternary into a `const` changed nothing. Without
+ the ternary (`${stamp}.wav`), as a ternary of two joins, or with the name handed to a function
+ from another module (the fix, `cacheFile` in `umtool/lib/paths.mjs`), it did not. The video
+ route's `.mp4` join did not reach the `.mp4` inside `.e2e-song`.
+ - **A pattern walk does not enter a symlinked directory**, in the root or out of it: a link at
+ `umtool/.e2e-song/data/planted` to `<primary>/transcripts/channels` gave 0 entries before and
+ after the fix, while the old route's walk listed the real files beside it (`data/mkvocals`);
+ one to the worktree's `common/` (675 files) gave 0 on the old route. That is what the
+ synthetic-HOME build showed, not that the env and home directory are unfollowed. A KNOWN
+ directory (slice Q's `<root>/transcripts/channels`) is different: it is walked through its
+ symlinks.
- **The guard is `scripts/next-build-trace.test.mjs`** (in `test:scripts`; it was
`umtool-build-trace.test.mjs` until release 14 widened it). It scans umtool's app, components,
lib, `report-to-video/*.mjs` and `song/paths.mjs`, and, since release 14 (F8), `homepage/app`,
@@ -7360,14 +7408,37 @@ Slices Q (`4855f70b`) and R (`ffdeb2cd`): [`release-12.md`](release-12.md), the
`process.cwd()`, `import.meta.url|dirname|filename`, `__dirname`, or a call to a function
declared in the file whose body carries one, must open with the opt-out.
- It is static and per module, as Turbopack's value analysis is: an imported binding is opaque to
- it.
+ it (to Turbopack it is an unknown, which is a dynamic part: see above).
- With the slice Q `paths.mjs` it fails on the defect's line.
+ - **Since release 15 (slice UT):**
+ - The scan set also follows every relative import out of those folders, which adds
+ `common/bin/_publicFile.ts`, `homepage/content/docs.ts` and seven `umtool/song` modules
+ (umtool 211 → 218 modules, the rest 895 → 897); a test pins them.
+ - The checked calls include `open`, `writeFile`, `appendFile`, `createWriteStream` (and their
+ Sync forms) and the `fs.promises.` / `fsPromises.` prefixes. No new finding.
+ - **It reads umtool's last build back.** Every `.nft.json` under `umtool/.next` but the
+ build's own `cache/` and `dev/` fails on an entry outside the repo, under `transcripts/`, or
+ through any name starting with a dot other than the build's own directory and
+ `node_modules/.pnpm`.
+ - **It skips, saying so,** when there is no build (`umtool/.next/server` or `BUILD_ID`
+ missing), and when `BUILD_ID` is older than `umtool/next.config.ts` or any umtool module it
+ scans (review M1). A merge or checkout gives the changed files new mtimes, and umtool runs
+ under `next dev`, which does not refresh `.next`, so a stale build skips until umtool is
+ rebuilt instead of failing on a call already fixed.
+ - **What it can see** (review L2). Without the excludes, the old clip-audio route failed it with
+ 1,704 entries (the primary's pre-fix build: 1,705, `test-results/.last-run.json` too). With
+ the excludes umtool's config now carries, such a pattern shows only through a name they
+ miss: `test-results/.last-run.json` after an e2e run, `.next-shots`, the corpus, a path
+ outside the repo. A checkout with no e2e run behind it is blind to it; the fix at the call
+ is what keeps the route clean.
- **The build gate** is run with the corpus visible and under a memory cap (the command is in
`plans/tools/implementer-rules.md`). Linking `<primary>/transcripts` into a worktree is for a
BUILD only. Remove the link afterwards: never run an app, an index or a fixture builder through it.
- **The other apps were safe by accident; since release 14 (F8) they are by rule:**
- `common/lib/paths.ts` builds every path on the repo root through one opted-out `under()`, and
- `findMonorepoRoot()`'s walk and its `process.cwd()` fallback are opted out.
+ `findMonorepoRoot()`'s walk and its `process.cwd()` fallback are opted out. Since release 15
+ `under(first, ...rest)` puts the marker before a named first argument, not a spread (the
+ release 14 review's L3); `getPaths()` is unchanged.
- `homepage/app/lib/source.ts`' directory join on `public` (the source mirror) and the homepage's
and export's other cwd joins carry the opt-out.
- The guard covers them. A homepage build with the published source measured the same with and
diff --git a/plans/export-header-first-search.md b/plans/export-header-first-search.md
@@ -330,6 +330,16 @@ Owns `common/components/theme*` (`themeConfig.ts`, `ThemeProvider.tsx`, `ThemeSc
## Slice S1 — a clear screen until the first Search (branch `r14/first-search`)
+As built (2026-09-29, on the rulings of that day): S1 only, the summaries download not deferred
+(S2 stays a candidate). A link carrying only a filter (`tg=`, a share link, the legacy filter keys),
+like one carrying `qt=`/`q=`, runs on load and shows its results. It runs no query, so a stored
+one stays held (release 8), on that load and on later mounts in the visit: the gate and the hold
+are two page-life flags (review M1). Step 0: the hub renders
+`SearchResults` (`HubHome` → `TranscriptSearch`), so it has the clear screen and `e2e:hub` joined
+the gate; Back restores the listing (measured in `release-14.md`, "Slice S1, as shipped"). Beyond
+the four specs this section owns, the ones that read the listing at load were `responsive` (two
+tests), `tag-chips` (three) and `e2e-hub/federated-search` (eight).
+
Owns `common/components/{SearchSessionContext,SearchResults,SearchBar}.tsx`,
`export/app/(workspace)/WorkspaceView.tsx`, `export/e2e/{browse-all,workspace-shell,charts,
restore-no-refire}.spec.ts`, a NEW `export/e2e/first-search.spec.ts`, `export/e2e/helpers.ts`
diff --git a/plans/release-14.md b/plans/release-14.md
@@ -20,10 +20,13 @@ slice HP added to it on the operator's ruling of the same day. Rules:
|---|---|---|---|
| HP | `homepage/social-visible` | The homepage's social row and one theme toggle in the header at every width, the wordmark's text dropped first on a very small screen and the row scrolling only as the last resort, one shared `SocialLinks` component, larger keys with a focus ring; Changelog in the footer only; the instance cards' names as the site's wordmark; the social icon checked by an allowlist on save and at render; a sized SVG with no viewBox gets one; `featured` ("Show in header") on a social link | `common/components/{SocialLinks,SocialScroll,ThemeRadios,ThemeToggle,ThemeScript,ThemeProvider,Wordmark}.tsx` + `themeConfig.ts`, `common/bin/doctor.ts`, the growth chart, `common/lib/{socialSvg,socialLinks}.ts` + tests, `common/lib/settingsSchema.ts` (the social-link type, parser, docs; the normalizer moved to `socialSvg.ts`), `common/lib/normalizeSocialSvg.test.ts`, `common/lib/{settings,site,homepage}.ts` (the save errors), `common/lib/{homepageSummary,siteColor}.ts`, `homepage/app/components/{Header,Footer,ArchiveCards}.tsx`, `homepage/app/lib/{nav,summary}.ts`, `homepage/app/not-found.tsx`, `homepage/e2e/**` (the fixtures, `helpers.ts`, the new and the rewritten specs), `homepage/playwright.config.ts`, `export/app/components/{MobileMenu,Footer}.tsx` (the `ThemeRadios` swap; the footer's read path), `editor/app/components/SocialLinksField.tsx` + `socialLinksJson{,.test}.ts`, `editor/app/{settings,sites}/actions.ts` (the save errors), `editor/e2e/settings.spec.ts`, `SETTINGS.md`, `SITE.md` |
| Lows, chart gap, T1, H1 (H3 folded in), H2 | `r14/two-grounds-headers` | The final review's Lows; the charts' surface gap; two grounds and each site in its own accent; the export and hub headers carry the social row and the toggle as one group, with an Archilyzer link to the homepage's `#instances` in place of the sites dropdown and the hub link; Changelog to the footer; after its review, the narrow header keeps the name and shows only the marked links (every header) | `common/components/{ThemeProvider,ThemeScript,ThemeToggle,SocialScroll}.tsx` + `themeConfig.ts` (and the deleted `ThemeMenu`, `ThemeRadios`), `common/components/charts/{ChartView,CrossSiteChart,surfaceGap}`, `common/styles/tokens.css`, `common/lib/{brand,accent,siteColor,paths,project,socialSvg,siteSchema,settingsSchema}.ts` + tests, `scripts/next-build-trace.test.mjs`, `export/app/components/{Header,MobileMenu,Footer}.tsx` (and the deleted `SiblingSwitcher`), `export/app/{layout.tsx,globals.css,changelog/page.tsx,lib/brand.ts}`, `export/e2e{,-hub}/**` (the theme, header and branding specs), `export/playwright.config.ts`, `editor/app/{layout.tsx,globals.css,sites/components/SiteForm.tsx}`, `editor/e2e/theme.spec.ts`, `homepage/app/{page.tsx,layout.tsx,globals.css,lib/*,changelog/page.tsx,components/{Header,ArchiveGrowthChart,ArchiveCards}.tsx}`, `homepage/e2e/**`, `homepage/content/docs/operate.md`, `SETTINGS.md`, `SITE.md` |
-| S1 | per the plan | per the plan | per the plan |
+| HS | `r14/hidden-sites` | Hidden sites: `site.json` `listed` (absent = listed). An unlisted site builds and deploys as before, and is left off the homepage (cards, chart, `/stats`), the hub (members, federated search, `corpus.json`, `llms.txt`), every other site's footer and the published id lists (`channel-sites.json`, the pooled `stats/`); the channels only it exposes count in no public total. One checkbox in the site form | `common/lib/{siteSchema,site,homepageSummary}.ts` + tests, `common/controller/{poolSummary,buildStats}.ts` + tests, `common/bin/{compose-homepage,compose-hub}.ts` + `compose-hub.test.ts`, `editor/app/sites/{actions.ts,components/SiteForm.tsx}`, `editor/e2e/{helpers.ts,sites-crud.spec.ts}`, `homepage/e2e/{fixture-summary.ts,unlisted-site.spec.ts}`, `SITE.md`, `plans/FACTS.md` (Naming hazards) |
+| S1 | `r14/first-search` | A clear screen until the first Search, on every site's search page and the hub's: no results area until the visitor asks (a Search, a profile load, or a link that carries a query or a filter), with the bar's "Press Enter or click Search to apply" line meanwhile; two page-life flags, the hold of release 8 unchanged and the gate new | `common/components/{SearchSessionContext,SearchResults,SearchBar}.tsx`, `export/e2e/first-search.spec.ts` (new), `export/e2e/{browse-all,workspace-shell,charts,restore-no-refire,responsive,tag-chips}.spec.ts`, `export/e2e/helpers.ts` (`showAll`), `export/e2e-hub/federated-search.spec.ts`, `export/CHANGELOG.md`, `plans/export-header-first-search.md` |
+| CF | `r14/chart-fold` | The homepage's growth chart folds two or more sites each under 5 % of its total into one Other band on top, in its own near-neutral grey (`--chart-other`); the kept sites keep their colours; the legend shows them and Other, the hover titles and the table every site | `homepage/app/lib/growthGaps.ts` + test, `homepage/app/components/ArchiveGrowthChart.tsx`, `common/styles/tokens.css` (`--chart-other`), `homepage/e2e/{fixture-summary.ts,growth-chart.spec.ts,instance-colours.spec.ts,marketing.spec.ts}`, `homepage/CHANGELOG.md`, `plans/FACTS.md` |
-**Order:** HP → `r14/two-grounds-headers` → S1. The shared files are the three changelogs'
-`[Unreleased]` sections and this record.
+**Order:** HP → `r14/two-grounds-headers` → HS (`r14/hidden-sites`) and S1 (`r14/first-search`),
+siblings → CF (`r14/chart-fold`), off `main` after HS, with `main` merged in after S1. The shared
+files are the three changelogs' `[Unreleased]` sections and this record.
## Record
@@ -1004,12 +1007,657 @@ There the bar overflows and the wide row scrolls while the name shows: measured
1100 px at 150 % and from 779 to 1300 px at 200 %. With the browser's own text size `md` moves too
and nothing overflows. The `md` layout is slice HP's and unchanged here.
+### Slice HS, as shipped — hidden sites: a site that builds and deploys, and is listed nowhere (2026-09-29)
+
+Branch `r14/hidden-sites` off `main` `99d4d76a`, worktree `~/Projects/homepage-social-visible`
+(block #3: editor test 3311, export 3310, homepage e2e 3340, hub e2e 3341), one Opus implementer.
+Scratch files `hs-*` in the job's `tmp`. The rulings (2026-09-29, not re-opened):
+1. A hidden site is absent from the homepage (cards, chart, `/stats`), the hub (members, federated
+ search, `corpus.json`, `llms.txt`), every other site's related-sites footer, and the published
+ id lists (`channel-sites.json`, the pooled `stats/`).
+2. Its hours and transcripts are in no public total.
+3. The editor has one checkbox in each site's settings, and no homepage settings page.
+4. The hidden site itself builds and deploys exactly as before.
+
+| sha | what |
+|---|---|
+| `122b8879` | `common:` `site.json` `listed` (schema, docs, parser, writer; `SITE.md`); `isListedSite` and `channelsOnlyOnUnlistedSites`; the footer drops an unlisted sibling |
+| `77aa1456` | `common:` the homepage summary (v6), `channel-sites.json` and the whole-pool stats leave out an unlisted site and the channels only it exposes |
+| `cbd12a8b` | `common:` `hub-sites.json` (and through it the hub's `corpus.json` and `llms.txt`) lists listed sites only |
+| `5836d6ec` | `sites:` the checkbox, and `saveSiteAction` carries the key; `sites-crud` e2e |
+| `64ae9b47` | `homepage:` the e2e fixture's seventh, unlisted site; `unlisted-site.spec.ts` |
+| `30b3f486` | `plans:` this record; the three changelogs |
+| `e3ee2eb1` | `homepage:` the fixture's second list unlists every site in it; the spec proves the site is in the input (review L1, L2) |
+| `65675373` | `common:` a shared channel stays the listed site's when the unlisted id sorts first (review L3) |
+| `20ff9ecf` | `plans:` FACTS' naming hazards (L4); this release's slices table, Order and Rollout carry HS (L6) |
+| `7ea35dfb` | merge of `main` `ccf90892` (release 15 IG); `buildStats.ts` and `editor/CHANGELOG.md` merged clean, the HS bullet under `[Unreleased]` |
+| _this_ | `plans:` the commit table, the review and the post-merge gates |
+
+The six commits up to `30b3f486` were rewritten after the review for the release's commit trailer
+(`git filter-branch --msg-filter`, trees unchanged); their first shas were `5918ba27`, `10378f7f`,
+`1491a7ab`, `ca22f0e6`, `e12515c2`, `4eeb80ee`.
+
+- **The key:** `listed?: boolean` in `site.json`, after `siteUrl`, in the type, `SITE_FIELD_DOCS`,
+ `siteFieldsSchema` and `siteToDisk`. Absent or anything but `false` reads `true`; only `false` is
+ written (the `archives` idiom). `SITE.md` regenerated.
+- **One predicate.** `isListedSite(site)` (`site.listed !== false`) and
+ `channelsOnlyOnUnlistedSites(sites)` — the channels at least one site exposes and no listed site
+ does. Both live beside the key in `common/lib/siteSchema.ts` and are exported from `lib/site`
+ (its `export *`), like `isValidSiteId`: the summary builder is pure and imports them without
+ `lib/site`'s file I/O. Every filter below calls them.
+- **What each public output does now:**
+
+ | Output | Code | An unlisted site | A channel only unlisted sites expose |
+ |---|---|---|---|
+ | Homepage summary: `sites`, `official`, `monthly`, `series`, `recent`, `channels` | `buildHomepageSummary`, the public-site filter | absent | absent |
+ | Homepage summary: `totals`, `availability` | the same, over the in-scope stats | — | not counted |
+ | `channel-sites.json` | `channelSitesOf` (`controller/poolSummary.ts`) | absent from every list | absent |
+ | The homepage's `stats/` (whole-pool bundle) | `buildStats`, the whole-pool block | — | its records and its manifest entry absent |
+ | `hub-summary.json` | `toHubSummary` of the same summary | absent | not counted |
+ | `hub-sites.json`, the hub's `corpus.json`, `llms.txt` | `compose-hub.ts`, the built-in pool | absent | — |
+ | Every other site's footer | `resolveRelatedSites` | not linked, even from a featured group | — |
+
+ - A channel a listed site also exposes is credited to the listed site: `primarySiteOf` runs over
+ listed public sites only.
+ - The summary's version is 6. No field was added or removed; the number marks the scope change.
+- **What stays, and why:**
+ - The unlisted site's own build and deploy: `buildAll` and the deploy paths
+ (`common/publish/build.ts`), its per-site stats and summaries (`buildStats`, `buildIndex`), its
+ archives and chart templates, its own `/site.json`, `corpus.json`, `llms.txt`, sitemap. Its own
+ footer still lists its listed siblings.
+ - The editor's pages (`/sites`, the channel page's memberships, the nav's site switcher, the
+ priority focus), which list every site.
+ - `siteChannelIndex`, `renameChannel`, `migrateToSites`, `listSiteIds` for the CLI's
+ site argument, the export's default site: none is a public list.
+ - The per-site staging in `buildIndex.ts` (another slice's file, and per site).
+- **Editor.** The site form has **List on the Archilyzer homepage and hub** after Public URL, on
+ by default, with a hint. `saveSiteAction` rebuilds the `Site` from the form; it now carries
+ `listed: false`, so a save of any other field keeps it. The other writers (`writeSite` from the
+ channel page, `renameChannel`, `migrateToSites`) spread the stored site or create a new one.
+- **Tests:**
+ - `siteSchema.test.ts`: absent and `true` read listed, only `false` unlists and only `false` is
+ written, through `writeSite`/`getSite`; the "everything" round-trip fixture carries
+ `listed: false`; `channelsOnlyOnUnlistedSites` keeps a shared channel with the listed site; the
+ footer drops an unlisted sibling named in a featured group, and an unlisted site's footer
+ still lists the rest.
+ - `homepageSummary.test.ts`: an unlisted site with its own channel and one shared with a listed
+ site gives a summary deep-equal to the one without it (every array, every total), and its id,
+ title and channel appear nowhere in the JSON; an unlisted site with no `siteUrl`, and one whose
+ channels are all shared, change nothing either; `listed: true` equals no key.
+ - `buildStats.test.ts` (k): the whole-pool bundle leaves out the unlisted-only channel (records,
+ manifest, count, log line) and keeps a pool-only one; the unlisted site's own bundle keeps all
+ three of its videos.
+ - `poolSummary.test.ts` (new): `channelSitesOf` names listed sites only.
+ - `compose-hub.test.ts`: `hub-sites.json`, `corpus.json` and `llms.txt` name the listed site and
+ not the unlisted one.
+ - Editor `sites-crud`: a `listed: false` file opens unticked; a save that changes only the title
+ keeps `false`; ticking removes the key; unticking writes it again.
+ - Homepage `unlisted-site.spec.ts`: the builder's input holds the unlisted site (`listed:
+ false`, its own records, the shared channel), and listed it would add a card and its records;
+ the summary the dev server reads equals the one built without it; `/` and `/stats/` (served
+ HTML and DOM) name all six listed sites and not the unlisted one.
+ - `homepageSummary.test.ts`, after the review: an unlisted id that sorts BEFORE the listed one
+ (`aaa-hidden` < `beta`) still leaves the shared channel with the listed site.
+- **The homepage e2e fixture** (`homepage/e2e/fixture-summary.ts`), for the slices that build on
+ it:
+ - `FIXTURE_SITES`: the six listed sites, unchanged.
+ - `FIXTURE_UNLISTED_SITE`: `fixture-unlisted`, "Fixture Unlisted", two channels of its own
+ (`fixture-unlisted-ch1/2`, six a day, like the rest); it also exposes `fixture-one-ch1`.
+ - `buildFixtureInputs(fixtureSites = FIXTURE_SITES, unlistedSites = [FIXTURE_UNLISTED_SITE])`
+ returns the builder's inputs (`stats`, `channelSites`, `sites`). The first list is listed;
+ every site in the second is written with `listed: false`, whatever it carries (`FixtureSite`
+ has no `listed` field), and shares the first listed site's first channel.
+ - `buildFixtureSummary(…)` is the real builder over those inputs. The unlisted sites' records are
+ generated last, after the megaspike, so every listed record is the same with them or without,
+ and `buildFixtureSummary()` equals `buildFixtureSummary(FIXTURE_SITES, [])`. Passed in the
+ FIRST list, the same site is listed: 7 sites and 39,599 transcripts instead of 6 and 37,199.
+
+#### Proof: a hidden fixture site through the real builds
+
+A throwaway corpus in the job's scratch dir (`$T/hs-proof/`, `make-corpus.mjs`): three channels
+of three captioned videos each, and two sites with public URLs — `fixture-listed` (the listed
+channel and the shared one) and `fixture-unlisted` (`listed: false`; the shared channel and one of
+its own). Every path the builds write was pinned there (`TRANSCRIPTS_DIR`, `SETTINGS_FILE`,
+`EXPORT_INDEX_DIR`, …) except the two apps' own `public/` and `out/`. The worktree's own gitignored
+`homepage/public` data and `export/out` were set aside first and put back after; the
+`export/public` links were dropped (never their targets) and re-seeded from the primary after. The
+primary's `export/public` was untouched (no entry newer than the slice's start). Each build was
+capped at 5 GB with no swap.
+
+| Build | Result | Time | Max RSS |
+|---|---|---|---|
+| `archilyzer index` | 0 | 4 s | — |
+| `archilyzer build homepage --no-source` | 0 | 20 s | 776 MB |
+| `archilyzer build hub` | 0 | 34 s | 1,005 MB |
+| `archilyzer build site fixture-unlisted --skip-archives` | 0 | 41 s | 962 MB |
+| `archilyzer build site fixture-listed --skip-archives` | 0 | 84 s | 970 MB |
+
+Counts (files holding the string / occurrences, `grep -rF`):
+
+| Tree | `fixture-unlisted` | its title | its own channel's slug | its own channel's name | `fixture-listed` |
+|---|---|---|---|---|---|
+| `homepage/public` (summary, `channel-sites.json`, `stats/`) | 0 / 0 | 0 / 0 | 0 / 0 | 0 / 0 | 2 / 25 |
+| `homepage/out` | 0 / 0 | 0 / 0 | 0 / 0 | 0 / 0 | 10 / 145 |
+| `export/public` (hub compose) | 0 / 0 | 0 / 0 | 0 / 0 | 0 / 0 | 4 / 10 |
+| `export/out` (hub) | 0 / 0 | 0 / 0 | 0 / 0 | 0 / 0 | 4 / 10 |
+| `export/out` (the listed site) | 0 / 0 | 0 / 0 | 0 / 0 | — | 9 / 30 (its URL) |
+
+- The compose lines: `compose-homepage: 2 channel(s) mapped across 1 listed site(s) (1 unlisted
+ left out); summary covers 6 transcription(s) / 6 download(s) across 1 public site(s)` and
+ `compose-hub: 1 built-in pool site(s) …; hub-summary.json covers 1 official instance(s)`.
+- The summary: version 6; `totals` 6 transcripts, 6 downloads, 1 site, 2 channels, 6 hours;
+ `official` the same; `availability.counted` 6. With the unlisted site counted they would have
+ been 9 transcripts and 9 hours.
+- The pooled `stats/` manifest: 6 records, channels `proof-listed-channel` and
+ `proof-shared-channel`. `channel-sites.json` maps both to `fixture-listed` alone.
+- **The unlisted site still builds:** its own stats bundle holds all six of its videos (both its
+ channels); its build ships its own pages (its id in 17 files), and its footer links
+ `https://fixture-listed.example`. The listed site's footer links nothing: its only sibling is
+ unlisted.
+
+#### Gates (at `64ae9b47`, the tree of the first `e12515c2`; logs `$T/hs-*.log`)
+
+- **tsc** was clean before every commit (69 s, 33 s, 44 s — the last over the tip's code).
+- **Unit:**
+
+ | Suite | Result |
+ |---|---|
+ | common | 2,218/2,218 (8 new) |
+ | editor unit | 87/87 |
+ | homepage unit | 12/12 |
+ | `test:scripts` | 191 passed, 1 skipped (192) |
+ | mcp | 271/271 |
+
+- **Docs:** `docs files --check`, `settings example --check` and `docs env --check` all exit **0**.
+- **e2e**, each detached and queued:
+
+ | Suite | Passed | Failed | Time |
+ |---|---|---|---|
+ | homepage, full (the new `unlisted-site.spec.ts` 3) | 97 | 0 | 3.6 min |
+ | hub, full | 36 | 0 | 1.4 min (after 3.5 min in the queue) |
+ | editor: `sites-crud` (the new listed round-trip 1) | 15 | 0 | 0.9 min (after 11 min in the queue) |
+
+- **Builds:** the five above, in the proof. The editor's and umtool's `next build` were not run (no
+ route or bundled path changed; tsc covers the form and the action).
+- **Numbers tool:** none.
+- **After the review and the merge of `main` (at `7ea35dfb`; `$T/hs-gates3.log`):** tsc clean
+ (183 s, the machine under load); common 2,229/2,229 (release 15 IG's 2,220, this slice's 8 and
+ the review's 1); homepage unit 12/12; homepage e2e `unlisted-site.spec.ts` 3 passed, 0 failed
+ (19 s). The fixture's second list unlisting a site that carries no key, and the listed variant's
+ 7 sites / 39,599 transcripts, were checked by a script (`$T/hs-fixture-check2.ts`).
+
+#### Review (verdict SHIP; `$T/hs-review.md`)
+
+| Finding | Fix |
+|---|---|
+| L1: the fixture's second list relied on each site's own `listed: false` | `e3ee2eb1`: `buildFixtureInputs` writes `listed: false` on every site in it; `FixtureSite` has no `listed` |
+| L2: the spec's first case would pass if the builder ignored the unlisted site | `e3ee2eb1`: it asserts the site is in the input (unlisted, its records, the shared channel) and that, listed, it adds a card and exactly its own records |
+| L3: every unlisted id sorted after the listed one | `65675373`: `aaa-hidden` sorts before `beta`; the summary is still the one without it |
+| L4: `unlisted` / `listed` already mean other things | `20ff9ecf`: a FACTS "Naming hazards" table of the three |
+| L5: the listed site's out not searched for the own channel's name; the export site suite and `e2e:2origin` not run | left: both changes are no-ops for a site without the key, and the unit test and the real build's footer cover them |
+| L6: this record's slices table, Order and Rollout did not name HS; the hub check's N | `20ff9ecf`: HS row and Order; the Rollout names HS, counts public LISTED sites and checks the new box |
+
+#### Found and left
+
+- **The unlisted site's own `/site.json` and `/corpus.json` still carry its `hubUrl`** (ruling 4:
+ it deploys exactly as before). A visitor who adds its origin to the hub by hand gets it as any
+ added origin, and the hub can read that `hubUrl` as a family member's.
+- **`listed` / `unlisted` mean three things:** a video's visibility, `useHubSites`' `listed`
+ flag (the hub's list has loaded), and a site the family lists. FACTS' "Naming hazards" has the
+ table since the review.
+- **The Rollout's hub check** ("`hub-summary.json covers N official instance(s)`") counts public
+ LISTED sites; the Rollout says so since the review.
+- **Unlisting takes effect at the next builds.** The homepage, the hub and every other site are
+ static: until each is rebuilt and deployed, it still lists the site.
+- **The editor's `/sites` list** shows no marker for an unlisted site; the form's checkbox is the
+ one place (ruling 3).
+
+#### Decisions the operator could overturn
+
+| What I assumed | The alternative |
+|---|---|
+| `totals` and `availability` keep counting pool-only channels and channels of sites with no public URL, as before; only a channel exposed by unlisted sites alone leaves them | `totals` count the listed public sites only, the same scope as `official` |
+| A channel an unlisted site shares with a listed site with NO public URL is not "only on unlisted sites", so `totals` count it | count a channel only when a listed PUBLIC site exposes it |
+| The pooled `stats/` keep pool-only channels, as before | the pooled `stats/` hold only channels a listed public site exposes |
+| `isListedSite` lives beside the key in `siteSchema.ts`, exported from `lib/site` | define it in `lib/site.ts` itself, and have the summary builder import `lib/site`'s file I/O |
+| The summary's version is 6 | stay at 5 (no field changed) |
+| An unlisted site's own footer still links its listed siblings | an unlisted site shows no related-sites footer |
+| The checkbox sits after Public URL, with a hint naming what it removes | at the end of the form, or without a hint |
+
+### Slice S1, as shipped — a clear screen until the first Search (2026-09-29)
+
+Branch `r14/first-search` off `main` `99d4d76a` (`r14/two-grounds-headers` merged), worktree
+`~/Projects/plans-export-header-first-search` (block #4: export e2e 3420, hub e2e 3441, editor test
+3411), one Opus implementer. Scratch files `s1-*` in the job's `tmp`.
+
+The rulings, 2026-09-29, not re-opened:
+1. S1 only. The summaries download is not deferred; S2 stays a candidate.
+2. A link carrying only a filter (`tg=`, `fv=`, `ch=`), like one carrying `qt=`/`q=`, runs on
+ load and shows its results.
+
+The parent's directions for the build (the plan's S1 steps):
+- The gate is in `SearchResults`, which the hub mounts too, so the hub gets the same behaviour and
+ `e2e:hub` joins the gate.
+- The module-level `ranThisPageLife` feeds a reactive `searchedThisPageLife`; the dead
+ `searchExecuted` goes. As built after the review, the flag the view reads is a second module
+ flag, `askedThisPageLife`, and `ranThisPageLife` keeps `main`'s rule (review M1, below).
+- A URL with `qt`, `q` or any filter key counts as asked, at hydration.
+- `SearchResults` renders nothing until the flag is set; `resultGroups` is still computed. The
+ page's intro (the transcript count, the Welcome card) stays.
+- The bar's "Press Enter or click Search to apply" line also shows before the first Search.
+
+My own choices are under "Decisions the operator could overturn".
+
+| sha | what |
+|---|---|
+| `93d72a80` | `common:` the results area renders nothing until the visitor asks in this page life; the bar's line shows meanwhile; `searchedThisPageLife`; `searchExecuted` deleted |
+| `90043b78` | `export:` `first-search.spec.ts`; one `showAll` helper, pressed where the export and hub specs read the listing at load |
+| `dbdaa31b` | `plans:` this record; the plan's S1 as built; the export changelog |
+| `9b01068c` | `common:` two page-life flags, one job each; a filter-only link no longer releases a stored query (review M1); the held-query comment (L1) |
+| `41dc1b4b` | `export:` `restore-no-refire` covers a second mount after a filter-only link (M1) |
+| _this_ | `plans:` the review's fixes recorded (L3, M1, L2 left); the S1 row of the slices table and the Rollout's checks (L4) |
+
+- **What a plain visit shows**, on a site's `/` and on the hub's:
+ - the search bar with its line, "Press Enter or click Search to apply";
+ - the page's intro (on a site the transcript count and the Welcome card; on the hub the shelf,
+ the figures and the archive chips);
+ - the footer, whole on the first screen at 1280×800 and 390×844 (the spec asserts it).
+ - No count, no Results/Chart toggle, no Copy for AI, no selection toolbar, no chart, no listing,
+ no `browse-hint`, and on the hub no "N of M archives answered" line.
+- **What counts as the first Search:** Enter, the Search button, a Filters Apply (all through
+ `commitSearch`), a profile load or Revert (`applySnapshot`), and a URL that asks.
+ - An empty Search shows "All videos (N)" and the listing, as before.
+ - A filter changed before it shows nothing; the line was already there for an unapplied edit.
+- **A URL that asks** (`urlAsks`, `SearchSessionContext.tsx`): any of `qt`, `q`, `tg`, the legacy
+ filter keys (`ch nov nol naa nar nav nd nu m tk`) or the share-link keys
+ (`fv fc ft fa fav fk fdf fdt`).
+ - Presence counts, not a valid value: `?tg=Not%20An%20Id` shows every video, which is what
+ `tag-chips.spec` already asserts.
+ - A video (`v`, `t`), a chart's shape (`view`, `cs`), `re` and `vm` do not ask. The Share button
+ always writes `fv=`, so a shared chart does run.
+- **Two flags, one job each** (module state: they survive client-side navigation and reset on a
+ reload):
+ - `ranThisPageLife` is the hold, exactly as on `main` (release 8). It is set by a commit, a
+ profile load, or an active URL query. While it is unset, a stored query is held, on this
+ mount and on every later one.
+ - `askedThisPageLife` is the gate: `ranThisPageLife`, or a URL that asks. `searchedThisPageLife`
+ starts from it and is set wherever it is set: at hydration, in `commitSearch` and in
+ `applySnapshot`.
+ - So a filter-only link shows its results and runs no query. A stored query stays held on that
+ load, and on a later mount in the same visit, which shows the listing under the held query.
+ - On the server neither module variable is set, so the first client render matches the static
+ HTML. The exported `index.html` now has the line and no results section.
+- **Step 0, settled:**
+ - **The hub renders `SearchResults`:** `HubHome.tsx` → `TranscriptSearch` (common) →
+ `SearchResults`. So the hub has the clear screen, and `e2e:hub` ran in the gate.
+ - **Back restores the listing.** Measured with a probe (not committed) on the 120-video fixture
+ at 1440×1200, scrolled to 3,000 px before leaving:
+
+ | Path | After Back | Scroll before → after |
+ |---|---|---|
+ | empty Search → **Use with AI** (header) → Back | the listing, "All videos (120)" | 2,838 → 2,676 px (the first card drawn: `0024` → `0025`) |
+ | `qt=` link → **Use with AI** → Back | the results | 2,952 → 2,904 px |
+ | empty Search or `qt=` → **Chat** (workspace nav) → Back | the listing | → 0 (the top) |
+ | a card's video (the modal) → Back | leaves the page | — |
+
+ - A route outside the workspace unmounts the session; Back remounts it with the flag already
+ set, and the scroll comes back to within a card.
+ - `/ask` hides the search pane in place, and the window is at the top when Back shows it again.
+ - The modal opens with `replaceState`, so there is no entry of the page's own to go Back to.
+ - The last two are unchanged by S1.
+- **Tests:**
+ - `first-search.spec.ts`, new (11):
+ - on load, no results area, the line, the intro and the whole footer, at 1280×800 and 390×844;
+ - Enter on an empty box shows every video, and the line goes;
+ - the Search button shows every video;
+ - a filter changed before the first Search shows nothing;
+ - `/` → `/ask` → `/` keeps the listing;
+ - Back from Use with AI keeps it;
+ - a reload clears it;
+ - a `qt=` link shows its results on load;
+ - `tg=` and `ch=` links show theirs;
+ - a restored query shows the clear screen and the filled form, and runs on Enter.
+ - `showAll(page)` in `export/e2e/helpers.ts`: wait for the query builder, press Search, wait for
+ `results-summary`.
+ - Where specs read the listing at load:
+ - `browse-all`: the first test rewritten to assert the clear screen, then the listing;
+ - `charts` (7 tests), `workspace-shell` (1), `responsive` (the filters sheet and the selection
+ toolbar), `tag-chips` (3; its `?tg=` tests unchanged);
+ - `restore-no-refire`: the held query now shows no results area. Two new tests cover a second
+ mount after a filter-only link, and each holds with no shard fetched: `?tg=` → Use with AI →
+ Back, and `?ch=` → Use with AI → the header's Search link;
+ - the hub's `federated-search` (8): its first test asserts the clear screen with both archives
+ in, then the listing.
+
+#### Gates (at `90043b78`; logs `$T/s1-*.log`)
+
+- **tsc** was clean before each commit and at the tip (36–38 s). The specs commit's own run
+ timed out at 100 s under the machine's load; it was run again on that commit, clean.
+- **Unit:** common **2,210/2,210** (75 s). Editor unit, `test:scripts` and mcp were not run: S1
+ touches no editor, mcp or scripts file, and the editor imports none of these components.
+- **Builds**, each capped at 5 GB with no swap, from a clean `.next`:
+
+ | Build | Time | Max RSS |
+ |---|---|---|
+ | export, site (the worktree's default) | 36 s | 957 MB |
+ | export, hub | 28 s | 1,034 MB |
+
+- **e2e**, each detached and queued:
+
+ | Suite | Passed | Failed | Time |
+ |---|---|---|---|
+ | export, the specs above first | 58 | 1 | 3.4 min |
+ | hub, full | 36 | 0 | 1.7 min |
+ | export, full | 254 | 0 | 11.4 min |
+ | `e2e:2origin` (`E2E_TWO_ORIGIN_REBUILD=1`, the `export/public` links in place) | 3 | 0 | 49 s |
+
+ The one failure in the first run was the new `/` → `/ask` test: that run's first visit to `/ask`,
+ compiled by the dev server, took longer than 5 s. The test now waits 20 s for the URL. After
+ `e2e:2origin`, the primary's `export/public` files kept their 2026-09-28 mtimes.
+- **Numbers tool:** none.
+
+#### Found and left
+
+- **Back from `/ask`** shows the listing at the top, not where the reader left it (the table
+ above). This is as before S1.
+- **A card's video opens without a history entry**, so Back from the modal leaves the site. This is
+ as before S1.
+- **Enter before the session hydrates (review L2).** Until the manifest is in, a `qt=` link shows the
+ line and not yet its results. Enter in that window commits the empty draft, and `commitSearch`
+ strips `qt` before hydration reads it. The race is on `main` too; the line now shows during it.
+ Gating the line's first-Search half on hydration would close it, at the cost of the line in the
+ static HTML. Left for the operator.
+
+#### Decisions the operator could overturn
+
+| What I assumed | The alternative |
+|---|---|
+| The line also shows under the bar on a fresh `/ask` (the bar is shared, and Search there applies the grounding) | show it only on the search view |
+| A link with only a chart's shape (`view=chart&cs=…`, copied from the address bar after an empty Search) opens on the clear screen | count `view`/`cs` as asking |
+| A video link (`?v=`) opens the video over the clear screen | count `v` as asking |
+| On the hub, the "N of M archives answered" line waits for the first Search with the rest of the results; each failed archive's chip says so, with its Retry, before that | show the line above the gate |
+| After a filter-only link, a later mount in the same visit (a page outside the workspace and back, the header's Search link) shows the listing, with the stored query held in the box: the gate is once per page life | the clear screen again on that mount |
+| A key's presence asks, not a valid value (`?tg=Not%20An%20Id`) | count only the keys the page applied |
+| The Search button keeps its outline look before the first Search; only the line says what to do | fill it as for an unapplied edit |
+
+#### Review (verdict SHIP AFTER FIXES; `$T/s1-review.md`)
+
+The reviewer also ran the editor suite's specs that drive the export's bar, results and `?v=`
+modal (`export-search`, `export-player-platform-cache`): 21 passed, 0 failed.
+
+| Finding | Fix |
+|---|---|
+| M1: a filter-only link set `ranThisPageLife`, so on a later mount in the same visit (a page outside the workspace and Back; the hub's `/` → `/ask`) a query stored by an earlier visit ran and fetched shards, where `main` held it. The reviewer's probe: `?tg=` → Use with AI → Back fetched 1 shard and showed "Matching videos"; `?ch=` → Use with AI → home did the same | `9b01068c`: two module flags. `ranThisPageLife` is set at hydration only by an active URL query, as on `main`, and `askedThisPageLife` (a run, or `urlAsks`) drives the gate. `41dc1b4b`: `restore-no-refire` seeds a stored query and checks both paths: 0 shards, the box filled, Search marked, and "All videos" |
+| L1: the held-query comment still said "the browse listing" | `9b01068c` |
+| L2: Enter before hydration on a `qt=` link commits the empty draft (a race on `main`; the line now shows during it) | left: "Found and left" |
+| L3: the rulings list mixed the rulings with the plan's steps and my choices | this commit: the rulings, the parent's directions, and my decisions, apart |
+| L4: the slices table's S1 row, and the Rollout's live checks | this commit: the row; three S1 checks under "Live checks" |
+
+#### Gates after the fixes (at `41dc1b4b`; logs `$T/s1-*2.log`, `$T/s1-fix-e2e.log`)
+
+The fix changes only which flag each place sets, so the full suites were not re-run.
+- **tsc** clean before each fix commit (182 s under the machine's load).
+- **common** 2,210/2,210.
+- **e2e**, detached and queued (spec lists in `$T/s1-fix-specs.txt`, `$T/s1-fix-hub-specs.txt`):
+
+ | Suite | Specs | Passed | Failed | Time |
+ |---|---|---|---|---|
+ | export | `first-search`, `restore-no-refire`, `browse-all`, `workspace-shell` | 20 | 0 | 5.1 min |
+ | hub | `federated-search` | 12 | 1 | 2.5 min |
+ | hub | `federated-search`, again | — | — | the dev server did not answer in 120 s |
+ | hub | `federated-search`, again | 13 | 0 | 1.7 min |
+
+ The hub's first run failed on its first test's first line, the archive chip still "loading"
+ after 5 s. That line is unchanged from `main` and comes before anything S1 changed; the machine's
+ load average was 33. On the second attempt the dev server did not start within 120 s. The third
+ run passed 13/13.
+
+### Slice CF, as shipped — the growth chart folds the small sites into Other (2026-09-29)
+
+Branch `r14/chart-fold` off `main` `69e058d6` (slice HS merged), worktree
+`~/Projects/homepage-social-visible` (block #3: homepage e2e 3340, homepage static 3331), one Opus
+implementer. Scratch files `cf-*` in the job's `tmp`. The rulings (2026-09-29, not re-opened):
+1. A site under **5 %** of the placed total — the chart's total over the whole plotted range, the
+ sum the bands are placed from — folds into ONE "Other" band, drawn on top of the stack, in a
+ neutral grey from the chart tokens that keeps ≥ 3:1 on the Light and the Dark ground.
+2. A fold of one is no fold: with a single site under the threshold, nothing folds.
+3. The legend shows the kept sites plus "Other"; the hover title and the "Numbers by year" table
+ still name every site.
+4. Colour follows the site, never its rank: a kept site's colour does not change because another
+ folded.
+5. The instance cards and `/stats` are unchanged; the surface gap draws correctly with Other on
+ top.
+6. (The review, ruled by the parent.) Other wears a dedicated `--chart-other` token in both chart
+ blocks of `tokens.css`: Light `#3e545c`, Dark `#62625c` (the near-neutral).
+
+| sha | what |
+|---|---|
+| `8351d24f` | `homepage:` the fold in `lib/growthGaps.ts` (`foldedSites`, `growthLayers`, `ownLayers`, `layerColors`; `growthStack` stacks layers); the chart draws the layers, its legend, label and caption; the unit tests; the e2e fixture's fifth site at 4 a day |
+| `6e0b652d` | `homepage:` e2e: `growth-chart.spec.ts` (the fold, the grey), `instance-colours.spec.ts`, `marketing.spec.ts` adjusted |
+| `e95a8058` | `plans:` this record, the slices table, Order and Rollout; FACTS; the homepage changelog |
+| `17832633` | `common, homepage:` `--chart-other` in both chart blocks with its numbers; `OTHER_COLOR` points at it; the chart's header comment; `growth-chart.spec.ts` resolves `OTHER_COLOR` and pins each base's value (review M1) |
+| `c42d2bbc` | `homepage:` `instance-colours.spec.ts` reads each site line's stroke on `/stats/`, the Vermilion site's the rust (review L2) |
+| `afddd66e` | `plans:` the review's fixes in this record (the grey, the Review table, the screenshots, Found and left, Decisions); the CF row's files; FACTS; the homepage changelog |
+| `9267ec08` | merge of `main` `721ed0eb` (slice S1); `plans/release-14.md` resolved by hand (below), the changelogs clean |
+| _this_ | `plans:` the post-merge gates; the commit table |
+
+**What shipped.**
+- **The rule** (`homepage/app/lib/growthGaps.ts`, pure, beside `growthStack`):
+ - `foldedSites(months, sites)`: each site's sum over the plotted months against the sum of all of
+ them. A site is under when `100 × its sum < FOLD_PERCENT × the total` (`FOLD_PERCENT = 5`), on
+ integers, so exactly 5 % keeps its band. The folded sites are returned only when two or more
+ are under; none when the total is 0.
+ - `growthLayers(months, sites)`: the kept sites in stack order (the summary's, `STACK_ORDER`
+ unchanged), then `{ key: "(other)", sites: [the folded indexes], other: true }` — a key no site
+ id can be (`[a-z0-9][a-z0-9-]*`). `ownLayers(sites)` is every site its own band.
+ - `growthStack(months, sites, layers = ownLayers(sites))`: one band per layer, a layer's value
+ in a month the sum of its sites'. It returns `layers` in place of `order` (the chart was
+ `order`'s only reader). Totals, peak, value scale and gridlines are over every site, so the
+ stack's top and the scale are the same folded or not.
+ - `layerColors(layers, sites)`: `siteChartColors` over EVERY site, and a kept site takes its own
+ entry, so a fold never repaints one (with no accents, the kept list alone would move a site
+ after a folded one to a lower slot; the unit test proves it); Other is `OTHER_COLOR`.
+- **The grey is its own token, `--chart-other`** (`common/styles/tokens.css`, beside `--chart-axis`
+ in both chart blocks, each with its numbers in a comment as `--chart-6` has; review M1): Light
+ `#3e545c` (OKLCH L 0.43, C 0.03), Dark `#62625c` (L 0.49, C 0.01). Contrast: 7.36:1 on the Light
+ ground and 7.99:1 on its chart surface; 3.22:1 on the Dark ground and 3.06:1 on its chart surface
+ (the Dark slots are 3.37–6.46:1 on the ground: Other is the dimmest mark there). Against each
+ chart slot it can sit on (the dataviz skill's validator's measures, OKLab ΔE × 100, normal / the
+ worse of protan and deutan):
+
+ | Slot | Light | Dark |
+ |---|---|---|
+ | blue (`--chart-1`) | 21.9 / 21.4 | 17.8 / 17.9 |
+ | green (`--chart-2`) | 17.4 / 15.2 | 19.2 / 15.7 |
+ | violet (`--chart-3`) | 16.2 / 13.4 | 23.4 / 21.5 |
+ | amber (`--chart-4`) | 22.1 / 18.2 | 20.3 / 18.1 |
+ | magenta (`--chart-5`) | 24.5 / 9.1 | 19.5 / 7.3 |
+ | rust (`--chart-6`) | 14.1 / 10.5 | 13.5 / 9.3 |
+
+ - Every CVD pair clears the floor (6) on both bases, and the target (8) on Light. Dark's magenta
+ (7.3) is in the 6–8 floor band, legal with the legend, the table and Other's place on top.
+ Rust is the one pair under the normal-vision 15, on both bases.
+ - A grey is under the validator's chroma floor by definition (it is the de-emphasis role, not a
+ categorical slot).
+ - Other touches only the kept site beneath it (the highest with a height that month). In
+ today's data that is Bonnellyzer, blue, in every month Other has data.
+ - The first build used `--chart-axis` (Light `#55646e`, Dark `#8a8170`: magenta at CVD 4.5 /
+ 4.2, and on Dark green at 4.1, under the floor). The review's sweep found in-band greys that
+ clear CVD 8 against every slot; the parent ruled the token above.
+- **The chart** (`ArchiveGrowthChart.tsx`):
+ - the legend is the layers: the kept sites' titles, then "Other";
+ - the areas and the gaps are keyed by layer;
+ - the hover title is unchanged: every site with data that month, by name, folded or not;
+ - the table is unchanged: a column per site;
+ - the image's `aria-label` gains, when something folds, "Hasanalyzer, Rekietalyzer and
+ Jasolyzer, each under 5% of the total, are drawn together as Other." (the legend is hidden
+ from assistive tech);
+ - the caption gains, when something folds, "Instances under 5% of the total are drawn together as
+ Other.";
+ - the header comment carries the grey's numbers.
+- **The gaps are unchanged code.** They work on bands, so Other is one band, the top one:
+ `bandAbove` of the top kept site is Other wherever Other has a height, and Other's own upper edge
+ has no gap (nothing sits on it). On today's summary the gap segments are the same folded and
+ unfolded, and none is drawn under Other: Bonnellyzer is under 3 px wherever Other sits on it, so
+ they touch, as the three small bands did before.
+
+ | Plot height | Segments | Runs | Under Other |
+ |---|---|---|---|
+ | 200 px | 73 | 7 | 0 |
+ | 260 px | 112 | 6 | 0 |
+ | 300 px | 119 | 7 | 0 |
+
+ On the fixture, gaps are drawn under Other, and no band is covered at any height (unit test).
+- **Unchanged:** the instance cards, `/stats`, `siteChartColors`, the hub; in `tokens.css` only
+ `--chart-other` is added.
+- **With today's data** (the primary's `homepage-summary.json` of 2026-09-28, 75,821 transcripts
+ on the chart): Jeralyzer 42.03 %, Anilyzer 38.40 %, Bonnellyzer 9.18 % keep their bands;
+ Hasanalyzer 4.33 %, Rekietalyzer 3.79 % and Jasolyzer 2.26 % are Other.
+
+**The e2e fixture** (`homepage/e2e/fixture-summary.ts`). Its six sites were 60.21, 12.90, 9.68,
+8.60, 5.38 and 3.23 % of the chart: one under 5 %, no fold. The smallest change that gives two:
+`fixture-five` transcribes 4 a day, not 5 (1,600 recordings, not 2,000). The shares are now 60.87,
+13.04, 9.78, 8.70, **4.35** and **3.26 %**, and `fixture-five` and `fixture-six` fold. The
+unlisted site stays out of the chart (it is out of the summary). Every changed expectation:
+- The fixture's listed totals: 36,799 transcripts, not 37,199; with the unlisted site listed,
+ 39,199, not 39,599 (HS's record has the old pair). No spec reads either: `unlisted-site.spec.ts`
+ computes both sides.
+- `growth-chart.spec.ts`, the pixel test: one colour per band (four kept sites and Other), not per
+ site.
+- `instance-colours.spec.ts`: the legend has five swatches, not six (the four kept sites in their
+ slots among all six, then Other's grey); `fixture-five`'s card is no longer compared with a
+ legend swatch (it has none); `fixture-six`'s hue check reads its chart colour, the rust, not a
+ legend swatch.
+- `marketing.spec.ts`: the caption's check is `/all official instances\.( |$)/i`, not `/…\.$/i`,
+ since the Other sentence can follow.
+
+**Tests.**
+- `growthGaps.test.ts`: the fixture test now runs the chart's layers (the fold is `[4, 5]`; a gap is
+ drawn between the top kept site and Other; none along Other's top; no band covered at any
+ height, folded or not). New:
+ - today's proportions (42.03 / 38.40 / 9.18 / 4.33 / 3.79 / 2.26 %, the family's accents): the
+ three largest keep their bands, Other holds the other three, its top is every month's total,
+ the colours are amber, magenta, blue and the grey;
+ - the edge: 999 of 20,000 (4.995 %) folds, 1,000 (exactly 5 %) does not; two sites at exactly
+ 5 % fold nothing;
+ - a single site under 5 %: no fold;
+ - no site under: `growthLayers` is `ownLayers`, and the stack is `growthStack`'s default;
+ - all but one under: one kept band and Other;
+ - a site with nothing in the range folds with another; nothing plotted folds nothing;
+ - 25 equal sites: every site folds (see "Decisions");
+ - colour stability: the kept sites' colours equal the unfolded run's, and differ from what the
+ kept list alone would give.
+- `growth-chart.spec.ts`, new:
+ - two sites under 5 % are one Other band on top: the layers are the four kept sites and
+ `(other)`; the legend reads the four titles and "Other"; the areas' fills, in paint order, are
+ the four slots and `OTHER_COLOR` (`var(--chart-other)`) last; the label names "Fixture Five and Fixture Six";
+ the caption says what Other is; every month's title names every site with data that month and
+ no title says "Other"; the table's header is Year, the six titles, Total;
+ - Other is `OTHER_COLOR` on both grounds, rendered `rgb(62, 84, 92)` on Light and `rgb(98, 98, 92)`
+ on Dark (so each base declares its own), not the axis colour, at least 3:1 on the ground, and no
+ kept site's colour (review M1).
+- `instance-colours.spec.ts`, new (review L2): on `/stats/`, on both bases, the six site lines'
+ rendered strokes are the six sites' chart colours in the summary's order; the Vermilion site's is
+ the rust (`--chart-6`) and within 25° of Vermilion's hue. The card test's own site-5 check is the
+ card's hue against its chart colour; the tautology it replaced (`chart[5]` against
+ `var(--chart-6)` twice) is gone.
+
+**They bite** (each change made by hand, the unit tests run, the change reverted):
+- `<=` for `<` in the rule: the edge test fails.
+- A fold of one allowed: the single-site test fails.
+- Colours from `siteChartColors` over the kept sites alone: the colour-stability test fails (the
+ today's-proportions test does not: its sites' slots come from their accents).
+- Other at the bottom of the stack: four tests fail (the fixture's, today's, all-but-one, colour).
+
+#### Gates (at `6e0b652d`; logs `$T/cf-*.log`)
+
+- **tsc** clean in every package on the tree of `6e0b652d`, run before the first commit (79 s);
+ `8351d24f`'s tree has `main`'s versions of the three spec files, which import nothing it changed.
+- **Unit:** common **2,229/2,229**; homepage **20/20** (12 before, 8 new).
+- **Build**, capped at 5 GB with no swap, from a clean `.next`, with the primary's summary copied
+ into the worktree's `homepage/public` for the screenshots (the worktree's own put back after):
+ `pnpm --filter homepage exec next build` **ok**, 39 s, max RSS 779,504 KB. `main`'s chart,
+ built the same way for the "before" shots: 80 s, 754,160 KB. After the review (`17832633`, for
+ the retaken shots): 20 s, 809,752 KB.
+- **Homepage e2e, full** (97 at `main` after HS; 2 new):
+
+ | Run | Passed | Failed | Time |
+ |---|---|---|---|
+ | first (`$T/cf-e2e-homepage.log`, after 9.5 min in the queue) | 98 | 1 | 4.2 min |
+ | again (`$T/cf-e2e-homepage2.log`, after 5.5 min in the queue) | **99** | **0** | 3.9 min |
+
+ The first run's failure was `marketing.spec.ts`' "Changelog is reachable from the footer on
+ every page": the 30 s test timeout, reached on the seventh of its seven pages (the dev server
+ compiling each on first visit, the machine's load average 11–17 with other suites running). It
+ passed in 5.3 s in the second run and in the three-spec run below; nothing in it reads the
+ chart.
+- **Along the way:** `growth-chart`, `instance-colours` and `marketing` specs, 21 passed, 0 failed
+ (1.8 min).
+- **Numbers tool:** none.
+- Not run: the export, hub and editor suites and their builds (no file of theirs changed).
+
+**Screenshots** (`~/reports/release-14/shots/chart3/`, 2×, the static server on 3331, today's
+summary):
+- `after-{390,1280}-{light,dark}.png`: the chart with `--chart-other` (retaken after the review;
+ the operator judges the Dark one); `…-2019-2026.png`: the plot from 2019 on, where Other lies;
+- `after-axis-…`: the same from the first build, Other in `--chart-axis`, for comparison;
+- `before-…`: the same from `main`'s chart and the same summary (six bands);
+- `after-1280-{light,dark}-table.png`: "Numbers by year" open, a column per site.
+
+**Found and left:**
+- **The pairs short of the validator's targets** are Dark's magenta (CVD 7.3, the 6–8 floor band)
+ and rust on both bases (normal 14.1 / 13.5). They matter only when that site is the top kept one under Other; today it
+ is blue.
+- **`/stats`' channel breakdown has an "Other (N)" of its own**, in `--muted-foreground`
+ (`common/lib/homepageChart.ts` `OTHER_COLOR`), not `--chart-other`. `/stats` is unchanged, as
+ ruled.
+- **`--chart-other` is not in `REQUIRED_TOKENS`** (`common/components/themeConfig.ts`, the list
+ `themeTokens.test.ts` checks every base declares): that file is outside this slice.
+ `growth-chart.spec.ts` pins each base's rendered value instead (a Dark block without it would
+ paint the Light value from `:root`).
+- **The fold is the homepage chart's only.** `/stats`, the hub and the sites' charts draw every
+ site, as ruled.
+
+#### Decisions the operator could overturn
+
+| What I assumed | The alternative |
+|---|---|
+| The caption gains one sentence saying what Other is, only when something folds | the caption unchanged |
+| The image's label names the folded sites | name only how many |
+| The legend reads "Other", with no count or names | "Other (3)" |
+| The hover title lists every site with data in the summary's order, with no Other subtotal | an "Other N" line, its sites under it |
+| When every site is under 5 % (21 or more sites), every site folds: one Other band | keep the largest, or fold nothing |
+| A site with nothing in the plotted range is under 5 % and folds with another | leave it out of the chart |
+
+#### Review (verdict SHIP AFTER FIXES; `$T/cf-review.md`)
+
+| Finding | Fix |
+|---|---|
+| M1: the axis grey was under the CVD floor against magenta on both bases (4.5 / 4.2) and against green on Dark (4.1); the record's sweep sentence overstated the case against a better grey | `17832633`: `--chart-other` in both chart blocks (Light `#3e545c`, Dark `#62625c`, as ruled), `OTHER_COLOR` points at it, the chart's comment and the spec follow; the sweep sentence is withdrawn and the token's measured numbers stated ("What shipped"); the labels' shared colour is gone from Found and left; the four `after-*` shots retaken, the first build's kept as `after-axis-*` |
+| L1: the CF row omitted `plans/FACTS.md` | `afddd66e`: the row lists it, and `common/styles/tokens.css` |
+| L2: `instance-colours.spec.ts` checked the sixth site's colour against itself | `c42d2bbc`: a rendered check of every site line's stroke on `/stats/`, the sixth the rust |
+| L3: "Found and left" left rust out of the weak pairs | moot with M1; the line names Dark's magenta and rust |
+| L4: the merge of `main` (`721ed0eb`, S1 merged) conflicts in this record only | `9267ec08`: S1's row, then CF's; S1's section, then CF's, before "## Rollout"; the Order line and the Rollout's intro name S1 and CF; both sets of live checks. The changelogs merged clean, every bullet under `[Unreleased]` (checked by eye) |
+| L5: the changelog's "at this release's numbers" will drift | left, as a release note |
+
+The caption and the image-label sentences are kept (the review: acceptable additions).
+
+**Gates after the review and the merge of `main`** (at `9267ec08`; logs `$T/cf-tsc2.log`,
+`$T/cf-tsc3.log`, `$T/cf-gates2.log`, `$T/cf-e2e3.log`):
+- **tsc** clean in every package before the fix commits (44 s) and on the merged tree before the
+ merge commit (164 s).
+- **Unit:** common **2,229/2,229**; homepage **20/20**.
+- **e2e**, detached and queued: `growth-chart`, `instance-colours` and `marketing`, **22 passed, 0
+ failed** (1.1 min; the new `/stats/` line test among them). The full suite was not re-run, as
+ directed.
+- **Build** (for the retaken shots, at `17832633`): 20 s, 809,752 KB, capped.
+
## Rollout
-Release 14 is slice HP (merged, `bfa1ff3c`) and `r14/two-grounds-headers` (after the parent's
-merge). Every command below is typed **from the primary checkout's root**. There is no
-`archilyzer` on PATH, so it is `pnpm archilyzer …`. The command forms are the ones verified in
-`plans/stats-cache-key.md`'s rollout.
+Release 14 is slice HP (merged, `bfa1ff3c`), `r14/two-grounds-headers`, slice HS
+(`r14/hidden-sites`), slice S1 (`r14/first-search`) and slice CF (`r14/chart-fold`), each after the
+parent's merge. Every command below is typed **from the primary checkout's root**. There is no `archilyzer` on PATH, so it is `pnpm archilyzer …`. The
+command forms are the ones verified in `plans/stats-cache-key.md`'s rollout.
**Preconditions.**
1. `main` carries `r14/two-grounds-headers`.
@@ -1038,8 +1686,9 @@ hub goes before the sites, because in basic mode they share `export/out`. The si
- `pnpm ops build-hub --wait`, then `pnpm ops deploy-hub --wait`.
**Between the two,** the build's `compose-hub: …` line must end with `hub-summary.json covers
- N official instance(s)`, where N is the number of public sites. If it says `hub-summary.json
- skipped: …`, stop and fix what it names.
+ N official instance(s)`, where N is the number of public LISTED sites (a site whose
+ **List on the Archilyzer homepage and hub** is unticked is not counted). If it says
+ `hub-summary.json skipped: …`, stop and fix what it names.
4. **The six sites:** `pnpm ops build-deploy --json '{"all":true}' --wait`.
**Live checks.**
@@ -1057,3 +1706,19 @@ hub goes before the sites, because in basic mode they share `export/out`. The si
and `localStorage.getItem("ytdlp-tb:base")` now reads `"light"`.
- The homepage's growth chart has no slash in the page colour through any band. `/stats` in
Area mode has coloured top lines.
+- The growth chart's legend lists the sites with 5 % or more of the chart's total, then
+ **Other** (a grey band on top); the caption ends "Instances under 5% of the total are drawn
+ together as Other."; **Numbers by year** has a column for every site. `/stats` still has a line
+ per site.
+- Every site's settings in the editor show **List on the Archilyzer homepage and hub**, ticked. With
+ none unticked, the homepage, the hub and every footer list the same sites as before, and
+ `https://archilyzer.pages.dev/homepage-summary.json` reads `"version":6`.
+- **S1, a clear screen until the first Search**, is in every site's build and the hub's, so steps 3
+ and 4 above (every site and the hub rebuilt and deployed) carry it. On one site:
+ - a plain visit to `/` shows the search bar, the transcript count and the whole footer, with no
+ scrolling and no listing;
+ - a `qt=` link shows its results on load;
+ - on a site that publishes tags (Anilyzer): Search for a word, then open a `tg=` link in the
+ same tab. Its listing shows with the word held in the box. After another page and Back the
+ word is still held: the box is filled, the Search button is marked, and the results say "All
+ videos", not "Matching videos".
diff --git a/plans/release-15.md b/plans/release-15.md
@@ -216,6 +216,253 @@ checkout's code, so they hold from the moment `main` has this branch. The editor
**Build index** button runs its built bundle, so it holds only after the editor is rebuilt and
restarted.
+### Slice UT, as shipped — umtool's build stops tracing its dot-directories (2026-09-29)
+
+Branch `r15/umtool-trace` off `main` `ccf90892`, worktree `~/Projects/r12-source-mirror` (block
+#13: editor 4301, test 4311, export 4310), one Opus implementer. Scratch files `ut-*` in the job's
+`tmp`. The ruling: find the one expression that widens the clip-audio route's trace and fix it
+there; exclude the fixture, the e2e build and env files as a second line; narrow or drop the
+`ignoreIssue`; make the trace guard read a build back and close the release 14 review's L1 and L3.
+
+**What was wrong.** With the primary's e2e fixture in place, umtool's
+`app/api/clip/[key]/audio/route.js.nft.json` listed 2,167 files: the 463 its sibling routes list,
+178 under `.e2e-song/` (the fixture, where `make-fixture.mjs` links the song data), 1,525 under
+`.next-e2e/` (the e2e dev server's build directory, 1.1 GB) and `.env.local`. Turbopack's warning
+for it ("Encountered unexpected file in NFT list", the "whole project was traced" text) was silenced
+by the config's `ignoreIssue`. The traces are not consumed while `output: "standalone"` stays off,
+so nothing broke; a fixture with more in it, or a standalone build, would have carried it.
+
+**The bisect.** One change per build, in this worktree with four probe files planted in
+`.e2e-song/probe/` and `.next-e2e/probe/`; the audio route's trace, total / under dot-directories.
+The route as on `main`: **467 / 4**.
+
+| Change (line on `main`) | Entries |
+|---|---|
+| `existsSync(file)` :51 stubbed | 467 / 4 |
+| **the join `path.join(CACHE_DIR, `${stamp}.${asMp3 ? "mp3" : "wav"}`)` :58 written as a string concatenation** | **463 / 0** |
+| `existsSync(cached)` :61, `readFile(cached)` :62, `mkdir(CACHE_DIR)` :78, `readFile(tmpMp3)` :89, `rename(tmpMp3, cached)` :90, `writeFile(tmp)` :93 or `rename(tmp, cached)` :94 stubbed, each alone | 467 / 4 each |
+| the `tmpWav` / `tmpMp3` joins :82-83 as concatenations; `writeFile(tmpWav)` :84 stubbed; both `writeFile`s stubbed; either `writeFile` opted out | 467 / 4 each |
+| the opt-out on the :58 join | 467 / 4 |
+| the :58 ternary hoisted into a `const ext` | 467 / 4 |
+| **:58 without the ternary** (`${stamp}.wav`) | **463 / 0** |
+| :58 as `path.join(CACHE_DIR, asMp3 ? `${stamp}.mp3` : `${stamp}.wav`)` | 463 / 0 |
+| :58 as a ternary of two joins | 463 / 0 |
+| opt-outs on `existsSync(cached)` and `readFile(cached)`, or either alone; both stubbed | 467 / 4 each |
+| all five readers and writers of `cached` stubbed | 467 / 4 |
+| **all five stubbed, and the opt-out on the :58 join** | **463 / 0** |
+| all five stubbed, and the join as a concatenation | 463 / 0 |
+
+With the primary's fixture (the table's 4 are 1,704 there), on top of the last-but-one row:
+
+| Change | Entries |
+|---|---|
+| one `existsSync(path.join(/* opt-out */ CACHE_DIR, …))` | 2,167 / 1,704 |
+| the same `existsSync` opted out as well | 463 / 0 |
+| `existsSync(/* opt-out */ cached)` as the only reader of the opted-out join | 463 / 0 |
+
+**The expression** is the :58 join, and in it the ternary inside the template literal. The join and
+the fs calls on its value each trace the pattern (the join alone with every reader stubbed; the
+readers alone with the join opted out; `existsSync` is one such reader), so no single opt-out or
+stub cleared it. Without the ternary, the pattern stays out of the dot-directories.
+
+**The fix** (`a305b956`). `umtool/lib/paths.mjs` gains `cacheFile(name)`, `path.join(/* opt-out */
+CACHE_DIR, name)`, re-exported by `lib/paths.ts`. The audio route names all four of its cache files
+through it (`cached`, `tmpWav`, `tmpMp3`, and `tmp`, now `cacheFile(`${stamp}.wav.tmp`)`, the same
+path as `${cached}.tmp` in that branch). A value returned by a function from another module is
+opaque to the tracer, so the call site traces nothing: the route lists 463, as its siblings do. The
+video and face-frame routes join `CACHE_DIR` with a fixed extension; they were measured clean (the
+video route's `.mp4` join did not reach the `.mp4` in `.e2e-song`) and name their cache files the
+same way, so no route joins `CACHE_DIR` itself. The run-time paths are unchanged.
+
+**The second line** (`f18fa034`). `outputFileTracingExcludes: { "/*": ["./.e2e-song/**/*",
+"./.next-e2e/**/*", "./.env*"] }` (Next 16.2.3's `05-config/01-next-config-js/output.md`: route
+globs to globs from the project root; Turbopack reads the key natively, `collect-build-traces.js`
+is the webpack path). Measured alone, with `main`'s route: 2,167 → 463.
+
+**The warning stays silenced, narrow as it was (path + title), with its measured reason.** Dropping
+it shows one warning on every build, and after the fix it is still true:
+- 66 of the 68 routes trace umtool's whole tree outside dot-directories: its 361 files,
+ `next.config.ts` (the file the warning names) among them. Only `_global-error` and `_not-found`
+ do not.
+- Path and fs calls on env, home-directory and parameter values do it. Opting out every path op in
+ `song/paths.mjs`, `lib/paths.mjs` and `lib/paths.ts` left 31 of the 68 routes clean (the audio
+ route 463 → 102). Opting out all 319 path ops in the 53 modules that have one left 49 clean. The
+ two clean before are among them. The rest come through fs calls; the next import trace the
+ warning names is `lib/report/snapshots.mjs`.
+- That walk skips dot-directories and does not enter symlinks. The warning names the same file for
+ it as for the audio route's, so it cannot tell the two apart; the second line and the guard below
+ cover the dot-directories instead. The config's comment says all of this.
+
+**A symlinked directory is not entered by these patterns.** The planted link
+`umtool/.e2e-song/data/planted` → `<primary>/transcripts/channels` gave 0 entries before and after
+the fix, and an in-root link to `common/` (675 files) gave 0 on `main`'s route. What the pattern
+reached was the fixture's real files.
+
+**The guard** (`scripts/next-build-trace.test.mjs`, `9a375ecd`, and after the review `a4d100b4`,
+`c3e8a2c7`), 6 → 10 tests:
+- **(a) It reads umtool's last build back.** Every `.nft.json` under `umtool/.next` but the
+ build's own `cache/` and `dev/` fails on an entry outside the repo, under `transcripts/`, or
+ through any name starting with a dot but the build's own directory and `node_modules/.pnpm`.
+ Unit tests pin `forbiddenTrace` and which directories are read.
+ - **It skips, saying so,** with no build, and (review M1) when `umtool/.next/BUILD_ID` is older
+ than `umtool/next.config.ts` or any umtool module the guard scans. The message gives the
+ build's time, the first newer file and how to rebuild. A merge or checkout gives the changed
+ files new mtimes, and umtool runs under `next dev`, which does not refresh `.next`, so a stale
+ build skips instead of failing on a call already fixed.
+ - **What it can see** (review L2). Without the excludes, `main`'s route failed it with 1,704
+ entries (the first 20 listed); the primary's pre-fix build fails it with 1,705 (the review:
+ `test-results/.last-run.json` too). With the excludes in place, a pattern like that one shows
+ only through a name they miss: `test-results/.last-run.json` after an e2e run, `.next-shots`,
+ the corpus, a path outside the repo. A checkout with no e2e run behind it is blind to it; the
+ fix at the call is what keeps the route clean.
+- **(b) The scan set follows relative imports** out of the listed folders, to any depth. It adds
+ the review's L1 modules and no others: `common/bin/_publicFile.ts`, `homepage/content/docs.ts`
+ (895 → 897) and seven `umtool/song` modules, `reasons`, `archive-url`, `pitch`, `flatness`,
+ `clipwindow`, `deplosive`, `orderfeat` (211 → 218; `song/paths.mjs` was listed by hand before and
+ is now reached). A test pins them, and that a song CLI and a common CLI stay out. The comments
+ that called them CLI-only are gone.
+- **(c) The checked calls** add `open`, `writeFile`, `appendFile`, `createWriteStream` and their
+ Sync forms, and the `fs.promises.` / `fsPromises.` prefixes. No new finding.
+- **(d)** `common/lib/paths.ts` `under(first, ...rest)` (`8b3409c2`): the opt-out sits before a
+ named first argument. `getPaths()` hashed identical, old module against new, from the repo root,
+ `editor/` and a directory outside the repo (56 keys). The guard passes; every caller type-checks.
+- The header no longer says the env and home directory are unfollowed or that the opt-out is
+ documented. The nested-call exemption's comment says it is a simplification (below).
+
+**FACTS**, "A path joined from `process.cwd()` …", corrected:
+- "documented": the Next docs list `turbopackIgnore` only for `import()`, `require()`,
+ `require.resolve()` and `new Worker()`. The path form is Turbopack's own advice, in the warning's
+ text in the 16.2.3 binary, and Next's own server uses it on the join and on the fs call around
+ it (`next/dist/server/next-server.js:620`; review L4).
+- "an outer fs call on an opted-out `path.join` is covered": it is not (the second bisect table).
+- "Not followed by the tracer: `os.homedir()` and `process.env.*`": such values are dynamic parts,
+ and make patterns over the app directory. The synthetic-HOME build showed that a pattern walk
+ does not enter symlinks.
+- The guard's entry, the worktree caveat (no fixture either) and `under()`'s shape.
+
+**Commits**
+
+| Commit | What |
+|---|---|
+| `a305b956` | `umtool:` `cacheFile`; the audio, video and face-frame routes name their cache files through it |
+| `f18fa034` | `umtool:` `outputFileTracingExcludes`; the `ignoreIssue` kept, its comment the measured reason |
+| `8b3409c2` | `common:` `under(first, ...rest)` |
+| `9a375ecd` | `scripts:` the post-build check, the relative-import scan set, the opens and writes, the header |
+| `750fc850` | `plans:` this section; FACTS; the editor changelog |
+| `a4d100b4` | `scripts:` review M1: the post-build check skips a build older than the code it judges |
+| `c3e8a2c7` | `scripts:` review L1 (only the build's own `cache/` and `dev/` unread, pinned), L2 (the check's comment says what it can see), L3 |
+| `b4d6607d` | `umtool:` review L3 in the `ignoreIssue` comment |
+| `31bb0f8c` | `plans:` the review's findings to their commits; FACTS (M1, L2, L3, L4); the changelog (M1); the rollout note |
+| `7c6d4969` | merge of `main` (`721ed0eb`, release 14 HS and S1); clean, the changelog bullet still under `[Unreleased]` |
+| this commit | `plans:` the gates after the review and the merge |
+
+#### Gates (logs `$T/ut-*.log`)
+
+- **tsc** clean over the combined tree before the first commit, 241 s (load average ~26). The
+ commits are independent pieces of that tree; since then only a comment changed in a
+ type-checked file (`cacheFile`'s, in `umtool/lib/paths.mjs`).
+- **`test:scripts`:** 194 passed, 1 skipped (195): `main`'s 191 + 1 and the guard's three new
+ tests. Before the final build it failed exactly the post-build test, on a build of `main`'s route.
+- **common:** 2,220/2,220, 107 s.
+- **umtool's build, capped at 5 GB with no swap, with the primary's `transcripts/` linked in, the
+ primary's `.e2e-song` and `.next-e2e` hard-linked in, an empty `.env.local`, and the planted
+ link** (all removed afterwards; none committed):
+
+ | Tree | Wall | User | Max RSS | Audio route | `.e2e-song` | `.next-e2e` | `.env*` | `transcripts` / planted |
+ |---|---|---|---|---|---|---|---|---|
+ | `main`'s route and config | 23 s | 57 s | 809 MB | 2,167 | 178 | 1,525 | 1 | 0 / 0 |
+ | the branch | 34 s | 64 s | 765 MB | 463 | 0 | 0 | 0 | 0 / 0 |
+
+ The wall times swing with the machine's load (another slice's e2e and builds ran alongside);
+ compile was 7.5 s and 10.7 s, TypeScript 12 s and 19 s.
+- **The editor's build**, capped, with the corpus linked in (`paths.ts` changed): 97 s wall, 159 s
+ user, max RSS 1,576 MB (IG's: 64 s / 1,642 MB, under less load); 0 of its 81 traces' entries
+ under `transcripts/`, and none that `forbiddenTrace` refuses.
+- **e2e** (umtool's own filter, `SONG_DIR=~/reports/quartering-uh-song/data`, queued):
+ `find.spec.ts` and `triage.spec.ts` fetch the audio route, `faces.spec.ts` the face-frame route,
+ and `find.spec.ts` names the video route: **6 passed, 29 skipped, 0 failed, 27 s** (6.3 min with
+ the queue). The skips are the fixture's: this machine has no `wav48/`, `asr/` or `media/`, so
+ every spec that fetches one of the three routes skipped, and they are not exercised at run time
+ here. What stands for that: calling `cacheFile` gives the same path as the old expression for
+ all six names (the four audio names, and the video and frame temporaries).
+- **After the e2e run** (which built this worktree its own fixture and `.next-e2e`), a last capped
+ build with the corpus linked: the audio route 463, none under a dot-directory; `test:scripts`
+ 194 passed, 1 skipped. The first run of that `test:scripts` failed `queue-lock.test.mjs`'s FIFO
+ case once (`S1E1S3E3S2E2`) under a load average of about 26; it passed on the rerun, and this
+ slice does not touch the queue lock.
+- **Numbers tool:** none.
+- **After the review and the merge of `main`** (at `7c6d4969`):
+ - tsc clean, 98 s;
+ - common **2,229/2,229** (`main`'s 2,229), 108 s;
+ - `test:scripts` **195 passed, 1 skipped (196)**: `main`'s 191 + 1 and the guard's four new
+ tests. The two runs before it each failed `queue-lock.test.mjs`'s "prints a banner naming the
+ holder while waiting" under a load average of about 26, the known flake (alone, 11/11 twice);
+ this slice does not touch the queue lock;
+ - umtool's build, capped, with the corpus linked and this worktree's own e2e fixture present:
+ 48 s wall, max RSS 796 MB, the audio route 463, none under a dot-directory; the post-build
+ check passes on it;
+ - the same build with its `BUILD_ID` set back to 2026-09-01 (in place, then put back; nothing
+ committed): the check skips, `umtool/.next was built 2026-09-01T04:00:00.000Z, before
+ umtool/next.config.ts (219 changed since); rebuild umtool (…) to check its traces`. With the
+ mtime put back it passes again.
+
+#### Found and left
+
+- **The whole-folder trace in 66 routes** (above). Bounded to umtool's own files; cleaning it means
+ opt-outs on hundreds of path and fs calls on unknown values, which no static check can find.
+- **The guard's nested-call exemption.** Without it, four calls would need an outer opt-out:
+ `common/lib/paths.ts:311` (`existsSync` of one file), `export/app/changelog/page.tsx:9` and
+ `homepage/app/changelog/page.tsx:24` (one file each; other slices own them), and
+ `umtool/report-to-video/brand.mjs:82` (the brand kits, which a standalone build needs). Each
+ traces the file or files it reads. Left, and the comment says so.
+- **Which fs calls Turbopack traces is not established per call.** `existsSync` does (the second
+ table); the writes were added to the guard without a measurement, since an extra opt-out costs
+ nothing.
+- **The post-build check reads umtool only.** In this worktree the editor's 81 traces pass the same
+ rule. The review's Info saw `editor/.env` in the primary's; a worktree has none, so it was not
+ re-measured.
+- **The gate command in `implementer-rules.md`, in a worktree that has a `transcripts/` directory,**
+ makes `transcripts/transcripts` and builds without the corpus where the paths point. This worktree
+ had one (an `index.mdb` from 2026-09-28), and my first two corpus-linked builds ran like that. The
+ numbers above are from builds that set it aside and put it back. `ln -sT` would refuse instead.
+- **A checkout whose umtool build predates its code skips the post-build check** until umtool is
+ rebuilt (review M1). The primary's `umtool/.next` is from before this slice, so the check skips
+ there until the rollout rebuilds it.
+
+#### Decisions the operator could overturn
+
+| What I did | The alternative |
+|---|---|
+| A `cacheFile` helper in `lib/paths.mjs`, used by all three routes that join `CACHE_DIR`. **Ruled at review: keep; one way to name a cache file.** | Opt-outs on the audio route's join and on every fs call on its value, in that route only |
+| The `ignoreIssue` stays, narrow, with the measured reason in its comment | Drop it: one warning on every build, naming one route's import trace |
+| The post-build check refuses any dot-named path but the build's own and `node_modules/.pnpm` | Refuse only `.e2e-song`, `.next-e2e`, `.env*` and `.git` |
+| The post-build check covers umtool only. **Ruled at review: umtool only.** In the primary the editor's traces would fail it (75 of 81, `editor/.env` among them) and the export's `.export-index` entries would be misjudged. | Also read the editor's, the export's and the homepage's builds |
+| The static check keeps its nested-call exemption, documented as a simplification. **Ruled at review: it stays; the four sites are safe.** | Require the outer opt-out: four new findings, two in files other slices own |
+| A stale build skips the post-build check (review M1) | Fail on it, as first shipped |
+
+#### Review
+
+**Verdict: SHIP AFTER FIXES** (`ut-review.md` in the job's scratch). No High. The reviewer found
+the fix's nine call-site paths byte-identical, the bisect logs in agreement with the tables, the
+excludes' key and shape right, and 0 dot entries across all 70 traces of a fresh build.
+
+| Finding | Where |
+|---|---|
+| M1: a stale umtool build turns `test:scripts` red, pointing at a call already fixed | `a4d100b4`: the check skips a build older than `next.config.ts` or any module it scans; the changelog, FACTS and "Found and left" say so; the rollout note below |
+| L1: `cache`/`dev` skipped at any depth | `c3e8a2c7`: only directly under `.next`; a test with a route directory named each |
+| L2: with the excludes in place the check cannot see the original defect | `c3e8a2c7` (the test's comment), guard (a) above and FACTS: it sees only a name the excludes miss |
+| L3: "cleaned 31 / 49" | `c3e8a2c7`, `b4d6607d`, this commit: "left 31 / 49 of the 68 clean" |
+| L4: FACTS cited the native binary's string as Next's runtime | this commit: `next-server.js:620` |
+| L5: the build-gate command's `ln -s` | The parent's (`implementer-rules.md`) |
+| Info: the editor's and export's primary builds carry the same class of widening | Recorded in the decisions table; a later slice's |
+
+**For the rollout.** Rebuild umtool in the primary first, under the cap
+(`timeout -s KILL 240 systemd-run --user --scope -q -p MemoryMax=5G -p MemorySwapMax=0 pnpm
+--filter umtool exec next build`), then run `pnpm run test:scripts` there. That is the only proof
+on the real fixture, which carries `.env.local`, `.next-shots` and `test-results/.last-run.json`,
+two of them names the excludes do not cover. Until that rebuild the post-build check skips in the
+primary, saying why. umtool's code changes nothing at run time.
+
### Slice DS, as shipped — a stalled drive does not stop the editor answering (2026-09-29)
Branch `r15/drive-stall` off `main` `ccf90892` (slice IG merged), worktree `~/Projects/r12-paths-fix`
diff --git a/scripts/next-build-trace.test.mjs b/scripts/next-build-trace.test.mjs
@@ -15,17 +15,28 @@
// per module; an imported binding is opaque to it): every path or fs call whose
// arguments carry a value derived IN THAT FILE from `process.cwd()`,
// `import.meta.url`, `import.meta.dirname|filename` or `__dirname` must open its
-// argument list with the documented opt-out, `/* turbopackIgnore: true */`.
+// argument list with Turbopack's opt-out, `/* turbopackIgnore: true */` (the
+// form its own "whole project was traced" warning advises; the Next docs list
+// the comment only for import(), require(), require.resolve() and new Worker()).
// The comment changes nothing at run time. A function declared in the file whose
// body carries a source is a source too (`const ROOT = findMonorepoRoot()`).
-// `os.homedir()` is NOT a source: the
-// tracer does not follow it (a build with HOME pointed at a synthetic home full
-// of out-of-root symlinks inside the project succeeds), and neither is
-// `process.env.*`.
+//
+// A value Turbopack cannot know -- `process.env.*`, `os.homedir()`, a
+// parameter, an imported binding -- is not one of these sources, and it is not
+// ignored either: it is a dynamic part, and a path or fs call on it becomes a
+// PATTERN over the app's own directory. Measured in umtool (release 15, slice
+// UT): the path ops on env and home-directory values in its path modules took
+// in the app's whole tree outside dot-directories (opting them out left 31 of
+// the 68 routes clean), and the clip-audio route's join with a dynamic extension took
+// in the dot-directories too, the e2e fixture and `.env.local` among them.
+// Neither walk entered a symlinked directory. No static check here can tell
+// such a pattern from a harmless one, so the last test reads a build's traces
+// back instead.
//
// Run with: pnpm test:scripts
import assert from "node:assert/strict";
-import { readdirSync, readFileSync } from "node:fs";
+import { existsSync, mkdirSync, mkdtempSync, readdirSync, readFileSync, rmSync, statSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
import path from "node:path";
import test from "node:test";
import { fileURLToPath } from "node:url";
@@ -38,10 +49,13 @@ const MARK = "__TURBOPACK_IGNORE__";
const IGNORE_COMMENT = /\/\*\s*turbopackIgnore\s*:\s*true\s*\*\//g;
// path ops, and fs calls either bare (`existsSync(`) or on a namespace
-// (`fs.readdir(`, `fsp.stat(`). A method on anything else (`obj.stat(`) is not
-// one.
+// (`fs.readdir(`, `fsp.stat(`, `fs.promises.readFile(`). A method on anything
+// else (`obj.stat(`) is not one. The opens and writes are checked too
+// (`open`, `writeFile`, `appendFile`, `createWriteStream`): which fs calls
+// Turbopack traces is not documented, and an opt-out on one it does not trace
+// costs nothing.
const SINK =
- /(?:\bpath\.(?:join|resolve|dirname|relative)|(?<![\w$.])(?:fs\.|fsp\.|promises\.)?(?:existsSync|readFileSync|readdirSync|statSync|lstatSync|realpathSync|opendirSync|readFile|readdir|stat|lstat|opendir|createReadStream))\s*\(/g;
+ /(?:\bpath\.(?:join|resolve|dirname|relative)|(?<![\w$.])(?:fs\.promises\.|fsPromises\.|fs\.|fsp\.|promises\.)?(?:existsSync|readFileSync|readdirSync|statSync|lstatSync|realpathSync|opendirSync|openSync|writeFileSync|appendFileSync|readFile|readdir|stat|lstat|opendir|open|writeFile|appendFile|createReadStream|createWriteStream))\s*\(/g;
/** Comments out, except the opt-out, which becomes a marker. Strings stay. */
function prepare(text) {
@@ -154,9 +168,15 @@ export function untracedCalls(text) {
const carries = (args) =>
SOURCE.test(args) ||
[...names].some((n) => new RegExp(`(?<![\\w$.])${n.replace(/\$/g, "\\$")}\\b(?!\\s*:)`).test(args));
- // A call nested in these arguments that opts out is opaque to this one too:
+ // A call nested in these arguments that opts out is let through:
// `readFileSync(path.join(/* turbopackIgnore: true */ HERE, "a.json"))`. Its
// own arguments are cut out before asking whether this call carries a source.
+ // That is a simplification, not how Turbopack reads it: the outer call traces
+ // the join's value all the same (release 15, slice UT, measured). On a
+ // cwd-derived value that value is a known path, so the outer call traces the
+ // one file it names (a changelog), or the files of one known directory (the
+ // brand kits). Where the join names a directory, or a dynamic part could
+ // reach past the files the call reads, give the outer call its own opt-out.
const withoutOptedOut = (args) => {
let out = args;
for (let i = out.search(new RegExp(`\\(\\s*${MARK}`)); i !== -1; i = out.search(new RegExp(`\\(\\s*${MARK}`))) {
@@ -187,24 +207,75 @@ function modulesUnder(dir, out = []) {
return out;
}
-/** umtool's modules that a Next build can reach: the app, its libs, the pipeline. */
+const MODULE_EXT = [".ts", ".tsx", ".mjs", ".js", ".cjs"];
+const isModule = (p) => MODULE_EXT.some((x) => p.endsWith(x)) && !/\.test\./.test(p) && !p.endsWith(".d.ts");
+const isFile = (p) => {
+ try {
+ return statSync(p).isFile();
+ } catch {
+ return false;
+ }
+};
+
+/** The module a relative specifier names, the way the bundler resolves it, or null. */
+function resolveRelative(from, spec) {
+ const base = path.resolve(path.dirname(from), spec);
+ const candidates = [base, ...MODULE_EXT.map((x) => base + x), ...MODULE_EXT.map((x) => path.join(base, "index" + x))];
+ return candidates.find((c) => isModule(c) && isFile(c)) ?? null;
+}
+
+// `import … from "./x"`, `export … from "../x"`, `import "./x"`, `import("./x")`,
+// `require("./x")`: the relative specifiers only. A package import (`next`,
+// `yt-dlp-transcript-common/…`) is covered by the package's own directory in
+// the set, or is not this repo's code.
+const RELATIVE_IMPORT = /(?:\bfrom\s*|\bimport\s*\(?\s*|\brequire\s*\(\s*)(["'])(\.{1,2}\/[^"'\n]+)\1/g;
+
+/**
+ * `files` plus every module of this repo they reach by relative imports, to
+ * any depth. A directory list alone misses a module one app imports from a
+ * folder the list treats as CLI-only (`common/bin/_publicFile.ts`, imported by
+ * `common/publish/source.ts`; umtool's `song/pitch.mjs`, imported by
+ * `lib/verdict.ts`) or keeps outside `app/` (`homepage/content/docs.ts`).
+ */
+export function withRelativeImports(files) {
+ const seen = new Set(files);
+ const queue = [...files];
+ while (queue.length) {
+ const file = queue.pop();
+ for (const m of readFileSync(file, "utf8").matchAll(RELATIVE_IMPORT)) {
+ const target = resolveRelative(file, m[2]);
+ if (!target || seen.has(target) || target.includes(`${path.sep}node_modules${path.sep}`)) continue;
+ if (path.relative(REPO, target).startsWith("..")) continue;
+ seen.add(target);
+ queue.push(target);
+ }
+ }
+ return [...seen];
+}
+
+/**
+ * umtool's modules that a Next build can reach: the app, its components and
+ * libs, the report pipeline, and whatever of `song/` they import (`paths.mjs`
+ * through `lib/paths.mjs`, `pitch.mjs`, `reasons.mjs` and the rest through the
+ * libs; the other song scripts are CLIs nothing in the app imports).
+ */
function umtoolModules() {
const out = [];
for (const d of ["app", "components", "lib"]) modulesUnder(path.join(UMTOOL, d), out);
for (const e of readdirSync(path.join(UMTOOL, "report-to-video"))) {
if (e.endsWith(".mjs") && !e.includes(".test.")) out.push(path.join(UMTOOL, "report-to-video", e));
}
- // song/paths.mjs is imported by lib/paths.mjs; the other song scripts are CLIs.
- out.push(path.join(UMTOOL, "song", "paths.mjs"));
- return out;
+ return withRelativeImports(out);
}
/**
* The editor's, the export's and the homepage's modules a Next build can
* reach: each app's `app/` (the editor's `lib/` and `instrumentation.ts` too),
- * and every module of common/ but its CLIs (`bin/`, which no app imports). The
- * common set is wider than what the apps import today, on purpose: a module
- * that starts being imported is already covered.
+ * every module of common/ but its CLIs in `bin/`, and every module those reach
+ * by a relative import — which brings in the few `bin/` modules an app does
+ * import (`_publicFile.ts`) and the homepage's `content/docs.ts`. The common set
+ * is wider than what the apps import today, on purpose: a module that starts
+ * being imported is already covered.
*/
function nextAppModules() {
const out = [];
@@ -214,7 +285,7 @@ function nextAppModules() {
if (!e.isDirectory() || e.name === "bin" || e.name === "node_modules" || e.name.startsWith(".")) continue;
modulesUnder(path.join(REPO, "common", e.name), out);
}
- return out;
+ return withRelativeImports(out);
}
function untracedIn(files) {
@@ -298,6 +369,135 @@ test("no umtool module the app can import joins a cwd-derived path without optin
);
});
+test("the scan set follows relative imports out of the listed folders", () => {
+ const rel = (files) => new Set(files.map((f) => path.relative(REPO, f)));
+ const um = rel(umtoolModules());
+ for (const m of ["paths", "reasons", "archive-url", "pitch", "flatness", "clipwindow", "deplosive", "orderfeat"]) {
+ assert.ok(um.has(`umtool/song/${m}.mjs`), `umtool/song/${m}.mjs is not scanned`);
+ }
+ // A song CLI nothing in the app imports stays out.
+ assert.ok(!um.has("umtool/song/build-um.mjs"));
+ const apps = rel(nextAppModules());
+ assert.ok(apps.has("common/bin/_publicFile.ts"), "common/bin/_publicFile.ts is not scanned");
+ assert.ok(apps.has("homepage/content/docs.ts"), "homepage/content/docs.ts is not scanned");
+ assert.ok(!apps.has("common/bin/compose-site.ts"), "a common CLI no app imports is scanned");
+});
+
+/**
+ * Every `.nft.json` a build wrote under its directory `dist`, its cache and
+ * dev-server output (`dist/cache`, `dist/dev`) aside. Only those two: a route
+ * directory named `cache` or `dev` deeper down is read like any other.
+ */
+function traceFilesUnder(dist, dir = dist, out = []) {
+ for (const e of readdirSync(dir, { withFileTypes: true })) {
+ const p = path.join(dir, e.name);
+ if (e.isDirectory()) {
+ if (dir !== dist || (e.name !== "cache" && e.name !== "dev")) traceFilesUnder(dist, p, out);
+ } else if (e.name.endsWith(".nft.json")) out.push(p);
+ }
+ return out;
+}
+
+/**
+ * Why `abs`, a file a build traced, must not be in the trace, or null.
+ *
+ * The build's own directory (`dist`) holds the chunks every trace lists, and
+ * `node_modules/.pnpm` is where pnpm keeps the packages; any other path through
+ * a directory or file whose name starts with a dot is something no server
+ * needs at run time — a fixture (`.e2e-song`, where the e2e fixture links the
+ * song data), another build (`.next-e2e`), a secret (`.env.local`), `.git` —
+ * and is the mark of a pattern Turbopack could not bound. So are the corpus
+ * and anything outside the repo.
+ */
+export function forbiddenTrace(abs, dist) {
+ const rel = path.relative(REPO, abs);
+ if (rel === "" || rel.startsWith("..") || path.isAbsolute(rel)) return "outside the repo";
+ if (abs.startsWith(dist + path.sep)) return null;
+ const parts = rel.split(path.sep);
+ if (parts[0] === "transcripts") return "the corpus";
+ for (let i = 0; i < parts.length; i += 1) {
+ if (!parts[i].startsWith(".")) continue;
+ if (parts[i] === ".pnpm" && parts[i - 1] === "node_modules") continue;
+ return `under ${parts.slice(0, i + 1).join("/")}`;
+ }
+ return null;
+}
+
+test("traceFilesUnder: only the build's own cache/ and dev/ are left out", () => {
+ const dist = mkdtempSync(path.join(tmpdir(), "next-build-trace-"));
+ try {
+ for (const f of ["cache/a.nft.json", "dev/b.nft.json", "server/app/api/cache/route.js.nft.json", "server/app/dev/page.js.nft.json"]) {
+ mkdirSync(path.dirname(path.join(dist, f)), { recursive: true });
+ writeFileSync(path.join(dist, f), '{"files":[]}');
+ }
+ const found = traceFilesUnder(dist).map((f) => path.relative(dist, f)).sort();
+ assert.deepEqual(found, ["server/app/api/cache/route.js.nft.json", "server/app/dev/page.js.nft.json"]);
+ } finally {
+ rmSync(dist, { recursive: true, force: true });
+ }
+});
+
+test("forbiddenTrace: a fixture, another build, a secret, the corpus, outside the repo", () => {
+ const dist = path.join(UMTOOL, ".next");
+ const at = (p) => forbiddenTrace(path.join(REPO, p), dist);
+ assert.equal(at("umtool/.next/server/chunks/ssr/a.js"), null);
+ assert.equal(at("node_modules/.pnpm/next@16.2.3/node_modules/next/dist/server/next.js"), null);
+ assert.equal(at("umtool/lib/paths.mjs"), null);
+ assert.equal(at("umtool/.e2e-song/data/planted/x/config.json"), "under umtool/.e2e-song");
+ assert.equal(at("umtool/.next-e2e/dev/server/a.js"), "under umtool/.next-e2e");
+ assert.equal(at("umtool/.env.local"), "under umtool/.env.local");
+ assert.equal(at(".git/config"), "under .git");
+ assert.equal(at("transcripts/channels/x/config.json"), "the corpus");
+ assert.equal(forbiddenTrace(path.resolve(REPO, "..", "elsewhere", "a.json"), dist), "outside the repo");
+});
+
+// The static checks above cannot see a value Turbopack reads through an
+// import: `path.join(CACHE_DIR, `${stamp}.${asMp3 ? "mp3" : "wav"}`)` in
+// umtool's clip-audio route took umtool's dot-directories into that route's
+// trace, the fixture and the e2e build's directory included (plans/release-15.md,
+// slice UT). So the last build's traces are read back, when there is one.
+//
+// What this can see: umtool/next.config.ts now excludes `.e2e-song`,
+// `.next-e2e` and `.env*` from every trace, so a pattern like that one shows
+// here only through a name the excludes miss -- `test-results/.last-run.json`
+// after an e2e run, `.next-shots`, the corpus, a path outside the repo. A
+// checkout with no e2e run behind it is blind to it; the fix at the call is
+// what keeps the route clean.
+test("umtool's last build traced no dot-directory, no corpus file and nothing outside the repo", (t) => {
+ const dist = path.join(UMTOOL, ".next");
+ const id = path.join(dist, "BUILD_ID");
+ if (!existsSync(path.join(dist, "server")) || !existsSync(id)) {
+ t.skip("no umtool build to read (umtool/.next/server); `pnpm --filter umtool exec next build` makes one");
+ return;
+ }
+ // A build older than the code that decides its traces judges code that is
+ // gone: after a merge or a checkout it would fail on a call already fixed.
+ // umtool runs under `next dev` day to day, so nothing else refreshes it.
+ const built = statSync(id).mtimeMs;
+ const newer = [path.join(UMTOOL, "next.config.ts"), ...umtoolModules()].filter((f) => statSync(f).mtimeMs > built);
+ if (newer.length) {
+ t.skip(
+ `umtool/.next was built ${new Date(built).toISOString()}, before ${path.relative(REPO, newer[0])}` +
+ ` (${newer.length} changed since); rebuild umtool (\`pnpm --filter umtool exec next build\`) to check its traces`,
+ );
+ return;
+ }
+ const bad = [];
+ for (const nft of traceFilesUnder(dist)) {
+ const { files } = JSON.parse(readFileSync(nft, "utf8"));
+ for (const f of files) {
+ const why = forbiddenTrace(path.resolve(path.dirname(nft), f), dist);
+ if (why) bad.push(`${path.relative(dist, nft)}: ${f} (${why})`);
+ }
+ }
+ assert.deepEqual(
+ bad.slice(0, 20),
+ [],
+ `${bad.length} traced file(s) no server needs; find the fs or path call whose value Turbopack could not bound (this file's header):\n` +
+ bad.slice(0, 20).join("\n"),
+ );
+});
+
test("no module the editor, the export or the homepage can bundle joins a cwd-derived path without opting out", () => {
const files = nextAppModules();
assert.ok(files.length > 500, `only ${files.length} modules found`);
diff --git a/umtool/app/api/clip/[key]/audio/route.ts b/umtool/app/api/clip/[key]/audio/route.ts
@@ -1,10 +1,9 @@
import { createHash } from "node:crypto";
import { existsSync } from "node:fs";
import { mkdir, readFile, writeFile, rename } from "node:fs/promises";
-import path from "node:path";
import { execFile } from "node:child_process";
import { promisify } from "node:util";
-import { CACHE_DIR } from "@/lib/paths";
+import { CACHE_DIR, cacheFile } from "@/lib/paths";
import { sourceWav } from "@/lib/clips";
import { readWavWindow, encodeWav, peakOver } from "@/lib/wav";
@@ -55,7 +54,8 @@ export async function GET(request: Request, ctx: { params: Promise<{ key: string
.update(`${video}|${from.toFixed(3)}|${to.toFixed(3)}|${asMp3 ? "mp3" : "wav"}`)
.digest("hex")
.slice(0, 16);
- const cached = path.join(CACHE_DIR, `${stamp}.${asMp3 ? "mp3" : "wav"}`);
+ // Through cacheFile, never a join here: see its comment in lib/paths.mjs.
+ const cached = cacheFile(`${stamp}.${asMp3 ? "mp3" : "wav"}`);
const type = asMp3 ? "audio/mpeg" : "audio/wav";
if (existsSync(cached)) {
@@ -79,8 +79,8 @@ export async function GET(request: Request, ctx: { params: Promise<{ key: string
const wav = encodeWav(x, region.sampleRate);
let body: Buffer = wav;
if (asMp3) {
- const tmpWav = path.join(CACHE_DIR, `${stamp}.in.wav`);
- const tmpMp3 = path.join(CACHE_DIR, `${stamp}.out.mp3`);
+ const tmpWav = cacheFile(`${stamp}.in.wav`);
+ const tmpMp3 = cacheFile(`${stamp}.out.mp3`);
await writeFile(tmpWav, wav);
await run("ffmpeg", [
"-nostdin", "-v", "error", "-y", "-i", tmpWav,
@@ -89,7 +89,7 @@ export async function GET(request: Request, ctx: { params: Promise<{ key: string
body = await readFile(tmpMp3);
await rename(tmpMp3, cached).catch(() => {});
} else {
- const tmp = `${cached}.tmp`;
+ const tmp = cacheFile(`${stamp}.wav.tmp`);
await writeFile(tmp, wav);
await rename(tmp, cached).catch(() => {});
}
diff --git a/umtool/app/api/clip/[key]/video/route.ts b/umtool/app/api/clip/[key]/video/route.ts
@@ -1,10 +1,9 @@
import { createHash } from "node:crypto";
import { existsSync } from "node:fs";
import { mkdir, readFile, rename } from "node:fs/promises";
-import path from "node:path";
import { execFile } from "node:child_process";
import { promisify } from "node:util";
-import { CACHE_DIR } from "@/lib/paths";
+import { CACHE_DIR, cacheFile } from "@/lib/paths";
import { sourceVideo } from "@/lib/clips";
const run = promisify(execFile);
@@ -44,7 +43,8 @@ export async function GET(request: Request, ctx: { params: Promise<{ key: string
.update(`v1|${video}|${from.toFixed(3)}|${to.toFixed(3)}`)
.digest("hex")
.slice(0, 16);
- const cached = path.join(CACHE_DIR, `${stamp}.mp4`);
+ // Through cacheFile, never a join here: see its comment in lib/paths.mjs.
+ const cached = cacheFile(`${stamp}.mp4`);
if (existsSync(cached)) {
return new Response(new Uint8Array(await readFile(cached)), {
headers: { "content-type": "video/mp4", "cache-control": "no-store" },
@@ -52,7 +52,7 @@ export async function GET(request: Request, ctx: { params: Promise<{ key: string
}
await mkdir(CACHE_DIR, { recursive: true });
- const tmp = `${cached}.tmp.mp4`;
+ const tmp = cacheFile(`${stamp}.mp4.tmp.mp4`);
// -ss BEFORE -i for the fast seek, then -t for the length. Re-encoded rather
// than copied because a stream copy starts at the previous keyframe, which
// would slide the picture against the audio by up to several seconds.
diff --git a/umtool/app/api/face/frame/route.ts b/umtool/app/api/face/frame/route.ts
@@ -1,11 +1,10 @@
import { createHash } from "node:crypto";
import { existsSync } from "node:fs";
import { mkdir, readFile, rename } from "node:fs/promises";
-import path from "node:path";
import { execFile } from "node:child_process";
import { promisify } from "node:util";
import { sourceVideo } from "@/lib/clips";
-import { CACHE_DIR } from "@/lib/paths";
+import { CACHE_DIR, cacheFile } from "@/lib/paths";
const run = promisify(execFile);
@@ -55,11 +54,12 @@ export async function GET(request: Request) {
.update(`face1|${video}|${at.toFixed(3)}|${w ?? "native"}`)
.digest("hex")
.slice(0, 16);
- const cached = path.join(CACHE_DIR, `${stamp}.jpg`);
+ // Through cacheFile, never a join here: see its comment in lib/paths.mjs.
+ const cached = cacheFile(`${stamp}.jpg`);
if (!existsSync(cached)) {
await mkdir(CACHE_DIR, { recursive: true });
- const tmp = `${cached}.tmp.jpg`;
+ const tmp = cacheFile(`${stamp}.jpg.tmp.jpg`);
// -ss BEFORE -i for the fast seek. When a width is asked for it is scaled to
// an even one with the aspect preserved; when it is not, the frame comes out
// at the source's own size and no mapping is needed at all.
diff --git a/umtool/lib/paths.mjs b/umtool/lib/paths.mjs
@@ -18,6 +18,21 @@ export { SONG_DATA, SONG_REPORTS };
// the data, not in the repo, and is safe to delete at any time.
export const CACHE_DIR = path.join(SONG_DATA, ".cache", "umtool");
+/**
+ * A file in CACHE_DIR, by name. A route names its cache files through this
+ * rather than joining CACHE_DIR itself. Turbopack reads a path it can see as a
+ * pattern of files to trace, in the join and in every fs call its value
+ * reaches, and CACHE_DIR is unknown to it (an env var or the home directory).
+ * So `path.join(CACHE_DIR, `${stamp}.${asMp3 ? "mp3" : "wav"}`)` in the
+ * clip-audio route was a pattern that reached into umtool's dot-directories:
+ * that route's trace listed the e2e fixture, the e2e server's build directory
+ * and `.env.local` (plans/release-15.md, slice UT). A value returned by a
+ * function from another module is opaque to it, so a call site traces nothing.
+ */
+export function cacheFile(name) {
+ return path.join(/* turbopackIgnore: true */ CACHE_DIR, name);
+}
+
// Render scratch: the body render and the cut background sit in the job temp
// dir ABOVE SONG_DATA, not inside it, because render-poly.mjs writes them next
// to its logs.
@@ -105,8 +120,10 @@ const dedupe = (list) => [...new Set(list.map((p) => path.resolve(p)))];
* REFERENCE: `next build` walked the whole corpus (hundreds of GB, `data/`
* symlinked to another drive) and was OOM-killed, or died on the first symlink
* out of the root. A worktree with no transcripts/ builds fine, which is how it
- * shipped. The comment is the documented per-expression opt-out; the values at
- * run time are unchanged. scripts/next-build-trace.test.mjs holds the line.
+ * shipped. The comment is Turbopack's per-expression opt-out (the form its own
+ * "whole project was traced" warning advises; the Next docs list the comment
+ * for import(), require(), require.resolve() and new Worker() only); the values
+ * at run time are unchanged. scripts/next-build-trace.test.mjs holds the line.
*/
export function findRepoRoot(start) {
let dir = path.resolve(/* turbopackIgnore: true */ start);
diff --git a/umtool/lib/paths.ts b/umtool/lib/paths.ts
@@ -19,6 +19,7 @@ import path from "node:path";
// ---------------------------------------------------------------------------
export {
CACHE_DIR,
+ cacheFile,
INDEX_DIR,
MEDIA_ROOTS,
MIX_CACHE,
diff --git a/umtool/next.config.ts b/umtool/next.config.ts
@@ -18,19 +18,37 @@ const nextConfig: NextConfig = {
// fail the whole module graph. Every page importing lib/projects then 500s
// with "Can't resolve 'cbor-x'", which names a package nothing here uses.
serverExternalPackages: ["lmdb"],
+ // No route's trace may list the e2e fixture (.e2e-song, where
+ // e2e/fixtures/make-fixture.mjs links the song data), the e2e server's own
+ // build directory (.next-e2e) or an env file: none is a run-time input. The
+ // clip-audio route's trace listed 1,704 such files (plans/release-15.md, slice
+ // UT). That was fixed at the call (lib/paths.mjs `cacheFile`); this is the
+ // second line, measured on its own: with the old route it takes the trace
+ // back to what the sibling routes list. scripts/next-build-trace.test.mjs
+ // reads the last build's traces back.
+ outputFileTracingExcludes: {
+ "/*": ["./.e2e-song/**/*", "./.next-e2e/**/*", "./.env*"],
+ },
turbopack: {
// Same reasoning as editor/next.config.ts: Turbopack infers the workspace
// root by walking up for the outermost lockfile, and a stray pnpm-lock.yaml
// above the checkout silently relocates it. Nothing here lives above the
// monorepo root, so pinning it costs nothing.
root: path.join(__dirname, ".."),
- // Suppress the harmless "whole project was traced unintentionally" NFT
- // warning, exactly as editor/next.config.ts does. It fires because the
- // server genuinely does runtime-dynamic fs reads it cannot statically bound
- // -- lib/paths.ts resolves SONG_DATA from an env var and the routes read
- // wav48/<video>.wav by name. We don't use `output: 'standalone'`, so the
- // .nft.json traces are never consumed and the over-tracing is cosmetic.
- // Scoped to this exact issue (path + title) so other warnings still surface.
+ // Silences the "whole project was traced unintentionally" warning, and only
+ // it (path + title), for one measured reason: 66 of the 68 routes trace
+ // umtool's own tree, its 361 files outside dot-directories, next.config.ts
+ // (the file the warning names) among them. A path or fs call on a value
+ // Turbopack cannot know (an env var, the home directory, a parameter) is a
+ // pattern over the project, and umtool has hundreds. Opting out every path
+ // op in the three path modules left 31 of the 68 routes clean; opting out
+ // all 319 in the 53 modules that have one left 49 clean, and the rest come
+ // through fs calls (lib/report/snapshots.mjs, among others). That walk skips
+ // dot-directories and does not enter symlinks, and the traces are not
+ // consumed while `output: "standalone"` stays off. The warning cannot tell
+ // that walk from one that does reach a dot-directory (both name
+ // next.config.ts), so that case is excluded above and checked after the
+ // build by scripts/next-build-trace.test.mjs instead.
ignoreIssue: [
{
path: "**/next.config.ts",
