commit 7f23509456b148853e9d0c33e95d3c50a1d2ffa3
parent 310189c1176aa0c8416bbf7e56d1166d089b1cc2
Author: I Mean I'm Just Saying <imeanimjustsaying@kiwifarms.st>
Date: Sat, 26 Sep 2026 03:36:27 -0400
common: compose-hub and compose-site never write through a link in public/ (review fix 2)
In a git worktree the gitignored entries of export/public are symlinks into
the primary checkout, and writeFile / cp write through them.
e2e:2origin's build:hub therefore wrote the worktree's empty hub pool into
the primary's live hub-sites.json, corpus.json, llms.txt, robots.txt and
_headers. Every worktree run had to swap copies in first.
common/bin/_publicFile.ts:
- writePublicFile / copyPublicFile rm the path first. rm uses lstat, so
only a link is removed, never its target.
- ownDir replaces a linked directory with an empty real one before
anything is written into or pruned from it.
compose-hub routes all its writes through them. compose-site routes every
top-level file (_headers, site.json, corpus.json, llms.txt, robots.txt,
sitemap.xml, search-aliases.json, tags, duplicates) and the per-site
subs/posts/digests manifests through them, and reconcileChannelTree owns
its tree first. summaries/stats (replaceDir), archives, sw.js and
chart-templates already removed before writing.
Where the paths are real, as in the primary and a docker build's
/site/public, the result is what a plain write gave: the primary's
export/public holds no links, and per-site build dirs are real.
Tests: _publicFile (4). compose-hub: a worktree compose leaves the
primary's seven files' content and mtime as they were (red on the old
compose-hub). compose-site: a linked tree is owned and the primary's
untouched.
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Diffstat:
6 files changed, 272 insertions(+), 24 deletions(-)
diff --git a/common/bin/_publicFile.test.ts b/common/bin/_publicFile.test.ts
@@ -0,0 +1,112 @@
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import {
+ existsSync,
+ lstatSync,
+ mkdirSync,
+ mkdtempSync,
+ readFileSync,
+ readdirSync,
+ rmSync,
+ statSync,
+ symlinkSync,
+ utimesSync,
+ writeFileSync,
+} from "node:fs";
+import { tmpdir } from "node:os";
+import path from "node:path";
+import { copyPublicFile, ownDir, writePublicFile } from "./_publicFile";
+
+// Run with:
+// pnpm --filter yt-dlp-transcript-common test
+//
+// A worktree's export/public entries are symlinks into the primary checkout; a
+// compose run there must write into the worktree and leave the primary's files
+// exactly as they were.
+
+const OLD = new Date("2026-09-25T20:47:37Z");
+
+// A "primary" dir holding the real file and dir, and a "worktree" public dir
+// whose entries are links to them.
+function fixture() {
+ const root = mkdtempSync(path.join(tmpdir(), "public-file-"));
+ const primary = path.join(root, "primary");
+ const worktree = path.join(root, "worktree");
+ mkdirSync(path.join(primary, "subs"), { recursive: true });
+ mkdirSync(worktree, { recursive: true });
+ writeFileSync(path.join(primary, "hub-sites.json"), "[live pool]");
+ writeFileSync(path.join(primary, "subs", "manifest.json"), "live subs");
+ utimesSync(path.join(primary, "hub-sites.json"), OLD, OLD);
+ symlinkSync(path.join(primary, "hub-sites.json"), path.join(worktree, "hub-sites.json"));
+ symlinkSync(path.join(primary, "subs"), path.join(worktree, "subs"));
+ return { root, primary, worktree, cleanup: () => rmSync(root, { recursive: true, force: true }) };
+}
+
+test("writePublicFile replaces a link with a file and leaves its target as it was", async () => {
+ const { primary, worktree, cleanup } = fixture();
+ try {
+ await writePublicFile(path.join(worktree, "hub-sites.json"), "[]");
+ const at = path.join(worktree, "hub-sites.json");
+ assert.ok(lstatSync(at).isFile());
+ assert.equal(readFileSync(at, "utf8"), "[]");
+ const target = path.join(primary, "hub-sites.json");
+ assert.equal(readFileSync(target, "utf8"), "[live pool]");
+ assert.equal(statSync(target).mtimeMs, OLD.getTime());
+ } finally {
+ cleanup();
+ }
+});
+
+test("writePublicFile over a real file or a dangling link just writes it", async () => {
+ const { root, worktree, cleanup } = fixture();
+ try {
+ const real = path.join(worktree, "corpus.json");
+ writeFileSync(real, "old");
+ await writePublicFile(real, "new");
+ assert.equal(readFileSync(real, "utf8"), "new");
+
+ const dangling = path.join(worktree, "llms.txt");
+ symlinkSync(path.join(root, "nowhere", "llms.txt"), dangling);
+ await writePublicFile(dangling, "text");
+ assert.ok(lstatSync(dangling).isFile());
+ assert.equal(readFileSync(dangling, "utf8"), "text");
+ assert.ok(!existsSync(path.join(root, "nowhere")));
+ } finally {
+ cleanup();
+ }
+});
+
+test("copyPublicFile replaces a link with a copy and leaves its target as it was", async () => {
+ const { root, primary, worktree, cleanup } = fixture();
+ try {
+ const src = path.join(root, "sw-hub.js");
+ writeFileSync(src, "hub worker");
+ await copyPublicFile(src, path.join(worktree, "hub-sites.json"));
+ assert.ok(lstatSync(path.join(worktree, "hub-sites.json")).isFile());
+ assert.equal(readFileSync(path.join(worktree, "hub-sites.json"), "utf8"), "hub worker");
+ assert.equal(readFileSync(path.join(primary, "hub-sites.json"), "utf8"), "[live pool]");
+ } finally {
+ cleanup();
+ }
+});
+
+test("ownDir replaces a linked dir with an empty real one, keeps a real one, creates a missing one", async () => {
+ const { primary, worktree, cleanup } = fixture();
+ try {
+ const linked = path.join(worktree, "subs");
+ await ownDir(linked);
+ assert.ok(lstatSync(linked).isDirectory());
+ assert.deepEqual(readdirSync(linked), []);
+ assert.equal(readFileSync(path.join(primary, "subs", "manifest.json"), "utf8"), "live subs");
+
+ writeFileSync(path.join(linked, "keep.json"), "{}");
+ await ownDir(linked);
+ assert.deepEqual(readdirSync(linked), ["keep.json"]);
+
+ const missing = path.join(worktree, "posts");
+ await ownDir(missing);
+ assert.ok(lstatSync(missing).isDirectory());
+ } finally {
+ cleanup();
+ }
+});
diff --git a/common/bin/_publicFile.ts b/common/bin/_publicFile.ts
@@ -0,0 +1,39 @@
+// Writes into a SERVED public dir (export/public) that never follow a link.
+//
+// In a git worktree the gitignored entries of export/public are symlinks into
+// the primary checkout (plans/tools/implementer-rules.md), and writeFile / cp
+// write THROUGH a link: a worktree's compose — e2e:2origin's `build:hub` —
+// wrote its empty hub pool into the primary's hub-sites.json, corpus.json,
+// llms.txt, robots.txt and _headers. Removing the path first (fs.rm reads it
+// with lstat) removes only the link, so the write lands in this checkout and
+// the link's target is untouched. Where the path is a real file — the primary
+// checkout, a docker build's /site/public — the result is what a plain write
+// gave: the same path with the new content.
+//
+// ownDir is the same rule for a directory a compose writes INTO: a linked
+// directory is replaced by an empty real one (its target untouched), a real one
+// is kept as it is, and a missing one is created.
+
+import { cp, lstat, mkdir, rm, unlink, writeFile } from "node:fs/promises";
+
+export async function writePublicFile(
+ file: string,
+ data: string | Uint8Array,
+): Promise<void> {
+ await rm(file, { force: true });
+ await writeFile(file, data);
+}
+
+export async function copyPublicFile(src: string, dest: string): Promise<void> {
+ await rm(dest, { force: true });
+ await cp(src, dest);
+}
+
+export async function ownDir(dir: string): Promise<void> {
+ try {
+ if ((await lstat(dir)).isSymbolicLink()) await unlink(dir);
+ } catch (err) {
+ if ((err as NodeJS.ErrnoException).code !== "ENOENT") throw err;
+ }
+ await mkdir(dir, { recursive: true });
+}
diff --git a/common/bin/compose-hub.test.ts b/common/bin/compose-hub.test.ts
@@ -1,6 +1,17 @@
import { test } from "node:test";
import assert from "node:assert/strict";
-import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
+import {
+ existsSync,
+ lstatSync,
+ mkdirSync,
+ mkdtempSync,
+ readFileSync,
+ rmSync,
+ statSync,
+ symlinkSync,
+ utimesSync,
+ writeFileSync,
+} from "node:fs";
import { tmpdir } from "node:os";
import path from "node:path";
import { getPaths, type Paths } from "../lib/paths";
@@ -81,3 +92,41 @@ test("hub-sites.json publishes every accent as a hex: an id becomes its on-dark
rmSync(root, { recursive: true, force: true });
}
});
+
+test("in a worktree, compose-hub writes its own files and never through the links into the primary", async () => {
+ const root = mkdtempSync(path.join(tmpdir(), "compose-hub-"));
+ const log = console.log;
+ try {
+ const paths = fixturePaths(root);
+ // The primary checkout's composed hub files, and the worktree's public/
+ // entries linking to them (as the worktree seed does).
+ const primary = path.join(root, "primary-public");
+ mkdirSync(primary, { recursive: true });
+ const old = new Date("2026-09-25T20:47:37Z");
+ const files = ["hub-sites.json", "corpus.json", "llms.txt", "robots.txt", "_headers", "sw.js", "hub-summary.json"];
+ for (const f of files) {
+ writeFileSync(path.join(primary, f), `live ${f}`);
+ utimesSync(path.join(primary, f), old, old);
+ symlinkSync(path.join(primary, f), path.join(paths.exportPublicDir, f));
+ }
+ console.log = () => {};
+ await main({ paths });
+ console.log = log;
+ for (const f of files) {
+ const target = path.join(primary, f);
+ assert.equal(readFileSync(target, "utf8"), `live ${f}`, `${f} in the primary`);
+ assert.equal(statSync(target).mtimeMs, old.getTime(), `${f}'s mtime in the primary`);
+ }
+ // The worktree has its own: an empty pool, and no summary (no index).
+ const at = (f: string) => path.join(paths.exportPublicDir, f);
+ assert.ok(lstatSync(at("hub-sites.json")).isFile());
+ assert.equal(readFileSync(at("hub-sites.json"), "utf8"), "[]");
+ for (const f of ["corpus.json", "llms.txt", "robots.txt", "_headers"]) {
+ assert.ok(lstatSync(at(f)).isFile(), `${f} is the worktree's own file`);
+ }
+ assert.ok(!existsSync(at("hub-summary.json")));
+ } finally {
+ console.log = log;
+ rmSync(root, { recursive: true, force: true });
+ }
+});
diff --git a/common/bin/compose-hub.ts b/common/bin/compose-hub.ts
@@ -16,7 +16,7 @@
import path from "node:path";
import { existsSync } from "node:fs";
-import { cp, rm, writeFile, access } from "node:fs/promises";
+import { cp, rm, access } from "node:fs/promises";
import { getPaths, type Paths } from "../lib/paths";
import { accentHex } from "../lib/accent";
import { listSites, resolveHubUrl } from "../lib/site";
@@ -32,6 +32,7 @@ import { HUB_CORS_PATHS, renderHeadersFile } from "../lib/archive/headers";
import { buildPoolSummary } from "../controller/poolSummary";
import { HUB_SUMMARY_FILE, toHubSummary } from "../lib/hubSummary";
import { runIfEntryPoint } from "./_cli";
+import { writePublicFile } from "./_publicFile";
async function exists(p: string): Promise<boolean> {
try {
@@ -66,7 +67,7 @@ async function composeHubSummary(
statsDir: path.join(paths.exportIndexDir, "hub-stats"),
});
const hubSummary = toHubSummary(summary);
- await writeFile(dest, JSON.stringify(hubSummary));
+ await writePublicFile(dest, JSON.stringify(hubSummary));
return `hub-summary.json covers ${hubSummary.sites.length} official instance(s)`;
} catch (err) {
await rm(dest, { force: true });
@@ -99,7 +100,10 @@ export async function main(opts: { paths?: Paths } = {}): Promise<void> {
contract: SITE_DESCRIPTOR_VERSION,
});
}
- await writeFile(
+ // Every file below is written with writePublicFile, never through a link:
+ // in a worktree these paths are links into the primary checkout, whose live
+ // hub files a worktree compose (e2e:2origin's build:hub) used to overwrite.
+ await writePublicFile(
path.join(publicDir, "hub-sites.json"),
JSON.stringify(builtins),
);
@@ -113,20 +117,20 @@ export async function main(opts: { paths?: Paths } = {}): Promise<void> {
hubUrl: hub.siteUrl,
generatedAt: new Date().toISOString(),
});
- await writeFile(
+ await writePublicFile(
path.join(publicDir, "corpus.json"),
JSON.stringify(hubCorpus),
);
- await writeFile(
+ await writePublicFile(
path.join(publicDir, "llms.txt"),
renderHubLlmsTxt(hubCorpus),
);
- await writeFile(
+ await writePublicFile(
path.join(publicDir, "robots.txt"),
renderRobotsTxt({ siteUrl: hub.siteUrl }),
);
- await writeFile(
+ await writePublicFile(
path.join(publicDir, "_headers"),
renderHeadersFile("compose-hub.ts", HUB_CORS_PATHS),
);
diff --git a/common/bin/compose-site.test.ts b/common/bin/compose-site.test.ts
@@ -1,6 +1,16 @@
import { test } from "node:test";
import assert from "node:assert/strict";
-import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
+import {
+ existsSync,
+ lstatSync,
+ mkdirSync,
+ mkdtempSync,
+ readdirSync,
+ readFileSync,
+ rmSync,
+ symlinkSync,
+ writeFileSync,
+} from "node:fs";
import { tmpdir } from "node:os";
import path from "node:path";
import { MANIFEST_ONLY_SIGNATURE, reconcileChannelTree } from "./compose-site";
@@ -106,3 +116,28 @@ test("a member with no source (not even a manifest) is removed, and a non-member
cleanup();
}
});
+
+test("a linked tree (a worktree's public/<tree>) becomes this checkout's own; the primary's is untouched", async () => {
+ const { src, dest, cleanup } = fixture();
+ try {
+ // The primary's composed tree, and the worktree's public/transcripts
+ // linking to it.
+ const primaryTree = path.join(path.dirname(dest), "primary-transcripts");
+ mkdirSync(path.join(primaryTree, "live-chan"), { recursive: true });
+ writeFileSync(path.join(primaryTree, "live-chan", "page-0000.json"), "live");
+ symlinkSync(primaryTree, dest);
+ mkdirSync(path.join(src, "chan"));
+ writeFileSync(path.join(src, "chan", "manifest.json"), MANIFEST);
+ writeFileSync(path.join(src, "chan", "page-0000.json"), "[]");
+
+ await reconcileChannelTree("transcripts", src, dest, ["chan"], {}, quiet);
+
+ assert.ok(lstatSync(dest).isDirectory() && !lstatSync(dest).isSymbolicLink());
+ assert.deepEqual(readdirSync(dest), ["chan"]);
+ // Nothing was copied into, or pruned from, the primary's tree.
+ assert.deepEqual(readdirSync(primaryTree), ["live-chan"]);
+ assert.equal(readFileSync(path.join(primaryTree, "live-chan", "page-0000.json"), "utf8"), "live");
+ } finally {
+ cleanup();
+ }
+});
diff --git a/common/bin/compose-site.ts b/common/bin/compose-site.ts
@@ -61,15 +61,21 @@ import {
type ArchiveManifestEntry,
} from "../lib/archiveOptions";
import { runIfEntryPoint } from "./_cli";
+import { copyPublicFile, ownDir, writePublicFile } from "./_publicFile";
// Emit the public federation contract: /site.json (branding + channels +
// freshness) and the CORS _headers file. Emitted for EVERY site regardless of
// whether it ships a PWA — a dumb instance is still federatable.
+//
+// Every file this compose puts in public/ is written with writePublicFile /
+// copyPublicFile, and every tree it writes into is ownDir'd first, so nothing
+// is written THROUGH a link: in a git worktree public/'s entries are links into
+// the primary checkout (_publicFile.ts).
async function emitFederationFiles(
site: Site,
paths: ReturnType<typeof getPaths>,
): Promise<void> {
- await writeFile(
+ await writePublicFile(
path.join(paths.exportPublicDir, "_headers"),
renderHeadersFile("compose-site.ts"),
);
@@ -81,7 +87,7 @@ async function emitFederationFiles(
pwa: shipsPwa(site),
hubUrl: resolveHubUrl(site),
});
- await writeFile(
+ await writePublicFile(
path.join(paths.exportPublicDir, "site.json"),
JSON.stringify(descriptor),
);
@@ -156,15 +162,15 @@ async function emitAiFiles(paths: ReturnType<typeof getPaths>): Promise<void> {
digestCounts,
hasTags,
});
- await writeFile(
+ await writePublicFile(
path.join(paths.exportPublicDir, "corpus.json"),
JSON.stringify(corpus),
);
- await writeFile(
+ await writePublicFile(
path.join(paths.exportPublicDir, "llms.txt"),
renderSiteLlmsTxt(corpus),
);
- await writeFile(
+ await writePublicFile(
path.join(paths.exportPublicDir, "robots.txt"),
renderRobotsTxt({ siteUrl: descriptor.siteUrl }),
);
@@ -178,7 +184,7 @@ async function emitAiFiles(paths: ReturnType<typeof getPaths>): Promise<void> {
if (await exists(path.join(paths.exportPublicDir, DUPLICATES_FILENAME))) {
routes.push("/duplicates");
}
- await writeFile(
+ await writePublicFile(
sitemapPath,
renderSitemapXml({ siteUrl: descriptor.siteUrl, routes }),
);
@@ -600,7 +606,9 @@ export async function reconcileChannelTree(
prev: Record<string, string>,
log: (m: string) => void,
): Promise<Record<string, string>> {
- await mkdir(destRoot, { recursive: true });
+ // A linked tree (a worktree's public/<tree> → the primary's) becomes this
+ // checkout's own before anything is copied into or pruned from it.
+ await ownDir(destRoot);
const next: Record<string, string> = {};
const memberSet = new Set(memberSlugs);
let copied = 0;
@@ -732,7 +740,8 @@ export async function main(
"manifest.json",
);
if (await exists(subsManifestSrc)) {
- await cp(subsManifestSrc, path.join(paths.exportSubsDir, "manifest.json"));
+ await ownDir(paths.exportSubsDir);
+ await copyPublicFile(subsManifestSrc, path.join(paths.exportSubsDir, "manifest.json"));
}
// Same for the per-site posts manifest (which channels carry posts).
const postsManifestSrc = path.join(
@@ -742,8 +751,8 @@ export async function main(
"manifest.json",
);
if (await exists(postsManifestSrc)) {
- await mkdir(paths.exportPostsDir, { recursive: true });
- await cp(postsManifestSrc, path.join(paths.exportPostsDir, "manifest.json"));
+ await ownDir(paths.exportPostsDir);
+ await copyPublicFile(postsManifestSrc, path.join(paths.exportPostsDir, "manifest.json"));
}
// Same for the per-site digests manifest (which channels carry digests).
const digestsManifestSrc = path.join(
@@ -753,8 +762,8 @@ export async function main(
"manifest.json",
);
if (await exists(digestsManifestSrc)) {
- await mkdir(paths.exportDigestsDir, { recursive: true });
- await cp(
+ await ownDir(paths.exportDigestsDir);
+ await copyPublicFile(
digestsManifestSrc,
path.join(paths.exportDigestsDir, "manifest.json"),
);
@@ -777,7 +786,7 @@ export async function main(
// staging step. Always emitted — a fresh install ships the seeded defaults so
// the viewer's suggestion chip works out of the box.
const aliases = effectiveSiteAliases(paths, siteId);
- await writeFile(
+ await writePublicFile(
path.join(paths.exportPublicDir, "search-aliases.json"),
JSON.stringify({ aliases }),
);
@@ -808,7 +817,7 @@ export async function main(
const publishedTags = publishedTagsFrom(tagDefs, tagCounts);
const tagsDest = path.join(paths.exportPublicDir, TAGS_FILENAME);
if (publishedTags.tags.length > 0) {
- await writeFile(tagsDest, JSON.stringify(publishedTags));
+ await writePublicFile(tagsDest, JSON.stringify(publishedTags));
} else {
await rm(tagsDest, { force: true });
}
@@ -900,7 +909,7 @@ export async function main(
},
clusters,
};
- await writeFile(dupDest, JSON.stringify(filtered));
+ await writePublicFile(dupDest, JSON.stringify(filtered));
wrote = true;
}
}
