commit 7314e90ff1e950806f1c3a486eb1bb414619adf8
parent d7fb767d7f16f9df8c78e31343b64ca4c47d4a3b
Author: I Mean I'm Just Saying <imeanimjustsaying@kiwifarms.st>
Date: Fri, 25 Sep 2026 11:17:50 -0400
compose: a manifest-only channel tree is copied, not deleted
reconcileChannelTree's signature ignores manifest.json, so a member whose
source is only its manifest — a social channel's transcripts tree, written with
pageCount 0 because it never enters the video scan — signed as "" and was
removed, while corpus.json advertises manifests.transcripts for every channel:
jeralyzer's thequartering-X 404. Such a tree now copies under a constant
signature; the contract, spec 4 and the readers are unchanged.
compose-site.test.ts (reconcileChannelTree exported): manifest-only copied,
unchanged skipped, no source removed.
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Diffstat:
2 files changed, 124 insertions(+), 2 deletions(-)
diff --git a/common/bin/compose-site.test.ts b/common/bin/compose-site.test.ts
@@ -0,0 +1,108 @@
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import path from "node:path";
+import { MANIFEST_ONLY_SIGNATURE, reconcileChannelTree } from "./compose-site";
+
+// Run with:
+// pnpm --filter yt-dlp-transcript-common test
+//
+// reconcileChannelTree materializes a site's member subset of a shared
+// per-channel tree (transcripts, subs, posts, digests) into public/. Importing
+// it runs nothing: compose-site's main() only runs as the entry point.
+//
+// The case that motivated this file: a social channel's transcripts tree is a
+// manifest.json with `pageCount: 0` and nothing else (it never enters the video
+// scan), and the signature ignores manifest.json — so it signed as "" and was
+// DELETED, while corpus.json advertised it. jeralyzer's thequartering-X 404.
+
+function fixture(): { src: string; dest: string; cleanup: () => void } {
+ const root = mkdtempSync(path.join(tmpdir(), "compose-site-"));
+ const src = path.join(root, "shared");
+ const dest = path.join(root, "public");
+ mkdirSync(src, { recursive: true });
+ return { src, dest, cleanup: () => rmSync(root, { recursive: true, force: true }) };
+}
+
+const MANIFEST = JSON.stringify({ pageCount: 0, count: 0 });
+const quiet = () => {};
+
+test("a manifest-only member is copied, not removed", async () => {
+ const { src, dest, cleanup } = fixture();
+ try {
+ mkdirSync(path.join(src, "posts-only"));
+ writeFileSync(path.join(src, "posts-only", "manifest.json"), MANIFEST);
+ const sigs = await reconcileChannelTree("transcripts", src, dest, ["posts-only"], {}, quiet);
+ assert.deepEqual(sigs, { "posts-only": MANIFEST_ONLY_SIGNATURE });
+ assert.equal(
+ readFileSync(path.join(dest, "posts-only", "manifest.json"), "utf8"),
+ MANIFEST,
+ );
+ } finally {
+ cleanup();
+ }
+});
+
+test("an unchanged member is skipped, a changed one re-copied", async () => {
+ const { src, dest, cleanup } = fixture();
+ try {
+ mkdirSync(path.join(src, "chan"));
+ writeFileSync(path.join(src, "chan", "manifest.json"), MANIFEST);
+ writeFileSync(path.join(src, "chan", "page-0000.json"), "[1]");
+ const logs: string[] = [];
+ const log = (m: string) => logs.push(m);
+ const first = await reconcileChannelTree("transcripts", src, dest, ["chan"], {}, log);
+ assert.ok(first.chan && first.chan !== MANIFEST_ONLY_SIGNATURE);
+
+ // Unchanged source: the dest is left alone — a marker written into it
+ // survives, which a re-copy would have wiped.
+ writeFileSync(path.join(dest, "chan", "marker"), "x");
+ const second = await reconcileChannelTree("transcripts", src, dest, ["chan"], first, log);
+ assert.deepEqual(second, first);
+ assert.ok(existsSync(path.join(dest, "chan", "marker")));
+ assert.match(logs[1], /0 copied, 1 unchanged/);
+
+ // The same holds for a manifest-only member on its second compose.
+ mkdirSync(path.join(src, "social"));
+ writeFileSync(path.join(src, "social", "manifest.json"), MANIFEST);
+ const third = await reconcileChannelTree("transcripts", src, dest, ["chan", "social"], second, log);
+ writeFileSync(path.join(dest, "social", "marker"), "x");
+ await reconcileChannelTree("transcripts", src, dest, ["chan", "social"], third, log);
+ assert.ok(existsSync(path.join(dest, "social", "marker")));
+
+ // Pages arriving change the signature, so the tree is re-copied.
+ writeFileSync(path.join(src, "social", "page-0000.json"), "[2]");
+ const fourth = await reconcileChannelTree("transcripts", src, dest, ["chan", "social"], third, log);
+ assert.notEqual(fourth.social, MANIFEST_ONLY_SIGNATURE);
+ assert.ok(existsSync(path.join(dest, "social", "page-0000.json")));
+ assert.ok(!existsSync(path.join(dest, "social", "marker")));
+ } finally {
+ cleanup();
+ }
+});
+
+test("a member with no source (not even a manifest) is removed, and a non-member pruned", async () => {
+ const { src, dest, cleanup } = fixture();
+ try {
+ mkdirSync(path.join(dest, "gone"), { recursive: true });
+ writeFileSync(path.join(dest, "gone", "manifest.json"), MANIFEST);
+ mkdirSync(path.join(dest, "former"), { recursive: true });
+ mkdirSync(path.join(src, "empty-dir"));
+ mkdirSync(path.join(dest, "empty-dir"), { recursive: true });
+ const sigs = await reconcileChannelTree(
+ "transcripts",
+ src,
+ dest,
+ ["gone", "empty-dir"],
+ { gone: "old" },
+ quiet,
+ );
+ assert.deepEqual(sigs, {});
+ assert.ok(!existsSync(path.join(dest, "gone")));
+ assert.ok(!existsSync(path.join(dest, "empty-dir")));
+ assert.ok(!existsSync(path.join(dest, "former")));
+ } finally {
+ cleanup();
+ }
+});
diff --git a/common/bin/compose-site.ts b/common/bin/compose-site.ts
@@ -581,7 +581,18 @@ async function dirSignature(
// the previous "rm -rf the whole tree then cp every member" with an in-place
// reconcile: only changed channels are re-copied, and channels no longer members
// are pruned. Returns the new per-slug signature map for the compose cache.
-async function reconcileChannelTree(
+//
+// A MANIFEST-ONLY TREE IS A CHANNEL WITH NOTHING IN IT, NOT A MISSING CHANNEL.
+// The signature ignores manifest.json (it churns), so a member whose source is
+// JUST the manifest — a social channel's transcripts tree, which buildIndex
+// writes with `pageCount: 0` because such a channel never enters the video scan
+// — used to sign as "" and be deleted, while corpus.json still advertised its
+// transcripts manifest to every reader: a 404 (jeralyzer's thequartering-X).
+// The manifest is exactly what "0 transcripts" means, so that tree is copied
+// under a constant signature and the contract stays unconditional.
+export const MANIFEST_ONLY_SIGNATURE = "manifest-only";
+
+export async function reconcileChannelTree(
kind: string,
srcRoot: string,
destRoot: string,
@@ -599,7 +610,10 @@ async function reconcileChannelTree(
const dest = path.join(destRoot, slug);
// Exclude the per-channel manifest.json — its `generatedAt` churns every
// mutation build; the page files capture real content changes.
- const sig = await dirSignature(src, "manifest.json");
+ let sig = await dirSignature(src, "manifest.json");
+ if (sig === "" && (await exists(path.join(src, "manifest.json")))) {
+ sig = MANIFEST_ONLY_SIGNATURE;
+ }
if (sig === "") {
// No source for this member — ensure no stale dest survives.
await rm(dest, { recursive: true, force: true });
