import { test } from "node:test"; import assert from "node:assert/strict"; import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import path from "node:path"; import type { Paths } from "../lib/paths"; import { DUPLICATES_FILENAME, DUPLICATE_REPORT_VERSION, type DuplicateCluster, type DuplicateVideoRef, } from "../lib/duplicates"; import { buildDigestClusterPlan, dirBySlugForCluster, planSlugForDir, } from "./digestSharing"; // Run with: // pnpm --filter yt-dlp-transcript-common exec tsx --test common/controller/digestSharing.test.ts async function withPaths(fn: (paths: Paths) => Promise): Promise { const dir = await mkdtemp(path.join(tmpdir(), "ttb-digest-sharing-")); const paths = { transcriptsDir: dir, channelsDir: path.join(dir, "channels"), } as Paths; await mkdir(paths.transcriptsDir, { recursive: true }); try { await fn(paths); } finally { await rm(dir, { recursive: true, force: true }); } } function ref( channelSlug: string, id: string, extra: Partial = {}, ): DuplicateVideoRef { return { slug: `${channelSlug}/${id}`, channelSlug, channel: channelSlug, platform: "youtube", id, title: "t", duration: 600, uploadDate: "20240101", hasTranscript: true, ...extra, }; } async function writeReport( paths: Paths, clusters: DuplicateCluster[], ): Promise { await writeFile( path.join(paths.transcriptsDir, DUPLICATES_FILENAME), JSON.stringify({ version: DUPLICATE_REPORT_VERSION, generatedAt: new Date(0).toISOString(), runConfig: { thresholdSeconds: null, durationToleranceSeconds: 2, nearThreshold: 0.35, containmentThreshold: 0.8, shingleSize: 5, }, totals: { videosScanned: 2, clusters: clusters.length, videosInClusters: 2 }, clusters, }), ); } function cluster( clusterId: string, videoRefs: DuplicateVideoRef[], extra: Partial = {}, ): DuplicateCluster { return { clusterId, matchKind: "transcript-near", score: 0.9, contained: false, durationBucket: 600, crossPlatform: true, crossChannel: true, videoRefs, canonicalSlug: videoRefs[0].slug, ...extra, }; } // The bug this whole mapping exists for: a slug is `${channelSlug}/${id}` while // the batch runner enumerates DIRECTORIES, and those disagree for 14.5% of the // real corpus (every Rumble re-upload). Before the mapping, the lookup below // missed and every such mirror was regenerated instead of shared. test("plan maps a directory name to its slug when the two differ", async () => { await withPaths(async (paths) => { await writeReport(paths, [ cluster("c1", [ ref("chan-yt", "canon1"), // The Rumble shape: on-disk dir `v1007ay`, metadata id `vxe1ae`. ref("chan-rumble", "vxe1ae", { videoDir: "v1007ay" }), ]), ]); const plan = await buildDigestClusterPlan(paths, { overrides: null }); assert.equal( planSlugForDir(plan, "chan-rumble", "v1007ay"), "chan-rumble/vxe1ae", "the directory name must resolve to the metadata-id slug", ); assert.equal( plan.bySlug.get(planSlugForDir(plan, "chan-rumble", "v1007ay"))?.kind, "mirror", "and that slug must then hit the plan as a mirror", ); assert.equal( plan.dirBySlug.get("chan-rumble/vxe1ae"), "v1007ay", "the reverse direction is what lets the sharing pass open the mirror's files", ); }); }); test("a member whose dir equals its id is absent from the mapping", async () => { await withPaths(async (paths) => { await writeReport(paths, [ cluster("c1", [ref("chan-a", "same1"), ref("chan-b", "same2")]), ]); const plan = await buildDigestClusterPlan(paths, { overrides: null }); assert.equal(plan.dirBySlug.size, 0); assert.equal(plan.slugByDir.size, 0); // The fallback is the old behaviour, and it is the correct one here. assert.equal(planSlugForDir(plan, "chan-a", "same1"), "chan-a/same1"); }); }); // A report written before `videoDir` existed must not regress: it keeps the // pre-mapping behaviour rather than resolving to nothing. test("an old report with no videoDir falls back to the id", async () => { await withPaths(async (paths) => { await writeReport(paths, [ cluster("c1", [ref("chan-a", "aaa"), ref("chan-b", "bbb")]), ]); const plan = await buildDigestClusterPlan(paths, { overrides: null }); assert.equal(planSlugForDir(plan, "chan-b", "bbb"), "chan-b/bbb"); assert.equal(planSlugForDir(null, "chan-b", "bbb"), "chan-b/bbb"); }); }); test("dirBySlugForCluster carries only the members that differ", () => { const c = cluster("c1", [ ref("chan-yt", "canon1"), ref("chan-rumble", "vxe1ae", { videoDir: "v1007ay" }), ref("chan-odysee", "plain", { videoDir: "plain" }), ]); const map = dirBySlugForCluster(c); assert.deepEqual([...map], [["chan-rumble/vxe1ae", "v1007ay"]]); }); // The directory mapping is recorded for EVERY cluster, including ones that // share nothing — their members are still generated, and a caller holding the // plan still needs a correct path for them. test("an unshareable cluster still contributes its directory mapping", async () => { await withPaths(async (paths) => { await writeReport(paths, [ cluster( "c1", [ ref("chan-yt", "canon1"), ref("chan-rumble", "vxe1ae", { videoDir: "v1007ay" }), ], // needsReview and unconfirmed → clusterMaySharePartial fails closed. { needsReview: true }, ), ]); const plan = await buildDigestClusterPlan(paths, { overrides: null }); assert.equal(plan.bySlug.size, 0, "it shares nothing"); assert.equal(plan.independentClusters, 1); assert.equal(plan.dirBySlug.get("chan-rumble/vxe1ae"), "v1007ay"); }); });