"use client"; import { useEffect, useMemo, useRef, useState } from "react"; import { useWindowVirtualizer, type VirtualItem, } from "yt-dlp-transcript-common/components/virtualizer"; import { usePlayer } from "yt-dlp-transcript-common/components/PlayerProvider"; import { useMediaQuery } from "yt-dlp-transcript-common/lib/useMediaQuery"; import { Checkbox } from "yt-dlp-transcript-common/components/ui/checkbox"; import { VirtualRow } from "yt-dlp-transcript-common/components/VirtualRow"; import { fetchTranscript } from "yt-dlp-transcript-common/components/transcriptCache"; import { fetchDuplicateReport } from "yt-dlp-transcript-common/components/duplicatesCache"; import { formatDate } from "yt-dlp-transcript-common/lib/format"; import type { DuplicateCluster, DuplicateReport, DuplicateVideoRef, } from "yt-dlp-transcript-common/lib/duplicates"; type Detail = { title: string; webpageUrl: string }; type DetailMap = ReadonlyMap; type LoadState = | { status: "loading" } | { status: "ready"; report: DuplicateReport | null }; const PLATFORM_LABEL: Record = { youtube: "YouTube", rumble: "Rumble", odysee: "Odysee", twitch: "Twitch", kick: "Kick", archiveorg: "archive.org", bitchute: "BitChute", }; const MATCH_LABEL: Record = { "transcript-exact": "exact transcript", "transcript-near": "near transcript", "title-duration": "title + runtime", }; // MATCH_KINDS is not just a label list — it seeds the default filter state, and // the visible-cluster filter tests membership in it. A kind missing from here is // invisibly dropped from the page with no checkbox to turn it back on, so every // variant of DuplicateMatchKind must appear. (`title-duration` clusters are // filtered out at compose time unless a human confirmed them, so in practice // this checkbox only has anything to show once that happens — which is exactly // the case that must not silently vanish.) const MATCH_KINDS: DuplicateCluster["matchKind"][] = [ "transcript-exact", "transcript-near", "title-duration", ]; const RELATIONSHIPS = [ "same-channel", "cross-channel", "cross-platform", "clip-of-longer", ] as const; type Relationship = (typeof RELATIONSHIPS)[number]; function hasSameChannelDup(cluster: DuplicateCluster): boolean { return ( new Set(cluster.videoRefs.map((r) => r.channelSlug)).size < cluster.videoRefs.length ); } // Keep only members passing `keep`, recomputing the cross-* flags over the // survivors. A cluster with fewer than two members left is no longer a visible // duplicate, so it is dropped (returns null). function filterClusterMembers( cluster: DuplicateCluster, keep: (ref: DuplicateVideoRef) => boolean, ): DuplicateCluster | null { const videoRefs = cluster.videoRefs.filter(keep); if (videoRefs.length < 2) return null; return { ...cluster, videoRefs, crossPlatform: new Set(videoRefs.map((r) => r.platform)).size > 1, crossChannel: new Set(videoRefs.map((r) => r.channelSlug)).size > 1, }; } function relationshipsOf(cluster: DuplicateCluster): Set { const s = new Set(); if (hasSameChannelDup(cluster)) s.add("same-channel"); if (cluster.crossChannel) s.add("cross-channel"); if (cluster.crossPlatform) s.add("cross-platform"); if (cluster.contained) s.add("clip-of-longer"); return s; } export function DuplicatesClient() { const [state, setState] = useState({ status: "loading" }); const [details, setDetails] = useState(new Map()); // The site-filtered report is baked into the static build at /duplicates.json // (see common/bin/compose-site.ts). A missing file (404) or parse failure // means no detection has been run for this site — fall back to the empty // state rather than an error. useEffect(() => { let alive = true; fetchDuplicateReport().then((report) => { if (alive) setState({ status: "ready", report }); }); return () => { alive = false; }; }, []); const report = state.status === "ready" ? state.report : null; // Resolve every member's real title + source URL up front (the report's // stored titles are unreliable — VideoStat carries none). The same cached // fetch the modal uses, deduped per channel page, so opening a member is // instant afterwards and title-based filtering has data to work with. useEffect(() => { if (!report) return; const slugs = [ ...new Set(report.clusters.flatMap((c) => c.videoRefs.map((r) => r.slug))), ]; let alive = true; Promise.allSettled( slugs.map((slug) => fetchTranscript(slug).then( (d) => [slug, { title: d.title, webpageUrl: d.webpageUrl }] as const, ), ), ).then((results) => { if (!alive) return; const m = new Map(); for (const r of results) { if (r.status === "fulfilled") m.set(r.value[0], r.value[1]); } setDetails(m); }); return () => { alive = false; }; }, [report]); return ( ); } function DuplicatesView({ loading, report, details, }: { loading: boolean; report: DuplicateReport | null; details: DetailMap; }) { const clusters = useMemo(() => report?.clusters ?? [], [report]); // Platforms / channels present across the report, for the toggle lists. const presentPlatforms = useMemo( () => [...new Set(clusters.flatMap((c) => c.videoRefs.map((r) => r.platform)))], [clusters], ); const presentChannels = useMemo(() => { const m = new Map(); for (const c of clusters) for (const r of c.videoRefs) m.set(r.channelSlug, r.channel || r.channelSlug); return [...m.entries()].map(([slug, name]) => ({ slug, name })); }, [clusters]); // Filter state. `null` means "untouched" → fall back to the default derived // below: YouTube on (others off), all channels on. The first toggle // materializes the set from that default. const [platforms, setPlatforms] = useState | null>(null); const [channels, setChannels] = useState | null>(null); const [matchKinds, setMatchKinds] = useState>( () => new Set(MATCH_KINDS), ); const [relationships, setRelationships] = useState>( () => new Set(), ); const [query, setQuery] = useState(""); const platformSet = useMemo( () => platforms ?? new Set( presentPlatforms.includes("youtube") ? ["youtube"] : presentPlatforms, ), [platforms, presentPlatforms], ); const channelSet = useMemo( () => channels ?? new Set(presentChannels.map((c) => c.slug)), [channels, presentChannels], ); const q = query.trim().toLowerCase(); const visible = useMemo(() => { return clusters .map((c) => filterClusterMembers( c, (r) => platformSet.has(r.platform) && channelSet.has(r.channelSlug), ), ) .filter((c): c is DuplicateCluster => c !== null) .filter((c) => matchKinds.has(c.matchKind)) .filter((c) => { if (relationships.size === 0) return true; const rel = relationshipsOf(c); return [...relationships].some((r) => rel.has(r)); }) .filter((c) => { if (!q) return true; return c.videoRefs.some((r) => { const title = details.get(r.slug)?.title || r.title || r.slug; return ( title.toLowerCase().includes(q) || (r.channel || r.channelSlug).toLowerCase().includes(q) ); }); }); }, [clusters, platformSet, channelSet, matchKinds, relationships, q, details]); const sameChannelCount = useMemo( () => visible.filter(hasSameChannelDup).length, [visible], ); if (loading) { return

Loading duplicates…

; } return (
{report && (

{report.runConfig.thresholdSeconds === null ? "all durations" : `shorts ≤${report.runConfig.thresholdSeconds}s`} {" · "} {visible.length} {visible.length === 1 ? "cluster" : "clusters"} {" · "} {sameChannelCount} with same-channel duplicates {" · "} generated {new Date(report.generatedAt).toLocaleDateString()}

)} {report && clusters.length > 0 && ( setPlatforms(toggled(platformSet, p))} matchKinds={matchKinds} onToggleMatch={(k) => setMatchKinds(toggled(matchKinds, k))} relationships={relationships} onToggleRelationship={(r) => setRelationships(toggled(relationships, r)) } presentChannels={presentChannels} channels={channelSet} onToggleChannel={(c) => setChannels(toggled(channelSet, c))} /> )} {clusters.length === 0 ? (

No duplicate shorts have been detected for this site yet.

) : visible.length === 0 ? (

No duplicates match the current filters.

) : ( )}
); } function toggled(set: Set, value: T): Set { const next = new Set(set); if (next.has(value)) next.delete(value); else next.add(value); return next; } function FilterPanel({ query, onQuery, presentPlatforms, platforms, onTogglePlatform, matchKinds, onToggleMatch, relationships, onToggleRelationship, presentChannels, channels, onToggleChannel, }: { query: string; onQuery: (v: string) => void; presentPlatforms: string[]; platforms: Set; onTogglePlatform: (p: string) => void; matchKinds: Set; onToggleMatch: (k: string) => void; relationships: Set; onToggleRelationship: (r: Relationship) => void; presentChannels: { slug: string; name: string }[]; channels: Set; onToggleChannel: (c: string) => void; }) { const platformLabel = (n: number) => `platform${n === 1 ? "" : "s"}`; // Five filter rows above the clusters is most of a phone screen. Open where // there is room, closed where there isn't, and whatever the reader last chose // once they choose. const wide = useMediaQuery("(min-width: 1024px)"); const [userOpen, setUserOpen] = useState(null); const narrowed = (query.trim() !== "" ? 1 : 0) + (platforms.size !== presentPlatforms.length ? 1 : 0) + (matchKinds.size !== MATCH_KINDS.length ? 1 : 0) + (relationships.size > 0 ? 1 : 0) + (channels.size > 0 ? 1 : 0); return (
setUserOpen((e.currentTarget as HTMLDetailsElement).open)} className="text-sm text-muted-foreground" > Filters {platforms.size} of {presentPlatforms.length}{" "} {platformLabel(presentPlatforms.length)} {narrowed > 0 && ( · {narrowed} narrowed )}
{presentPlatforms.map((p) => ( onTogglePlatform(p)} > {PLATFORM_LABEL[p] ?? p} ))} {MATCH_KINDS.map((k) => ( onToggleMatch(k)} > {MATCH_LABEL[k]} ))} {RELATIONSHIPS.map((r) => ( onToggleRelationship(r)} > {r} ))} {presentChannels.length > 1 && ( {presentChannels.map((c) => ( onToggleChannel(c.slug)} > {c.name} ))} )}
); } function FilterRow({ label, children, }: { label: string; children: React.ReactNode; }) { return (
{label} {children}
); } function Check({ checked, onChange, children, }: { checked: boolean; onChange: () => void; children: React.ReactNode; }) { return ( // py-1.5 so the whole row is a tap target, not just the 16px box. ); } // Mirrors the search ResultCard chrome (bordered rounded card, tinted header // bar, divided rows) so duplicate clusters read like a familiar group of // search hits — the header bar carries the duplicate evidence instead of a // search query, and each member row is a hit-style title + channel·date meta. function DuplicateClusterCard({ cluster, details, }: { cluster: DuplicateCluster; details: DetailMap; }) { return (
{MATCH_LABEL[cluster.matchKind]} {cluster.score !== null && score {cluster.score.toFixed(2)}} {cluster.contained && clip-of-longer} {cluster.crossPlatform && cross-platform} {cluster.crossChannel && cross-channel} {hasSameChannelDup(cluster) && same-channel} {cluster.videoRefs.length} videos · ~{cluster.durationBucket}s
    {cluster.videoRefs.map((ref) => ( ))}
); } // The cluster list, windowed against the page scroll — a site with a real // duplicate report renders tens of thousands of pixels of cards, and all of it // used to be in the DOM at once. Same shape as SearchResults' list: a spacer // div of the full height, absolutely-positioned rows, and a `scrollMargin` that // tracks where the list starts on the page. const CLUSTER_ESTIMATE_PX = 140; const CLUSTER_GAP_PX = 12; function ClusterList({ clusters, details, }: { clusters: DuplicateCluster[]; details: DetailMap; }) { const listRef = useRef(null); const [scrollMargin, setScrollMargin] = useState(0); useEffect(() => { if (typeof window === "undefined") return; const update = () => { const el = listRef.current; if (!el) return; setScrollMargin(el.getBoundingClientRect().top + window.scrollY); }; update(); const ro = new ResizeObserver(update); if (listRef.current) ro.observe(listRef.current); ro.observe(document.body); window.addEventListener("resize", update); return () => { ro.disconnect(); window.removeEventListener("resize", update); }; }, []); const virtualizer = useWindowVirtualizer({ count: clusters.length, estimateSize: () => CLUSTER_ESTIMATE_PX + CLUSTER_GAP_PX, overscan: 4, scrollMargin, }); return (
    {virtualizer.getVirtualItems().map((vi: VirtualItem) => { const cluster = clusters[vi.index]; return ( ); })}
); } // The title is the primary clickable (opens the modal); the channel·date meta // sits alongside it (search-hit style), and the original-source URL sits on its // own row below so it's obvious the members are genuinely different videos. function MemberRow({ member, detail, }: { member: DuplicateVideoRef; detail: Detail | undefined; }) { const { openTranscript } = usePlayer(); const title = detail?.title || member.title || member.slug; return (
  • {member.channel || member.channelSlug} · {member.platform} ·{" "} {formatDate(member.uploadDate)}
    {detail?.webpageUrl && ( {detail.webpageUrl} )}
  • ); } function Badge({ children }: { children: React.ReactNode }) { return ( {children} ); }