commit 103b21f4e2e2bd5eaf081d2d05eec6f8f0e1533a
parent ed3957260bcc623fc8bf1051b81a8a31273e8557
Author: I Mean I'm Just Saying <imeanimjustsaying@kiwifarms.st>
Date: Thu, 16 Apr 2026 22:13:52 -0400
optimize for bigger dataset
Diffstat:
6 files changed, 141 insertions(+), 144 deletions(-)
diff --git a/app/PlayerProvider.tsx b/app/PlayerProvider.tsx
@@ -13,6 +13,7 @@ import {
import type ReactPlayerType from "react-player";
import { fetchTranscript } from "./transcriptCache";
import { useUrlParams, writeUrlParams } from "./urlState";
+import { formatDate, formatDuration } from "@/lib/format";
const ReactPlayer = dynamic(() => import("react-player/youtube"), {
ssr: false,
@@ -20,7 +21,7 @@ const ReactPlayer = dynamic(() => import("react-player/youtube"), {
export type Cue = { start: number; end: number; text: string };
-export type TranscriptSummary = {
+export type TranscriptData = {
slug: string;
id: string;
title: string;
@@ -28,9 +29,6 @@ export type TranscriptSummary = {
date: string;
duration: string;
channel: string;
-};
-
-export type TranscriptData = TranscriptSummary & {
description?: string;
cues?: Cue[];
};
@@ -40,6 +38,11 @@ export type DisplayMode = "modal" | "mini" | "hidden";
type Detail = {
slug: string;
+ id: string;
+ title: string;
+ uploadDate: string;
+ duration: number;
+ channel: string;
description?: string;
cues?: Cue[];
};
@@ -88,10 +91,8 @@ function toHMS(seconds: number): string {
export function PlayerProvider({
children,
- transcripts,
}: {
children: React.ReactNode;
- transcripts: TranscriptSummary[];
}) {
const { v: urlSlug, t: urlTime } = useUrlParams();
const [detail, setDetail] = useState<Detail | null>(null);
@@ -110,25 +111,25 @@ export function PlayerProvider({
const pendingSeekRef = useRef<number | null>(null);
const readyForSlugRef = useRef<string | null>(null);
- const bySlug = useMemo(() => {
- const m = new Map<string, TranscriptSummary>();
- for (const t of transcripts) m.set(t.slug, t);
- return m;
- }, [transcripts]);
-
const activeSlug = urlSlug;
const displayMode: DisplayMode =
displayState?.slug === activeSlug ? displayState.mode : "modal";
const modalOpen = activeSlug !== null && displayMode === "modal";
- const summary = activeSlug ? bySlug.get(activeSlug) ?? null : null;
const detailMatches = detail?.slug === activeSlug;
const data: TranscriptData | null = useMemo(() => {
- if (!summary) return null;
- if (detailMatches && detail) {
- return { ...summary, description: detail.description, cues: detail.cues };
- }
- return { ...summary };
- }, [summary, detail, detailMatches]);
+ if (!detailMatches || !detail) return null;
+ return {
+ slug: detail.slug,
+ id: detail.id,
+ title: detail.title,
+ uploadDate: detail.uploadDate,
+ date: formatDate(detail.uploadDate),
+ duration: formatDuration(detail.duration),
+ channel: detail.channel,
+ description: detail.description,
+ cues: detail.cues,
+ };
+ }, [detail, detailMatches]);
const status: Status = !activeSlug
? "idle"
: detailMatches
@@ -213,7 +214,6 @@ export function PlayerProvider({
useEffect(() => {
if (!urlSlug) return;
- if (!bySlug.has(urlSlug)) return;
readyForSlugRef.current = null;
pendingSeekRef.current = urlTime ?? 0;
let cancelled = false;
@@ -222,20 +222,25 @@ export function PlayerProvider({
if (cancelled) return;
setDetail({
slug: urlSlug,
+ id: full.id,
+ title: full.title,
+ uploadDate: full.uploadDate,
+ duration: full.duration,
+ channel: full.channel,
description: full.description,
cues: full.cues,
});
})
.catch(() => {
if (cancelled) return;
- setDetail({ slug: urlSlug });
+ setDetail(null);
});
return () => {
cancelled = true;
};
// urlTime intentionally omitted — t changes shouldn't refetch
// eslint-disable-next-line react-hooks/exhaustive-deps
- }, [urlSlug, bySlug]);
+ }, [urlSlug]);
useEffect(() => {
if (urlTime === null) return;
diff --git a/app/TranscriptSearch.tsx b/app/TranscriptSearch.tsx
@@ -3,18 +3,11 @@
import { useEffect, useMemo, useState } from "react";
import { usePlayer } from "./PlayerProvider";
import { fetchTranscript } from "./transcriptCache";
+import { fetchSummaries } from "./summariesCache";
import { useUrlParams, writeUrlParams } from "./urlState";
-import type { TranscriptDetail } from "@/lib/transcripts";
+import type { DisplaySummary, TranscriptDetail } from "@/lib/transcripts";
-type Summary = {
- slug: string;
- id: string;
- title: string;
- uploadDate: string;
- date: string;
- duration: string;
- channel: string;
-};
+type Summary = DisplaySummary;
type Hit = {
start: number;
@@ -31,11 +24,7 @@ type HitGroup = {
const MAX_HITS = 1000;
-export default function TranscriptSearch({
- transcripts,
-}: {
- transcripts: Summary[];
-}) {
+export default function TranscriptSearch() {
const {
q: query,
re: useRegex,
@@ -52,13 +41,35 @@ export default function TranscriptSearch({
return !(params.has("v") && (params.get("q") ?? "") !== "");
});
+ const [transcripts, setTranscripts] = useState<Summary[] | null>(null);
const [fetched, setFetched] = useState<Record<string, TranscriptDetail>>({});
+ const [hitLimit, setHitLimit] = useState(MAX_HITS);
const { openTranscript } = usePlayer();
const trimmed = searchExecuted ? query.trim() : "";
useEffect(() => {
+ setHitLimit(MAX_HITS);
+ }, [trimmed, useRegex]);
+
+ useEffect(() => {
if (!trimmed) return;
+ if (transcripts) return;
+ let cancelled = false;
+ fetchSummaries()
+ .then((list) => {
+ if (cancelled) return;
+ setTranscripts(list);
+ })
+ .catch(() => {});
+ return () => {
+ cancelled = true;
+ };
+ }, [trimmed, transcripts]);
+
+ useEffect(() => {
+ if (!trimmed) return;
+ if (!transcripts) return;
let cancelled = false;
for (const t of transcripts) {
if (fetched[t.slug]) continue;
@@ -77,8 +88,10 @@ export default function TranscriptSearch({
}, [trimmed, transcripts, fetched]);
const loadedCount = Object.keys(fetched).length;
- const totalCount = transcripts.length;
- const fetching = Boolean(trimmed) && loadedCount < totalCount;
+ const totalCount = transcripts?.length ?? 0;
+ const indexLoading = Boolean(trimmed) && transcripts === null;
+ const fetching =
+ Boolean(trimmed) && transcripts !== null && loadedCount < totalCount;
const regex = useMemo<{ re: RegExp | null; error: string | null }>(() => {
if (!trimmed || !useRegex) return { re: null, error: null };
@@ -89,31 +102,13 @@ export default function TranscriptSearch({
}
}, [trimmed, useRegex]);
- const matches = (haystack: string): boolean => {
- if (!trimmed) return true;
- if (useRegex) return regex.re ? regex.re.test(haystack) : false;
- return haystack.toLowerCase().includes(trimmed.toLowerCase());
- };
-
- const summaryBySlug = useMemo(() => {
- const m = new Map<string, Summary>();
- for (const t of transcripts) m.set(t.slug, t);
- return m;
- }, [transcripts]);
-
- const titleMatches = useMemo(() => {
- if (!trimmed) return transcripts;
- if (useRegex && !regex.re) return [];
- return transcripts.filter((t) => matches(t.title));
- // eslint-disable-next-line react-hooks/exhaustive-deps
- }, [trimmed, transcripts, useRegex, regex.re]);
-
const { groups, totalHits, capped } = useMemo<{
groups: HitGroup[];
totalHits: number;
capped: boolean;
}>(() => {
if (!trimmed) return { groups: [], totalHits: 0, capped: false };
+ if (!transcripts) return { groups: [], totalHits: 0, capped: false };
if (useRegex && !regex.re)
return { groups: [], totalHits: 0, capped: false };
const groups: HitGroup[] = [];
@@ -147,23 +142,23 @@ export default function TranscriptSearch({
text: crosses ? windowText : cur.text,
});
total++;
- if (total >= MAX_HITS) break;
+ if (total >= hitLimit) break;
}
if (hits.length > 0) {
groups.push({
slug: t.slug,
title: t.title,
channel: t.channel,
- date: summaryBySlug.get(t.slug)?.date ?? "",
+ date: t.date,
hits,
});
}
- if (total >= MAX_HITS) {
+ if (total >= hitLimit) {
return { groups, totalHits: total, capped: true };
}
}
return { groups, totalHits: total, capped: false };
- }, [trimmed, transcripts, fetched, summaryBySlug, useRegex, regex.re]);
+ }, [trimmed, transcripts, fetched, useRegex, regex.re, hitLimit]);
return (
<div className="flex flex-col gap-6">
@@ -182,7 +177,7 @@ export default function TranscriptSearch({
placeholder={
useRegex
? "Regex pattern (case-insensitive)…"
- : "Search titles and transcripts..."
+ : "Search transcripts..."
}
className={`w-full rounded-lg border ${
regex.error
@@ -214,49 +209,37 @@ export default function TranscriptSearch({
</form>
{!trimmed && (
- <TranscriptGrid
- items={transcripts}
- onOpen={(slug) => openTranscript(slug)}
- activeSlug={activeVideo}
- />
+ <p className="text-sm text-zinc-500">
+ Enter a query to search transcripts.
+ </p>
)}
{trimmed && (
<>
<section>
- <h2 className="text-sm font-medium text-zinc-500 mb-2">
- Titles ({titleMatches.length})
- </h2>
- {titleMatches.length > 0 ? (
- <TranscriptGrid
- items={titleMatches}
- onOpen={(slug) => openTranscript(slug)}
- activeSlug={activeVideo}
- />
- ) : (
- <p className="text-sm text-zinc-500">No title matches.</p>
- )}
- </section>
-
- <section>
<h2 className="text-sm font-medium text-zinc-500 mb-2 flex items-center gap-2">
<span>
Transcript mentions{" "}
{totalHits > 0
- ? `(${capped ? `${MAX_HITS}+` : totalHits} mention${
+ ? `(${totalHits}${capped ? "+" : ""} mention${
totalHits === 1 ? "" : "s"
} across ${groups.length} video${
groups.length === 1 ? "" : "s"
})`
: "(0)"}
</span>
+ {indexLoading && (
+ <span className="text-xs text-zinc-400 font-normal">
+ loading index…
+ </span>
+ )}
{fetching && (
<span className="text-xs text-zinc-400 font-normal">
loaded {loadedCount}/{totalCount}…
</span>
)}
</h2>
- {groups.length === 0 && !fetching && (
+ {!indexLoading && groups.length === 0 && !fetching && (
<p className="text-sm text-zinc-500">No transcript mentions.</p>
)}
{groups.length > 0 && (
@@ -310,6 +293,17 @@ export default function TranscriptSearch({
))}
</div>
)}
+ {capped && (
+ <div className="mt-3 flex justify-center">
+ <button
+ type="button"
+ onClick={() => setHitLimit((l) => l + MAX_HITS)}
+ className="px-4 py-2 rounded-md border border-zinc-300 dark:border-zinc-700 bg-white dark:bg-zinc-900 hover:bg-zinc-50 dark:hover:bg-zinc-800 text-sm"
+ >
+ Load more
+ </button>
+ </div>
+ )}
</section>
</>
)}
@@ -317,49 +311,6 @@ export default function TranscriptSearch({
);
}
-function TranscriptGrid({
- items,
- onOpen,
- activeSlug,
-}: {
- items: Summary[];
- onOpen: (slug: string) => void;
- activeSlug?: string | null;
-}) {
- return (
- <ul className="flex flex-col divide-y divide-zinc-200 dark:divide-zinc-800 border border-zinc-200 dark:border-zinc-800 rounded-lg overflow-hidden bg-white dark:bg-zinc-900">
- {items.map((t) => {
- const isActive = activeSlug === t.slug;
- return (
- <li key={t.slug}>
- <button
- type="button"
- onClick={() => onOpen(t.slug)}
- className={`w-full text-left flex items-baseline gap-3 px-4 py-2 hover:bg-zinc-50 dark:hover:bg-zinc-800 ${
- isActive
- ? "bg-amber-100/70 dark:bg-amber-500/10 ring-1 ring-inset ring-amber-400/60"
- : ""
- }`}
- >
- <span className="text-xs font-mono text-zinc-500 shrink-0 w-24">
- {t.date}
- </span>
- <span className="flex-1 min-w-0 font-medium truncate">
- {t.title}
- </span>
- {t.duration && (
- <span className="text-xs font-mono text-zinc-500 shrink-0">
- {t.duration}
- </span>
- )}
- </button>
- </li>
- );
- })}
- </ul>
- );
-}
-
function findFirstMatchInRange(
haystack: string,
rangeStart: number,
diff --git a/app/page.tsx b/app/page.tsx
@@ -1,4 +1,4 @@
-import { listTranscripts, formatDate, formatDuration } from "@/lib/transcripts";
+import { countTranscripts } from "@/lib/transcripts";
import { getSettings } from "@/lib/settings";
import { PlayerProvider } from "./PlayerProvider";
import TranscriptModal from "./TranscriptModal";
@@ -6,29 +6,18 @@ import TranscriptSearch from "./TranscriptSearch";
export default async function Home() {
const settings = getSettings();
- const transcripts = await listTranscripts();
- const summaries = transcripts.map((t) => ({
- slug: t.slug,
- id: t.id,
- title: t.title,
- uploadDate: t.uploadDate,
- date: formatDate(t.uploadDate),
- duration: formatDuration(t.duration),
- channel: t.channel,
- }));
+ const count = await countTranscripts();
return (
- <PlayerProvider transcripts={summaries}>
+ <PlayerProvider>
<div className="flex flex-col gap-6">
<div>
- <h1 className="text-2xl font-semibold">
- {summaries.length} transcripts
- </h1>
+ <h1 className="text-2xl font-semibold">{count} transcripts</h1>
{settings.homeTagline && (
<p className="text-sm text-zinc-500 mt-1">{settings.homeTagline}</p>
)}
</div>
- <TranscriptSearch transcripts={summaries} />
+ <TranscriptSearch />
<section className="border-t border-zinc-200 dark:border-zinc-800 pt-4 text-sm text-zinc-500 flex gap-4">
<span>Downloads:</span>
<a
diff --git a/app/summaries/route.ts b/app/summaries/route.ts
@@ -0,0 +1,7 @@
+import { listDisplaySummaries } from "@/lib/transcripts";
+
+export const dynamic = "force-static";
+
+export async function GET() {
+ return Response.json(await listDisplaySummaries());
+}
diff --git a/app/summariesCache.ts b/app/summariesCache.ts
@@ -0,0 +1,17 @@
+"use client";
+
+import type { DisplaySummary } from "@/lib/transcripts";
+
+let promise: Promise<DisplaySummary[]> | null = null;
+
+export function fetchSummaries(): Promise<DisplaySummary[]> {
+ if (promise) return promise;
+ promise = fetch("/summaries").then((r) => {
+ if (!r.ok) throw new Error(`Failed to fetch summaries: ${r.status}`);
+ return r.json() as Promise<DisplaySummary[]>;
+ });
+ promise.catch(() => {
+ promise = null;
+ });
+ return promise;
+}
diff --git a/lib/transcripts.ts b/lib/transcripts.ts
@@ -1,6 +1,7 @@
import path from "node:path";
import { parseVtt, type Cue } from "./vtt";
import { readFile, readdir } from "fs-extra";
+import { formatDate, formatDuration } from "./format";
const ROOT = path.join(process.cwd(), "transcripts", "data");
@@ -14,6 +15,16 @@ export type TranscriptSummary = {
description: string;
};
+export type DisplaySummary = {
+ slug: string;
+ id: string;
+ title: string;
+ uploadDate: string;
+ date: string;
+ duration: string;
+ channel: string;
+};
+
export type TranscriptDetail = TranscriptSummary & {
cues: Cue[] | undefined;
};
@@ -73,6 +84,23 @@ export async function listTranscripts(): Promise<TranscriptSummary[]> {
);
}
+export async function countTranscripts(): Promise<number> {
+ return (await listSlugs()).length;
+}
+
+export async function listDisplaySummaries(): Promise<DisplaySummary[]> {
+ const summaries = await listTranscripts();
+ return summaries.map((t) => ({
+ slug: t.slug,
+ id: t.id,
+ title: t.title,
+ uploadDate: t.uploadDate,
+ date: formatDate(t.uploadDate),
+ duration: formatDuration(t.duration),
+ channel: t.channel,
+ }));
+}
+
export async function loadTranscript(slug: string): Promise<TranscriptDetail> {
const meta = await readMetadata(slug);
const cues = await readTranscript(slug);
