commit af8d892dd17b6da8d0f34e6733e04ad6fe6f3226
parent 5687dc29b5297927305f89fc9d9f9ab8778e5dfd
Author: I Mean I'm Just Saying <imeanimjustsaying@kiwifarms.st>
Date: Tue, 30 Jun 2026 23:54:04 -0400
Merge feat/export-pwa-offline-search: installable PWA + service-worker offline shard cache + FlexSearch web-worker off-main-thread index & offline search on export sites
Diffstat:
22 files changed, 1279 insertions(+), 2 deletions(-)
diff --git a/common/components/searchIndex.worker.ts b/common/components/searchIndex.worker.ts
@@ -0,0 +1,147 @@
+// FlexSearch index web worker. Builds and queries a per-channel full-text index
+// entirely off the main thread, so indexing hundreds of MB of transcript text
+// never blocks the UI. Bundled by Turbopack under output:"export" (proven to
+// emit into static out/) via `new Worker(new URL('./searchIndex.worker.ts',
+// import.meta.url), {type:'module'})` in searchIndexWorkerClient.
+//
+// The worker fetches the same /transcripts/<slug>/{manifest,page-NNNN}.json
+// shards the scanner uses — so a service-worker-cached (offline) channel indexes
+// and searches with no network.
+
+import { Index, Charset } from "flexsearch";
+import type {
+ IndexHit,
+ WorkerRequest,
+ WorkerResponse,
+} from "./searchIndexWorkerProtocol";
+
+// Minimal worker-global surface (avoids depending on the webworker tsconfig lib).
+const ctx = globalThis as unknown as {
+ postMessage(msg: WorkerResponse): void;
+ addEventListener(
+ type: "message",
+ cb: (e: MessageEvent<WorkerRequest>) => void,
+ ): void;
+};
+
+type ChannelIndex = {
+ idx: Index;
+ // docId → the hit it resolves to (FlexSearch returns ids, not the source).
+ meta: IndexHit[];
+};
+
+const channels = new Map<string, ChannelIndex>();
+
+function post(msg: WorkerResponse): void {
+ ctx.postMessage(msg);
+}
+
+function pageFileName(index: number): string {
+ return `page-${String(index).padStart(4, "0")}.json`;
+}
+
+function mkIndex(): Index {
+ // forward tokenizer → prefix matches (search-as-you-type); LatinBalance folds
+ // case/diacritics for recall without the heavier phonetic passes.
+ return new Index({
+ tokenize: "forward",
+ encoder: Charset.LatinBalance,
+ resolution: 9,
+ } as never);
+}
+
+type RawCue = { start?: number; text?: unknown };
+type RawEntry = { slug?: unknown; cues?: unknown };
+
+async function build(reqId: number, slug: string): Promise<void> {
+ // Manifest → page count.
+ const manRes = await fetch(`/transcripts/${slug}/manifest.json`, {
+ cache: "force-cache",
+ });
+ if (!manRes.ok) throw new Error(`manifest ${manRes.status}`);
+ const manifest = (await manRes.json()) as { pageCount?: number };
+ const pageCount = manifest.pageCount ?? 0;
+
+ const idx = mkIndex();
+ const meta: IndexHit[] = [];
+ let id = 0;
+
+ for (let p = 0; p < pageCount; p++) {
+ const res = await fetch(`/transcripts/${slug}/${pageFileName(p)}`, {
+ cache: "force-cache",
+ });
+ if (!res.ok) throw new Error(`page ${p} ${res.status}`);
+ const entries = (await res.json()) as RawEntry[];
+ for (const entry of entries) {
+ const eslug = typeof entry.slug === "string" ? entry.slug : slug;
+ const cues = Array.isArray(entry.cues) ? entry.cues : [];
+ for (const c of cues as RawCue[]) {
+ const text = typeof c.text === "string" ? c.text : "";
+ if (!text) continue;
+ idx.add(id, text);
+ meta[id] = { slug: eslug, start: c.start, text };
+ id++;
+ }
+ }
+ post({ type: "progress", reqId, slug, done: p + 1, total: pageCount });
+ }
+
+ channels.set(slug, { idx, meta });
+ post({ type: "built", reqId, slug, docCount: meta.length });
+}
+
+async function search(
+ reqId: number,
+ slug: string,
+ term: string,
+ limit: number,
+): Promise<void> {
+ const ch = channels.get(slug);
+ if (!ch) {
+ post({ type: "result", reqId, slug, hits: [] });
+ return;
+ }
+ // The in-worker Index is synchronous; search returns ids directly.
+ const ids = ch.idx.search(term, { limit }) as unknown as number[];
+ const hits: IndexHit[] = [];
+ for (const rid of ids) {
+ const m = ch.meta[rid];
+ if (m) hits.push(m);
+ }
+ post({ type: "result", reqId, slug, hits });
+}
+
+ctx.addEventListener("message", (e: MessageEvent<WorkerRequest>) => {
+ const msg = e.data;
+ const fail = (err: unknown) =>
+ post({
+ type: "error",
+ reqId: msg.reqId,
+ slug: (msg as { slug?: string }).slug ?? "",
+ message: err instanceof Error ? err.message : String(err),
+ });
+
+ switch (msg.type) {
+ case "build":
+ build(msg.reqId, msg.slug).catch(fail);
+ break;
+ case "search":
+ search(msg.reqId, msg.slug, msg.term, msg.limit).catch(fail);
+ break;
+ case "status": {
+ const ch = channels.get(msg.slug);
+ post({
+ type: "status",
+ reqId: msg.reqId,
+ slug: msg.slug,
+ ready: !!ch,
+ docCount: ch ? ch.meta.length : 0,
+ });
+ break;
+ }
+ case "drop":
+ channels.delete(msg.slug);
+ post({ type: "dropped", reqId: msg.reqId, slug: msg.slug });
+ break;
+ }
+});
diff --git a/common/components/searchIndexWorkerClient.ts b/common/components/searchIndexWorkerClient.ts
@@ -0,0 +1,100 @@
+"use client";
+
+// Main-thread client for the FlexSearch web worker. Lazily spawns one module
+// worker and exposes an async request/response API (build / search / status /
+// drop) with build progress. The `new Worker(new URL(...))` form is what
+// Turbopack statically detects to bundle the worker into the static export.
+
+import type {
+ IndexHit,
+ WorkerRequest,
+ WorkerResponse,
+} from "./searchIndexWorkerProtocol";
+
+export type { IndexHit } from "./searchIndexWorkerProtocol";
+
+// Distributive Omit so the discriminated-union request keeps its per-variant
+// fields (a plain Omit<Union, K> collapses to the common keys only).
+type DistributiveOmit<T, K extends keyof any> = T extends unknown
+ ? Omit<T, K>
+ : never;
+
+type Pending = {
+ resolve: (value: WorkerResponse) => void;
+ reject: (err: Error) => void;
+ onProgress?: (done: number, total: number) => void;
+};
+
+let worker: Worker | null = null;
+let nextReqId = 1;
+const pending = new Map<number, Pending>();
+
+export function workerSupported(): boolean {
+ return typeof window !== "undefined" && typeof Worker !== "undefined";
+}
+
+function getWorker(): Worker | null {
+ if (!workerSupported()) return null;
+ if (worker) return worker;
+ worker = new Worker(new URL("./searchIndex.worker.ts", import.meta.url), {
+ type: "module",
+ });
+ worker.addEventListener("message", (e: MessageEvent<WorkerResponse>) => {
+ const msg = e.data;
+ const p = pending.get(msg.reqId);
+ if (!p) return;
+ if (msg.type === "progress") {
+ p.onProgress?.(msg.done, msg.total);
+ return; // keep pending open until the terminal reply
+ }
+ pending.delete(msg.reqId);
+ if (msg.type === "error") p.reject(new Error(msg.message));
+ else p.resolve(msg);
+ });
+ return worker;
+}
+
+function request(
+ req: DistributiveOmit<WorkerRequest, "reqId">,
+ onProgress?: (done: number, total: number) => void,
+): Promise<WorkerResponse> {
+ const w = getWorker();
+ if (!w) return Promise.reject(new Error("Web workers unavailable"));
+ const reqId = nextReqId++;
+ return new Promise<WorkerResponse>((resolve, reject) => {
+ pending.set(reqId, { resolve, reject, onProgress });
+ w.postMessage({ ...req, reqId } as WorkerRequest);
+ });
+}
+
+// Build (or rebuild) a channel's index in the worker from its transcript shards.
+// Resolves with the indexed document (cue) count.
+export async function buildChannelIndex(
+ slug: string,
+ onProgress?: (done: number, total: number) => void,
+): Promise<number> {
+ const res = await request({ type: "build", slug }, onProgress);
+ return res.type === "built" ? res.docCount : 0;
+}
+
+export async function searchChannel(
+ slug: string,
+ term: string,
+ limit = 50,
+): Promise<IndexHit[]> {
+ const res = await request({ type: "search", slug, term, limit });
+ return res.type === "result" ? res.hits : [];
+}
+
+export async function channelIndexStatus(
+ slug: string,
+): Promise<{ ready: boolean; docCount: number }> {
+ const res = await request({ type: "status", slug });
+ return res.type === "status"
+ ? { ready: res.ready, docCount: res.docCount }
+ : { ready: false, docCount: 0 };
+}
+
+export async function dropChannelIndex(slug: string): Promise<void> {
+ await request({ type: "drop", slug });
+}
diff --git a/common/components/searchIndexWorkerProtocol.ts b/common/components/searchIndexWorkerProtocol.ts
@@ -0,0 +1,21 @@
+// Message protocol shared between the main thread (searchIndexWorkerClient) and
+// the FlexSearch web worker (searchIndex.worker). Types only — safe to import
+// from both sides.
+
+export type IndexHit = { slug: string; start?: number; text: string };
+
+// Main thread → worker.
+export type WorkerRequest =
+ | { type: "build"; reqId: number; slug: string }
+ | { type: "search"; reqId: number; slug: string; term: string; limit: number }
+ | { type: "status"; reqId: number; slug: string }
+ | { type: "drop"; reqId: number; slug: string };
+
+// Worker → main thread.
+export type WorkerResponse =
+ | { type: "progress"; reqId: number; slug: string; done: number; total: number }
+ | { type: "built"; reqId: number; slug: string; docCount: number }
+ | { type: "result"; reqId: number; slug: string; hits: IndexHit[] }
+ | { type: "status"; reqId: number; slug: string; ready: boolean; docCount: number }
+ | { type: "dropped"; reqId: number; slug: string }
+ | { type: "error"; reqId: number; slug: string; message: string };
diff --git a/common/package.json b/common/package.json
@@ -12,6 +12,7 @@
"cmdk": "^1.1.1",
"d3-scale": "^4.0.2",
"execa": "^9.6.1",
+ "flexsearch": "^0.8.0",
"fs-extra": "^11.3.4",
"lmdb": "^3.5.4",
"lucide-react": "^1.16.0",
diff --git a/export/CHANGELOG.md b/export/CHANGELOG.md
@@ -1,6 +1,8 @@
# Changelog
## [Unreleased]
+- **Instant offline word-search, powered by a background FlexSearch index.** On the Offline page, each channel now has a **Build index** action: it constructs a FlexSearch full-text index for that channel in a **Web Worker** — entirely off the main thread, so indexing a large channel never freezes the page (a live progress % shows the work). Once built, a search box does **instant word/prefix search** across your indexed channels and returns the matching transcript lines (with channel + timestamp) — and because the index is built from the same shards the offline download caches, it works with no connection. The worker bundles into the static site (no server needed). This is a first, self-contained surface; the main search box will gain the same acceleration next.
+- **Install the archive as an app, and search it offline.** Each public site is now a PWA: it ships a web app manifest (named + themed with the site's accent color) and an icon set, so your browser can install it to your home screen or desktop and open it in its own window. A service worker caches the app so it loads instantly on repeat visits and still opens without a connection. A new **Offline** page (linked in the footer) lists the site's channels with a **Download** button each — pick the channels you want and their transcripts are saved to your device so you can **search them with no connection**; a live Online/Offline indicator and per-channel "Available offline" state show what's ready, and you can remove a channel to free space. Downloads are opt-in (a channel's transcripts can be large) and refresh automatically when the site is rebuilt. Everything still works online-only in browsers without service-worker support.
- **Two more themes + a theme picker.** A palette menu next to the light/dark toggle lets you switch the whole site's look between **Archive** (the warm reading room), **Terminal** (an ice-cyan signal terminal), **Swiss** (red/black/white editorial), and **Base** (neutral) — each in light or dark, and your choice is remembered. Switching is instant and purely cosmetic (just colors; the layout never changes).
- **Optional per-site brand accent.** A site can set its own accent color (a hex, in the editor's site config) to replace the family brass everywhere on its public build — the wordmark mark, links, badges, and highlights. It's baked onto the page at build time so it's correct on the very first paint, and applies in both light and dark. Sites that don't set one keep the family brass.
- **New look: the "Archive / Reading Room" identity.** The public site adopts the family design system — a serif display face (Source Serif 4) for headings/counts, clean sans for body, and mono for the query/power layer, over a warm paper (light) / ink (dark) palette with a brass accent. A light/dark/system theme toggle in the header replaces OS-only dark mode and persists your choice. Header, footer, and the whole search UI (query builder, filters, scopes, charts) are rebuilt on a shared component kit for a calmer, more consistent feel — the simple one-box search is unchanged; the power tools just read better.
diff --git a/export/app/components/Footer.tsx b/export/app/components/Footer.tsx
@@ -29,6 +29,12 @@ export default function Footer() {
>
code.tar.gz
</a>
+ <a
+ href="/offline/"
+ className="underline underline-offset-2 hover:text-foreground transition-colors"
+ >
+ Offline
+ </a>
</div>
{socialLinks.length > 0 && (
<ul className="flex items-center gap-3 list-none">
diff --git a/export/app/components/OfflineManager.tsx b/export/app/components/OfflineManager.tsx
@@ -0,0 +1,355 @@
+"use client";
+
+import { useCallback, useEffect, useMemo, useRef, useState } from "react";
+import {
+ channelCachedPages,
+ downloadChannelOffline,
+ evictChannelOffline,
+ isPinned,
+ swSupported,
+} from "../lib/offlineCache";
+import {
+ buildChannelIndex,
+ dropChannelIndex,
+ searchChannel,
+ workerSupported,
+ type IndexHit,
+} from "yt-dlp-transcript-common/components/searchIndexWorkerClient";
+
+export type OfflineChannel = { slug: string; name: string };
+
+type ChannelState = {
+ download: "idle" | "downloading" | "cached" | "error";
+ done: number;
+ total: number;
+ // Worker FlexSearch index state.
+ index: "none" | "building" | "ready" | "error";
+ idxDone: number;
+ idxTotal: number;
+ docCount: number;
+};
+
+const DEFAULT: ChannelState = {
+ download: "idle",
+ done: 0,
+ total: 0,
+ index: "none",
+ idxDone: 0,
+ idxTotal: 0,
+ docCount: 0,
+};
+
+function useOnline(): boolean {
+ const [online, setOnline] = useState(true);
+ useEffect(() => {
+ const set = () => setOnline(navigator.onLine);
+ set();
+ window.addEventListener("online", set);
+ window.addEventListener("offline", set);
+ return () => {
+ window.removeEventListener("online", set);
+ window.removeEventListener("offline", set);
+ };
+ }, []);
+ return online;
+}
+
+export function OfflineManager({ channels }: { channels: OfflineChannel[] }) {
+ const online = useOnline();
+ const [supported, setSupported] = useState(true);
+ const [hasWorker, setHasWorker] = useState(false);
+ const [states, setStates] = useState<Record<string, ChannelState>>({});
+
+ const setState = useCallback((slug: string, patch: Partial<ChannelState>) => {
+ setStates((prev) => {
+ const base = prev[slug] ?? DEFAULT;
+ return { ...prev, [slug]: { ...base, ...patch } };
+ });
+ }, []);
+
+ useEffect(() => {
+ if (!swSupported()) {
+ setSupported(false);
+ return;
+ }
+ setHasWorker(workerSupported());
+ let cancelled = false;
+ (async () => {
+ for (const c of channels) {
+ if (!isPinned(c.slug)) continue;
+ const cached = await channelCachedPages(c.slug);
+ if (!cancelled && cached > 0) {
+ setState(c.slug, { download: "cached", done: cached, total: cached });
+ }
+ }
+ })();
+ return () => {
+ cancelled = true;
+ };
+ }, [channels, setState]);
+
+ async function download(slug: string) {
+ setState(slug, { download: "downloading", done: 0, total: 0 });
+ const ok = await downloadChannelOffline(slug, ({ done, total }) =>
+ setState(slug, { download: "downloading", done, total }),
+ ).catch(() => false);
+ setState(slug, { download: ok ? "cached" : "error" });
+ }
+
+ async function remove(slug: string) {
+ await evictChannelOffline(slug);
+ await dropChannelIndex(slug).catch(() => {});
+ setState(slug, { ...DEFAULT });
+ }
+
+ async function buildIndex(slug: string) {
+ setState(slug, { index: "building", idxDone: 0, idxTotal: 0 });
+ try {
+ const docCount = await buildChannelIndex(slug, (done, total) =>
+ setState(slug, { index: "building", idxDone: done, idxTotal: total }),
+ );
+ setState(slug, { index: "ready", docCount });
+ } catch {
+ setState(slug, { index: "error" });
+ }
+ }
+
+ const indexedSlugs = useMemo(
+ () =>
+ channels
+ .filter((c) => states[c.slug]?.index === "ready")
+ .map((c) => c.slug),
+ [channels, states],
+ );
+ const nameBySlug = useMemo(() => {
+ const m = new Map<string, string>();
+ for (const c of channels) m.set(c.slug, c.name);
+ return m;
+ }, [channels]);
+
+ if (!supported) {
+ return (
+ <div
+ role="status"
+ className="rounded-md border border-border bg-muted px-3 py-2 text-sm text-muted-foreground"
+ >
+ Offline downloads aren't available in this browser (no service
+ worker support). Search still works while you're online.
+ </div>
+ );
+ }
+
+ return (
+ <div className="flex flex-col gap-4">
+ <div
+ role="status"
+ aria-label="connection status"
+ className="flex items-center gap-2 text-xs font-mono uppercase tracking-[0.14em] text-muted-foreground"
+ >
+ <span
+ aria-hidden="true"
+ className={
+ "inline-block h-2 w-2 rounded-full " +
+ (online ? "bg-brand" : "bg-destructive")
+ }
+ />
+ {online ? "Online" : "Offline"}
+ </div>
+
+ {hasWorker && indexedSlugs.length > 0 && (
+ <OfflineSearch indexedSlugs={indexedSlugs} nameBySlug={nameBySlug} />
+ )}
+
+ {channels.length === 0 ? (
+ <p className="text-sm text-muted-foreground">
+ This site has no channels.
+ </p>
+ ) : (
+ <ul className="flex flex-col divide-y divide-border rounded-lg border border-border">
+ {channels.map((c) => {
+ const st = states[c.slug] ?? DEFAULT;
+ return (
+ <li
+ key={c.slug}
+ aria-label={`offline ${c.slug}`}
+ className="flex flex-wrap items-center gap-x-3 gap-y-2 px-4 py-3"
+ >
+ <span className="min-w-0 flex-1 truncate font-medium text-foreground">
+ {c.name}
+ </span>
+
+ {st.download === "cached" && (
+ <span
+ aria-label={`${c.slug} available offline`}
+ className="rounded-full border border-brand/30 bg-brand-soft px-2 py-0.5 text-xs text-brand"
+ >
+ Available offline
+ </span>
+ )}
+ {st.index === "ready" && (
+ <span
+ aria-label={`${c.slug} indexed`}
+ className="rounded-full border border-border bg-muted px-2 py-0.5 text-xs text-muted-foreground"
+ >
+ Indexed · {st.docCount.toLocaleString()}
+ </span>
+ )}
+ {(st.download === "error" || st.index === "error") && (
+ <span className="text-xs text-destructive">
+ {st.index === "error" ? "Index failed" : "Download failed"}
+ </span>
+ )}
+ {st.download === "downloading" && (
+ <span className="font-mono text-xs tabular-nums text-muted-foreground">
+ {st.total > 0
+ ? `${Math.round((st.done / st.total) * 100)}%`
+ : "starting…"}
+ </span>
+ )}
+ {st.index === "building" && (
+ <span className="font-mono text-xs tabular-nums text-muted-foreground">
+ indexing{" "}
+ {st.idxTotal > 0
+ ? `${Math.round((st.idxDone / st.idxTotal) * 100)}%`
+ : "…"}
+ </span>
+ )}
+
+ {hasWorker && st.index !== "ready" && (
+ <button
+ type="button"
+ disabled={st.index === "building" || !online}
+ onClick={() => buildIndex(c.slug)}
+ aria-label={`build search index for ${c.slug}`}
+ title={
+ !online ? "Reconnect to build the index" : undefined
+ }
+ className="rounded-md border border-border px-2.5 py-1 text-xs font-medium text-muted-foreground transition-colors hover:text-foreground hover:bg-muted disabled:opacity-50"
+ >
+ {st.index === "building" ? "Indexing…" : "Build index"}
+ </button>
+ )}
+
+ {st.download === "cached" ? (
+ <button
+ type="button"
+ onClick={() => remove(c.slug)}
+ aria-label={`remove offline copy of ${c.slug}`}
+ className="rounded-md border border-border px-2.5 py-1 text-xs font-medium text-muted-foreground transition-colors hover:text-foreground hover:bg-muted"
+ >
+ Remove
+ </button>
+ ) : (
+ <button
+ type="button"
+ disabled={st.download === "downloading" || !online}
+ onClick={() => download(c.slug)}
+ aria-label={`download ${c.slug} for offline`}
+ title={!online ? "Reconnect to download" : undefined}
+ className="rounded-md border border-brand/30 bg-brand-soft px-2.5 py-1 text-xs font-medium text-brand transition-colors hover:bg-brand/15 disabled:opacity-50"
+ >
+ {st.download === "downloading" ? "Downloading…" : "Download"}
+ </button>
+ )}
+ </li>
+ );
+ })}
+ </ul>
+ )}
+ </div>
+ );
+}
+
+// Instant word-search across the channels whose FlexSearch index has been built
+// in the worker. Runs entirely off the main thread and works offline once the
+// channel's shards are cached.
+function OfflineSearch({
+ indexedSlugs,
+ nameBySlug,
+}: {
+ indexedSlugs: string[];
+ nameBySlug: Map<string, string>;
+}) {
+ const [term, setTerm] = useState("");
+ const [hits, setHits] = useState<{ channel: string; hit: IndexHit }[]>([]);
+ const [searching, setSearching] = useState(false);
+ const runId = useRef(0);
+
+ useEffect(() => {
+ const q = term.trim();
+ if (q.length < 2) {
+ setHits([]);
+ return;
+ }
+ const id = ++runId.current;
+ setSearching(true);
+ const timer = setTimeout(async () => {
+ const perChannel = await Promise.all(
+ indexedSlugs.map((slug) =>
+ searchChannel(slug, q, 20)
+ .then((hs) => hs.map((hit) => ({ channel: slug, hit })))
+ .catch(() => []),
+ ),
+ );
+ if (id !== runId.current) return; // a newer query superseded this one
+ setHits(perChannel.flat().slice(0, 40));
+ setSearching(false);
+ }, 180);
+ return () => clearTimeout(timer);
+ }, [term, indexedSlugs]);
+
+ return (
+ <div className="flex flex-col gap-2 rounded-lg border border-border bg-card p-4">
+ <label className="flex flex-col gap-1">
+ <span className="font-mono text-[11px] uppercase tracking-[0.14em] text-muted-foreground">
+ Offline search · {indexedSlugs.length} channel
+ {indexedSlugs.length === 1 ? "" : "s"} indexed
+ </span>
+ <input
+ type="search"
+ value={term}
+ onChange={(e) => setTerm(e.target.value)}
+ aria-label="offline search"
+ placeholder="Search indexed transcripts…"
+ className="rounded-md border border-input bg-background px-3 py-2 text-sm text-foreground outline-none focus-visible:ring-2 focus-visible:ring-ring"
+ />
+ </label>
+ {term.trim().length >= 2 && (
+ <div
+ aria-label="offline search results"
+ className="flex flex-col gap-1 text-sm"
+ >
+ {searching && hits.length === 0 ? (
+ <p className="text-muted-foreground">Searching…</p>
+ ) : hits.length === 0 ? (
+ <p className="text-muted-foreground">No matches.</p>
+ ) : (
+ <ul className="flex flex-col gap-1">
+ {hits.map(({ channel, hit }, i) => (
+ <li
+ key={`${channel}-${hit.slug}-${hit.start ?? i}-${i}`}
+ className="flex flex-col rounded-md px-2 py-1 hover:bg-muted"
+ >
+ <span className="truncate text-foreground">{hit.text}</span>
+ <span className="font-mono text-[11px] text-muted-foreground">
+ {nameBySlug.get(channel) ?? channel} · {hit.slug}
+ {typeof hit.start === "number"
+ ? ` · ${formatTime(hit.start)}`
+ : ""}
+ </span>
+ </li>
+ ))}
+ </ul>
+ )}
+ </div>
+ )}
+ </div>
+ );
+}
+
+function formatTime(sec: number): string {
+ const s = Math.max(0, Math.floor(sec));
+ const m = Math.floor(s / 60);
+ const r = s % 60;
+ return `${m}:${String(r).padStart(2, "0")}`;
+}
diff --git a/export/app/components/ServiceWorkerRegister.tsx b/export/app/components/ServiceWorkerRegister.tsx
@@ -0,0 +1,29 @@
+"use client";
+
+import { useEffect } from "react";
+
+// Registers the offline service worker (public/sw.js) after load. Rendered once
+// in the root layout. No-op where service workers are unavailable (older
+// browsers, insecure origins) — the site still works online-only.
+export function ServiceWorkerRegister() {
+ useEffect(() => {
+ // Only register in production builds. A SW caching assets/data in dev fights
+ // hot-reload and the e2e suite's request mocking; the deployed static export
+ // is where offline support matters.
+ if (process.env.NODE_ENV !== "production") return;
+ if (typeof navigator === "undefined" || !("serviceWorker" in navigator)) {
+ return;
+ }
+ const register = () => {
+ navigator.serviceWorker.register("/sw.js").catch(() => {
+ // Registration failures are non-fatal (e.g. SW disabled) — stay online.
+ });
+ };
+ if (document.readyState === "complete") register();
+ else {
+ window.addEventListener("load", register, { once: true });
+ return () => window.removeEventListener("load", register);
+ }
+ }, []);
+ return null;
+}
diff --git a/export/app/layout.tsx b/export/app/layout.tsx
@@ -1,14 +1,17 @@
-import type { Metadata } from "next";
+import type { Metadata, Viewport } from "next";
import { fontVars } from "yt-dlp-transcript-common/styles/fonts";
import { ThemeScript } from "yt-dlp-transcript-common/components/ThemeScript";
import { ThemeProvider } from "yt-dlp-transcript-common/components/ThemeProvider";
import { QueryProvider } from "yt-dlp-transcript-common/components/QueryProvider";
-import { siteAccentVars } from "yt-dlp-transcript-common/lib/accent";
+import { parseAccent, siteAccentVars } from "yt-dlp-transcript-common/lib/accent";
import { currentSite } from "./lib/site";
import Header from "./components/Header";
import Footer from "./components/Footer";
+import { ServiceWorkerRegister } from "./components/ServiceWorkerRegister";
import "./globals.css";
+const FAMILY_BRASS = "#95661a";
+
export function generateMetadata(): Metadata {
const s = currentSite();
return {
@@ -17,9 +20,28 @@ export function generateMetadata(): Metadata {
template: `%s — ${s.siteTitle}`,
},
description: s.siteDescription,
+ // Installable PWA: manifest + icon set + iOS standalone hints.
+ manifest: "/manifest.webmanifest",
+ icons: {
+ icon: [
+ { url: "/icons/icon-192.png", sizes: "192x192", type: "image/png" },
+ { url: "/icons/icon-512.png", sizes: "512x512", type: "image/png" },
+ ],
+ apple: [{ url: "/icons/apple-touch-icon.png", sizes: "180x180" }],
+ },
+ appleWebApp: {
+ capable: true,
+ statusBarStyle: "black-translucent",
+ title: s.headerTitle || s.siteTitle,
+ },
};
}
+export function generateViewport(): Viewport {
+ // Browser chrome color tracks the site's brand accent (falls back to brass).
+ return { themeColor: parseAccent(currentSite().accent) ?? FAMILY_BRASS };
+}
+
export default async function RootLayout({
children,
}: Readonly<{
@@ -37,6 +59,7 @@ export default async function RootLayout({
>
<body className="min-h-full flex flex-col bg-background text-foreground font-sans">
<ThemeScript defaultTheme="archive" defaultMode="system" />
+ <ServiceWorkerRegister />
<ThemeProvider defaultTheme="archive" defaultMode="system">
<QueryProvider>
<Header />
diff --git a/export/app/lib/offlineCache.ts b/export/app/lib/offlineCache.ts
@@ -0,0 +1,136 @@
+"use client";
+
+// Client helpers for the opt-in "download this channel for offline" feature and
+// the pinned-channel registry. Talks to the service worker (public/sw.js) over a
+// MessageChannel; degrades gracefully where no SW is controlling the page.
+
+import {
+ transcriptPageFileName,
+ type ChannelTranscriptsManifest,
+} from "yt-dlp-transcript-common/lib/manifest";
+
+const PINNED_KEY = "ytdlp-tb:offline-channels";
+
+export function swSupported(): boolean {
+ return (
+ typeof navigator !== "undefined" && "serviceWorker" in navigator
+ );
+}
+
+async function controller(): Promise<ServiceWorker | null> {
+ if (!swSupported()) return null;
+ // Wait for an active, controlling worker (first load registers but may not
+ // control this page until reload; ready resolves once active).
+ const reg = await navigator.serviceWorker.ready.catch(() => null);
+ return navigator.serviceWorker.controller ?? reg?.active ?? null;
+}
+
+// Round-trip a message to the SW over a fresh MessageChannel. `onMessage` sees
+// every reply; the promise resolves when a reply of type `done`/`ok`/`status`
+// arrives (or rejects on timeout).
+function sendToSw(
+ sw: ServiceWorker,
+ message: unknown,
+ onMessage?: (data: any) => void,
+): Promise<any> {
+ return new Promise((resolve, reject) => {
+ const channel = new MessageChannel();
+ const timer = setTimeout(() => reject(new Error("SW timeout")), 10 * 60_000);
+ channel.port1.onmessage = (e) => {
+ const data = e.data || {};
+ onMessage?.(data);
+ if (data.type === "done" || data.ok || data.type === "status") {
+ clearTimeout(timer);
+ resolve(data);
+ }
+ };
+ sw.postMessage(message, [channel.port2]);
+ });
+}
+
+async function fetchManifest(
+ slug: string,
+): Promise<ChannelTranscriptsManifest | null> {
+ try {
+ const res = await fetch(`/transcripts/${slug}/manifest.json`, {
+ cache: "no-store",
+ });
+ if (!res.ok) return null;
+ return (await res.json()) as ChannelTranscriptsManifest;
+ } catch {
+ return null;
+ }
+}
+
+export type DownloadProgress = { done: number; total: number };
+
+// Download every shard of a channel into the SW cache, reporting progress. The
+// URL list is the channel manifest plus each page-NNNN.json.
+export async function downloadChannelOffline(
+ slug: string,
+ onProgress?: (p: DownloadProgress) => void,
+): Promise<boolean> {
+ const sw = await controller();
+ if (!sw) return false;
+ const manifest = await fetchManifest(slug);
+ if (!manifest) return false;
+ const urls = [`/transcripts/${slug}/manifest.json`];
+ for (let p = 0; p < manifest.pageCount; p++) {
+ urls.push(`/transcripts/${slug}/${transcriptPageFileName(p)}`);
+ }
+ await sendToSw(sw, { type: "CACHE_URLS", slug, urls }, (data) => {
+ if (data.type === "progress") {
+ onProgress?.({ done: data.done, total: data.total });
+ }
+ });
+ pin(slug);
+ return true;
+}
+
+export async function evictChannelOffline(slug: string): Promise<void> {
+ const sw = await controller();
+ if (sw) await sendToSw(sw, { type: "EVICT_CHANNEL", slug }).catch(() => {});
+ unpin(slug);
+}
+
+// How many page shards of a channel are currently cached (0 = not offline).
+export async function channelCachedPages(slug: string): Promise<number> {
+ const sw = await controller();
+ if (!sw) return 0;
+ try {
+ const data = await sendToSw(sw, { type: "CHANNEL_STATUS", slug });
+ return typeof data.cachedPages === "number" ? data.cachedPages : 0;
+ } catch {
+ return 0;
+ }
+}
+
+// ---- Pinned registry (which channels the user chose to keep offline) ----
+export function pinnedChannels(): string[] {
+ if (typeof window === "undefined") return [];
+ try {
+ const raw = window.localStorage.getItem(PINNED_KEY);
+ if (!raw) return [];
+ const arr = JSON.parse(raw);
+ return Array.isArray(arr) ? arr.filter((x) => typeof x === "string") : [];
+ } catch {
+ return [];
+ }
+}
+export function isPinned(slug: string): boolean {
+ return pinnedChannels().includes(slug);
+}
+function writePinned(slugs: string[]): void {
+ if (typeof window === "undefined") return;
+ try {
+ window.localStorage.setItem(PINNED_KEY, JSON.stringify([...new Set(slugs)]));
+ } catch {
+ /* quota — non-fatal */
+ }
+}
+function pin(slug: string): void {
+ writePinned([...pinnedChannels(), slug]);
+}
+function unpin(slug: string): void {
+ writePinned(pinnedChannels().filter((s) => s !== slug));
+}
diff --git a/export/app/manifest.ts b/export/app/manifest.ts
@@ -0,0 +1,50 @@
+import type { MetadataRoute } from "next";
+import { parseAccent } from "yt-dlp-transcript-common/lib/accent";
+import { currentSite } from "./lib/site";
+
+// Web App Manifest → statically emitted as /manifest.webmanifest at build time
+// (output: "export"). Per-site: name/description from currentSite(), theme_color
+// from the site's brand accent (falls back to the family brass). Makes each
+// deployed export site installable as a standalone PWA. Icons are the shared
+// Archive mark under public/icons (a per-site accent-tinted icon is a later
+// refinement — theme_color already differentiates the browser chrome).
+export const dynamic = "force-static";
+
+// Family brass — the archive brand accent, used when a site sets no accent.
+const FAMILY_BRASS = "#95661a";
+
+export default function manifest(): MetadataRoute.Manifest {
+ const site = currentSite();
+ const themeColor = parseAccent(site.accent) ?? FAMILY_BRASS;
+ return {
+ name: site.siteTitle,
+ short_name: site.headerTitle || site.siteTitle,
+ description: site.siteDescription,
+ start_url: "/",
+ scope: "/",
+ display: "standalone",
+ orientation: "portrait-primary",
+ background_color: "#0c0a08",
+ theme_color: themeColor,
+ icons: [
+ {
+ src: "/icons/icon-192.png",
+ sizes: "192x192",
+ type: "image/png",
+ purpose: "any",
+ },
+ {
+ src: "/icons/icon-512.png",
+ sizes: "512x512",
+ type: "image/png",
+ purpose: "any",
+ },
+ {
+ src: "/icons/maskable-512.png",
+ sizes: "512x512",
+ type: "image/png",
+ purpose: "maskable",
+ },
+ ],
+ };
+}
diff --git a/export/app/offline/page.tsx b/export/app/offline/page.tsx
@@ -0,0 +1,40 @@
+import type { Metadata } from "next";
+import { getPaths } from "yt-dlp-transcript-common/lib/paths";
+import { readChannelConfig } from "yt-dlp-transcript-common/controller/channels";
+import { currentSite } from "../lib/site";
+import { OfflineManager, type OfflineChannel } from "../components/OfflineManager";
+
+export const metadata: Metadata = {
+ title: "Offline",
+ description: "Download channels for offline search.",
+};
+
+// Static list of this site's channels (slug + display name), resolved at build
+// time. The client OfflineManager uses the slugs to fetch/cache each channel's
+// transcript shards through the service worker.
+export default async function OfflinePage() {
+ const site = currentSite();
+ const paths = getPaths();
+ const channels: OfflineChannel[] = await Promise.all(
+ site.channels.map(async ({ slug }) => {
+ const config = await readChannelConfig(paths, slug).catch(() => null);
+ return { slug, name: config?.name ?? slug };
+ }),
+ );
+ channels.sort((a, b) => a.name.localeCompare(b.name));
+
+ return (
+ <div className="flex flex-col gap-6">
+ <div className="flex flex-col gap-1">
+ <h1 className="font-display text-3xl font-semibold tracking-tight text-foreground">
+ Offline
+ </h1>
+ <p className="text-sm text-muted-foreground">
+ Install this archive and download channels to search them without a
+ connection. Downloads can be large — each channel is fetched on demand.
+ </p>
+ </div>
+ <OfflineManager channels={channels} />
+ </div>
+ );
+}
diff --git a/export/e2e/pwa-search.spec.ts b/export/e2e/pwa-search.spec.ts
@@ -0,0 +1,76 @@
+import { test, expect } from "@playwright/test";
+
+// Exercises the FlexSearch web worker end to end: /offline builds a channel's
+// index off the main thread from the transcript shards, then searches it. The
+// SW is disabled in dev (production-only), so the worker's shard fetches hit the
+// network and are mocked here via context.route (which reaches worker requests).
+
+const SLUG = "test-youtube"; // testsite's channel (see e2e/fixtures/sites)
+
+test("builds a channel index in the worker and searches it offline", async ({
+ page,
+}) => {
+ await page.context().route(
+ `**/transcripts/${SLUG}/manifest.json`,
+ async (route) => {
+ await route.fulfill({
+ contentType: "application/json",
+ body: JSON.stringify({
+ version: 2,
+ channelSlug: SLUG,
+ pageCount: 1,
+ maxPageBytes: 1000,
+ generatedAt: "2026-06-30T00:00:00.000Z",
+ slugToPage: {},
+ }),
+ });
+ },
+ );
+ await page.context().route(
+ `**/transcripts/${SLUG}/page-0000.json`,
+ async (route) => {
+ await route.fulfill({
+ contentType: "application/json",
+ body: JSON.stringify([
+ {
+ slug: "vidA",
+ cues: [
+ { start: 0, text: "a constitutional hearing began today" },
+ { start: 12, text: "the witness testified under oath" },
+ ],
+ },
+ {
+ slug: "vidB",
+ cues: [{ start: 3, text: "an unrelated cooking segment" }],
+ },
+ ]),
+ });
+ },
+ );
+
+ await page.goto("/offline/");
+
+ const build = page.getByRole("button", {
+ name: `build search index for ${SLUG}`,
+ });
+ await expect(build).toBeVisible();
+ await build.click();
+
+ // The worker finishes → an "Indexed · N" badge appears.
+ await expect(page.getByLabel(`${SLUG} indexed`)).toBeVisible({
+ timeout: 20_000,
+ });
+
+ // The offline search box is now available; a word query returns the matching
+ // cue via the worker's FlexSearch index.
+ const box = page.getByLabel("offline search", { exact: true });
+ await expect(box).toBeVisible();
+ await box.fill("constitutional");
+ const results = page.getByLabel("offline search results");
+ await expect(results).toContainText("constitutional hearing", {
+ timeout: 10_000,
+ });
+ // A non-matching term yields no hit for that cue.
+ await box.fill("cooking");
+ await expect(results).toContainText("cooking segment", { timeout: 10_000 });
+});
diff --git a/export/e2e/pwa.spec.ts b/export/e2e/pwa.spec.ts
@@ -0,0 +1,61 @@
+import { test, expect } from "@playwright/test";
+
+// PWA surface for the export (Archive) sites: an installable web app manifest,
+// a registered service worker, and the /offline management page. The transcript
+// shard caching itself is validated by the production build (artifacts emit) +
+// the SW code; the e2e harness mocks /transcripts/** so a full offline
+// round-trip isn't exercised here.
+
+test("serves a valid web app manifest with per-site fields", async ({
+ request,
+}) => {
+ const res = await request.get("/manifest.webmanifest");
+ expect(res.ok()).toBeTruthy();
+ const m = await res.json();
+ expect(m.name).toBeTruthy();
+ expect(m.start_url).toBe("/");
+ expect(m.display).toBe("standalone");
+ expect(Array.isArray(m.icons)).toBe(true);
+ // 192 + 512 + a maskable icon (installability).
+ const sizes = m.icons.map((i: { sizes: string }) => i.sizes);
+ expect(sizes).toContain("192x192");
+ expect(sizes).toContain("512x512");
+ expect(
+ m.icons.some((i: { purpose?: string }) => i.purpose === "maskable"),
+ ).toBe(true);
+ // theme_color is a hex string (from the site accent or the family brass).
+ expect(m.theme_color).toMatch(/^#[0-9a-f]{6}$/i);
+});
+
+test("serves the service worker and the icon set", async ({ request }) => {
+ const sw = await request.get("/sw.js");
+ expect(sw.ok()).toBeTruthy();
+ for (const icon of [
+ "/icons/icon-192.png",
+ "/icons/icon-512.png",
+ "/icons/maskable-512.png",
+ "/icons/apple-touch-icon.png",
+ ]) {
+ const r = await request.get(icon);
+ expect(r.ok(), icon).toBeTruthy();
+ }
+});
+
+// Note: the service worker registers in PRODUCTION builds only (it caches
+// assets/data and would fight dev hot-reload + this suite's request mocking), so
+// live registration isn't asserted here — the production build proves /sw.js,
+// the manifest, and the icons emit into out/. See ServiceWorkerRegister.tsx.
+
+test("the offline page lists channels with a connection status and download control", async ({
+ page,
+}) => {
+ await page.goto("/offline/");
+ await expect(
+ page.getByRole("heading", { name: "Offline", level: 1 }),
+ ).toBeVisible();
+ await expect(page.getByLabel("connection status")).toBeVisible();
+ // testsite exposes at least one channel → a Download control is present.
+ await expect(
+ page.getByRole("button", { name: /download .* for offline/ }).first(),
+ ).toBeVisible();
+});
diff --git a/export/public/icons/apple-touch-icon.png b/export/public/icons/apple-touch-icon.png
Binary files differ.
diff --git a/export/public/icons/icon-192.png b/export/public/icons/icon-192.png
Binary files differ.
diff --git a/export/public/icons/icon-512.png b/export/public/icons/icon-512.png
Binary files differ.
diff --git a/export/public/icons/icon.svg b/export/public/icons/icon.svg
@@ -0,0 +1,6 @@
+<svg xmlns="http://www.w3.org/2000/svg" width="512" height="512" viewBox="0 0 512 512" role="img" aria-label="Archive">
+ <rect width="512" height="512" rx="112" fill="#0c0a08"/>
+ <polygon points="256,92 420,256 256,420 92,256" fill="#e3b15c"/>
+ <polygon points="256,150 362,256 256,362 150,256" fill="#0c0a08"/>
+ <polygon points="256,196 316,256 256,316 196,256" fill="#e3b15c"/>
+</svg>
diff --git a/export/public/icons/maskable-512.png b/export/public/icons/maskable-512.png
Binary files differ.
diff --git a/export/public/icons/maskable.svg b/export/public/icons/maskable.svg
@@ -0,0 +1,6 @@
+<svg xmlns="http://www.w3.org/2000/svg" width="512" height="512" viewBox="0 0 512 512" role="img" aria-label="Archive">
+ <rect width="512" height="512" fill="#0c0a08"/>
+ <polygon points="256,140 372,256 256,372 140,256" fill="#e3b15c"/>
+ <polygon points="256,182 330,256 256,330 182,256" fill="#0c0a08"/>
+ <polygon points="256,214 298,256 256,298 214,256" fill="#e3b15c"/>
+</svg>
diff --git a/export/public/sw.js b/export/public/sw.js
@@ -0,0 +1,210 @@
+/* Service worker for the export (Archive) search sites.
+ *
+ * Offline model (see plan): browse-cache + opt-in per-channel download. Nothing
+ * huge is precached on install — the transcript corpus is hundreds of MB. We
+ * runtime-cache the app shell + whatever the user actually opens, and expose a
+ * message API so the page can bulk-download a channel's shards for offline use.
+ *
+ * Caches:
+ * SHELL — app shell: hashed /_next/static/* (immutable, cache-first) + HTML
+ * navigations (network-first, cache fallback) + icons/manifest.
+ * PAGES — transcript/subs/summaries JSON shards (manifest.json + page-NNNN.json).
+ * Pages are cache-first; manifests are network-first so a rebuild is
+ * seen. Invalidation is keyed off manifest.generatedAt: when a channel's
+ * manifest generatedAt changes, that channel's cached page shards are
+ * evicted (the shards have stable, non-content-hashed URLs, so this is
+ * the only reliable staleness signal).
+ * META — tiny synthetic Responses storing each channel's last-seen generatedAt
+ * (avoids IndexedDB inside the SW).
+ */
+
+const VERSION = "v1";
+const SHELL = `shell-${VERSION}`;
+const PAGES = `pages-${VERSION}`;
+const META = `meta-${VERSION}`;
+
+// Matches /transcripts/<slug>/... and the parallel /subs, /summaries trees.
+const SHARD_RE = /^\/(transcripts|subs|summaries)\/([^/]+)\/(.+)$/;
+
+self.addEventListener("install", () => {
+ // Activate immediately — no precache list (corpus is too large to bundle).
+ self.skipWaiting();
+});
+
+self.addEventListener("activate", (event) => {
+ event.waitUntil(
+ (async () => {
+ // Drop caches from older SW versions.
+ const keep = new Set([SHELL, PAGES, META]);
+ const names = await caches.keys();
+ await Promise.all(names.map((n) => (keep.has(n) ? null : caches.delete(n))));
+ await self.clients.claim();
+ })(),
+ );
+});
+
+self.addEventListener("fetch", (event) => {
+ const req = event.request;
+ if (req.method !== "GET") return;
+ const url = new URL(req.url);
+ if (url.origin !== self.location.origin) return;
+
+ const shard = SHARD_RE.exec(url.pathname);
+ if (shard) {
+ const [, , slug, rest] = shard;
+ if (rest === "manifest.json") {
+ event.respondWith(handleManifest(req, slug));
+ } else {
+ event.respondWith(cacheFirst(req, PAGES));
+ }
+ return;
+ }
+
+ // App shell.
+ if (url.pathname.startsWith("/_next/static/") || url.pathname.startsWith("/icons/")) {
+ event.respondWith(cacheFirst(req, SHELL));
+ return;
+ }
+ if (req.mode === "navigate" || req.destination === "document") {
+ event.respondWith(networkFirstDoc(req));
+ return;
+ }
+});
+
+// Cache-first: serve from cache, else fetch and store.
+async function cacheFirst(req, cacheName) {
+ const cache = await caches.open(cacheName);
+ const hit = await cache.match(req);
+ if (hit) return hit;
+ try {
+ const res = await fetch(req);
+ if (res.ok) cache.put(req, res.clone());
+ return res;
+ } catch (err) {
+ // Offline miss — nothing cached.
+ return hit || Response.error();
+ }
+}
+
+// Navigations: network-first (fresh HTML after deploys), fall back to cache, then
+// to any cached document so the installed app still opens offline.
+async function networkFirstDoc(req) {
+ const cache = await caches.open(SHELL);
+ try {
+ const res = await fetch(req);
+ if (res.ok) cache.put(req, res.clone());
+ return res;
+ } catch (err) {
+ const hit = await cache.match(req);
+ if (hit) return hit;
+ const root = await cache.match("/");
+ if (root) return root;
+ return Response.error();
+ }
+}
+
+// Channel manifest: network-first. On a fresh manifest, compare generatedAt to
+// the stored value and evict that channel's page shards if it changed. Offline →
+// serve the cached manifest.
+async function handleManifest(req, slug) {
+ const pages = await caches.open(PAGES);
+ try {
+ const res = await fetch(req);
+ if (res.ok) {
+ const clone = res.clone();
+ try {
+ const body = await clone.json();
+ const gen = String(body?.generatedAt ?? "");
+ const prev = await readMeta(slug);
+ if (prev !== null && prev !== gen) {
+ await evictChannelPages(slug);
+ }
+ await writeMeta(slug, gen);
+ } catch {
+ // Unparseable manifest — still serve it.
+ }
+ pages.put(req, res.clone());
+ }
+ return res;
+ } catch (err) {
+ const hit = await pages.match(req);
+ return hit || Response.error();
+ }
+}
+
+async function evictChannelPages(slug) {
+ const pages = await caches.open(PAGES);
+ const keys = await pages.keys();
+ const prefixes = [
+ `/transcripts/${slug}/`,
+ `/subs/${slug}/`,
+ `/summaries/${slug}/`,
+ ];
+ await Promise.all(
+ keys.map((k) => {
+ const p = new URL(k.url).pathname;
+ const inChannel = prefixes.some(
+ (pre) => p.startsWith(pre) && !p.endsWith("manifest.json"),
+ );
+ return inChannel ? pages.delete(k) : null;
+ }),
+ );
+}
+
+// Per-channel generatedAt stored as a synthetic Response in the META cache.
+async function readMeta(slug) {
+ const meta = await caches.open(META);
+ const hit = await meta.match(`/__gen__/${slug}`);
+ return hit ? await hit.text() : null;
+}
+async function writeMeta(slug, gen) {
+ const meta = await caches.open(META);
+ await meta.put(`/__gen__/${slug}`, new Response(gen));
+}
+
+// ---- Message API (the page drives bulk download / eviction / status) ----
+self.addEventListener("message", (event) => {
+ const data = event.data || {};
+ const port = event.ports && event.ports[0];
+ if (data.type === "CACHE_URLS") {
+ event.waitUntil(cacheUrls(data.slug, data.urls || [], port));
+ } else if (data.type === "EVICT_CHANNEL") {
+ event.waitUntil(
+ evictChannelPages(data.slug).then(() => port && port.postMessage({ ok: true })),
+ );
+ } else if (data.type === "CHANNEL_STATUS") {
+ event.waitUntil(channelStatus(data.slug, port));
+ }
+});
+
+// Fetch + cache a list of shard URLs, reporting progress over the message port.
+async function cacheUrls(slug, urls, port) {
+ const cache = await caches.open(PAGES);
+ let done = 0;
+ for (const u of urls) {
+ try {
+ const res = await fetch(u, { cache: "reload" });
+ if (res.ok) await cache.put(u, res.clone());
+ } catch {
+ // Skip failures; the channel just won't be fully offline.
+ }
+ done++;
+ if (port) port.postMessage({ type: "progress", slug, done, total: urls.length });
+ }
+ if (port) port.postMessage({ type: "done", slug, done, total: urls.length });
+}
+
+// Report how many of a channel's page shards are currently cached.
+async function channelStatus(slug, port) {
+ const cache = await caches.open(PAGES);
+ const keys = await cache.keys();
+ const prefix = `/transcripts/${slug}/`;
+ let cached = 0;
+ for (const k of keys) {
+ const p = new URL(k.url).pathname;
+ if (p.startsWith(prefix) && p.endsWith(".json") && !p.endsWith("manifest.json")) {
+ cached++;
+ }
+ }
+ if (port) port.postMessage({ type: "status", slug, cachedPages: cached });
+}
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
@@ -38,6 +38,9 @@ importers:
execa:
specifier: ^9.6.1
version: 9.6.1
+ flexsearch:
+ specifier: ^0.8.0
+ version: 0.8.212
fs-extra:
specifier: ^11.3.4
version: 11.3.4
@@ -2552,6 +2555,9 @@ packages:
flatted@3.4.2:
resolution: {integrity: sha512-PjDse7RzhcPkIJwy5t7KPWQSZ9cAbzQXcafsetQoD7sOJRQlGikNbx7yZp2OotDnJyrDcbyRq3Ttb18iYOqkxA==}
+ flexsearch@0.8.212:
+ resolution: {integrity: sha512-wSyJr1GUWoOOIISRu+X2IXiOcVfg9qqBRyCPRUdLMIGJqPzMo+jMRlvE83t14v1j0dRMEaBbER/adQjp6Du2pw==}
+
for-each@0.3.5:
resolution: {integrity: sha512-dKx12eRCVIzqCxFGplyFKJMPvLEWgmNtUrpTiJIR5u97zEhRG8ySrtboPHZXx7daLxQVrl643cTzbab2tkQjxg==}
engines: {node: '>= 0.4'}
@@ -6195,6 +6201,8 @@ snapshots:
flatted@3.4.2: {}
+ flexsearch@0.8.212: {}
+
for-each@0.3.5:
dependencies:
is-callable: 1.2.7
