commit 3ccdc7280d42a9eb8aa97516838584af35c0a7f4
parent 24294e60f987bc3c979a77856c1e8b7789545f0a
Author: I Mean I'm Just Saying <imeanimjustsaying@kiwifarms.st>
Date: Thu, 2 Jul 2026 02:28:09 -0400
Phase 7: hub service worker + origin-aware offline cache
- New export/service-worker/sw-hub.js: cross-origin variant of the site SW.
Caches transcript/subs/summaries shards from ANY origin (federation), always
fetching readable `cors` responses (the added origins are CORS-open). Staleness
(generatedAt) META, channel eviction, and status are scoped per (origin, slug)
keyed off the full request URL; the app shell stays same-origin. compose-hub
already prefers sw-hub.js → the hub build now ships it (verified in out/sw.js).
- export/app/lib/offlineCache.ts is origin-aware: downloadChannelOffline/
evictChannelOffline/channelCachedPages take an origin ("" = same-origin);
shard URLs are idBaseUrl(origin)-prefixed; the pinned registry keys on
makeId(origin, slug); SW messages carry origin + absolute URLs. OfflineManager
passes "" at every call site.
Single-site path is byte-identical (idBaseUrl("")==="" → root-relative URLs;
makeId("",slug)===slug → same pinned keys; the site SW ignores the origin field).
Type-checks clean; hub build ships the hub SW.
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Diffstat:
3 files changed, 272 insertions(+), 24 deletions(-)
diff --git a/export/app/components/OfflineManager.tsx b/export/app/components/OfflineManager.tsx
@@ -76,8 +76,8 @@ export function OfflineManager({ channels }: { channels: OfflineChannel[] }) {
let cancelled = false;
(async () => {
for (const c of channels) {
- if (!isPinned(c.slug)) continue;
- const cached = await channelCachedPages(c.slug);
+ if (!isPinned("", c.slug)) continue;
+ const cached = await channelCachedPages("", c.slug);
if (!cancelled && cached > 0) {
setState(c.slug, { download: "cached", done: cached, total: cached });
}
@@ -90,14 +90,14 @@ export function OfflineManager({ channels }: { channels: OfflineChannel[] }) {
async function download(slug: string) {
setState(slug, { download: "downloading", done: 0, total: 0 });
- const ok = await downloadChannelOffline(slug, ({ done, total }) =>
+ const ok = await downloadChannelOffline("", slug, ({ done, total }) =>
setState(slug, { download: "downloading", done, total }),
).catch(() => false);
setState(slug, { download: ok ? "cached" : "error" });
}
async function remove(slug: string) {
- await evictChannelOffline(slug);
+ await evictChannelOffline("", slug);
await dropChannelIndex(slug).catch(() => {});
setState(slug, { ...DEFAULT });
}
diff --git a/export/app/lib/offlineCache.ts b/export/app/lib/offlineCache.ts
@@ -3,11 +3,19 @@
// Client helpers for the opt-in "download this channel for offline" feature and
// the pinned-channel registry. Talks to the service worker (public/sw.js) over a
// MessageChannel; degrades gracefully where no SW is controlling the page.
+//
+// Origin-aware: every entry point takes an `origin` ("" = same-origin, the
+// single-site case; a full origin like "https://x.com" for a federated hub
+// channel). idBaseUrl("") === "" so same-origin URLs stay root-relative and the
+// pinned-registry key stays the bare slug — the single-site path is unchanged.
+// The hub SW (sw-hub.js) reads the `origin` in each message to scope caching
+// per (origin, slug); the site SW ignores it (same-origin only).
import {
transcriptPageFileName,
type ChannelTranscriptsManifest,
} from "yt-dlp-transcript-common/lib/manifest";
+import { idBaseUrl, makeId } from "yt-dlp-transcript-common/components/originId";
const PINNED_KEY = "ytdlp-tb:offline-channels";
@@ -49,10 +57,11 @@ function sendToSw(
}
async function fetchManifest(
+ origin: string,
slug: string,
): Promise<ChannelTranscriptsManifest | null> {
try {
- const res = await fetch(`/transcripts/${slug}/manifest.json`, {
+ const res = await fetch(`${idBaseUrl(origin)}/transcripts/${slug}/manifest.json`, {
cache: "no-store",
});
if (!res.ok) return null;
@@ -65,40 +74,51 @@ async function fetchManifest(
export type DownloadProgress = { done: number; total: number };
// Download every shard of a channel into the SW cache, reporting progress. The
-// URL list is the channel manifest plus each page-NNNN.json.
+// URL list is the channel manifest plus each page-NNNN.json, prefixed with the
+// channel's origin so a hub can cache cross-origin channels.
export async function downloadChannelOffline(
+ origin: string,
slug: string,
onProgress?: (p: DownloadProgress) => void,
): Promise<boolean> {
const sw = await controller();
if (!sw) return false;
- const manifest = await fetchManifest(slug);
+ const manifest = await fetchManifest(origin, slug);
if (!manifest) return false;
- const urls = [`/transcripts/${slug}/manifest.json`];
+ const base = idBaseUrl(origin);
+ const urls = [`${base}/transcripts/${slug}/manifest.json`];
for (let p = 0; p < manifest.pageCount; p++) {
- urls.push(`/transcripts/${slug}/${transcriptPageFileName(p)}`);
+ urls.push(`${base}/transcripts/${slug}/${transcriptPageFileName(p)}`);
}
- await sendToSw(sw, { type: "CACHE_URLS", slug, urls }, (data) => {
+ await sendToSw(sw, { type: "CACHE_URLS", origin, slug, urls }, (data) => {
if (data.type === "progress") {
onProgress?.({ done: data.done, total: data.total });
}
});
- pin(slug);
+ pin(origin, slug);
return true;
}
-export async function evictChannelOffline(slug: string): Promise<void> {
+export async function evictChannelOffline(
+ origin: string,
+ slug: string,
+): Promise<void> {
const sw = await controller();
- if (sw) await sendToSw(sw, { type: "EVICT_CHANNEL", slug }).catch(() => {});
- unpin(slug);
+ if (sw) {
+ await sendToSw(sw, { type: "EVICT_CHANNEL", origin, slug }).catch(() => {});
+ }
+ unpin(origin, slug);
}
// How many page shards of a channel are currently cached (0 = not offline).
-export async function channelCachedPages(slug: string): Promise<number> {
+export async function channelCachedPages(
+ origin: string,
+ slug: string,
+): Promise<number> {
const sw = await controller();
if (!sw) return 0;
try {
- const data = await sendToSw(sw, { type: "CHANNEL_STATUS", slug });
+ const data = await sendToSw(sw, { type: "CHANNEL_STATUS", origin, slug });
return typeof data.cachedPages === "number" ? data.cachedPages : 0;
} catch {
return 0;
@@ -106,6 +126,10 @@ export async function channelCachedPages(slug: string): Promise<number> {
}
// ---- Pinned registry (which channels the user chose to keep offline) ----
+// Entries are origin-qualified ids (makeId): a bare slug for same-origin,
+// "origin\tslug" for a federated channel — so a hub can pin the same channel
+// name from different sites without collision, and single-site pins are
+// unchanged (bare slugs).
export function pinnedChannels(): string[] {
if (typeof window === "undefined") return [];
try {
@@ -117,20 +141,21 @@ export function pinnedChannels(): string[] {
return [];
}
}
-export function isPinned(slug: string): boolean {
- return pinnedChannels().includes(slug);
+export function isPinned(origin: string, slug: string): boolean {
+ return pinnedChannels().includes(makeId(origin, slug));
}
-function writePinned(slugs: string[]): void {
+function writePinned(ids: string[]): void {
if (typeof window === "undefined") return;
try {
- window.localStorage.setItem(PINNED_KEY, JSON.stringify([...new Set(slugs)]));
+ window.localStorage.setItem(PINNED_KEY, JSON.stringify([...new Set(ids)]));
} catch {
/* quota — non-fatal */
}
}
-function pin(slug: string): void {
- writePinned([...pinnedChannels(), slug]);
+function pin(origin: string, slug: string): void {
+ writePinned([...pinnedChannels(), makeId(origin, slug)]);
}
-function unpin(slug: string): void {
- writePinned(pinnedChannels().filter((s) => s !== slug));
+function unpin(origin: string, slug: string): void {
+ const id = makeId(origin, slug);
+ writePinned(pinnedChannels().filter((s) => s !== id));
}
diff --git a/export/service-worker/sw-hub.js b/export/service-worker/sw-hub.js
@@ -0,0 +1,223 @@
+/* Service worker for the Archilyzer HUB (INSTANCE_MODE=hub).
+ *
+ * Same offline model as the site SW (browse-cache + opt-in per-channel
+ * download), but CROSS-ORIGIN: the hub federates content from many origins, so
+ * this worker caches transcript/subs/summaries shards from ANY origin — not
+ * just its own. The only origins it ever reaches are ones the user added, whose
+ * JSON is CORS-open (Access-Control-Allow-Origin: *); a cross-origin fetch is
+ * therefore a readable `cors` response that can be cached (never `no-cors` —
+ * opaque responses are unreadable and quota-toxic).
+ *
+ * Differences from the site SW:
+ * - no same-origin early-return: shard requests to any origin are cached;
+ * - staleness (generatedAt) META and channel eviction/status are scoped per
+ * (origin, slug), keyed off the FULL request URL, so same-named channels on
+ * different sites don't collide;
+ * - the page's message API carries `origin` + absolute shard URLs.
+ */
+
+const VERSION = "v1";
+const SHELL = `shell-${VERSION}`;
+const PAGES = `pages-${VERSION}`;
+const META = `meta-${VERSION}`;
+
+// Matches /transcripts/<slug>/... and the parallel /subs, /summaries trees, on
+// any origin (we test url.pathname, so it's origin-independent).
+const SHARD_RE = /^\/(transcripts|subs|summaries)\/([^/]+)\/(.+)$/;
+
+self.addEventListener("install", () => {
+ self.skipWaiting();
+});
+
+self.addEventListener("activate", (event) => {
+ event.waitUntil(
+ (async () => {
+ const keep = new Set([SHELL, PAGES, META]);
+ const names = await caches.keys();
+ await Promise.all(names.map((n) => (keep.has(n) ? null : caches.delete(n))));
+ await self.clients.claim();
+ })(),
+ );
+});
+
+self.addEventListener("fetch", (event) => {
+ const req = event.request;
+ if (req.method !== "GET") return;
+ const url = new URL(req.url);
+
+ // Shard requests are cached for ANY origin (this is the federation change).
+ const shard = SHARD_RE.exec(url.pathname);
+ if (shard) {
+ const [, , slug, rest] = shard;
+ if (rest === "manifest.json") {
+ event.respondWith(handleManifest(req, url.origin, slug));
+ } else {
+ event.respondWith(cacheFirst(req, PAGES));
+ }
+ return;
+ }
+
+ // App shell is same-origin only (the hub's own bundle).
+ if (url.origin !== self.location.origin) return;
+ if (url.pathname.startsWith("/_next/static/") || url.pathname.startsWith("/icons/")) {
+ event.respondWith(cacheFirst(req, SHELL));
+ return;
+ }
+ if (req.mode === "navigate" || req.destination === "document") {
+ event.respondWith(networkFirstDoc(req));
+ return;
+ }
+});
+
+// Cache-first: serve from cache, else fetch (cors) and store.
+async function cacheFirst(req, cacheName) {
+ const cache = await caches.open(cacheName);
+ const hit = await cache.match(req);
+ if (hit) return hit;
+ try {
+ const res = await fetch(req);
+ if (res.ok) cache.put(req, res.clone());
+ return res;
+ } catch (err) {
+ return hit || Response.error();
+ }
+}
+
+async function networkFirstDoc(req) {
+ const cache = await caches.open(SHELL);
+ try {
+ const res = await fetch(req);
+ if (res.ok) cache.put(req, res.clone());
+ return res;
+ } catch (err) {
+ const hit = await cache.match(req);
+ if (hit) return hit;
+ const root = await cache.match("/");
+ if (root) return root;
+ return Response.error();
+ }
+}
+
+// Channel manifest: network-first. On a fresh manifest, compare generatedAt to
+// the stored value (keyed per origin) and evict that (origin, slug)'s page
+// shards if it changed. Offline → serve the cached manifest.
+async function handleManifest(req, origin, slug) {
+ const pages = await caches.open(PAGES);
+ try {
+ const res = await fetch(req);
+ if (res.ok) {
+ const clone = res.clone();
+ try {
+ const body = await clone.json();
+ const gen = String(body?.generatedAt ?? "");
+ const prev = await readMeta(origin, slug);
+ if (prev !== null && prev !== gen) {
+ await evictChannelPages(origin, slug);
+ }
+ await writeMeta(origin, slug, gen);
+ } catch {
+ // Unparseable manifest — still serve it.
+ }
+ pages.put(req, res.clone());
+ }
+ return res;
+ } catch (err) {
+ const hit = await pages.match(req);
+ return hit || Response.error();
+ }
+}
+
+// True when a cached entry belongs to (origin, slug) and is a page shard (not a
+// manifest). origin === "" means the hub's own same-origin content.
+function belongsToChannel(entryUrl, origin, slug) {
+ const u = new URL(entryUrl);
+ const wantOrigin = origin || self.location.origin;
+ if (u.origin !== wantOrigin) return false;
+ const prefixes = [
+ `/transcripts/${slug}/`,
+ `/subs/${slug}/`,
+ `/summaries/${slug}/`,
+ ];
+ return prefixes.some(
+ (pre) => u.pathname.startsWith(pre) && !u.pathname.endsWith("manifest.json"),
+ );
+}
+
+async function evictChannelPages(origin, slug) {
+ const pages = await caches.open(PAGES);
+ const keys = await pages.keys();
+ await Promise.all(
+ keys.map((k) => (belongsToChannel(k.url, origin, slug) ? pages.delete(k) : null)),
+ );
+}
+
+// Per-(origin, channel) generatedAt stored as a synthetic Response in META.
+function metaKey(origin, slug) {
+ return `/__gen__/${encodeURIComponent(origin || "")}/${slug}`;
+}
+async function readMeta(origin, slug) {
+ const meta = await caches.open(META);
+ const hit = await meta.match(metaKey(origin, slug));
+ return hit ? await hit.text() : null;
+}
+async function writeMeta(origin, slug, gen) {
+ const meta = await caches.open(META);
+ await meta.put(metaKey(origin, slug), new Response(gen));
+}
+
+// ---- Message API (the page drives bulk download / eviction / status) ----
+self.addEventListener("message", (event) => {
+ const data = event.data || {};
+ const port = event.ports && event.ports[0];
+ const origin = data.origin || "";
+ if (data.type === "CACHE_URLS") {
+ event.waitUntil(cacheUrls(data.slug, data.urls || [], port));
+ } else if (data.type === "EVICT_CHANNEL") {
+ event.waitUntil(
+ evictChannelPages(origin, data.slug).then(
+ () => port && port.postMessage({ ok: true }),
+ ),
+ );
+ } else if (data.type === "CHANNEL_STATUS") {
+ event.waitUntil(channelStatus(origin, data.slug, port));
+ }
+});
+
+// Fetch + cache a list of (absolute or root-relative) shard URLs.
+async function cacheUrls(slug, urls, port) {
+ const cache = await caches.open(PAGES);
+ let done = 0;
+ for (const u of urls) {
+ try {
+ const res = await fetch(u, { cache: "reload" });
+ if (res.ok) await cache.put(u, res.clone());
+ } catch {
+ // Skip failures; the channel just won't be fully offline.
+ }
+ done++;
+ if (port) port.postMessage({ type: "progress", slug, done, total: urls.length });
+ }
+ if (port) port.postMessage({ type: "done", slug, done, total: urls.length });
+}
+
+// Report how many of a channel's transcript page shards are cached, scoped to
+// (origin, slug).
+async function channelStatus(origin, slug, port) {
+ const cache = await caches.open(PAGES);
+ const keys = await cache.keys();
+ const wantOrigin = origin || self.location.origin;
+ const prefix = `/transcripts/${slug}/`;
+ let cached = 0;
+ for (const k of keys) {
+ const u = new URL(k.url);
+ if (
+ u.origin === wantOrigin &&
+ u.pathname.startsWith(prefix) &&
+ u.pathname.endsWith(".json") &&
+ !u.pathname.endsWith("manifest.json")
+ ) {
+ cached++;
+ }
+ }
+ if (port) port.postMessage({ type: "status", slug, cachedPages: cached });
+}
