commit 20b533f2a9f19b3bd4d3ad498a1d2a5149188449
parent ff43837c4aafab58729486df34a6a8518d03c9e9
Author: I Mean I'm Just Saying <imeanimjustsaying@kiwifarms.st>
Date: Wed, 27 May 2026 23:15:59 -0400
Twitch support
Diffstat:
18 files changed, 199 insertions(+), 32 deletions(-)
diff --git a/common/components/PlayerProvider.tsx b/common/components/PlayerProvider.tsx
@@ -20,6 +20,7 @@ import { formatDate, formatDuration } from "../lib/format";
import type { Platform } from "../lib/transcripts";
import type { RumblePlayerHandle } from "./RumblePlayer";
import type { OdyseePlayerHandle } from "./OdyseePlayer";
+import type { TwitchPlayerHandle } from "./TwitchPlayer";
const ReactPlayer = dynamic(() => import("react-player/youtube"), {
ssr: false,
@@ -33,10 +34,15 @@ const OdyseePlayer = dynamic(() => import("./OdyseePlayer"), {
ssr: false,
});
+const TwitchPlayer = dynamic(() => import("./TwitchPlayer"), {
+ ssr: false,
+});
+
type PlayerHandle =
| Pick<ReactPlayerType, "seekTo">
| RumblePlayerHandle
- | OdyseePlayerHandle;
+ | OdyseePlayerHandle
+ | TwitchPlayerHandle;
export type Cue = { start: number; end: number; text: string };
@@ -524,12 +530,16 @@ export function PlayerProvider({
setCurrentTime(urlTime);
}, [urlSlug, urlTime]);
- // Rumble / Odysee have no onReady or progress events; seed currentTime from
- // the URL so clip-mark buttons and active-cue highlighting have a baseline,
- // and mark the slug ready so later ?t= changes can seek via the imperative
- // handle.
+ // Rumble / Odysee / Twitch have no onReady or progress events; seed
+ // currentTime from the URL so clip-mark buttons and active-cue highlighting
+ // have a baseline, and mark the slug ready so later ?t= changes can seek via
+ // the imperative handle.
useEffect(() => {
- if (detail?.platform === "rumble" || detail?.platform === "odysee") {
+ if (
+ detail?.platform === "rumble" ||
+ detail?.platform === "odysee" ||
+ detail?.platform === "twitch"
+ ) {
readyForSlugRef.current = detail.slug;
pendingSeekRef.current = null;
setCurrentTime(urlTime ?? 0);
@@ -654,6 +664,15 @@ export function PlayerProvider({
videoId={data.id}
startSeconds={urlTime ?? 0}
/>
+ ) : data.platform === "twitch" ? (
+ <TwitchPlayer
+ key={data.id}
+ ref={(p: TwitchPlayerHandle | null) => {
+ playerRef.current = p;
+ }}
+ videoId={data.id}
+ startSeconds={urlTime ?? 0}
+ />
) : (
<ReactPlayer
ref={(p: ReactPlayerType | null) => {
diff --git a/common/components/TwitchPlayer.tsx b/common/components/TwitchPlayer.tsx
@@ -0,0 +1,65 @@
+"use client";
+
+import { forwardRef, useImperativeHandle, useState } from "react";
+
+export type TwitchPlayerHandle = {
+ seekTo: (seconds: number, unit?: "seconds") => void;
+};
+
+type Props = {
+ videoId: string;
+ startSeconds?: number;
+};
+
+// Twitch's `time` param wants an `XhYmZs` string, not raw seconds.
+function toTwitchTime(totalSeconds: number): string {
+ const s = Math.max(0, Math.floor(totalSeconds));
+ const h = Math.floor(s / 3600);
+ const m = Math.floor((s % 3600) / 60);
+ return `${h}h${m}m${s % 60}s`;
+}
+
+// Twitch's iframe embed — like Rumble/Odysee there's no outside JS API, so we
+// seek by reloading with a new `time` query param.
+//
+// Twitch requires a `parent=` param naming every domain the player is embedded
+// under. We read `window.location.hostname` at runtime (this is a client-only
+// dynamic import, ssr:false), which covers the normal case. Caveat: Twitch
+// rejects bare IP-address parents and some reverse-proxy hosts — on those the
+// embed will refuse to load.
+const TwitchPlayer = forwardRef<TwitchPlayerHandle, Props>(function TwitchPlayer(
+ { videoId, startSeconds = 0 },
+ ref,
+) {
+ const [start, setStart] = useState(() =>
+ Math.max(0, Math.floor(startSeconds)),
+ );
+
+ useImperativeHandle(
+ ref,
+ () => ({
+ seekTo(seconds: number) {
+ setStart(Math.max(0, Math.floor(seconds)));
+ },
+ }),
+ [],
+ );
+
+ const parent =
+ typeof window !== "undefined" ? window.location.hostname : "localhost";
+ const src = `https://player.twitch.tv/?video=${encodeURIComponent(
+ videoId,
+ )}&parent=${encodeURIComponent(parent)}&time=${toTwitchTime(start)}&autoplay=true`;
+
+ return (
+ <iframe
+ key={start}
+ src={src}
+ allow="autoplay; fullscreen; encrypted-media; picture-in-picture"
+ allowFullScreen
+ style={{ border: 0, width: "100%", height: "100%" }}
+ />
+ );
+});
+
+export default TwitchPlayer;
diff --git a/common/components/charts/FilterBar.tsx b/common/components/charts/FilterBar.tsx
@@ -1,12 +1,10 @@
"use client";
-import type { Platform } from "../../lib/platform";
+import { PLATFORM_VALUES, type Platform } from "../../lib/platform";
import type { ChartFilters } from "../../lib/chartConfig";
export type ChannelOption = { slug: string; name: string };
-const PLATFORMS: Platform[] = ["youtube", "rumble", "odysee"];
-
// "20260513" <-> "2026-05-13" for <input type="date">.
function ymdToInput(ymd?: string): string {
if (!ymd || !/^\d{8}$/.test(ymd)) return "";
@@ -68,7 +66,7 @@ export function FilterBar({
<div className="flex flex-col gap-1">
<span className={label}>Platform</span>
<div className="flex gap-1">
- {PLATFORMS.map((p) => {
+ {PLATFORM_VALUES.map((p) => {
const on = filters.platforms?.includes(p) ?? false;
return (
<button
diff --git a/common/lib/chartShare.ts b/common/lib/chartShare.ts
@@ -3,7 +3,7 @@
// validated on the way back in with safe fallbacks. The encoded string is
// stored verbatim in a URL param (?chart= for one chart, ?dash= for a board).
-import type { Platform } from "./platform";
+import { PLATFORM_VALUES, type Platform } from "./platform";
import {
newChart,
nextChartId,
@@ -20,7 +20,6 @@ import {
const TYPES: ChartType[] = ["line", "bar", "area", "stackedBar", "pie"];
const BINS: TimeBin[] = ["week", "month", "quarter", "year"];
const GROUPS: SeriesGroupBy[] = ["none", "channel", "platform", "language"];
-const PLATFORMS: Platform[] = ["youtube", "rumble", "odysee"];
type SC = Record<string, unknown>;
@@ -52,7 +51,7 @@ function decFilters(raw: unknown): ChartFilters {
if (Array.isArray(r.ch)) out.channels = r.ch.filter((c): c is string => typeof c === "string");
if (Array.isArray(r.pl)) {
out.platforms = r.pl.filter((p): p is Platform =>
- (PLATFORMS as string[]).includes(p as string),
+ (PLATFORM_VALUES as readonly string[]).includes(p as string),
);
}
if (r.ht === 1) out.hasTranscriptOnly = true;
diff --git a/common/lib/platform.ts b/common/lib/platform.ts
@@ -1,9 +1,10 @@
-export type Platform = "youtube" | "rumble" | "odysee";
+export type Platform = "youtube" | "rumble" | "odysee" | "twitch";
export const PLATFORM_VALUES: ReadonlyArray<Platform> = [
"youtube",
"rumble",
"odysee",
+ "twitch",
];
export function detectPlatform(
@@ -15,6 +16,7 @@ export function detectPlatform(
if (host.endsWith("youtube.com") || host === "youtu.be") return "youtube";
if (host.endsWith("rumble.com")) return "rumble";
if (host.endsWith("odysee.com")) return "odysee";
+ if (host.endsWith("twitch.tv")) return "twitch";
} catch {
/* fall through */
}
@@ -27,6 +29,35 @@ export function platformQueueKey(
return `platform:${platform ?? "unknown"}`;
}
+// Best-effort registrable domain for an arbitrary URL, used to route
+// unrecognized sources to their own job queue. Uses a dependency-free
+// "last two labels" heuristic (clips.twitch.tv -> twitch.tv,
+// player.vimeo.com -> vimeo.com). Multi-part TLDs coarsen (example.co.uk
+// -> co.uk), which is harmless for a queue key — at worst two unrelated
+// sources share a serial queue. Returns null when the URL can't be parsed.
+export function registrableDomain(url: string | undefined | null): string | null {
+ if (!url) return null;
+ try {
+ const host = new URL(url).hostname.toLowerCase();
+ if (!host) return null;
+ const labels = host.split(".").filter(Boolean);
+ if (labels.length <= 2) return labels.join(".") || null;
+ return labels.slice(-2).join(".");
+ } catch {
+ return null;
+ }
+}
+
+// Queue key for a channel/video URL. Known platforms get their canonical
+// `platform:<name>` queue; unrecognized hosts fall back to a per-domain queue
+// (e.g. `platform:vimeo.com`) instead of all colliding in `platform:unknown`.
+export function queueKeyForUrl(url: string | undefined | null): string {
+ const detected = detectPlatform(url);
+ if (detected) return platformQueueKey(detected);
+ const host = registrableDomain(url);
+ return host ? `platform:${host}` : platformQueueKey(null);
+}
+
// System-wide queue for resource-bound local jobs (whisper, ffmpeg).
// Channels share this queue so two heavy local jobs never run in parallel.
export const TRANSCRIPTION_QUEUE = "transcription";
diff --git a/common/lib/transcripts-server.ts b/common/lib/transcripts-server.ts
@@ -33,12 +33,14 @@ function detectPlatform(meta: RawMetadata): Platform {
const key = meta.extractor_key ?? meta.extractor ?? "";
if (/^rumble/i.test(key)) return "rumble";
if (/^lbry/i.test(key)) return "odysee";
+ if (/^twitch/i.test(key)) return "twitch";
return "youtube";
}
function defaultWebpageUrl(platform: Platform, id: string): string {
if (platform === "rumble") return `https://rumble.com/${id}`;
if (platform === "odysee") return `https://odysee.com/${id}`;
+ if (platform === "twitch") return `https://www.twitch.tv/videos/${id}`;
return `https://www.youtube.com/watch?v=${id}`;
}
diff --git a/common/ytdlp/runYtdlp.ts b/common/ytdlp/runYtdlp.ts
@@ -914,6 +914,17 @@ export function extractVideoId(url: string): string | null {
if (lastColon > 0) return decoded.slice(lastColon + 1);
return null;
}
+ if (host.endsWith("twitch.tv")) {
+ // VODs: /videos/<id> or legacy /<channel>/v/<id>; clips:
+ // clips.twitch.tv/<slug> or /<channel>/clip/<slug>. Grab the segment
+ // after the marker; otherwise fall back to the last path segment.
+ const segs = u.pathname.split("/").filter(Boolean);
+ for (const marker of ["videos", "v", "clip", "clips"]) {
+ const idx = segs.indexOf(marker);
+ if (idx >= 0 && segs[idx + 1]) return segs[idx + 1];
+ }
+ return segs.pop() ?? null;
+ }
const seg = u.pathname.split("/").filter(Boolean).pop();
return seg ?? null;
} catch {
diff --git a/editor/CHANGELOG.md b/editor/CHANGELOG.md
@@ -1,6 +1,8 @@
# Changelog
## [Unreleased]
+- **Twitch.tv support.** Twitch is now a first-class platform: Twitch channel/VOD URLs are auto-detected, "Twitch" is selectable in the channel form's platform dropdown and the charts platform filter, videos play via an in-browser Twitch embed, and downloads are queued on a `platform:twitch` queue like the other platforms.
+- **Smarter queues for unrecognized sources.** When a channel's platform can't be detected, its jobs are now queued per-domain (e.g. `platform:vimeo.com`, with subdomains stripped) instead of all sharing the single `platform:unknown` queue — so unrelated unknown sources no longer block one another.
- **Charts authoring + stats dataset.** A new **Charts** tab lets you author the default chart dashboard that ships to the viewer — add/edit/remove charts and configure axes, metrics, series, filters, and search-derived series with a live preview; edits save automatically and bake into the next export build. Search-derived charts now use the full layered query builder (AND/OR/NOT, nesting, per-layer scope/regex), matching the search page. A new **Build stats dataset** action on the Build page extracts per-video stats (views, likes, comments, follower count, duration, categories, language, cue counts) into `export/public/stats/` for the charts to read; it's incremental (mtime short-circuit) and also runs automatically as part of the static export build.
- **Per-operation progress bars on `/jobs/active`.** Each running download/transcription now shows its own live progress bar parsed from the tool's shell output — yt-dlp's download percent (and fragment count) and whisper's transcribed position against the audio length. Batch jobs (Transcribe missing, Download from playlist, Sync, …) list a bar per in-flight video underneath the batch's overall bar; standalone single-video jobs get one too. The screen now polls about once a second so the bars advance live.
- **"Drain" (soft-cancel) for batch jobs.** Alongside the existing Cancel (which immediately kills everything), running batches now offer **Drain**: it lets the in-flight operations finish, starts no new ones, then completes the job normally and releases the queue so the next batch can run. Hard Cancel still works at any time, including mid-drain.
diff --git a/editor/app/channels/[slug]/availabilityActions.ts b/editor/app/channels/[slug]/availabilityActions.ts
@@ -3,8 +3,8 @@
import { revalidatePath } from "next/cache";
import { getPaths } from "yt-dlp-transcript-common/lib/paths";
import {
- detectPlatform,
platformQueueKey,
+ queueKeyForUrl,
} from "yt-dlp-transcript-common/lib/platform";
import { readChannelConfig } from "yt-dlp-transcript-common/controller/channels";
import {
@@ -38,9 +38,9 @@ export async function checkAvailabilityAction(
const config = await readChannelConfig(paths, slug);
// Default to the platform queue so availability checks share the per-source
// rate-limit budget with downloads instead of fighting them.
- const defaultQueue = platformQueueKey(
- config?.platform ?? detectPlatform(config?.url),
- );
+ const defaultQueue = config?.platform
+ ? platformQueueKey(config.platform)
+ : queueKeyForUrl(config?.url);
const limit = sanitizeConcurrency(concurrency);
return runManagedFunction({
kind: "check-availability",
diff --git a/editor/app/channels/[slug]/bulkVideoActions.ts b/editor/app/channels/[slug]/bulkVideoActions.ts
@@ -6,8 +6,8 @@ import { revalidatePath } from "next/cache";
import type { ChannelConfig } from "yt-dlp-transcript-common/lib/channelConfig";
import { getPaths } from "yt-dlp-transcript-common/lib/paths";
import {
- detectPlatform,
platformQueueKey,
+ queueKeyForUrl,
} from "yt-dlp-transcript-common/lib/platform";
import { readChannelConfig } from "yt-dlp-transcript-common/controller/channels";
import {
@@ -38,7 +38,9 @@ function resolveQueueKey(
if (override === undefined) return undefined;
const trimmed = override.trim();
if (!trimmed) {
- return platformQueueKey(config.platform ?? detectPlatform(config.url));
+ return config.platform
+ ? platformQueueKey(config.platform)
+ : queueKeyForUrl(config.url);
}
return trimmed;
}
diff --git a/editor/app/channels/[slug]/page.tsx b/editor/app/channels/[slug]/page.tsx
@@ -23,8 +23,8 @@ import { loadDownloadOutcome } from "yt-dlp-transcript-common/lib/downloadOutcom
import { getPaths } from "yt-dlp-transcript-common/lib/paths";
import { getSettings } from "yt-dlp-transcript-common/lib/settings";
import {
- detectPlatform,
platformQueueKey,
+ queueKeyForUrl,
TRANSCRIPTION_QUEUE,
} from "yt-dlp-transcript-common/lib/platform";
import { getRegistry } from "yt-dlp-transcript-common/jobs/registry";
@@ -138,9 +138,9 @@ export default async function ChannelDetailPage({
j.channelSlug === slug &&
(j.status === "running" || j.status === "queued"),
);
- const platformDefaultQueueKey = platformQueueKey(
- config.platform ?? detectPlatform(config.url),
- );
+ const platformDefaultQueueKey = config.platform
+ ? platformQueueKey(config.platform)
+ : queueKeyForUrl(config.url);
const existing = await readChannelSnapshot(paths, slug);
const snapshot = existing ?? (await generateChannelSnapshot(paths, slug));
// The retry-failures panel is interactive (a click mutates the file), so
diff --git a/editor/app/channels/[slug]/pipelineActions.ts b/editor/app/channels/[slug]/pipelineActions.ts
@@ -8,8 +8,8 @@ import {
} from "yt-dlp-transcript-common/lib/channelConfig";
import { getPaths } from "yt-dlp-transcript-common/lib/paths";
import {
- detectPlatform,
platformQueueKey,
+ queueKeyForUrl,
} from "yt-dlp-transcript-common/lib/platform";
import {
countNotYetDownloaded,
@@ -25,7 +25,9 @@ import {
import { makeTaskTracker } from "yt-dlp-transcript-common/jobs/taskHooks";
function defaultQueueKey(config: ChannelConfig): string {
- return platformQueueKey(config.platform ?? detectPlatform(config.url));
+ return config.platform
+ ? platformQueueKey(config.platform)
+ : queueKeyForUrl(config.url);
}
async function runPipelineAction(
diff --git a/editor/app/channels/[slug]/videos/[id]/page.tsx b/editor/app/channels/[slug]/videos/[id]/page.tsx
@@ -8,8 +8,8 @@ import { readChannelConfig } from "yt-dlp-transcript-common/controller/channels"
import { loadDownloadOutcome } from "yt-dlp-transcript-common/lib/downloadOutcome-server";
import { getPaths } from "yt-dlp-transcript-common/lib/paths";
import {
- detectPlatform,
platformQueueKey,
+ queueKeyForUrl,
} from "yt-dlp-transcript-common/lib/platform";
import { getRegistry } from "yt-dlp-transcript-common/jobs/registry";
import { RunningJobsList } from "../../../../jobs/components/RunningJobsList";
@@ -115,9 +115,9 @@ export default async function VideoDetailPage({
channelSlug: j.channelSlug,
videoId: j.videoId,
}));
- const defaultQueueKey = platformQueueKey(
- config.platform ?? detectPlatform(config.url),
- );
+ const defaultQueueKey = config.platform
+ ? platformQueueKey(config.platform)
+ : queueKeyForUrl(config.url);
return (
<div className="flex flex-col gap-6">
diff --git a/editor/app/channels/[slug]/videos/[id]/videoActions.ts b/editor/app/channels/[slug]/videos/[id]/videoActions.ts
@@ -11,8 +11,8 @@ import type {
import { AUDIO_FORMAT_VALUES } from "yt-dlp-transcript-common/lib/channelConfig";
import { getPaths } from "yt-dlp-transcript-common/lib/paths";
import {
- detectPlatform,
platformQueueKey,
+ queueKeyForUrl,
} from "yt-dlp-transcript-common/lib/platform";
import { readChannelConfig } from "yt-dlp-transcript-common/controller/channels";
import { pruneFailedTranscriptions } from "yt-dlp-transcript-common/controller/failedTranscriptions";
@@ -30,7 +30,9 @@ import { makeTaskTracker } from "yt-dlp-transcript-common/jobs/taskHooks";
function videoQueueKey(config: ChannelConfig, override: string | undefined): string {
if (override === undefined) {
- return platformQueueKey(config.platform ?? detectPlatform(config.url));
+ return config.platform
+ ? platformQueueKey(config.platform)
+ : queueKeyForUrl(config.url);
}
return override.trim();
}
diff --git a/editor/app/channels/components/ChannelForm.tsx b/editor/app/channels/components/ChannelForm.tsx
@@ -132,6 +132,7 @@ export function ChannelForm({
<option value="youtube">YouTube</option>
<option value="rumble">Rumble</option>
<option value="odysee">Odysee</option>
+ <option value="twitch">Twitch</option>
</select>
<span className="text-xs text-zinc-500">
Used as the default job queue, so all channels on the same platform
diff --git a/editor/e2e/channels.spec.ts b/editor/e2e/channels.spec.ts
@@ -27,6 +27,20 @@ test("creates a youtube channel", async ({ page }) => {
});
});
+test("auto-detects twitch platform from URL", async ({ page }) => {
+ await resetData("empty");
+ await page.goto("/channels/new");
+ await page.getByLabel(/^name/i).fill("Twitch Test Channel");
+ await page.getByLabel(/^slug/i).fill("twitch-test");
+ await page.getByLabel(/^url/i).fill("https://www.twitch.tv/sometwitchuser");
+ await page.getByRole("button", { name: /create channel/i }).click();
+ await page.waitForURL("**/channels/twitch-test", { timeout: 10_000 });
+ const config = await readJson<{ platform?: string; url?: string }>(
+ "test-transcripts/channels/twitch-test/config.json",
+ );
+ expect(config.platform).toBe("twitch");
+});
+
test("auto-derives slug from name when blank", async ({ page }) => {
await resetData("empty");
await page.goto("/channels/new");
diff --git a/editor/e2e/queues.spec.ts b/editor/e2e/queues.spec.ts
@@ -30,6 +30,24 @@ test("default queue is platform:<platform>", async ({ page }) => {
await expect(firstRow).toContainText("platform:youtube");
});
+test("unknown platform infers its queue from the URL domain", async ({
+ page,
+}) => {
+ await resetData("empty");
+ await page.goto("/channels/new");
+ await page.getByLabel(/^name/i).fill("Vimeo Channel");
+ await page.getByLabel(/^slug/i).fill("vimeo-test");
+ await page.getByLabel(/^url/i).fill("https://player.vimeo.com/video/12345");
+ await page.getByRole("button", { name: /create channel/i }).click();
+ await page.waitForURL("**/channels/vimeo-test", { timeout: 10_000 });
+
+ // No known platform → queue keyed by the registrable domain (subdomain
+ // stripped), not the catch-all platform:unknown.
+ await expect(
+ page.getByLabel("queue for Sync", { exact: true }),
+ ).toHaveValue("platform:vimeo.com");
+});
+
test("queues a second job in the same queue, runs sequentially", async ({
page,
}) => {
diff --git a/export/CHANGELOG.md b/export/CHANGELOG.md
@@ -1,6 +1,7 @@
# Changelog
## [Unreleased]
+- **Twitch.tv videos.** Twitch is now a supported platform: Twitch videos play inline via a Twitch embed, and "Twitch" appears as an option in the charts platform filter and series grouping.
- **Advanced query builder when charting search matches.** A chart's search source now uses the same layered query builder as the main search page — combine any number of layers with AND/OR/NOT, nest groups, and set each layer's scope and regex — instead of the old single term box. Simple one-term searches still collapse to a single input.
- **"Chart this search."** A new control beside "Share current search" turns the query you're looking at into a chart in one click: pick a starting template (mentions over time / matching videos per month / matches by channel) and jump to the Charts tab with the query pre-filled and ready to edit. Choose whether it's added to your existing board or opens as a single-chart board, and your selected channels carry over as the chart's channel filter.
- **Charts.** A new **Charts** tab (a prominent primary nav item alongside Search) visualises the library. Build charts from video metadata — upload date (binned by week/month/quarter/year, with optional cumulative running totals), engagement metrics (views, likes, comments, follower count), duration distributions, categories, languages — or from **search matches**, plotting how many videos match a term (or total hits) over time. Each chart has labelled axes and hover tooltips. Pick the chart type (line/bar/stacked/area/pie), split any chart into series by channel/platform/language, and apply a dashboard-wide filter bar (date range, platform, channel, has-transcript). The editor sets the data source with a clear Metadata/Search toggle, and search terms can be plain text or **regex**. A chart's date/channel/platform filters narrow the search scope (not just the displayed results), so a date range keeps a search chart light. The default board leads with example search charts, date-limited to the recent ~2 years so they load quickly out of the box. Charts open with an editor-authored default dashboard plus a grouped gallery of templates — including ready-made search examples — that you can fork and customise. Your board auto-saves locally and survives reloads; **Reset to default** restores the editor's dashboard. **Share** copies a link that restores the whole board, and every chart can be exported as a PNG (download or copy to clipboard) or its data as CSV. Engagement metrics come from a new stats dataset built alongside the search index.
