commit a353c5523f64965b3c2214914f0e38a2146e5704
parent fe6ed6df139e33fc0cc8cfa9d46c47bf516451d1
Author: I Mean I'm Just Saying <imeanimjustsaying@kiwifarms.st>
Date: Tue, 26 May 2026 22:50:27 -0400
more chart improvements
Diffstat:
8 files changed, 120 insertions(+), 38 deletions(-)
diff --git a/common/components/charts/ChartConfigEditor.tsx b/common/components/charts/ChartConfigEditor.tsx
@@ -31,14 +31,20 @@ const METRIC_FIELDS: { value: MetricField; label: string }[] = [
{ value: "cueCount", label: "Cue count" },
];
-// Pull the first leaf's query/scope out of a serialized tree for editing.
-function firstLeaf(tree?: string): { query: string; scope: LayerScope } {
- if (!tree) return { query: "", scope: "transcripts" };
+// Pull the first leaf's query/scope/regex out of a serialized tree for editing.
+function firstLeaf(tree?: string): {
+ query: string;
+ scope: LayerScope;
+ useRegex: boolean;
+} {
+ const fallback = { query: "", scope: "transcripts" as LayerScope, useRegex: false };
+ if (!tree) return fallback;
const root = parseRoot(tree);
- let found = { query: "", scope: "transcripts" as LayerScope };
+ let found = fallback;
if (root)
forEachLeaf(root, (l) => {
- if (!found.query && l.query) found = { query: l.query, scope: l.scope };
+ if (!found.query && l.query)
+ found = { query: l.query, scope: l.scope, useRegex: l.useRegex };
});
return found;
}
@@ -59,6 +65,7 @@ export function ChartConfigEditor({
);
const [term, setTerm] = useState(seed.query);
const [scope, setScope] = useState<LayerScope>(seed.scope);
+ const [useRegex, setUseRegex] = useState(seed.useRegex);
const patch = (p: Partial<ChartConfig>) => setDraft((d) => ({ ...d, ...p }));
const patchX = (x: XAxisField) => patch({ x });
@@ -69,7 +76,9 @@ export function ChartConfigEditor({
if (mode === "search") {
// Keep the search block even with an empty term, so the card prompts the
// user to fill one in rather than silently reverting to a metadata chart.
- const root = newGroup({ children: [newLeaf({ query: term.trim(), scope })] });
+ const root = newGroup({
+ children: [newLeaf({ query: term.trim(), scope, useRegex })],
+ });
search = {
queryTree: stringifyRoot(root),
metric: draft.search?.metric ?? "matchedVideos",
@@ -285,10 +294,18 @@ export function ChartConfigEditor({
<input
className={sel}
value={term}
- placeholder="e.g. bitcoin"
+ placeholder={useRegex ? "e.g. tim\\s+pool" : "e.g. bitcoin"}
onChange={(e) => setTerm(e.target.value)}
/>
</label>
+ <label className="flex items-center gap-2 self-end pb-1 text-sm">
+ <input
+ type="checkbox"
+ checked={useRegex}
+ onChange={(e) => setUseRegex(e.target.checked)}
+ />
+ Regex
+ </label>
<label className={lbl}>
In
<select
diff --git a/common/components/charts/useSearchSeries.ts b/common/components/charts/useSearchSeries.ts
@@ -40,16 +40,28 @@ export function useSearchSeries(
const root = useMemo(() => parseRoot(search.queryTree), [search.queryTree]);
+ // Narrow the search to videos passing the chart's filters (date range,
+ // channel, platform, …) instead of the whole corpus. This is what makes a
+ // date limit actually reduce load — the engine only fetches transcripts for
+ // in-scope videos. A stable key avoids re-running on unrelated renders.
+ const scopeSlugs = useMemo(() => {
+ const summarySet = new Set(summaries.map((s) => s.slug));
+ return applyFilters(stats, config.filters)
+ .map((s) => s.slug)
+ .filter((slug) => summarySet.has(slug));
+ }, [stats, summaries, config.filters]);
+ const scopeKey = scopeSlugs.length + ":" + (scopeSlugs[0] ?? "") + (scopeSlugs[scopeSlugs.length - 1] ?? "");
+
useEffect(() => {
setProgress(null);
setError(null);
controllerRef.current?.cancel();
controllerRef.current = null;
- if (!enabled || !root || summaries.length === 0) return;
+ if (!enabled || !root || scopeSlugs.length === 0) return;
try {
const controller = runQueryTree({
root,
- globalScope: summaries.map((s) => s.slug),
+ globalScope: scopeSlugs,
summaries,
initialHitLimit: 50000,
concurrency: 8,
@@ -64,7 +76,9 @@ export function useSearchSeries(
controllerRef.current?.cancel();
controllerRef.current = null;
};
- }, [enabled, root, summaries]);
+ // scopeKey captures filter/scope changes without churning on identity.
+ // eslint-disable-next-line react-hooks/exhaustive-deps
+ }, [enabled, root, summaries, scopeKey]);
const statBySlug = useMemo(() => {
const m = new Map<string, VideoStat>();
diff --git a/common/lib/chartConfig.ts b/common/lib/chartConfig.ts
@@ -80,10 +80,13 @@ export type GalleryGroup = {
templates: GalleryTemplate[];
};
+// Only the default dashboard is editor-authored/persisted. The gallery of
+// add-able templates (including the search-derived starters) always comes from
+// the code-level PRESET_GROUPS below, so it can never go stale against a saved
+// file.
export type ChartTemplates = {
version: 1;
defaultDashboard: Dashboard;
- gallery: GalleryGroup[];
};
// Deterministic id counter (no Date.now/Math.random) so server + client render
@@ -169,8 +172,14 @@ export function yAxisLabel(config: Pick<ChartConfig, "y" | "search">): string {
// A one-leaf search tree seeded with an (optionally empty) term; search-starter
// templates leave it empty so adding one opens the editor to fill in a term.
-function searchTree(query = "", scope: "transcripts" | "chat" | "metadata" = "transcripts"): string {
- return stringifyRoot(newGroup({ children: [newLeaf({ query, scope })] }));
+function searchTree(
+ query = "",
+ scope: "transcripts" | "chat" | "metadata" = "transcripts",
+ useRegex = false,
+): string {
+ return stringifyRoot(
+ newGroup({ children: [newLeaf({ query, scope, useRegex })] }),
+ );
}
const OVERVIEW_TEMPLATES: GalleryTemplate[] = [
@@ -281,9 +290,48 @@ const SEARCH_TEMPLATES: GalleryTemplate[] = [
},
];
+// Search examples first — search-derived charts are the headline use case.
export const PRESET_GROUPS: GalleryGroup[] = [
- { id: "overview", title: "Overview", templates: OVERVIEW_TEMPLATES },
{ id: "search", title: "Search examples", templates: SEARCH_TEMPLATES },
+ { id: "overview", title: "Overview", templates: OVERVIEW_TEMPLATES },
+];
+
+// YYYYMMDD for `yearsBack` years before today. Used to scope the default
+// search charts to a recent window so they don't search the whole corpus on
+// load. Computed at build time (baked into chart-templates.json).
+export function recentCutoffYmd(yearsBack: number): string {
+ const d = new Date();
+ d.setFullYear(d.getFullYear() - yearsBack);
+ const y = d.getFullYear();
+ const m = String(d.getMonth() + 1).padStart(2, "0");
+ const day = String(d.getDate()).padStart(2, "0");
+ return `${y}${m}${day}`;
+}
+
+export const DEFAULT_SEARCH_YEARS = 2;
+
+// Default search charts seeded with example terms so the board demonstrates the
+// search feature out of the box. Date-limited to the recent window so the
+// on-load search stays light; widen the date range to search further back.
+const DEFAULT_SEARCH_CHARTS: ChartConfig[] = [
+ newChart({
+ id: "default-tim-pool",
+ title: '“Tim Pool” mentions over time',
+ type: "line",
+ x: { kind: "time", bin: "month" },
+ y: { agg: "count" },
+ filters: { dateFrom: recentCutoffYmd(DEFAULT_SEARCH_YEARS) },
+ search: { queryTree: searchTree("Tim Pool"), metric: "totalHits" },
+ }),
+ newChart({
+ id: "default-under-attack",
+ title: 'Videos mentioning “under attack”',
+ type: "area",
+ x: { kind: "time", bin: "month" },
+ y: { agg: "count" },
+ filters: { dateFrom: recentCutoffYmd(DEFAULT_SEARCH_YEARS) },
+ search: { queryTree: searchTree("under attack"), metric: "matchedVideos" },
+ }),
];
export function defaultDashboard(): Dashboard {
@@ -291,7 +339,11 @@ export function defaultDashboard(): Dashboard {
version: 1,
title: "Overview",
filters: {},
- charts: OVERVIEW_TEMPLATES.slice(0, 4).map((t) => t.config),
+ // Search charts first, then the metadata overview charts.
+ charts: [
+ ...DEFAULT_SEARCH_CHARTS,
+ ...OVERVIEW_TEMPLATES.slice(0, 4).map((t) => t.config),
+ ],
};
}
@@ -299,6 +351,5 @@ export function defaultTemplates(): ChartTemplates {
return {
version: 1,
defaultDashboard: defaultDashboard(),
- gallery: PRESET_GROUPS,
};
}
diff --git a/common/lib/chartsStore.ts b/common/lib/chartsStore.ts
@@ -9,8 +9,6 @@ import {
defaultTemplates,
type ChartTemplates,
type Dashboard,
- type GalleryGroup,
- type GalleryTemplate,
} from "./chartConfig";
const EXPORT_TEMPLATES_FILE = "chart-templates.json";
@@ -19,21 +17,6 @@ export function exportTemplatesPath(paths: Paths): string {
return path.join(paths.exportPublicDir, EXPORT_TEMPLATES_FILE);
}
-// Accept the current grouped gallery; migrate a legacy flat array (items with
-// `config`) by wrapping it in a single group.
-function coerceGallery(raw: unknown, fallback: GalleryGroup[]): GalleryGroup[] {
- if (!Array.isArray(raw) || raw.length === 0) return fallback;
- const isGroup = (g: unknown): g is GalleryGroup =>
- !!g && typeof g === "object" && Array.isArray((g as GalleryGroup).templates);
- const isTemplate = (t: unknown): t is GalleryTemplate =>
- !!t && typeof t === "object" && "config" in (t as object);
- if (raw.every(isGroup)) return raw as GalleryGroup[];
- if (raw.every(isTemplate)) {
- return [{ id: "gallery", title: "Templates", templates: raw as GalleryTemplate[] }];
- }
- return fallback;
-}
-
function coerce(raw: unknown): ChartTemplates {
if (!raw || typeof raw !== "object") return defaultTemplates();
const r = raw as Partial<ChartTemplates>;
@@ -44,7 +27,6 @@ function coerce(raw: unknown): ChartTemplates {
r.defaultDashboard && Array.isArray(r.defaultDashboard.charts)
? (r.defaultDashboard as Dashboard)
: base.defaultDashboard,
- gallery: coerceGallery(r.gallery, base.gallery),
};
}
diff --git a/editor/app/charts/EditorChartsClient.tsx b/editor/app/charts/EditorChartsClient.tsx
@@ -5,7 +5,11 @@ import { QueryProvider } from "yt-dlp-transcript-common/components/QueryProvider
import { useStats } from "yt-dlp-transcript-common/components/statsCache";
import { useSummaries } from "yt-dlp-transcript-common/components/summariesCache";
import { ChartsDashboard } from "yt-dlp-transcript-common/components/charts/ChartsDashboard";
-import type { ChartTemplates, Dashboard } from "yt-dlp-transcript-common/lib/chartConfig";
+import {
+ PRESET_GROUPS,
+ type ChartTemplates,
+ type Dashboard,
+} from "yt-dlp-transcript-common/lib/chartConfig";
import { saveDefaultDashboardAction } from "./actions";
function Inner({ templates }: { templates: ChartTemplates }) {
@@ -41,7 +45,7 @@ function Inner({ templates }: { templates: ChartTemplates }) {
return (
<ChartsDashboard
initialDashboard={templates.defaultDashboard}
- gallery={templates.gallery}
+ gallery={PRESET_GROUPS}
stats={stats}
summaries={summaries}
channels={channels}
diff --git a/export/CHANGELOG.md b/export/CHANGELOG.md
@@ -1,7 +1,7 @@
# Changelog
## [Unreleased]
-- **Charts.** A new **Charts** tab (a prominent primary nav item alongside Search) visualises the library. Build charts from video metadata — upload date (binned by week/month/quarter/year, with optional cumulative running totals), engagement metrics (views, likes, comments, follower count), duration distributions, categories, languages — or from **search matches**, plotting how many videos match a term (or total hits) over time. Each chart has labelled axes and hover tooltips. Pick the chart type (line/bar/stacked/area/pie), split any chart into series by channel/platform/language, and apply a dashboard-wide filter bar (date range, platform, channel, has-transcript). The editor sets the data source with a clear Metadata/Search toggle. Charts open with an editor-authored default dashboard plus a grouped gallery of templates — including ready-made search examples — that you can fork and customise. Your board auto-saves locally and survives reloads; **Reset to default** restores the editor's dashboard. **Share** copies a link that restores the whole board, and every chart can be exported as a PNG (download or copy to clipboard) or its data as CSV. Engagement metrics come from a new stats dataset built alongside the search index.
+- **Charts.** A new **Charts** tab (a prominent primary nav item alongside Search) visualises the library. Build charts from video metadata — upload date (binned by week/month/quarter/year, with optional cumulative running totals), engagement metrics (views, likes, comments, follower count), duration distributions, categories, languages — or from **search matches**, plotting how many videos match a term (or total hits) over time. Each chart has labelled axes and hover tooltips. Pick the chart type (line/bar/stacked/area/pie), split any chart into series by channel/platform/language, and apply a dashboard-wide filter bar (date range, platform, channel, has-transcript). The editor sets the data source with a clear Metadata/Search toggle, and search terms can be plain text or **regex**. A chart's date/channel/platform filters narrow the search scope (not just the displayed results), so a date range keeps a search chart light. The default board leads with example search charts, date-limited to the recent ~2 years so they load quickly out of the box. Charts open with an editor-authored default dashboard plus a grouped gallery of templates — including ready-made search examples — that you can fork and customise. Your board auto-saves locally and survives reloads; **Reset to default** restores the editor's dashboard. **Share** copies a link that restores the whole board, and every chart can be exported as a PNG (download or copy to clipboard) or its data as CSV. Engagement metrics come from a new stats dataset built alongside the search index.
## [0.2.2] - 2026-05-26
- **Browse all videos without searching.** With no search query entered, the results list now shows every video matching the current filters (newest-first), instead of a "add a layer to search" prompt. Typing a query narrows the list as before; clearing it returns to the full browse view. This replaces the old workaround of searching regex `.` on titles.
diff --git a/export/app/charts/ChartsClient.tsx b/export/app/charts/ChartsClient.tsx
@@ -6,6 +6,7 @@ import { useSummaries } from "yt-dlp-transcript-common/components/summariesCache
import { ChartsDashboard } from "yt-dlp-transcript-common/components/charts/ChartsDashboard";
import {
defaultTemplates,
+ PRESET_GROUPS,
type ChartTemplates,
} from "yt-dlp-transcript-common/lib/chartConfig";
@@ -55,7 +56,7 @@ export function ChartsClient() {
return (
<ChartsDashboard
initialDashboard={templates.defaultDashboard}
- gallery={templates.gallery}
+ gallery={PRESET_GROUPS}
stats={stats}
summaries={summaries}
channels={channels}
diff --git a/export/e2e/charts.spec.ts b/export/e2e/charts.spec.ts
@@ -71,6 +71,19 @@ test.describe("charts", () => {
});
});
+ test("regex search term matches across alternation", async ({ page }) => {
+ await page.goto("/charts");
+ await page.getByRole("button", { name: "New blank" }).click();
+ await page.getByRole("button", { name: "Search matches" }).click();
+ await page.getByRole("checkbox", { name: "Regex" }).check();
+ await page.getByPlaceholder(/tim/).fill("alpha|beta");
+ await page.getByRole("button", { name: "Save", exact: true }).click();
+ // "alpha" and "beta" each appear once per fixture transcript → 3 videos, 6 hits.
+ await expect(page.getByText(/3 videos · 6 hits/)).toBeVisible({
+ timeout: 20_000,
+ });
+ });
+
test("charts render axis labels", async ({ page }) => {
await page.goto("/charts");
await expect(page.locator(".recharts-surface").first()).toBeVisible();
