commit 9fed0dd7170efe759505fd5a750913eaae884421
parent 3dc9b95c1387303ebce7294a5e4d1d78429e516a
Author: I Mean I'm Just Saying <imeanimjustsaying@kiwifarms.st>
Date: Thu, 21 May 2026 15:42:48 -0400
layered search improvements
Diffstat:
7 files changed, 298 insertions(+), 11 deletions(-)
diff --git a/common/components/QueryBuilder.tsx b/common/components/QueryBuilder.tsx
@@ -16,6 +16,7 @@ import {
newLeaf,
removeNode,
replaceNode,
+ unwrapGroup,
wrapInGroup,
type GroupNode,
type LeafNode,
@@ -50,6 +51,8 @@ export default function QueryBuilder({
onChange(insertChild(root, groupId, child));
const onWrapLeaf = (leafId: string) =>
onChange(wrapInGroup(root, leafId, "AND"));
+ const onUnwrapGroup = (groupId: string) =>
+ onChange(unwrapGroup(root, groupId));
// Compact mode: root AND group with one non-negated leaf, no negation
// applied to the root either. Anything else falls back to the full
@@ -123,6 +126,7 @@ export default function QueryBuilder({
onRemoveNode={onRemoveNode}
onAddChild={onAddChild}
onWrapLeaf={onWrapLeaf}
+ onUnwrapGroup={onUnwrapGroup}
/>
</div>
);
diff --git a/common/components/QueryGroupView.tsx b/common/components/QueryGroupView.tsx
@@ -27,6 +27,7 @@ type Props = {
onRemoveNode: (id: string) => void;
onAddChild: (groupId: string, child: QueryNode) => void;
onWrapLeaf: (leafId: string) => void;
+ onUnwrapGroup: (groupId: string) => void;
};
export default function QueryGroupView({
@@ -39,6 +40,7 @@ export default function QueryGroupView({
onRemoveNode,
onAddChild,
onWrapLeaf,
+ onUnwrapGroup,
}: Props) {
const state = groupStates.get(group.id);
const onlyLeafCount = group.children.filter(isLeaf).length;
@@ -86,15 +88,30 @@ export default function QueryGroupView({
</span>
)}
{!isRoot && (
- <button
- type="button"
- onClick={() => onRemoveNode(group.id)}
- aria-label="Remove group"
- data-testid={`group-delete-${group.id}`}
- className="ml-auto text-xs rounded border border-zinc-300 dark:border-zinc-700 w-6 h-6 inline-flex items-center justify-center hover:bg-zinc-100 dark:hover:bg-zinc-800"
- >
- ×
- </button>
+ <div className="ml-auto flex items-center gap-1">
+ <button
+ type="button"
+ onClick={() => onUnwrapGroup(group.id)}
+ className="text-xs underline text-zinc-500 hover:text-zinc-700 dark:hover:text-zinc-300"
+ title={
+ group.negate
+ ? "Flatten this group into its parent (each child gets NOT applied)"
+ : "Flatten this group into its parent"
+ }
+ data-testid={`group-unwrap-${group.id}`}
+ >
+ unwrap
+ </button>
+ <button
+ type="button"
+ onClick={() => onRemoveNode(group.id)}
+ aria-label="Remove group"
+ data-testid={`group-delete-${group.id}`}
+ className="text-xs rounded border border-zinc-300 dark:border-zinc-700 w-6 h-6 inline-flex items-center justify-center hover:bg-zinc-100 dark:hover:bg-zinc-800"
+ >
+ ×
+ </button>
+ </div>
)}
</div>
@@ -128,6 +145,7 @@ export default function QueryGroupView({
onRemoveNode={onRemoveNode}
onAddChild={onAddChild}
onWrapLeaf={onWrapLeaf}
+ onUnwrapGroup={onUnwrapGroup}
/>
);
}
diff --git a/common/components/TranscriptSearch.tsx b/common/components/TranscriptSearch.tsx
@@ -789,7 +789,21 @@ export default function TranscriptSearch() {
const current = loadStoredState() ?? emptyStoredState();
current.working = nextSnapshot;
current.profiles = profiles;
- current.activeProfileName = activeProfileName;
+ // If the user committed changes that diverge from the active profile,
+ // drop the profile pointer. Otherwise hydration would prefer the
+ // (unchanged) profile snapshot on reload and the user's commit would
+ // silently revert.
+ const activeProfileSnap =
+ activeProfileName != null ? profiles[activeProfileName] : null;
+ const profileNowDirty =
+ activeProfileSnap != null &&
+ !snapshotsEqual(activeProfileSnap, nextSnapshot);
+ if (profileNowDirty) {
+ setActiveProfileName(null);
+ current.activeProfileName = null;
+ } else {
+ current.activeProfileName = activeProfileName;
+ }
saveStoredState(current);
promoteDraftsToCommitted();
// Write the URL: `qt=` is canonical. Strip legacy `q`/`m`/`re` so old
diff --git a/common/lib/searchQuery.ts b/common/lib/searchQuery.ts
@@ -118,7 +118,10 @@ export function canonicalHash(n: QueryNode): string {
function canonicalString(n: QueryNode): string {
if (isLeaf(n)) {
const q = n.useRegex ? n.query : n.query.toLowerCase();
- return `L|${n.scope}|${n.useRegex ? "r" : "p"}|${n.negate ? "n" : "y"}|${q}`;
+ // contributeHits MUST be in the hash: the cached payload's hit map is
+ // empty when contributeHits=false, so a collision lets a hits-off run
+ // poison the cache for a later hits-on run with the same query/scope.
+ return `L|${n.scope}|${n.useRegex ? "r" : "p"}|${n.negate ? "n" : "y"}|${n.contributeHits ? "h" : "H"}|${q}`;
}
const childStrs = n.children.filter(isNodeActive).map(canonicalString);
if (n.op === "OR") childStrs.sort();
@@ -369,6 +372,45 @@ export function wrapInGroup(
return replaceNode(root, id, newGroup({ op, children: [target] }));
}
+// Inverse of `wrapInGroup`: replace a group with its children spliced into
+// the parent at the same position. If the group is negated, flip each
+// promoted child's `negate` so the meaning of the subtree is preserved
+// locally (De Morgan applied per-child). Root is left alone since the tree
+// invariant requires a GroupNode at the top.
+export function unwrapGroup(root: GroupNode, id: string): GroupNode {
+ if (root.id === id) return root;
+ const out = unwrapInNode(root, id);
+ if (!out || !isGroup(out)) return root;
+ return out;
+}
+
+function unwrapInNode(node: QueryNode, id: string): QueryNode | null {
+ if (!isGroup(node)) return null;
+ let changed = false;
+ const newChildren: QueryNode[] = [];
+ for (const c of node.children) {
+ if (isGroup(c) && c.id === id) {
+ changed = true;
+ const promoted = c.negate ? c.children.map(flipNegate) : c.children;
+ newChildren.push(...promoted);
+ continue;
+ }
+ const r = unwrapInNode(c, id);
+ if (r) {
+ changed = true;
+ newChildren.push(r);
+ } else {
+ newChildren.push(c);
+ }
+ }
+ if (!changed) return null;
+ return { ...node, children: newChildren };
+}
+
+function flipNegate(n: QueryNode): QueryNode {
+ return { ...n, negate: !n.negate };
+}
+
// Walk every leaf in the tree, depth-first. Used by the UI to compute live
// counts, colour swatches, and the "contributing leaves" list.
export function forEachLeaf(
diff --git a/export/CHANGELOG.md b/export/CHANGELOG.md
@@ -4,8 +4,11 @@
### Added
- Composable layered search. The single search bar is now a query builder: any number of layers can be combined with AND / OR / NOT and arbitrary nesting. Each layer targets a scope (transcripts, live chat, or title/channel), and matches from every contributing layer are surfaced in the result list with a per-layer colour swatch. Per-layer results are memoised in IndexedDB so editing a deeper leaf only re-runs that layer against its already-narrowed scope. Simple one-keyword search still looks like a single input — the builder collapses to compact mode when there's only one layer. The composite query serialises into a new `qt=` URL parameter; legacy `?q=&m=&re=` links auto-migrate to a one-layer tree.
+- "Unwrap" button on every non-root group in the query builder. Flattens the group's children into the parent at the group's position, giving users a fast way back out of a sub-group they no longer want. If the group has NOT set, each promoted child inherits a flipped `negate` so the meaning of the subtree doesn't silently flip.
- New `/changelog` page that renders the export's `CHANGELOG.md`. Linked from the right side of the sticky header.
- Per-heading copy-link buttons on the changelog page for permalinks to any section.
### Fixed
+- Layered search no longer drops transcript hit rows for AND-chained leaves after certain edit sequences. The per-leaf result cache was keyed without `contributeHits`, so a leaf evaluated with "Show hits in results" off would write an empty-hits cache entry that a later same-query/scope hits-on leaf would silently inherit — the result list surfaced the video but no hit timestamps. Adding `contributeHits` to the canonical hash separates the two payload shapes. Toggling regex (which already changed the hash) was the existing workaround.
+- Filter changes survive a refresh again. When the user had any saved profile active, `commitSearch` was writing the new state to the working snapshot but leaving the profile pointer set; hydration then preferred the unchanged profile snapshot and silently reverted the commit. Committing changes that diverge from the active profile now clears the profile pointer (the selector falls back to "(unsaved)") so the next reload reads the working snapshot.
- `build:index` no longer runs out of memory on large datasets. Both the transcripts and the subs page writers now stream each entry directly to disk and hash it incrementally instead of materialising the joined page body in memory, which previously OOMed when a single live-chat track encoded to hundreds of MB. The post-processing "is this video deleted?" pass is now an in-memory LMDB scan (the availability check is cached in the mtime record at mutation time, schema bumped to 8 to invalidate the old cache) instead of ~28k sequential `availability.json` reads on every build. The `build:index` script also pre-sets `--max-old-space-size=8192` as a backstop.
diff --git a/export/e2e/filter-profile-persistence.spec.ts b/export/e2e/filter-profile-persistence.spec.ts
@@ -0,0 +1,86 @@
+import { expect, test } from "@playwright/test";
+import { installRoutes } from "./helpers";
+
+// Regression: when a profile is active and the user commits filter edits
+// that diverge from that profile, commitSearch must drop the profile
+// pointer. Otherwise hydration on reload prefers the (unchanged) profile
+// snapshot over the just-committed `working` snapshot and the user sees
+// their commit silently revert.
+//
+// The Save-as profile UI is only rendered when there are multiple channels
+// to choose between, but the underlying behavior is about
+// commitSearch + hydration. We pre-seed localStorage directly via
+// page.evaluate (not addInitScript — that would re-fire on reload and
+// clobber the commit we're trying to test) so the test isn't coupled to
+// fixture channel counts.
+
+const STORAGE_KEY = "ytdlp-tb:export-filters";
+
+test.describe("filter persistence — diverging commit clears active profile", () => {
+ test.beforeEach(async ({ page }) => {
+ await installRoutes(page);
+ });
+
+ test("commit-with-divergence survives refresh", async ({ page }) => {
+ // First navigation: just to give us a window to write localStorage on.
+ // Hydration here runs against empty storage; we throw it away.
+ await page.goto("/");
+ await page.evaluate(
+ ({ key, value }) => {
+ window.localStorage.setItem(key, value);
+ },
+ {
+ key: STORAGE_KEY,
+ value: JSON.stringify({
+ v: 1,
+ // Profile "p1": Livestreams off (nol=true), Deleted on.
+ working: { channels: { included: [], excluded: [] }, nol: true },
+ profiles: {
+ p1: { channels: { included: [], excluded: [] }, nol: true },
+ },
+ activeProfileName: "p1",
+ }),
+ },
+ );
+
+ // Reload so hydration picks up the pre-seeded state.
+ await page.reload();
+ const livestreams = page.getByRole("checkbox", { name: "Livestreams" });
+ const deleted = page.getByRole("checkbox", { name: "Deleted" });
+ await expect(livestreams).not.toBeChecked();
+ await expect(deleted).toBeChecked();
+
+ // Diverge from the profile: also uncheck Deleted, then commit.
+ await deleted.uncheck();
+ await page.getByPlaceholder("Search transcripts...").press("Enter");
+
+ // The commit should clear activeProfileName and persist the new
+ // divergent state into `working` (nd=true added).
+ await expect
+ .poll(
+ async () =>
+ await page.evaluate((key) => {
+ const raw = window.localStorage.getItem(key);
+ if (!raw) return null;
+ const parsed = JSON.parse(raw);
+ return {
+ activeProfileName: parsed.activeProfileName,
+ workingNd: parsed.working?.nd ?? false,
+ workingNol: parsed.working?.nol ?? false,
+ };
+ }, STORAGE_KEY),
+ { timeout: 5_000 },
+ )
+ .toEqual({ activeProfileName: null, workingNd: true, workingNol: true });
+
+ // Reload. Without the fix, hydration would re-read profile p1 (Deleted
+ // still on) and silently revert the commit.
+ await page.reload();
+ await expect(
+ page.getByRole("checkbox", { name: "Livestreams" }),
+ ).not.toBeChecked();
+ await expect(
+ page.getByRole("checkbox", { name: "Deleted" }),
+ ).not.toBeChecked();
+ });
+});
diff --git a/export/e2e/query-tree.spec.ts b/export/e2e/query-tree.spec.ts
@@ -235,6 +235,126 @@ test.describe("composite search — query tree", () => {
});
});
+ test("hit-contribution: toggling 'Show hits' on a leaf surfaces hits (no stale cache)", async ({
+ page,
+ }) => {
+ // Regression: canonicalString must include contributeHits in the leaf
+ // hash. If it doesn't, two leaves that differ only in `Show hits` share
+ // a cache key — the hits-off entry (empty hit map) poisons later
+ // hits-on reads, so the result list shows slugs but zero hit rows. We
+ // load a two-leaf AND where the first leaf is h:0, let the pipeline
+ // cache it, then flip the checkbox to h:1 and assert hit rows appear.
+ // (Two leaves to escape compact mode so the "Show hits" checkbox is
+ // accessible.)
+ const tree: SGroup = {
+ k: "g",
+ o: "AND",
+ c: [
+ { k: "l", q: "alpha", s: "transcripts", h: 0 },
+ { k: "l", q: "gamma", s: "transcripts" },
+ ],
+ };
+ await page.goto(`/?qt=${qt(tree)}`);
+ await expectResultSlugs(page, [
+ TRANSCRIPT_ONLY_SLUG,
+ CHAT_SMALL_SLUG,
+ CHAT_LARGE_SLUG,
+ ]);
+ // Only the gamma leaf contributes hits — 1 leaf-section per matched video.
+ await expect(
+ page.locator(
+ `[data-result-slug="${TRANSCRIPT_ONLY_SLUG}"] [data-leaf-section]`,
+ ),
+ ).toHaveCount(1);
+
+ // Flip the alpha leaf's "Show hits" checkbox on and re-commit.
+ const showHits = page.locator('input[data-testid^="leaf-hits-"]').first();
+ await showHits.check();
+ await page.getByTestId("search-submit").click();
+
+ // Now alpha contributes hits too — every matched video should show two
+ // leaf-section bars (one per contributing leaf).
+ await expect(
+ page.locator(
+ `[data-result-slug="${TRANSCRIPT_ONLY_SLUG}"] [data-leaf-section]`,
+ ),
+ ).toHaveCount(2, { timeout: 15_000 });
+ });
+
+ test("unwrap: wrapping then unwrapping a leaf restores the original tree", async ({
+ page,
+ }) => {
+ // Two leaves so the builder isn't in compact mode (the `wrap` button is
+ // only rendered in the full leaf card).
+ const tree: SGroup = {
+ k: "g",
+ o: "AND",
+ c: [
+ { k: "l", q: "alpha", s: "transcripts" },
+ { k: "l", q: "beta", s: "transcripts" },
+ ],
+ };
+ await page.goto(`/?qt=${qt(tree)}`);
+ await expectResultSlugs(page, [
+ TRANSCRIPT_ONLY_SLUG,
+ CHAT_SMALL_SLUG,
+ CHAT_LARGE_SLUG,
+ ]);
+
+ // Wrap the first leaf in an AND sub-group, then unwrap it.
+ await page.getByTestId(/^leaf-wrap-/).first().click();
+ const innerGroupUnwrap = page.getByTestId(/^group-unwrap-/);
+ await expect(innerGroupUnwrap).toBeVisible();
+ await innerGroupUnwrap.click();
+ await expect(page.getByTestId(/^group-unwrap-/)).toHaveCount(0);
+
+ // Both leaf inputs are now direct children of the root group again.
+ const leafInputs = page.locator('input[data-testid^="leaf-query-"]');
+ await expect(leafInputs).toHaveCount(2);
+ await expect(leafInputs.nth(0)).toHaveValue("alpha");
+ await expect(leafInputs.nth(1)).toHaveValue("beta");
+ });
+
+ test("unwrap (negated): each promoted child inherits the group's NOT", async ({
+ page,
+ }) => {
+ // Inner group `(metadata:'Transcript only')` flipped to NOT — wraps the
+ // leaf in an AND sub-group, then unwraps. The child leaf should come
+ // out with its own `negate` flipped.
+ const tree: SGroup = {
+ k: "g",
+ o: "AND",
+ c: [
+ { k: "l", q: "alpha", s: "transcripts" },
+ {
+ k: "g",
+ o: "AND",
+ n: 1,
+ c: [{ k: "l", q: "Transcript only", s: "metadata" }],
+ },
+ ],
+ };
+ await page.goto(`/?qt=${qt(tree)}`);
+ // alpha AND NOT (title contains "Transcript only") → small + large chat.
+ await expectResultSlugs(page, [CHAT_SMALL_SLUG, CHAT_LARGE_SLUG]);
+
+ // Unwrap the negated inner group.
+ const innerUnwrap = page.getByTestId(/^group-unwrap-/);
+ await expect(innerUnwrap).toHaveCount(1);
+ await innerUnwrap.click();
+
+ // The metadata leaf should now be a direct child of root with its
+ // `negate` checkbox checked (De Morgan applied locally).
+ await expect(page.getByTestId(/^group-unwrap-/)).toHaveCount(0);
+ const negateBoxes = page.locator('input[data-testid^="leaf-negate-"]');
+ // The transcripts leaf and the metadata leaf are now both root-level
+ // children. The metadata leaf is the negated one.
+ const checkedStates = await negateBoxes.evaluateAll((els) =>
+ (els as HTMLInputElement[]).map((e) => e.checked),
+ );
+ expect(checkedStates.filter(Boolean).length).toBe(1);
+ });
+
test("builder UI: + Add layer adds a second leaf and commits on Search", async ({
page,
}) => {
