import { readFileSync } from "node:fs"; import { expect, test, type Page } from "@playwright/test"; import { CHANNEL_SLUG, VIDEO_TRANSCRIPT_ONLY } from "./fixtures/data"; import { installRoutes } from "./helpers"; // ─── Live-search grounding helpers ────────────────────────────────────────── // The chat auto-grounds in the workspace's active search (no sessionStorage // hand-off). These tests therefore produce grounding by running a REAL search // on `/` (via the ?qt= query tree, same encoding as query-tree.spec) over the // mocked fixtures, then crossing to the chat via the Results⇄Chat nav so the // shared SearchSession stays mounted and the chat picks the results up live. type SLeaf = { k: "l"; q: string; s: "transcripts" | "chat" | "metadata" | "description" | "tags"; n?: 1; h?: 0; }; type SGroup = { k: "g"; o: "AND" | "OR"; n?: 1; c: (SLeaf | SGroup)[] }; function qt(root: SGroup): string { return encodeURIComponent(JSON.stringify(root)); } // The video whose fixture transcript has cues at 5s/50s/100s — grounding on it // lets "Load context"/fetch_context window past the matched 5s hit into the 50s // "beta line". const REAL_KEY = `${CHANNEL_SLUG}/${VIDEO_TRANSCRIPT_ONLY}`; // A single-video grounding: transcripts:"alpha" matches all three fixtures, and // the metadata leaf (filter-only, h:0 — contributes no hits) narrows to just // VIDEO_TRANSCRIPT_ONLY. So the chat grounds in ONE result carrying exactly the // one matched excerpt (the 5s "alpha line"). const ONE_RESULT_TREE: SGroup = { k: "g", o: "AND", c: [ { k: "l", q: "alpha", s: "transcripts" }, { k: "l", q: "Transcript only", s: "metadata", h: 0 }, ], }; // A two-video grounding: transcripts:"alpha" AND NOT chat:"message" keeps // VIDEO_TRANSCRIPT_ONLY (no chat) and VIDEO_CHAT_LARGE (chat without "message"), // dropping VIDEO_CHAT_SMALL. Only the transcripts leaf contributes hits, so each // grounded video carries its 5s "alpha line" excerpt. const TWO_RESULT_TREE: SGroup = { k: "g", o: "AND", c: [ { k: "l", q: "alpha", s: "transcripts" }, { k: "l", q: "message", s: "chat", n: 1 }, ], }; // A three-video grounding: transcripts:"alpha" alone matches all three fixtures. // Used by the corpus-sweep tests, which force a small batch size so three videos // span multiple batches. const ALL_RESULT_TREE: SGroup = { k: "g", o: "AND", c: [{ k: "l", q: "alpha", s: "transcripts" }], }; // Run a real search on `/` (via qt=), wait for it to settle, then cross to the // chat through the shared workspace nav so it auto-grounds in those results. async function searchThenChat(page: Page, tree: SGroup, cardCount: number) { await page.goto(`/?qt=${qt(tree)}`); await expect(page.locator("[data-card-header]")).toHaveCount(cardCount, { timeout: 15_000, }); await page .getByTestId("workspace-nav") .getByRole("link", { name: "Chat" }) .click(); } // The /ask agentic chat. Retrieval runs client-side over the mocked transcript // fixtures (installRoutes); the AI provider is mocked here. We drive the // provider-agnostic "scripted" mode so a single Anthropic SSE mock covers every // call (gather decisions via askOnce + the streamed answer): the mock reads the // request to decide what to return — SEARCH on a fresh question, DONE once // results are in or when the message is a reformat follow-up, and a Markdown // answer for the answer phase. const CORS = { "access-control-allow-origin": "*", "access-control-allow-headers": "*", "access-control-allow-methods": "*", }; function sse(text: string, stopReason?: string): string { const events = [ `data: ${JSON.stringify({ type: "content_block_delta", delta: { type: "text_delta", text }, })}\n\n`, ]; // The terminal message_delta carries stop_reason ("max_tokens" = truncated). if (stopReason) { events.push( `data: ${JSON.stringify({ type: "message_delta", delta: { stop_reason: stopReason } })}\n\n`, ); } events.push(`data: ${JSON.stringify({ type: "message_stop" })}\n\n`); return events.join(""); } function toolUse(name: string, input: Record) { return JSON.stringify({ content: [{ type: "tool_use", id: "tu_1", name, input }], stop_reason: "tool_use", }); } async function mockAnthropic(page: Page) { await page.route("https://api.anthropic.com/**", async (route) => { if (route.request().method() === "OPTIONS") { await route.fulfill({ status: 204, headers: CORS }); return; } const body = route.request().postDataJSON() as { system?: string; tools?: unknown[]; messages?: { role: string; content: unknown }[]; }; const system = body.system ?? ""; const msgs = body.messages ?? []; const lastUser = [...msgs].reverse().find((m) => m.role === "user"); const lastText = typeof lastUser?.content === "string" ? lastUser.content : ""; // Native tool-calling gather turn → respond with a tool_use JSON block. if (Array.isArray(body.tools) && !system.includes("Markdown")) { const searchedAlready = Array.isArray(lastUser?.content); // tool_result turn if (searchedAlready || /timeline|format/i.test(lastText)) { await route.fulfill({ status: 200, headers: { ...CORS, "content-type": "application/json" }, body: toolUse("finish", {}), }); } else { await route.fulfill({ status: 200, headers: { ...CORS, "content-type": "application/json" }, body: toolUse("search_transcripts", { query: "alpha" }), }); } return; } // Scripted gather + answer phase → stream SSE text. let text: string; if (system.includes("Markdown")) { text = "Here is the answer with a list:\n\n- point one [1]\n- point two"; // Truncation drill: a question containing "truncate" gets a cut-off answer // (terminal stop_reason "max_tokens"), so tests can exercise the notice. if (/truncate/i.test(lastText)) { await route.fulfill({ status: 200, headers: { ...CORS, "content-type": "text/event-stream" }, body: sse(text, "max_tokens"), }); return; } } else if (lastText.includes('Results for "')) { text = "DONE"; // we've searched once — stop } else if (/timeline|format/i.test(lastText)) { text = "DONE"; // reformat follow-up — no search needed } else { text = "SEARCH: alpha"; // fresh question — search a term in the fixtures } await route.fulfill({ status: 200, headers: { ...CORS, "content-type": "text/event-stream" }, body: sse(text), }); }); } async function setup(page: Page, mode: "Scripted" | "Native tools" = "Scripted") { await installRoutes(page); await mockAnthropic(page); await page.goto("/ask/"); // Pick the search mode while the settings panel is open (it starts open with no // key set, and no longer collapses while you type the key). await page.getByRole("button", { name: mode }).click(); await page.locator('input[placeholder^="sk-ant"]').fill("sk-ant-test"); // Guard against the one-time restore effect clearing the freshly-typed key. await expect(page.locator('input[placeholder^="sk-ant"]')).toHaveValue( "sk-ant-test", ); } async function ask(page: Page, text: string) { const box = page.getByPlaceholder(/Ask about the transcripts/); await box.fill(text); await page.getByRole("button", { name: "Ask", exact: true }).click(); } test.describe("ask chat", () => { test("searches, answers with citations, and renders Markdown", async ({ page, }) => { await setup(page); await ask(page, "tell me about the alpha discussion"); // The turn ran a search (persistent trace) for the query the model chose. await expect(page.getByText(/Searched:\s*alpha/)).toBeVisible(); // Citations reference the fixture videos. await expect(page.getByText("Transcript only", { exact: false }).first()).toBeVisible(); // The answer rendered as Markdown (a real list item). await expect(page.locator("li", { hasText: "point one" }).first()).toBeVisible(); }); test("clicking an inline [n] citation opens the transcript modal at the cited line", async ({ page, }) => { await setup(page); await ask(page, "tell me about the alpha discussion"); await expect(page.locator("li", { hasText: "point one" }).first()).toBeVisible(); // The [1] in the answer is a clickable anchor; its href stays a #cite- anchor // (a11y + hub fallback) even though the click opens the modal. const cite = page.getByRole("link", { name: "[1]", exact: true }); await expect(cite).toBeVisible(); await expect(cite).toHaveAttribute("href", /^#cite-/); // Clicking it opens the transcript modal seeked to the cited video — the URL // gains ?v=&t= (via replaceState on the same /ask route). // vm defaults to "transcript", which is intentionally not written to the URL. await cite.click(); await expect(page).toHaveURL(/[?&]v=/); await expect(page).toHaveURL(/[?&]t=\d/); const close = page.getByRole("button", { name: "Close player" }); await expect(close).toBeVisible(); // Dismissing (✕) returns to /ask with the chat intact (no ?v=). await close.click(); await expect(page).toHaveURL(/\/ask\/$/); await expect(page.locator("li", { hasText: "point one" }).first()).toBeVisible(); }); test("a source-list timestamp opens the transcript modal at its moment", async ({ page, }) => { await setup(page); await ask(page, "tell me about the alpha discussion"); await expect(page.locator("li", { hasText: "point one" }).first()).toBeVisible(); // Each source's timestamp is a button that opens the modal at that second. const ts = page.getByTitle("Open the transcript at this moment").first(); await expect(ts).toBeVisible(); await ts.click(); await expect(page).toHaveURL(/[?&]v=/); await expect(page.getByRole("button", { name: "Close player" })).toBeVisible(); }); test("a truncated answer (stop_reason max_tokens) shows the cut-off notice", async ({ page, }) => { await setup(page); // The mock returns a cut-off answer (stop_reason "max_tokens") for a question // containing "truncate". await ask(page, "tell me about alpha and truncate it"); await expect(page.locator("li", { hasText: "point one" }).first()).toBeVisible(); await expect(page.getByText(/Cut off at the model/)).toBeVisible(); }); test("Export debug JSON downloads a redacted snapshot (no API key)", async ({ page, }) => { await setup(page); await ask(page, "tell me about the alpha discussion"); await expect(page.locator("li", { hasText: "point one" }).first()).toBeVisible(); // The settings panel stays open after keying in; the debug export lives // behind its "Advanced" disclosure (provider, model and key are the only // top-level settings now). await page.getByText("Advanced", { exact: true }).click(); const [download] = await Promise.all([ page.waitForEvent("download"), page.getByRole("button", { name: "Download debug JSON" }).click(), ]); const path = await download.path(); const json = readFileSync(path, "utf8"); const obj = JSON.parse(json); expect(obj.meta.provider).toBeTruthy(); expect(obj.messages.length).toBeGreaterThan(0); // The key we typed must never appear in the export. expect(json).not.toContain("sk-ant-test"); }); test("a reformat follow-up reuses context without a junk search", async ({ page, }) => { await setup(page); await ask(page, "tell me about the alpha discussion"); // The "Searched:" trace only appears once the turn completes. await expect(page.getByText(/Searched:\s*alpha/)).toBeVisible(); await expect(page.locator("li", { hasText: "point one" })).toHaveCount(1); await ask(page, "format it in a timeline"); // A second answer rendered (turn 2 completed). await expect(page.locator("li", { hasText: "point one" })).toHaveCount(2); // …but the follow-up ran NO new search — only turn 1's trace exists. await expect(page.getByText(/^Searched:/)).toHaveCount(1); }); test("native function-calling drives the same search loop", async ({ page, }) => { await setup(page, "Native tools"); await ask(page, "tell me about the alpha discussion"); // The native tool_use loop searched and then answered. await expect(page.getByText(/Searched:\s*alpha/)).toBeVisible(); await expect(page.locator("li", { hasText: "point one" }).first()).toBeVisible(); }); test("the Format answers toggle switches to plain text", async ({ page }) => { await setup(page); await ask(page, "tell me about the alpha discussion"); await expect(page.locator("li", { hasText: "point one" }).first()).toBeVisible(); await page.getByLabel("Format answers").uncheck(); // Plain mode shows the raw Markdown source, dashes and all. await expect(page.getByText("- point one [1]", { exact: false })).toBeVisible(); }); test("Retry re-runs a failed answer reusing the gathered excerpts", async ({ page, }) => { await installRoutes(page); let answerAttempts = 0; await page.route("https://api.anthropic.com/**", async (route) => { if (route.request().method() === "OPTIONS") { await route.fulfill({ status: 204, headers: CORS }); return; } const body = route.request().postDataJSON() as { system?: string; messages?: { role: string; content: unknown }[]; }; const system = body.system ?? ""; const msgs = body.messages ?? []; const lastUser = [...msgs].reverse().find((m) => m.role === "user"); const lastText = typeof lastUser?.content === "string" ? lastUser.content : ""; if (system.includes("Markdown")) { answerAttempts += 1; if (answerAttempts === 1) { // First answer attempt fails transiently. await route.fulfill({ status: 503, headers: { ...CORS, "content-type": "application/json" }, body: JSON.stringify({ type: "error", error: { message: "overloaded" } }), }); return; } await route.fulfill({ status: 200, headers: { ...CORS, "content-type": "text/event-stream" }, body: sse("Recovered answer [1]"), }); return; } const text = lastText.includes('Results for "') ? "DONE" : "SEARCH: alpha"; await route.fulfill({ status: 200, headers: { ...CORS, "content-type": "text/event-stream" }, body: sse(text), }); }); await page.goto("/ask/"); await keyIn(page, "Scripted"); await ask(page, "tell me about the alpha discussion"); // The answer failed → an error turn with a Retry action. const retry = page.getByRole("button", { name: "Retry" }); await expect(retry).toBeVisible(); await retry.click(); // Retry succeeds… await expect(page.getByText(/Recovered answer/)).toBeVisible(); // …and it reused the excerpts — only the ONE original search ever ran. await expect(page.getByText(/Searched:\s*alpha/)).toHaveCount(1); }); test("the conversation persists across a reload", async ({ page }) => { await setup(page); await ask(page, "tell me about the alpha discussion"); await expect(page.locator("li", { hasText: "point one" }).first()).toBeVisible(); // Let the debounced persistence flush, then reload. await page.waitForTimeout(600); await page.reload(); // Restored from localStorage — question + answer are back without a new call. await expect(page.getByText("tell me about the alpha discussion")).toBeVisible(); await expect(page.getByText("point one", { exact: false }).first()).toBeVisible(); }); async function keyIn(page: Page, mode: "Scripted" | "Native tools" = "Scripted") { // Fill the key FIRST and confirm it sticks: a value that survives proves // React has hydrated and attached its handlers, so the mode-button click // that follows isn't lost to a pre-hydration no-op. The persistent workspace // search bar makes /ask heavier to hydrate under `next dev`, widening that // window — this mirrors the guard already used in setup(). const key = page.locator('input[placeholder^="sk-ant"]'); await key.fill("sk-ant-test"); await expect(key).toHaveValue("sk-ant-test"); await page.getByRole("button", { name: mode }).click(); } test("expand mode: the model reads more transcript via fetch_context", async ({ page, }) => { await installRoutes(page); let sawFetchTool = false; let answerSawWindow = false; let round = 0; await page.route("https://api.anthropic.com/**", async (route) => { if (route.request().method() === "OPTIONS") { await route.fulfill({ status: 204, headers: CORS }); return; } const body = route.request().postDataJSON() as { system?: string; tools?: { name?: string }[]; messages?: { role: string; content: unknown }[]; }; const system = body.system ?? ""; if (system.includes("Markdown")) { answerSawWindow = JSON.stringify(body.messages ?? []).includes("beta line"); await route.fulfill({ status: 200, headers: { ...CORS, "content-type": "text/event-stream" }, body: sse("Fetched answer [1]"), }); return; } // Native gather turn. if (Array.isArray(body.tools)) { if (body.tools.some((t) => t.name === "fetch_context")) sawFetchTool = true; const msgs = body.messages ?? []; const lastUser = [...msgs].reverse().find((m) => m.role === "user"); const isToolResult = Array.isArray(lastUser?.content); round += 1; await route.fulfill({ status: 200, headers: { ...CORS, "content-type": "application/json" }, body: isToolResult ? toolUse("finish", {}) : toolUse("fetch_context", { video: REAL_KEY, aroundSeconds: 5 }), }); return; } await route.fulfill({ status: 200, headers: { ...CORS, "content-type": "text/event-stream" }, body: sse("DONE"), }); }); await searchThenChat(page, ONE_RESULT_TREE, 1); await keyIn(page, "Native tools"); await expect(page.getByText(/Grounded in 1 result/)).toBeVisible(); // Expand mode → the fetch_context tool is offered and used. await page.getByLabel("Answer only from these results").uncheck(); await ask(page, "what surrounds the alpha moment"); await expect(page.getByText(/Fetched answer/)).toBeVisible(); expect(sawFetchTool).toBe(true); // The windowed line (50s "beta line") reached the answer's grounding. expect(answerSawWindow).toBe(true); expect(round).toBeGreaterThanOrEqual(2); // fetch round + finish round }); test("user 'Load context' enriches the pin and persists (strict, no search)", async ({ page, }) => { await installRoutes(page); let gatherCalls = 0; let answerSawWindow = false; await page.route("https://api.anthropic.com/**", async (route) => { if (route.request().method() === "OPTIONS") { await route.fulfill({ status: 204, headers: CORS }); return; } const body = route.request().postDataJSON() as { system?: string; messages?: { role: string; content: unknown }[]; }; const system = body.system ?? ""; if (system.includes("Markdown")) { answerSawWindow = JSON.stringify(body.messages ?? []).includes("beta line"); await route.fulfill({ status: 200, headers: { ...CORS, "content-type": "text/event-stream" }, body: sse("Answer over expanded excerpts [1]"), }); return; } gatherCalls += 1; // strict grounding → this must never fire await route.fulfill({ status: 200, headers: { ...CORS, "content-type": "text/event-stream" }, body: sse("SEARCH: alpha"), }); }); // Run the search and COMMIT it (click Search) so the query persists to // localStorage — the reload below lands on /ask/, whose URL carries no qt=, // and the live grounding must be restored from the persisted search. await page.goto(`/?qt=${qt(ONE_RESULT_TREE)}`); await expect(page.locator("[data-card-header]")).toHaveCount(1, { timeout: 15_000, }); await page.getByTestId("search-submit").click(); await page .getByTestId("workspace-nav") .getByRole("link", { name: "Chat" }) .click(); await keyIn(page); await expect(page.getByText(/Grounded in 1 result/)).toBeVisible(); await page.getByRole("button", { name: /Show the 1 video/ }).click(); // Starts with the single matched excerpt. await expect(page.getByText(/1 excerpt/)).toBeVisible(); await page.getByRole("button", { name: "context" }).click(); // Windowing pulled in the neighbouring cue → the count grows. await expect(page.getByText(/2 excerpts/)).toBeVisible(); // Persists across a reload (enrichment stored in the conversation; the live // search restored from localStorage). The key is only saved on send, so // re-enter it after reloading. A restored search loads into the bar but // does not run until the visitor submits it (release 8 slice E), so run it. await page.waitForTimeout(600); await page.reload(); await keyIn(page); await expect(page.getByTestId("search-submit")).toHaveAttribute( "data-dirty", "true", ); await expect(page.getByText(/Grounded in 1 result/)).toHaveCount(0); await page.getByTestId("search-submit").click(); await expect(page.getByText(/Grounded in 1 result/)).toBeVisible(); await page.getByRole("button", { name: /Show the 1 video/ }).click(); await expect(page.getByText(/2 excerpts/)).toBeVisible(); // Strict answer carries the enriched excerpts, with no gather search. await ask(page, "summarize these"); await expect(page.getByText(/Answer over expanded excerpts/)).toBeVisible(); expect(gatherCalls).toBe(0); expect(answerSawWindow).toBe(true); }); test("scripted transport degrades gracefully: no fetch tool, still answers", async ({ page, }) => { await installRoutes(page); let fetchToolEverOffered = false; await page.route("https://api.anthropic.com/**", async (route) => { if (route.request().method() === "OPTIONS") { await route.fulfill({ status: 204, headers: CORS }); return; } const body = route.request().postDataJSON() as { system?: string; tools?: { name?: string }[]; messages?: { role: string; content: unknown }[]; }; if (Array.isArray(body.tools) && body.tools.some((t) => t.name === "fetch_context")) { fetchToolEverOffered = true; } const system = body.system ?? ""; const msgs = body.messages ?? []; const lastUser = [...msgs].reverse().find((m) => m.role === "user"); const lastText = typeof lastUser?.content === "string" ? lastUser.content : ""; let text: string; if (system.includes("Markdown")) text = "Scripted answer [1]"; else if (lastText.includes('Results for "')) text = "DONE"; else text = "SEARCH: alpha"; await route.fulfill({ status: 200, headers: { ...CORS, "content-type": "text/event-stream" }, body: sse(text), }); }); await searchThenChat(page, ONE_RESULT_TREE, 1); await keyIn(page, "Scripted"); await expect(page.getByText(/Grounded in 1 result/)).toBeVisible(); await page.getByLabel("Answer only from these results").uncheck(); await ask(page, "what surrounds the alpha moment"); await expect(page.getByText(/Scripted answer/)).toBeVisible(); expect(fetchToolEverOffered).toBe(false); }); test("pinned strict grounding answers from the live search, no search", async ({ page, }) => { await installRoutes(page); let gatherCalls = 0; let answerSawExcerpts = false; await page.route("https://api.anthropic.com/**", async (route) => { if (route.request().method() === "OPTIONS") { await route.fulfill({ status: 204, headers: CORS }); return; } const body = route.request().postDataJSON() as { system?: string; messages?: { role: string; content: unknown }[]; }; const system = body.system ?? ""; if (system.includes("Markdown")) { // The real matched excerpts (the 5s "alpha line" cues) reach the answer. answerSawExcerpts = JSON.stringify(body.messages ?? []).includes( "alpha line", ); await route.fulfill({ status: 200, headers: { ...CORS, "content-type": "text/event-stream" }, body: sse("Grounded answer [1]"), }); return; } // Any non-answer call would be the gather phase — it must NOT happen. gatherCalls += 1; await route.fulfill({ status: 200, headers: { ...CORS, "content-type": "text/event-stream" }, body: sse("SEARCH: alpha"), }); }); await searchThenChat(page, TWO_RESULT_TREE, 2); await keyIn(page); await expect(page.getByText(/Grounded in 2 results/)).toBeVisible(); await ask(page, "summarize these"); await expect(page.getByText(/Grounded answer/)).toBeVisible(); expect(gatherCalls).toBe(0); expect(answerSawExcerpts).toBe(true); }); test("expanding the pin lets the assistant also search the archive", async ({ page, }) => { await installRoutes(page); let gatherCalls = 0; await page.route("https://api.anthropic.com/**", async (route) => { if (route.request().method() === "OPTIONS") { await route.fulfill({ status: 204, headers: CORS }); return; } const body = route.request().postDataJSON() as { system?: string; messages?: { role: string; content: unknown }[]; }; const system = body.system ?? ""; const msgs = body.messages ?? []; const lastUser = [...msgs].reverse().find((m) => m.role === "user"); const lastText = typeof lastUser?.content === "string" ? lastUser.content : ""; if (system.includes("Markdown")) { await route.fulfill({ status: 200, headers: { ...CORS, "content-type": "text/event-stream" }, body: sse("Expanded answer [1]"), }); return; } gatherCalls += 1; const text = lastText.includes('Results for "') ? "DONE" : "SEARCH: alpha"; await route.fulfill({ status: 200, headers: { ...CORS, "content-type": "text/event-stream" }, body: sse(text), }); }); await searchThenChat(page, TWO_RESULT_TREE, 2); await keyIn(page); await expect(page.getByText(/Grounded in 2 results/)).toBeVisible(); // Turn strict OFF → the assistant may search. await page.getByLabel("Answer only from these results").uncheck(); await ask(page, "what else is there"); await expect(page.getByText(/Expanded answer/)).toBeVisible(); // A gather search ran, and the grounded fixture video is among the citations. expect(gatherCalls).toBeGreaterThan(0); // Scope to the chat pane: the results rail (mounted but hidden in Chat view) // also renders this title, so an unscoped .first() would match the hidden one. await expect( page .getByTestId("pane-chat") .getByText("Transcript only", { exact: false }) .first(), ).toBeVisible(); }); test("detaching from the live search returns to a normal AI-decides-search chat", async ({ page, }) => { await installRoutes(page); await mockAnthropic(page); await searchThenChat(page, TWO_RESULT_TREE, 2); await keyIn(page); await expect(page.getByText(/Grounded in 2 results/)).toBeVisible(); // Detach → the grounding panel disappears… await page.getByRole("button", { name: "Detach" }).click(); await expect(page.getByText(/Grounded in 2 results/)).toHaveCount(0); // …and, with a live search still available, a re-attach affordance appears. await expect( page.getByRole("button", { name: "Ground in my search" }), ).toBeVisible(); // Back to the normal empty-state copy. await expect( page.getByText(/The assistant searches for what it needs/), ).toBeVisible(); // Asking now runs a normal AI-decided search (the scripted mock searches // "alpha", then answers) — grounding no longer short-circuits the gather. await ask(page, "tell me about the alpha discussion"); await expect(page.getByText(/Searched:\s*alpha/)).toBeVisible(); await expect( page.locator("li", { hasText: "point one" }).first(), ).toBeVisible(); }); test("report mode: the model maintains a report and the history is compacted", async ({ page, }) => { await installRoutes(page); let reportToolOffered = false; const answerBodies: string[] = []; await page.route("https://api.anthropic.com/**", async (route) => { if (route.request().method() === "OPTIONS") { await route.fulfill({ status: 204, headers: CORS }); return; } const body = route.request().postDataJSON() as { system?: string; tools?: { name?: string }[]; messages?: { role: string; content: unknown }[]; }; const system = body.system ?? ""; // Answer phase → capture the messages so we can assert compaction. if (system.includes("Markdown")) { answerBodies.push(JSON.stringify(body.messages ?? [])); await route.fulfill({ status: 200, headers: { ...CORS, "content-type": "text/event-stream" }, body: sse("Report-mode answer [1]"), }); return; } // Native gather turn: write a report section once, then finish. if (Array.isArray(body.tools)) { if (body.tools.some((t) => t.name === "update_report")) { reportToolOffered = true; } const msgs = body.messages ?? []; const lastUser = [...msgs].reverse().find((m) => m.role === "user"); const isToolResult = Array.isArray(lastUser?.content); await route.fulfill({ status: 200, headers: { ...CORS, "content-type": "application/json" }, body: isToolResult ? toolUse("finish", {}) : toolUse("update_report", { section: "Findings", content: "Alpha finding recorded from the archive.", }), }); return; } await route.fulfill({ status: 200, headers: { ...CORS, "content-type": "text/event-stream" }, body: sse("DONE"), }); }); await page.goto("/ask/"); await keyIn(page, "Native tools"); // Enable report mode (the toggle lives in the Report panel header). await page.getByLabel("Report mode").check(); // Turn 1 establishes the report + some history. await ask(page, "first question about the alpha topic"); await expect(page.getByText(/Report-mode answer/)).toBeVisible(); expect(reportToolOffered).toBe(true); // The Report panel shows the section the model wrote. await page.locator("summary", { hasText: "Report" }).click(); await expect( page.getByText("Alpha finding recorded from the archive."), ).toBeVisible(); // Turn 2: the answer request must be COMPACTED — it carries the "Report so // far" summary and does NOT replay turn 1's full question text. await ask(page, "second question about the beta topic"); await expect(page.getByText(/Report-mode answer/).nth(1)).toBeVisible(); const lastAnswer = answerBodies[answerBodies.length - 1]; expect(lastAnswer).toContain("Report so far"); expect(lastAnswer).toContain("Alpha finding recorded from the archive."); expect(lastAnswer).not.toContain("first question about the alpha topic"); }); test("corpus sweep: accumulates report sections across batches; extend keeps, fresh clears", async ({ page, }) => { await installRoutes(page); // Force a small batch size so the 3 fixtures span 2 batches ([2,1]). await page.addInitScript(() => localStorage.setItem("ytdlp-tb:ai:sweepchunk", "2"), ); // A monotonic write counter → each folded section is uniquely named, so we // can tell an EXTEND (prior sections survive) from a FRESH (prior cleared). let writeSeq = 0; const batches = new Set(); await page.route("https://api.anthropic.com/**", async (route) => { if (route.request().method() === "OPTIONS") { await route.fulfill({ status: 204, headers: CORS }); return; } const body = route.request().postDataJSON() as { system?: string; tools?: { name?: string }[]; messages?: { role: string; content: unknown }[]; }; const system = body.system ?? ""; const msgs = body.messages ?? []; const lastUser = [...msgs].reverse().find((m) => m.role === "user"); const isToolResult = Array.isArray(lastUser?.content); const text = typeof lastUser?.content === "string" ? lastUser.content : ""; // Sweep gather turn (native tools + the accumulation system prompt): write // one uniquely-numbered section per batch, then finish on the tool_result. if (Array.isArray(body.tools) && /running report/.test(system)) { if (isToolResult) { await route.fulfill({ status: 200, headers: { ...CORS, "content-type": "application/json" }, body: toolUse("finish", {}), }); return; } batches.add(/Batch (\d+) of/.exec(text)?.[1] ?? "?"); writeSeq += 1; await route.fulfill({ status: 200, headers: { ...CORS, "content-type": "application/json" }, body: toolUse("update_report", { section: `Finding ${writeSeq}`, content: `Recorded finding ${writeSeq}.`, }), }); return; } await route.fulfill({ status: 200, headers: { ...CORS, "content-type": "text/event-stream" }, body: sse("DONE"), }); }); await searchThenChat(page, ALL_RESULT_TREE, 3); await keyIn(page, "Native tools"); await expect(page.getByText(/Grounded in 3 results/)).toBeVisible(); // No report yet → a single primary "Build report from all 3 results". await page .getByRole("button", { name: /Build report from all 3 results/ }) .click(); // Resolves with a summary naming the batch count, and BOTH batches folded in. await expect( page.getByText(/Built a report from 3 results across 2 batches/), ).toBeVisible(); expect([...batches].sort()).toEqual(["1", "2"]); // The Report panel auto-opened and accumulated both batches' sections. await expect(page.getByText("Recorded finding 1.")).toBeVisible(); await expect(page.getByText("Recorded finding 2.")).toBeVisible(); // A report now exists → the action splits. EXTEND folds new sections in while // keeping the old ones. await page.getByRole("button", { name: "Add to report" }).click(); await expect(page.getByText("Recorded finding 3.")).toBeVisible(); await expect(page.getByText("Recorded finding 4.")).toBeVisible(); // Prior sections survived the extend. await expect(page.getByText("Recorded finding 1.")).toBeVisible(); // FRESH replaces: the prior sections are cleared before the new ones land. await page.getByRole("button", { name: "Start new report" }).click(); await expect(page.getByText("Recorded finding 5.")).toBeVisible(); await expect(page.getByText("Recorded finding 6.")).toBeVisible(); // The pre-fresh sections are gone. await expect(page.getByText("Recorded finding 1.")).toHaveCount(0); await expect(page.getByText("Recorded finding 4.")).toHaveCount(0); }); test("report citations are clickable and open the transcript at the cited moment", async ({ page, }) => { await installRoutes(page); await page.route("https://api.anthropic.com/**", async (route) => { if (route.request().method() === "OPTIONS") { await route.fulfill({ status: 204, headers: CORS }); return; } const body = route.request().postDataJSON() as { system?: string; tools?: { name?: string }[]; messages?: { role: string; content: unknown }[]; }; const system = body.system ?? ""; const msgs = body.messages ?? []; const lastUser = [...msgs].reverse().find((m) => m.role === "user"); const isToolResult = Array.isArray(lastUser?.content); // Sweep gather turn: fold in ONE section citing a precise moment, then finish. if (Array.isArray(body.tools) && /running report/.test(system)) { await route.fulfill({ status: 200, headers: { ...CORS, "content-type": "application/json" }, body: isToolResult ? toolUse("finish", {}) : toolUse("update_report", { section: "Findings", content: "The archive discusses this at length [1 @ 0:05].", }), }); return; } await route.fulfill({ status: 200, headers: { ...CORS, "content-type": "text/event-stream" }, body: sse("DONE"), }); }); // Sweep the whole search (3 fixtures) into a report in a single batch. await searchThenChat(page, ALL_RESULT_TREE, 3); await keyIn(page, "Native tools"); await expect(page.getByText(/Grounded in 3 results/)).toBeVisible(); await page .getByRole("button", { name: /Build report from all 3 results/ }) .click(); await expect(page.getByText(/Built a report from 3 results/)).toBeVisible(); // The report renders the [n @ mm:ss] marker as an in-page citation anchor, // with a numbered Sources list beneath the document. const cite = page.getByRole("link", { name: "[1 @ 0:05]", exact: true }); await expect(cite).toBeVisible(); await expect(cite).toHaveAttribute("href", /^#cite-/); await expect( page.getByRole("listitem").filter({ hasText: /alpha|Transcript|Chat/ }).first(), ).toBeVisible(); // Clicking it opens the transcript modal seeked to the cited second (5s) — // the same modal a chat citation opens, via replaceState on the /ask route. await cite.click(); await expect(page).toHaveURL(/[?&]v=/); await expect(page).toHaveURL(/[?&]t=5\b/); await expect(page.getByRole("button", { name: "Close player" })).toBeVisible(); }); test("corpus sweep: Stop aborts mid-sweep and keeps the partial report", async ({ page, }) => { await installRoutes(page); await page.addInitScript(() => localStorage.setItem("ytdlp-tb:ai:sweepchunk", "2"), ); // Batch 2 must still be in flight when Stop is clicked. A wall-clock delay // made that a race the test loses on a loaded box (the 500 ms timer expires // before Playwright gets round to the click, batch 2 lands, and the final // assertion sees its finding). The test holds the response open itself // instead: batch 2's route awaits this latch, and only the line after the // Stop click releases it. No timing assumption left to lose. let releaseBatch2 = () => {}; const batch2Gate = new Promise((resolve) => { releaseBatch2 = resolve; }); await page.route("https://api.anthropic.com/**", async (route) => { if (route.request().method() === "OPTIONS") { await route.fulfill({ status: 204, headers: CORS }); return; } const body = route.request().postDataJSON() as { system?: string; tools?: { name?: string }[]; messages?: { role: string; content: unknown }[]; }; const system = body.system ?? ""; const msgs = body.messages ?? []; const lastUser = [...msgs].reverse().find((m) => m.role === "user"); const isToolResult = Array.isArray(lastUser?.content); const text = typeof lastUser?.content === "string" ? lastUser.content : ""; if (Array.isArray(body.tools) && /running report/.test(system)) { if (isToolResult) { await route.fulfill({ status: 200, headers: { ...CORS, "content-type": "application/json" }, body: toolUse("finish", {}), }); return; } const b = /Batch (\d+) of/.exec(text)?.[1] ?? "1"; // Batch 1 answers at once; every later batch waits for the test. if (b !== "1") await batch2Gate; await route.fulfill({ status: 200, headers: { ...CORS, "content-type": "application/json" }, body: toolUse("update_report", { section: `Batch ${b} finding`, content: `Finding recorded in batch ${b}.`, }), }); return; } await route.fulfill({ status: 200, headers: { ...CORS, "content-type": "text/event-stream" }, body: sse("DONE"), }); }); await searchThenChat(page, ALL_RESULT_TREE, 3); await keyIn(page, "Native tools"); await expect(page.getByText(/Grounded in 3 results/)).toBeVisible(); await page .getByRole("button", { name: /Build report from all 3 results/ }) .click(); // Batch 1 folds in — its progress strip renders while batch 2 is delayed. await expect(page.getByText("Finding recorded in batch 1.")).toBeVisible({ timeout: 15_000, }); await expect(page.getByText(/Reading batch/)).toBeVisible(); // Stop the sweep from the pinned panel while batch 2 is still running. await page.getByRole("button", { name: "Stop the sweep" }).click(); // Stop has been asked for; let batch 2's response go. The sweep must // discard it, which is the thing under test. releaseBatch2(); // The sweep message resolves to a "stopped" line, and the partial report // (batch 1 only) survives — batch 2 never landed. await expect( page.getByText(/Stopped after 1 of 2 batches — the partial report is saved/), ).toBeVisible(); await expect(page.getByText("Finding recorded in batch 1.")).toBeVisible(); await expect(page.getByText("Finding recorded in batch 2.")).toHaveCount(0); }); test("corpus sweep is disabled on the scripted transport", async ({ page }) => { await installRoutes(page); await searchThenChat(page, ALL_RESULT_TREE, 3); await keyIn(page, "Scripted"); await expect(page.getByText(/Grounded in 3 results/)).toBeVisible(); // The sweep block appears but its action is disabled, with the report-mode // voice pointing at the Scripted transport. const build = page.getByRole("button", { name: /Build report from all 3 results/, }); await expect(build).toBeVisible(); await expect(build).toBeDisabled(); await expect( page.getByText(/Needs a tool-capable provider — switch the search mode off/), ).toBeVisible(); }); test("report mode is disabled on the scripted transport", async ({ page }) => { await installRoutes(page); await page.goto("/ask/"); // keyIn fills the key first (proving hydration) before switching to Scripted, // so the mode click isn't lost to a pre-hydration no-op. await keyIn(page, "Scripted"); // The toggle is present but disabled (no tool to maintain the report). const toggle = page.getByLabel("Report mode"); await expect(toggle).toBeVisible(); await expect(toggle).toBeDisabled(); }); test("editing the context changes what the next turn sends", async ({ page, }) => { await installRoutes(page); let sawMarker = false; await page.route("https://api.anthropic.com/**", async (route) => { if (route.request().method() === "OPTIONS") { await route.fulfill({ status: 204, headers: CORS }); return; } const body = route.request().postDataJSON() as { system?: string; messages?: { role: string; content: unknown }[]; }; if (JSON.stringify(body.messages ?? []).includes("PRUNED_MARKER")) { sawMarker = true; } const system = body.system ?? ""; const msgs = body.messages ?? []; const lastUser = [...msgs].reverse().find((m) => m.role === "user"); const lastText = typeof lastUser?.content === "string" ? lastUser.content : ""; let text: string; if (system.includes("Markdown")) text = "answer with a point [1]"; else if (lastText.includes('Results for "')) text = "DONE"; else text = "SEARCH: alpha"; await route.fulfill({ status: 200, headers: { ...CORS, "content-type": "text/event-stream" }, body: sse(text), }); }); await page.goto("/ask/"); await keyIn(page, "Scripted"); await ask(page, "first question"); await expect(page.getByText("answer with a point", { exact: false }).first()).toBeVisible(); // Open the Context panel, prune to a marker, and apply it as the new base. await page.locator("summary", { hasText: "Context" }).click(); const ta = page.locator("details textarea"); await ta.fill("USER:\nPRUNED_MARKER only this context"); await expect(ta).toHaveValue(/PRUNED_MARKER/); await page.getByRole("button", { name: "Apply as starting point" }).click(); // Wait for the override to take effect before sending (avoids a state race). await expect(page.getByText(/Continuing from an edited context/)).toBeVisible(); // The display reset to empty (messages cleared) → the suggestion chips return. await expect( page.getByRole("button", { name: "What are the main topics discussed?" }), ).toBeVisible(); // The next turn must carry the pruned context, not the original conversation. await ask(page, "second question"); // Scope to the rendered answer paragraph (the open context textarea also // contains this text). await expect(page.locator("p", { hasText: "answer with a point" }).first()).toBeVisible(); expect(sawMarker).toBe(true); }); });