// A read-only smoke test against a REAL corpus, driving the server the way the // registered MCP client does (`pnpm --filter … exec tsx src/index.ts --local …`) // rather than importing anything. // // The unit tests prove the logic; this proves the shipped thing works on 1.3 GB // of real shards — where the failures are the ones a stub cannot have: a layer // that isn't where the code thinks it is, a manifest shape that differs from the // fixture, a filter that silently matches nothing. // // pnpm --filter yt-dlp-transcript-mcp smoke // pnpm --filter yt-dlp-transcript-mcp smoke -- --local /path/to/public // // Exits non-zero if any check fails. import { Client } from "@modelcontextprotocol/client"; import { StdioClientTransport } from "@modelcontextprotocol/client/stdio"; import path from "node:path"; import { fileURLToPath } from "node:url"; const HERE = path.dirname(fileURLToPath(import.meta.url)); const REPO_ROOT = path.resolve(HERE, "..", ".."); const TIMEOUT = 900_000; const argv = process.argv.slice(2); const flag = (name: string): string | undefined => { const i = argv.indexOf(name); return i >= 0 && i + 1 < argv.length ? argv[i + 1] : undefined; }; const corpusDir = flag("--local") ?? path.join(REPO_ROOT, "export", "public"); const QUERY = flag("--query") ?? "lawsuit"; let failures = 0; function check(ok: boolean, label: string, detail = ""): void { if (ok) { console.log(` ✓ ${label}`); } else { failures++; console.log(` ✗ ${label}${detail ? `\n ${detail}` : ""}`); } } function textOf(result: unknown): string { const content = (result as { content?: { type: string; text?: string }[] }) .content; return (content ?? []) .filter((c) => c.type === "text") .map((c) => c.text ?? "") .join("\n"); } async function main(): Promise { const transport = new StdioClientTransport({ command: "pnpm", args: [ "-C", REPO_ROOT, "--filter", "yt-dlp-transcript-mcp", "exec", "tsx", "src/index.ts", "--local", corpusDir, ], cwd: REPO_ROOT, stderr: "pipe", }); const client = new Client({ name: "mcp-smoke", version: "1.0.0" }); await client.connect(transport); const call = (name: string, args: Record): Promise => client.callTool({ name, arguments: args }, { timeout: TIMEOUT }); console.log(`smoke: ${corpusDir}, query "${QUERY}"\n`); // ── every result names the corpus it read ── const channels = textOf(await call("list_channels", {})); check(/\(corpus: local:/.test(channels), "list_channels echoes (corpus: …)"); const firstChannel = /slug: ([^,)]+)/.exec(channels)?.[1]?.trim(); check(Boolean(firstChannel), "a channel slug is discoverable", channels.slice(0, 200)); const bogus = textOf(await call("get_transcript", { video_id: "___nope___" })); check(/\(corpus: local:/.test(bogus), "an ERROR result also echoes (corpus: …)"); // ── the invariant: enumerate and search agree on the deduped total ── const filters = { query: QUERY, content_types: ["video"], states: ["deleted", "private", "members_only", "unlisted", "maybe_missing"], }; const enumerated = textOf(await call("enumerate_matches", filters)); const searched = textOf(await call("search_transcripts", { ...filters, limit: 1 })); const enumTotal = /(\d+) match\(es\); \d+ batch/.exec(enumerated)?.[1]; const searchTotal = /\(total (\d+) match\(es\)/.exec(searched)?.[1]; check( enumTotal !== undefined && enumTotal === searchTotal, `enumerate_matches and search_transcripts agree on the total (${enumTotal} vs ${searchTotal})`, enumerated.slice(0, 400), ); // ── filter-first actually pruned ── check( /filter-pruned: planned \d+ of \d+ page\(s\)/.test(searched), "a states-filtered search reports filter-pruned page planning", /scanned [^;]+/.exec(searched)?.[0] ?? "", ); const planned = /filter-pruned: planned (\d+) of (\d+) page/.exec(searched); if (planned) { check( Number(planned[1]) < Number(planned[2]), `it planned fewer pages than the corpus has (${planned[1]} of ${planned[2]})`, ); } // ── the unfiltered path is untouched ── const plain = textOf( await call("search_transcripts", { query: QUERY, content_types: ["video"], limit: 2, }), ); check( !/filter-pruned/.test(plain), "an unfiltered search does NOT plan (no index read, no pruning)", ); // ── citations point at the archive, not the platform ── const withSnippets = textOf( await call("search_transcripts", { query: QUERY, content_types: ["video"], limit: 1, }), ); const link = /\]\((https?:\/\/[^)]+)\)/.exec(withSnippets)?.[1]; check( link !== undefined && /[?&]v=/.test(link) && /[?&]t=/.test(link), `a cited moment links to the archive viewer, not the platform (${link ?? "no link found"})`, ); // ── exclude ── const excluded = textOf( await call("enumerate_matches", { query: QUERY, content_types: ["video"], states: ["deleted", "private", "members_only", "unlisted", "maybe_missing"], exclude: [QUERY], }), ); check( /No matches for/.test(excluded) || /^0 match/.test(excluded), "excluding the query term itself yields nothing (the NOT is applied)", excluded.slice(0, 200), ); // ── an unknown filter token is reported, not swallowed ── const typo = textOf( await call("search_transcripts", { query: QUERY, content_types: ["video"], limit: 1, states: ["removed"], }), ); check( /unknown state\(s\) ignored: removed/.test(typo), "a typo'd state is named in the footer rather than silently widening the scan", ); // ── the joined layers on get_video_metadata ── const worklist = textOf( await call("enumerate_matches", { query: QUERY, content_types: ["video"], ...(firstChannel ? { channels: [firstChannel] } : {}), }), ); const someId = /^- (\S+) \| video \|/m.exec(worklist)?.[1]; if (someId) { const meta = textOf(await call("get_video_metadata", { video_id: someId })); check(/"title"/.test(meta), `get_video_metadata returns the base record (${someId})`); check(/## Stats/.test(meta), "…joined with stats/ (engagement, cue count, coverage)"); } else { check(false, "found an id to inspect with get_video_metadata"); } await client.close(); console.log(`\n${failures === 0 ? "ALL CHECKS PASSED" : `${failures} CHECK(S) FAILED`}`); if (failures > 0) process.exitCode = 1; } main().catch((e: unknown) => { console.error(e); process.exit(1); });