#!/usr/bin/env node import { spawn } from "node:child_process"; import os from "node:os"; import path from "node:path"; import { mkdirSync } from "node:fs"; import { fileURLToPath } from "node:url"; import { withQueue } from "./queue-lock.mjs"; const __dirname = path.dirname(fileURLToPath(import.meta.url)); const REPO_ROOT = path.resolve(__dirname, ".."); const EDITOR_DIR = path.join(REPO_ROOT, "editor"); const REPORT_DIR = path.join(EDITOR_DIR, "blob-report"); const IMAGE = process.env.E2E_IMAGE ?? "yt-dlp-transcript-browser-e2e"; // Flags this script consumes itself; everything else on the command line is // forwarded verbatim to each shard's `playwright test` (see parseArgs). // OWN_FLAGS take a value (and so skip the next argv entry); OWN_BOOL_FLAGS do // not — putting a boolean in the first set would silently eat the flag after it. const OWN_FLAGS = new Set(["--shards", "--retries"]); const OWN_BOOL_FLAGS = new Set(["--no-build"]); function parseIntArg(flag, envVar, fallback, min) { const argIdx = process.argv.indexOf(flag); if (argIdx !== -1 && process.argv[argIdx + 1]) { const n = Number(process.argv[argIdx + 1]); if (Number.isInteger(n) && n >= min) return n; } if (envVar && process.env[envVar]) { const n = Number(process.env[envVar]); if (Number.isInteger(n) && n >= min) return n; } return fallback; } function parseShards() { return parseIntArg( "--shards", "E2E_SHARDS", Math.min(Math.max(2, Math.floor(os.cpus().length / 2)), 8), 1, ); } // Retries default to 0 so a sharded run reports the same failures a serial run // does. Without this the container's CI=true would pick up playwright.config's // `retries: process.env.CI ? 2 : 0` and silently paper over flaky tests, which // makes the result impossible to compare against the recorded baseline. A CLI // --retries beats the config file, so this neutralizes it without unsetting CI. function parseRetries() { return parseIntArg("--retries", "E2E_RETRIES", 0, 0); } // Anything not consumed above is passed through to `playwright test` in every // shard, so a subset can be sharded: `pnpm e2e:sharded -- --grep "digest"` or // `node scripts/run-sharded-e2e.mjs --shards 4 e2e/deploy-page.spec.ts`. // An explicit `--` separator is honoured, but is not required: pnpm already // strips the first `--` before the script sees argv. function parsePassthrough() { const argv = process.argv.slice(2); const sepIdx = argv.indexOf("--"); if (sepIdx !== -1) return argv.slice(sepIdx + 1); const rest = []; for (let i = 0; i < argv.length; i++) { if (OWN_FLAGS.has(argv[i])) { i++; // skip the flag's value too continue; } if (OWN_BOOL_FLAGS.has(argv[i])) continue; // no value to skip rest.push(argv[i]); } return rest; } const SHARDS = parseShards(); const RETRIES = parseRetries(); const PASSTHROUGH = parsePassthrough(); const SKIP_BUILD = process.argv.includes("--no-build") || process.env.E2E_SKIP_BUILD === "1"; function run(cmd, args, opts = {}) { return new Promise((resolve, reject) => { const child = spawn(cmd, args, { stdio: "inherit", ...opts }); child.on("error", reject); child.on("exit", (code) => resolve(code ?? 0)); }); } // Playwright's list reporter ends with a summary block of lines like // " 3 failed", " 1 flaky", " 87 passed (1.2m)". Test titles under the // "failed" heading are indented further and start with "[", so anchoring on // exactly two leading spaces picks up only the tally lines. const TALLY_RE = /^ {2}(\d+) (passed|failed|flaky|skipped|interrupted|did not run)\b/gm; function parseTallies(output) { const counts = {}; for (const [, n, kind] of output.matchAll(TALLY_RE)) { counts[kind] = (counts[kind] ?? 0) + Number(n); } return counts; } function runShard(i, n) { const tag = `[shard ${i}/${n}]`; return new Promise((resolve) => { const args = [ "run", "--rm", "--init", // CI=true is load-bearing for playwright.config's // `reuseExistingServer: !process.env.CI` — it makes each container start // its own editor/export/ollama-stub servers instead of expecting one to // already be up. It is NOT a request for retries; --retries below wins. "-e", "CI=true", "-e", "E2E_MODE=start", "-v", `${REPORT_DIR}:/repo/editor/blob-report`, IMAGE, "pnpm", "exec", "playwright", "test", `--shard=${i}/${n}`, `--retries=${RETRIES}`, // `list` rides along with `blob` so each shard's tallies reach the // prefixer below and can be summed into a combined count. "--reporter=blob,list", ...PASSTHROUGH, ]; const child = spawn("docker", args); let captured = ""; const prefix = (chunk) => { const text = chunk.toString("utf8"); captured += text; const lines = text.split("\n"); const last = lines.pop(); for (const line of lines) process.stdout.write(`${tag} ${line}\n`); if (last) process.stdout.write(`${tag} ${last}`); }; child.stdout.on("data", prefix); child.stderr.on("data", prefix); child.on("error", (err) => { process.stderr.write(`${tag} spawn error: ${err.message}\n`); resolve({ code: 1, counts: {} }); }); child.on("exit", (code) => resolve({ code: code ?? 1, counts: parseTallies(captured) }), ); }); } async function cleanReportDir() { // Wipe blob-report via the test image so root-owned files from a prior run can be removed. await run("docker", [ "run", "--rm", "-v", `${EDITOR_DIR}:/host-editor`, IMAGE, "rm", "-rf", "/host-editor/blob-report", ]).catch(() => {}); mkdirSync(REPORT_DIR, { recursive: true }); } // The image tag is fixed, so two worktrees building at once would leave the tag // pointing at whichever finished last and the shards would run *another // checkout's* code. `COPY . .` also reads test-transcripts/ and // test-settings.json, which a concurrent host run rewrites on every resetData(). // Both are why the build lives inside the queue lock now instead of in a // `docker build && …` shell chain in package.json. A no-change rebuild is // layer-cached and costs seconds; --no-build skips it while iterating. async function buildImage() { console.log(`Building ${IMAGE} from Dockerfile.test ...`); const code = await run("docker", ["build", "-f", "Dockerfile.test", "-t", IMAGE, "."], { cwd: REPO_ROOT, }); if (code !== 0) { console.error(`docker build exited ${code}`); process.exit(code); } } async function runShards() { console.log(`Running ${SHARDS} shard(s) using image ${IMAGE}`); console.log(`Retries per shard: ${RETRIES}`); if (PASSTHROUGH.length) { console.log(`Forwarding to playwright: ${PASSTHROUGH.join(" ")}`); } const t0 = Date.now(); if (!SKIP_BUILD) await buildImage(); await cleanReportDir(); const shardPromises = []; for (let i = 1; i <= SHARDS; i++) shardPromises.push(runShard(i, SHARDS)); const results = await Promise.all(shardPromises); const tShards = Date.now(); console.log("\nShard exit codes:"); results.forEach(({ code }, idx) => { console.log(` shard ${idx + 1}/${SHARDS}: ${code === 0 ? "PASS" : `FAIL (${code})`}`); }); console.log(`Sharded test wall time: ${((tShards - t0) / 1000).toFixed(1)}s`); console.log("\nMerging blob reports..."); const mergeCode = await run( "pnpm", ["--filter", "editor", "exec", "playwright", "merge-reports", "--reporter=html", "blob-report"], { cwd: REPO_ROOT }, ); if (mergeCode !== 0) { console.error(`merge-reports exited ${mergeCode}`); } else { console.log("HTML report: editor/playwright-report/index.html"); } // Combined tallies, so the run reports what it actually did rather than just // whether each container exited non-zero. const total = {}; for (const { counts } of results) { for (const [kind, n] of Object.entries(counts)) { total[kind] = (total[kind] ?? 0) + n; } } const ORDER = ["passed", "failed", "flaky", "skipped", "interrupted", "did not run"]; const parts = ORDER.filter((k) => total[k]).map((k) => `${total[k]} ${k}`); const summary = parts.length ? parts.join(" / ") : "no tallies parsed"; console.log(`\nCombined: ${summary}`); if (RETRIES > 0) { console.log( `(--retries=${RETRIES}: 'flaky' means it failed then passed on retry — not comparable to a 0-retry baseline)`, ); } if (!parts.length) { console.log("(shard output did not contain a list-reporter summary; see the HTML report)"); } return results.some(({ code }) => code !== 0) ? 1 : 0; } // One e2e run at a time, machine-wide — N containers each running `next start` // would starve a concurrent host suite into flakes. No port preflight: the // shards have their own network namespaces and bind no host ports, so checking // 3011/3010 here would abort on an unrelated dev server for no reason. // Returning (rather than exiting) from runShards lets the lock be released // normally. See scripts/queue-lock.mjs. async function main() { const code = await withQueue({ name: "e2e", cmd: ["e2e:sharded"] }, runShards); process.exit(code); } main().catch((err) => { console.error(err); process.exit(1); });