#!/usr/bin/env node // PER-SPEC TIMINGS FOR AN E2E RUN, against the last run on the same branch. // // node scripts/e2e-timings.mjs [] [--suite ] [--top ] [--no-record] // // Every suite's Playwright config writes a `json` report beside `list` // (`/test-results/timings.json`; the hub's, report's and 2origin's in // their own test-results-* dirs). This totals it per spec FILE — every result // of every test, retries included, since that is what the run spent — prints // the files slowest first, each with its change against the last run of the // same suite on the same branch that ran it (the wall time against the last // run of the same spec files), and records the run so the next one has a // baseline. A spec that quietly grows a 60 s wait shows up here as a delta. // // THE HISTORY lives in the repository's git common dir // (`<.git>/e2e-timings//.json`, the last 30 runs): untracked by // construction, shared by every worktree of the checkout, so a branch's // baseline is found wherever its next run happens. A report already recorded // (the same start time) is not recorded twice, and is compared with the runs // before it. import { execFileSync } from "node:child_process"; import fs from "node:fs"; import path from "node:path"; import { fileURLToPath } from "node:url"; const SELF = fileURLToPath(import.meta.url); const REPO_ROOT = path.resolve(path.dirname(SELF), ".."); export const HISTORY_KEEP = 30; // --------------------------------------------------------------- the report /** * Per-spec totals from a Playwright JSON report: `{startTime, wallMs, totals, * specs: {[file]: {tests, passed, failed, flaky, skipped, ms}}}`. */ export function summarize(report) { const specs = {}; const totals = { tests: 0, passed: 0, failed: 0, flaky: 0, skipped: 0 }; const walk = (suite, file) => { const f = suite.file ?? file; for (const spec of suite.specs ?? []) { const name = spec.file ?? f; const row = (specs[name] ??= { tests: 0, passed: 0, failed: 0, flaky: 0, skipped: 0, ms: 0 }); for (const t of spec.tests ?? []) { row.tests += 1; totals.tests += 1; const kind = t.status === "expected" ? "passed" : t.status === "unexpected" ? "failed" : t.status === "flaky" ? "flaky" : "skipped"; row[kind] += 1; totals[kind] += 1; for (const r of t.results ?? []) row.ms += Number(r.duration) || 0; } } for (const child of suite.suites ?? []) walk(child, f); }; for (const s of report.suites ?? []) walk(s, s.file); return { startTime: report.stats?.startTime ?? null, wallMs: Number(report.stats?.duration) || 0, totals, specs, }; } // ------------------------------------------------------------------ history function gitOut(args, cwd) { try { return execFileSync("git", args, { cwd, encoding: "utf8", stdio: ["ignore", "pipe", "ignore"] }).trim(); } catch { return null; } } export function defaultHistoryDir(cwd = REPO_ROOT) { const common = gitOut(["rev-parse", "--path-format=absolute", "--git-common-dir"], cwd); return common ? path.join(common, "e2e-timings") : null; } const safe = (s) => String(s).replace(/[^A-Za-z0-9._-]+/g, "_"); export function historyFile(dir, suite, branch) { return path.join(dir, safe(suite), `${safe(branch)}.json`); } export function readHistory(file) { try { const h = JSON.parse(fs.readFileSync(file, "utf8")); return Array.isArray(h?.runs) ? h.runs : []; } catch { return []; } } /** * Where `run` goes: the runs recorded before it (`earlier`, oldest first) and * the history with it appended — or unchanged, when it is already there. */ export function placeRun(runs, run) { const idx = run.startTime == null ? -1 : runs.findIndex((r) => r.startTime === run.startTime); if (idx >= 0) return { earlier: runs.slice(0, idx), runs, recorded: false }; return { earlier: runs, runs: [...runs, run].slice(-HISTORY_KEEP), recorded: true }; } const specSet = (r) => Object.keys(r?.specs ?? {}).sort().join("\n"); /** * The baselines: the last earlier run of the SAME spec files (for the wall * time — a one-spec run is no baseline for a full suite), and per spec the last * earlier run that ran it. */ export function baselines(earlier, run) { const same = [...earlier].reverse().find((r) => specSet(r) === specSet(run)) ?? null; const perSpec = {}; for (const file of Object.keys(run.specs)) { const r = [...earlier].reverse().find((e) => e.specs?.[file]); if (r) perSpec[file] = r.specs[file]; } return { same, perSpec, any: earlier.length > 0 }; } export function writeHistory(file, runs) { fs.mkdirSync(path.dirname(file), { recursive: true }); const tmp = `${file}.${process.pid}.tmp`; fs.writeFileSync(tmp, `${JSON.stringify({ runs }, null, 1)}\n`); fs.renameSync(tmp, file); } // ------------------------------------------------------------------- output function secs(ms) { const s = ms / 1000; return s >= 600 ? `${Math.floor(s / 60)}m${String(Math.round(s % 60)).padStart(2, "0")}s` : `${s.toFixed(1)}s`; } function delta(ms) { const sign = ms > 0 ? "+" : ms < 0 ? "−" : "±"; return `${sign}${secs(Math.abs(ms))}`; } /** The printed table, as lines. */ export function render({ suite, branch, head, run, earlier = [], top = Infinity }) { const out = []; const stamp = (r) => (r.startTime ? `${r.startTime.replace("T", " ").slice(0, 16)} UTC` : "?"); out.push(`e2e timings — ${suite} on ${branch}${head ? ` (${head.slice(0, 8)})` : ""}, ${stamp(run)}, wall ${secs(run.wallMs)}`); const t = run.totals; out.push( ` ${t.tests} tests: ${t.passed} passed, ${t.failed} failed` + `${t.flaky ? `, ${t.flaky} flaky` : ""}, ${t.skipped} skipped`, ); const { same, perSpec, any } = baselines(earlier, run); if (same) { out.push(` against ${stamp(same)}${same.head ? ` (${same.head.slice(0, 8)})` : ""}, the same specs: wall ${delta(run.wallMs - same.wallMs)}`); } else if (any) { out.push(" no earlier run of these same specs on this branch: each spec against the last run that had it"); } else { out.push(" no earlier run of this suite on this branch: this one is the baseline"); } out.push(""); const rows = Object.entries(run.specs).sort((a, b) => b[1].ms - a[1].ms || a[0].localeCompare(b[0])); const width = Math.max(4, ...rows.map(([f]) => f.length)); out.push(` ${"spec".padEnd(width)} tests time${any ? " Δ" : ""}`); for (const [file, r] of rows.slice(0, top)) { const was = perSpec[file]; const d = !any ? "" : was ? delta(r.ms - was.ms).padStart(10) : " new"; const bad = r.failed ? ` (${r.failed} failed)` : ""; out.push(` ${file.padEnd(width)} ${String(r.tests).padStart(5)} ${secs(r.ms).padStart(8)}${d}${bad}`); } if (rows.length > top) out.push(` … ${rows.length - top} more`); return out; } // ---------------------------------------------------------------------- CLI function parseArgs(argv) { const opts = { report: null, suite: null, top: Infinity, record: true, historyDir: null, branch: null }; for (let i = 0; i < argv.length; i++) { const a = argv[i]; if (a === "--suite") opts.suite = argv[++i]; else if (a === "--top") opts.top = Number(argv[++i]); else if (a === "--no-record") opts.record = false; else if (a === "--history-dir") opts.historyDir = argv[++i]; else if (a === "--branch") opts.branch = argv[++i]; else if (a.startsWith("--")) throw new Error(`unknown flag '${a}'`); else opts.report = a; } return opts; } // `editor/test-results/timings.json` → "editor"; `export/test-results-hub/…` → "export-hub". export function suiteFromPath(file) { const dir = path.basename(path.dirname(file)); const pkg = path.basename(path.dirname(path.dirname(file))); const extra = dir.startsWith("test-results-") ? `-${dir.slice("test-results-".length)}` : ""; return `${pkg}${extra}`; } function main(argv) { const opts = parseArgs(argv); const report = path.resolve(opts.report ?? path.join(REPO_ROOT, "editor", "test-results", "timings.json")); let json; try { json = JSON.parse(fs.readFileSync(report, "utf8")); } catch (err) { process.stderr.write(`e2e-timings: cannot read ${report} (${err.code ?? err.message}) — run the suite first\n`); return 1; } const suite = opts.suite ?? suiteFromPath(report); const cwd = path.dirname(report); const branch = opts.branch ?? gitOut(["rev-parse", "--abbrev-ref", "HEAD"], cwd) ?? "unknown"; const head = gitOut(["rev-parse", "HEAD"], cwd); const run = { ...summarize(json), head }; const dir = opts.historyDir ?? defaultHistoryDir(cwd); let earlier = []; let note = ""; if (dir) { const file = historyFile(dir, suite, branch); const placed = placeRun(readHistory(file), run); earlier = placed.earlier; if (opts.record && placed.recorded) { writeHistory(file, placed.runs); note = `recorded in ${file}`; } else if (!placed.recorded) { note = "this run was already recorded"; } } const lines = render({ suite, branch, head, run, earlier, top: opts.top }); process.stdout.write(`${lines.join("\n")}\n${note ? `\n${note}\n` : ""}`); return 0; } if (process.argv[1] && fs.realpathSync(process.argv[1]) === fs.realpathSync(SELF)) { try { process.exit(main(process.argv.slice(2))); } catch (err) { process.stderr.write(`e2e-timings: ${err?.message ?? err}\n`); process.exit(2); } }