/** * Builds a run report from the evidence captured by passing tests. * * Playwright's own HTML report is a debugging tool: it shows what went wrong. * This one is the opposite -- a record of what the suite proved, with the * screenshots and the data each scenario observed, readable by somebody who * will never open a terminal. * * Output lands in `e2e/evidence-report/`: * index.html the report * report.json the same content, machine readable * report.md a text summary for pasting into a ticket or a PR * shots/ the screenshots */ import { copyFileSync, mkdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; import { join, relative, resolve, sep } from 'node:path'; import { pathToFileURL } from 'node:url'; import type { FullConfig, FullResult, Reporter, Suite, TestCase, TestResult } from '@playwright/test/reporter'; import { DATA_PREFIX, SHOT_PREFIX } from '../fixtures/evidence'; import { renderHtml } from './render-html'; import { ApiCall, RunMeta, Scenario, Step, Summary, areaFor, formatValue, friendly, group, label, sentenceCase, } from './report-model'; const OUT_DIR = resolve(__dirname, '..', 'evidence-report'); const SHOT_DIR = join(OUT_DIR, 'shots'); export default class EvidenceReporter implements Reporter { private readonly scenarios: Scenario[] = []; private startedAt = Date.now(); private rootDir = ''; private meta: RunMeta = { baseURL: 'unknown', projects: [], workers: 1 }; onBegin(config: FullConfig, _suite: Suite): void { this.startedAt = Date.now(); this.rootDir = config.rootDir; this.meta = { baseURL: String(config.projects[0]?.use?.baseURL ?? 'unknown'), projects: config.projects.map((project) => { const viewport = project.use?.viewport; const size = viewport ? `${viewport.width} x ${viewport.height}` : 'default size'; return `${project.name} (${size})`; }), workers: config.workers, }; rmSync(OUT_DIR, { recursive: true, force: true }); mkdirSync(SHOT_DIR, { recursive: true }); } onTestEnd(test: TestCase, result: TestResult): void { const path = test.titlePath().filter(Boolean); // titlePath is [project, file, ...describes, title] const project = path[0] ?? 'default'; const file = relative(this.rootDir, test.location.file).split(sep).join('/'); const title = path[path.length - 1] ?? test.title; const suite = path.slice(2, -1).join(' › '); const steps = this.collectSteps(test, result); const apiCalls = this.collectApiCalls(result); this.scenarios.push({ project, file, suite, title, status: result.status, expected: result.status === test.expectedStatus, durationMs: result.duration, steps, apiCalls, error: result.error?.message?.split('\n').slice(0, 4).join('\n'), }); } /** Reads the evidence-http attachment the apiTraffic fixture produced. */ private collectApiCalls(result: TestResult): ApiCall[] { const att = result.attachments.find((a) => a.name === 'evidence-http'); if (!att) { return []; } const raw = att.body ? att.body.toString('utf8') : att.path ? safeRead(att.path) : ''; if (!raw) { return []; } try { return JSON.parse(raw) as ApiCall[]; } catch { return []; } } /** Pairs `evidence:NN:name` screenshots with `evidence-data:NN:name` blobs. */ private collectSteps(test: TestCase, result: TestResult): Step[] { const bySeq = new Map(); for (const attachment of result.attachments) { const match = /^(evidence|evidence-data):(\d+):(.*)$/.exec(attachment.name); if (!match) { continue; } const [, kind, seq, name] = match; let step = bySeq.get(seq); if (!step) { step = { seq, name }; bySeq.set(seq, step); } if (kind === SHOT_PREFIX) { // An attachment made with `body` carries the bytes in memory; one made // with `path` points at a file on disk. Both shapes occur. const safe = `${slug(test.id)}-${seq}.png`; try { if (attachment.body) { writeFileSync(join(SHOT_DIR, safe), attachment.body); step.shot = `shots/${safe}`; } else if (attachment.path) { copyFileSync(attachment.path, join(SHOT_DIR, safe)); step.shot = `shots/${safe}`; } } catch { // A screenshot that could not be saved should not sink the report. } } if (kind === DATA_PREFIX) { const raw = attachment.body ? attachment.body.toString('utf8') : attachment.path ? safeRead(attachment.path) : ''; if (raw) { try { step.facts = JSON.parse(raw) as Record; } catch { step.facts = { raw }; } } } } const steps = [...bySeq.values()].sort((a, b) => a.seq.localeCompare(b.seq)); if (steps.length > 0) { return steps; } // Not every test is instrumented. Fall back to Playwright's own // end-of-test screenshot so the report still shows what the run looked // like, just without the narrated data. const auto = result.attachments.find( (a) => a.name === 'screenshot' && a.contentType === 'image/png', ); if (!auto) { return []; } const safe = `${slug(test.id)}-final.png`; try { if (auto.body) { writeFileSync(join(SHOT_DIR, safe), auto.body); } else if (auto.path) { copyFileSync(auto.path, join(SHOT_DIR, safe)); } else { return []; } } catch { return []; } return [{ seq: '01', name: 'Final state', shot: `shots/${safe}` }]; } async onEnd(result: FullResult): Promise { const durationMs = Date.now() - this.startedAt; const summary = { generatedAt: new Date().toISOString(), outcome: result.status, durationMs, total: this.scenarios.length, passed: this.scenarios.filter((s) => s.status === 'passed').length, failed: this.scenarios.filter((s) => s.status === 'failed' || s.status === 'timedOut').length, skipped: this.scenarios.filter((s) => s.status === 'skipped').length, narrated: this.scenarios.filter((s) => s.steps.some((step) => step.facts)).length, screenshots: this.scenarios.reduce( (n, s) => n + s.steps.filter((step) => step.shot).length, 0, ), apiCalls: this.scenarios.reduce((n, s) => n + s.apiCalls.length, 0), }; mkdirSync(OUT_DIR, { recursive: true }); writeFileSync( join(OUT_DIR, 'report.json'), JSON.stringify({ summary, scenarios: this.scenarios }, null, 2), 'utf8', ); writeFileSync(join(OUT_DIR, 'report.md'), renderMarkdown(summary, this.scenarios), 'utf8'); writeFileSync( join(OUT_DIR, 'index.html'), renderHtml(summary, this.scenarios, this.meta), 'utf8', ); // An absolute file:// URL, because a relative path is ambiguous: the suite // runs with its cwd in e2e/, but people launch it from the repo root. const file = join(OUT_DIR, 'index.html'); const url = pathToFileURL(file).href; console.log( [ '', 'Evidence report — what passed, with data and screenshots:', ` ${url}`, ` ${summary.passed}/${summary.total} passed · ${summary.screenshots} screenshots · ` + `${summary.apiCalls} API calls · ${summary.narrated} scenarios with recorded data`, '', ' Open it: npm run report', ' Markdown: ' + pathToFileURL(join(OUT_DIR, 'report.md')).href, '', " (`npx playwright show-report` opens Playwright's own report, which is a", ' different artifact — it is for debugging failures.)', ].join('\n'), ); } } // --- helpers --------------------------------------------------------------- function slug(value: string): string { return value.replace(/[^a-zA-Z0-9]+/g, '-').slice(0, 60); } function safeRead(path: string): string { try { return readFileSync(path, 'utf8'); } catch { return ''; } } // --- markdown -------------------------------------------------------------- function renderMarkdown(summary: Summary, scenarios: Scenario[]): string { const lines: string[] = [ '# TravelHub end-to-end run report', '', `Generated ${summary.generatedAt} · ${summary.passed}/${summary.total} passed · ` + `${friendly(summary.durationMs)}`, '', ]; for (const file of group(scenarios)) { lines.push(`## ${areaFor(file.name).title}`, '', `_${areaFor(file.name).blurb}_`, ''); for (const scenario of file.items) { const mark = scenario.status === 'passed' ? '✅' : scenario.status === 'skipped' ? '⏭️' : '❌'; const heading = sentenceCase(scenario.title); lines.push(`### ${mark} ${heading}`, ''); lines.push(`*${scenario.suite ? scenario.suite + ' · ' : ''}${scenario.project} · ${friendly(scenario.durationMs)}*`, ''); if (scenario.steps.length === 0 && scenario.apiCalls.length === 0) { lines.push('_No evidence captured._', ''); continue; } for (const step of scenario.steps) { lines.push(`**${step.seq}. ${step.name}**`, ''); if (step.facts) { lines.push('| Field | Value |', '| ----- | ----- |'); for (const [key, value] of Object.entries(step.facts)) { lines.push(`| ${label(key)} | ${formatValue(value)} |`); } lines.push(''); } if (step.shot) { lines.push(`![${step.name}](${step.shot})`, ''); } } if (scenario.apiCalls.length > 0) { lines.push(`**API calls (${scenario.apiCalls.length})**`, ''); for (const call of scenario.apiCalls) { lines.push(`- \`${call.method} ${call.path}\` → **${call.status}** (${call.durationMs} ms)`); if (call.requestBody) { lines.push('', ' Request:', '', ' ```json', indent(call.requestBody), ' ```'); } if (call.responseBody) { lines.push('', ' Response:', '', ' ```json', indent(call.responseBody), ' ```'); } } lines.push(''); } } } return lines.join('\n'); } /** Indents a block two spaces so it nests under a Markdown list item. */ function indent(text: string): string { return text .split('\n') .map((line) => ' ' + line) .join('\n'); }