nilesh.pandClaude Opus 4.8
45b5b34680
A complete loop for defining, generating, running and gating tests: - qa/: scenario-card template + examples (API/UI/binding), the Claude generation prompt, and a QA runbook tying it together - e2e/CLAUDE.md: conventions that keep AI-generated tests consistent - run-tests.ps1: one runner for every layer (backend/api/web/ui/binding), logs per run under .testlogs/, JUnit output, evidence report, and a re-seed guard against stale demo data - .githooks/ + scripts/install-hooks.ps1: pre-commit secret scan and pre-push unit tests (shift-left before CI) - evidence report now captures each API call's request, payload and response (Authorization redacted); JUnit reporter added for CI - scripts/db.mjs: db:check freshness guard Azure DevOps pipeline deferred; the runner already emits JUnit + the evidence report for any CI to publish. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
325 lines
10 KiB
TypeScript
325 lines
10 KiB
TypeScript
/**
|
||
* Builds a run report from the evidence captured by passing tests.
|
||
*
|
||
* Playwright's own HTML report is a debugging tool: it shows what went wrong.
|
||
* This one is the opposite -- a record of what the suite proved, with the
|
||
* screenshots and the data each scenario observed, readable by somebody who
|
||
* will never open a terminal.
|
||
*
|
||
* Output lands in `e2e/evidence-report/`:
|
||
* index.html the report
|
||
* report.json the same content, machine readable
|
||
* report.md a text summary for pasting into a ticket or a PR
|
||
* shots/ the screenshots
|
||
*/
|
||
import { copyFileSync, mkdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
|
||
import { join, relative, resolve, sep } from 'node:path';
|
||
import { pathToFileURL } from 'node:url';
|
||
|
||
import type { FullConfig, FullResult, Reporter, Suite, TestCase, TestResult } from '@playwright/test/reporter';
|
||
|
||
import { DATA_PREFIX, SHOT_PREFIX } from '../fixtures/evidence';
|
||
import { renderHtml } from './render-html';
|
||
import {
|
||
ApiCall,
|
||
RunMeta,
|
||
Scenario,
|
||
Step,
|
||
Summary,
|
||
areaFor,
|
||
formatValue,
|
||
friendly,
|
||
group,
|
||
label,
|
||
sentenceCase,
|
||
} from './report-model';
|
||
|
||
const OUT_DIR = resolve(__dirname, '..', 'evidence-report');
|
||
const SHOT_DIR = join(OUT_DIR, 'shots');
|
||
|
||
export default class EvidenceReporter implements Reporter {
|
||
private readonly scenarios: Scenario[] = [];
|
||
private startedAt = Date.now();
|
||
private rootDir = '';
|
||
private meta: RunMeta = { baseURL: 'unknown', projects: [], workers: 1 };
|
||
|
||
onBegin(config: FullConfig, _suite: Suite): void {
|
||
this.startedAt = Date.now();
|
||
this.rootDir = config.rootDir;
|
||
|
||
this.meta = {
|
||
baseURL: String(config.projects[0]?.use?.baseURL ?? 'unknown'),
|
||
projects: config.projects.map((project) => {
|
||
const viewport = project.use?.viewport;
|
||
const size = viewport ? `${viewport.width} x ${viewport.height}` : 'default size';
|
||
return `${project.name} (${size})`;
|
||
}),
|
||
workers: config.workers,
|
||
};
|
||
|
||
rmSync(OUT_DIR, { recursive: true, force: true });
|
||
mkdirSync(SHOT_DIR, { recursive: true });
|
||
}
|
||
|
||
onTestEnd(test: TestCase, result: TestResult): void {
|
||
const path = test.titlePath().filter(Boolean);
|
||
// titlePath is [project, file, ...describes, title]
|
||
const project = path[0] ?? 'default';
|
||
const file = relative(this.rootDir, test.location.file).split(sep).join('/');
|
||
const title = path[path.length - 1] ?? test.title;
|
||
const suite = path.slice(2, -1).join(' › ');
|
||
|
||
const steps = this.collectSteps(test, result);
|
||
const apiCalls = this.collectApiCalls(result);
|
||
|
||
this.scenarios.push({
|
||
project,
|
||
file,
|
||
suite,
|
||
title,
|
||
status: result.status,
|
||
expected: result.status === test.expectedStatus,
|
||
durationMs: result.duration,
|
||
steps,
|
||
apiCalls,
|
||
error: result.error?.message?.split('\n').slice(0, 4).join('\n'),
|
||
});
|
||
}
|
||
|
||
/** Reads the evidence-http attachment the apiTraffic fixture produced. */
|
||
private collectApiCalls(result: TestResult): ApiCall[] {
|
||
const att = result.attachments.find((a) => a.name === 'evidence-http');
|
||
if (!att) {
|
||
return [];
|
||
}
|
||
const raw = att.body ? att.body.toString('utf8') : att.path ? safeRead(att.path) : '';
|
||
if (!raw) {
|
||
return [];
|
||
}
|
||
try {
|
||
return JSON.parse(raw) as ApiCall[];
|
||
} catch {
|
||
return [];
|
||
}
|
||
}
|
||
|
||
/** Pairs `evidence:NN:name` screenshots with `evidence-data:NN:name` blobs. */
|
||
private collectSteps(test: TestCase, result: TestResult): Step[] {
|
||
const bySeq = new Map<string, Step>();
|
||
|
||
for (const attachment of result.attachments) {
|
||
const match = /^(evidence|evidence-data):(\d+):(.*)$/.exec(attachment.name);
|
||
if (!match) {
|
||
continue;
|
||
}
|
||
const [, kind, seq, name] = match;
|
||
|
||
let step = bySeq.get(seq);
|
||
if (!step) {
|
||
step = { seq, name };
|
||
bySeq.set(seq, step);
|
||
}
|
||
|
||
if (kind === SHOT_PREFIX) {
|
||
// An attachment made with `body` carries the bytes in memory; one made
|
||
// with `path` points at a file on disk. Both shapes occur.
|
||
const safe = `${slug(test.id)}-${seq}.png`;
|
||
try {
|
||
if (attachment.body) {
|
||
writeFileSync(join(SHOT_DIR, safe), attachment.body);
|
||
step.shot = `shots/${safe}`;
|
||
} else if (attachment.path) {
|
||
copyFileSync(attachment.path, join(SHOT_DIR, safe));
|
||
step.shot = `shots/${safe}`;
|
||
}
|
||
} catch {
|
||
// A screenshot that could not be saved should not sink the report.
|
||
}
|
||
}
|
||
|
||
if (kind === DATA_PREFIX) {
|
||
const raw = attachment.body
|
||
? attachment.body.toString('utf8')
|
||
: attachment.path
|
||
? safeRead(attachment.path)
|
||
: '';
|
||
if (raw) {
|
||
try {
|
||
step.facts = JSON.parse(raw) as Record<string, unknown>;
|
||
} catch {
|
||
step.facts = { raw };
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
const steps = [...bySeq.values()].sort((a, b) => a.seq.localeCompare(b.seq));
|
||
if (steps.length > 0) {
|
||
return steps;
|
||
}
|
||
|
||
// Not every test is instrumented. Fall back to Playwright's own
|
||
// end-of-test screenshot so the report still shows what the run looked
|
||
// like, just without the narrated data.
|
||
const auto = result.attachments.find(
|
||
(a) => a.name === 'screenshot' && a.contentType === 'image/png',
|
||
);
|
||
if (!auto) {
|
||
return [];
|
||
}
|
||
|
||
const safe = `${slug(test.id)}-final.png`;
|
||
try {
|
||
if (auto.body) {
|
||
writeFileSync(join(SHOT_DIR, safe), auto.body);
|
||
} else if (auto.path) {
|
||
copyFileSync(auto.path, join(SHOT_DIR, safe));
|
||
} else {
|
||
return [];
|
||
}
|
||
} catch {
|
||
return [];
|
||
}
|
||
|
||
return [{ seq: '01', name: 'Final state', shot: `shots/${safe}` }];
|
||
}
|
||
|
||
async onEnd(result: FullResult): Promise<void> {
|
||
const durationMs = Date.now() - this.startedAt;
|
||
|
||
const summary = {
|
||
generatedAt: new Date().toISOString(),
|
||
outcome: result.status,
|
||
durationMs,
|
||
total: this.scenarios.length,
|
||
passed: this.scenarios.filter((s) => s.status === 'passed').length,
|
||
failed: this.scenarios.filter((s) => s.status === 'failed' || s.status === 'timedOut').length,
|
||
skipped: this.scenarios.filter((s) => s.status === 'skipped').length,
|
||
narrated: this.scenarios.filter((s) => s.steps.some((step) => step.facts)).length,
|
||
screenshots: this.scenarios.reduce(
|
||
(n, s) => n + s.steps.filter((step) => step.shot).length,
|
||
0,
|
||
),
|
||
apiCalls: this.scenarios.reduce((n, s) => n + s.apiCalls.length, 0),
|
||
};
|
||
|
||
mkdirSync(OUT_DIR, { recursive: true });
|
||
writeFileSync(
|
||
join(OUT_DIR, 'report.json'),
|
||
JSON.stringify({ summary, scenarios: this.scenarios }, null, 2),
|
||
'utf8',
|
||
);
|
||
writeFileSync(join(OUT_DIR, 'report.md'), renderMarkdown(summary, this.scenarios), 'utf8');
|
||
writeFileSync(
|
||
join(OUT_DIR, 'index.html'),
|
||
renderHtml(summary, this.scenarios, this.meta),
|
||
'utf8',
|
||
);
|
||
|
||
// An absolute file:// URL, because a relative path is ambiguous: the suite
|
||
// runs with its cwd in e2e/, but people launch it from the repo root.
|
||
const file = join(OUT_DIR, 'index.html');
|
||
const url = pathToFileURL(file).href;
|
||
|
||
console.log(
|
||
[
|
||
'',
|
||
'Evidence report — what passed, with data and screenshots:',
|
||
` ${url}`,
|
||
` ${summary.passed}/${summary.total} passed · ${summary.screenshots} screenshots · ` +
|
||
`${summary.apiCalls} API calls · ${summary.narrated} scenarios with recorded data`,
|
||
'',
|
||
' Open it: npm run report',
|
||
' Markdown: ' + pathToFileURL(join(OUT_DIR, 'report.md')).href,
|
||
'',
|
||
" (`npx playwright show-report` opens Playwright's own report, which is a",
|
||
' different artifact — it is for debugging failures.)',
|
||
].join('\n'),
|
||
);
|
||
}
|
||
}
|
||
|
||
// --- helpers ---------------------------------------------------------------
|
||
|
||
function slug(value: string): string {
|
||
return value.replace(/[^a-zA-Z0-9]+/g, '-').slice(0, 60);
|
||
}
|
||
|
||
function safeRead(path: string): string {
|
||
try {
|
||
return readFileSync(path, 'utf8');
|
||
} catch {
|
||
return '';
|
||
}
|
||
}
|
||
|
||
|
||
|
||
|
||
// --- markdown --------------------------------------------------------------
|
||
|
||
function renderMarkdown(summary: Summary, scenarios: Scenario[]): string {
|
||
const lines: string[] = [
|
||
'# TravelHub end-to-end run report',
|
||
'',
|
||
`Generated ${summary.generatedAt} · ${summary.passed}/${summary.total} passed · ` +
|
||
`${friendly(summary.durationMs)}`,
|
||
'',
|
||
];
|
||
|
||
for (const file of group(scenarios)) {
|
||
lines.push(`## ${areaFor(file.name).title}`, '', `_${areaFor(file.name).blurb}_`, '');
|
||
for (const scenario of file.items) {
|
||
const mark = scenario.status === 'passed' ? '✅' : scenario.status === 'skipped' ? '⏭️' : '❌';
|
||
const heading = sentenceCase(scenario.title);
|
||
lines.push(`### ${mark} ${heading}`, '');
|
||
lines.push(`*${scenario.suite ? scenario.suite + ' · ' : ''}${scenario.project} · ${friendly(scenario.durationMs)}*`, '');
|
||
|
||
if (scenario.steps.length === 0 && scenario.apiCalls.length === 0) {
|
||
lines.push('_No evidence captured._', '');
|
||
continue;
|
||
}
|
||
|
||
for (const step of scenario.steps) {
|
||
lines.push(`**${step.seq}. ${step.name}**`, '');
|
||
if (step.facts) {
|
||
lines.push('| Field | Value |', '| ----- | ----- |');
|
||
for (const [key, value] of Object.entries(step.facts)) {
|
||
lines.push(`| ${label(key)} | ${formatValue(value)} |`);
|
||
}
|
||
lines.push('');
|
||
}
|
||
if (step.shot) {
|
||
lines.push(``, '');
|
||
}
|
||
}
|
||
|
||
if (scenario.apiCalls.length > 0) {
|
||
lines.push(`**API calls (${scenario.apiCalls.length})**`, '');
|
||
for (const call of scenario.apiCalls) {
|
||
lines.push(`- \`${call.method} ${call.path}\` → **${call.status}** (${call.durationMs} ms)`);
|
||
if (call.requestBody) {
|
||
lines.push('', ' Request:', '', ' ```json', indent(call.requestBody), ' ```');
|
||
}
|
||
if (call.responseBody) {
|
||
lines.push('', ' Response:', '', ' ```json', indent(call.responseBody), ' ```');
|
||
}
|
||
}
|
||
lines.push('');
|
||
}
|
||
}
|
||
}
|
||
|
||
return lines.join('\n');
|
||
}
|
||
|
||
/** Indents a block two spaces so it nests under a Markdown list item. */
|
||
function indent(text: string): string {
|
||
return text
|
||
.split('\n')
|
||
.map((line) => ' ' + line)
|
||
.join('\n');
|
||
}
|
||
|
||
|