A complete loop for defining, generating, running and gating tests: - qa/: scenario-card template + examples (API/UI/binding), the Claude generation prompt, and a QA runbook tying it together - e2e/CLAUDE.md: conventions that keep AI-generated tests consistent - run-tests.ps1: one runner for every layer (backend/api/web/ui/binding), logs per run under .testlogs/, JUnit output, evidence report, and a re-seed guard against stale demo data - .githooks/ + scripts/install-hooks.ps1: pre-commit secret scan and pre-push unit tests (shift-left before CI) - evidence report now captures each API call's request, payload and response (Authorization redacted); JUnit reporter added for CI - scripts/db.mjs: db:check freshness guard Azure DevOps pipeline deferred; the runner already emits JUnit + the evidence report for any CI to publish. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
216 lines
6.3 KiB
TypeScript
216 lines
6.3 KiB
TypeScript
/**
|
||
* The shape of a run report, and the vocabulary it is written in.
|
||
*
|
||
* Kept separate from both the collector and the renderers so neither depends on
|
||
* the other: `evidence.ts` gathers, `render-html.ts` presents, and both agree
|
||
* on what a scenario is by importing from here.
|
||
*/
|
||
import type { TestResult } from '@playwright/test/reporter';
|
||
|
||
/** One captured moment inside a scenario. */
|
||
export interface Step {
|
||
seq: string;
|
||
name: string;
|
||
shot?: string; // path relative to the report root
|
||
facts?: Record<string, unknown>;
|
||
}
|
||
|
||
/** One API request/response pair observed during a scenario. */
|
||
export interface ApiCall {
|
||
method: string;
|
||
path: string;
|
||
status: number;
|
||
statusText: string;
|
||
durationMs: number;
|
||
requestHeaders: Record<string, string>;
|
||
requestBody?: string;
|
||
responseBody?: string;
|
||
truncated?: boolean;
|
||
}
|
||
|
||
/** One test, as the report talks about it. */
|
||
export interface Scenario {
|
||
project: string;
|
||
file: string;
|
||
suite: string;
|
||
title: string;
|
||
status: TestResult['status'];
|
||
expected: boolean;
|
||
durationMs: number;
|
||
steps: Step[];
|
||
apiCalls: ApiCall[];
|
||
error?: string;
|
||
}
|
||
|
||
/** How the run was configured, for the cover page. */
|
||
export interface RunMeta {
|
||
baseURL: string;
|
||
projects: string[];
|
||
workers: number;
|
||
}
|
||
|
||
export interface Summary {
|
||
generatedAt: string;
|
||
outcome: string;
|
||
durationMs: number;
|
||
total: number;
|
||
passed: number;
|
||
failed: number;
|
||
skipped: number;
|
||
narrated: number;
|
||
screenshots: number;
|
||
apiCalls: number;
|
||
}
|
||
|
||
// --- areas -----------------------------------------------------------------
|
||
|
||
/**
|
||
* Spec filenames are an implementation detail. The report groups by what the
|
||
* tests actually cover, so somebody who has never opened the codebase can read
|
||
* it without a glossary.
|
||
*/
|
||
export interface Area {
|
||
title: string;
|
||
blurb: string;
|
||
}
|
||
|
||
export const AREAS: Record<string, Area> = {
|
||
'auth.spec.ts': {
|
||
title: 'Signing in and sessions',
|
||
blurb:
|
||
'Agents sign in against their own agency. Credentials issued by one agency must not work at another, and a session should survive a page reload but not outlive a sign-out.',
|
||
},
|
||
'search.spec.ts': {
|
||
title: 'Searching for flights',
|
||
blurb:
|
||
'Filtering the agency inventory by route, date, cabin and party size; ordering results by fare, journey time or departure; and paging through them.',
|
||
},
|
||
'booking.spec.ts': {
|
||
title: 'Booking and cancelling',
|
||
blurb:
|
||
'Selling a ticket end to end: the fare carried into checkout, traveller details, confirmation with a booking reference, and cancellation returning the seats to inventory.',
|
||
},
|
||
'tenant-isolation.spec.ts': {
|
||
title: 'Keeping agencies separate',
|
||
blurb:
|
||
'The central promise of the platform. Two agencies share one deployment and one database, and neither may see the other’s inventory, customers or revenue — not even by guessing a web address.',
|
||
},
|
||
'dashboard.spec.ts': {
|
||
title: 'Agency dashboard',
|
||
blurb:
|
||
'Trading figures for the signed-in agency — revenue, seats sold, bookings and best-selling routes — and that they move correctly when a sale is made and when it is cancelled.',
|
||
},
|
||
'simulation.spec.ts': {
|
||
title: 'Behaviour when things go wrong',
|
||
blurb:
|
||
'Here the backend is simulated so it can misbehave on demand: server errors, slow responses, a dropped network, an expired session, a seat sold mid-purchase, and awkward data such as very long names.',
|
||
},
|
||
'responsive.spec.ts': {
|
||
title: 'Phones and small screens',
|
||
blurb:
|
||
'The console is used at a desk and on a phone. No screen may scroll sideways at phone width, and wide tables must scroll inside their own frame rather than stretching the page.',
|
||
},
|
||
};
|
||
|
||
/** Reading order, rather than alphabetical: a reader follows the user journey. */
|
||
export const AREA_ORDER = [
|
||
'auth.spec.ts',
|
||
'search.spec.ts',
|
||
'booking.spec.ts',
|
||
'tenant-isolation.spec.ts',
|
||
'dashboard.spec.ts',
|
||
'simulation.spec.ts',
|
||
'responsive.spec.ts',
|
||
];
|
||
|
||
export function areaFor(file: string): Area {
|
||
const name = file.split('/').pop() ?? file;
|
||
return AREAS[name] ?? { title: name, blurb: '' };
|
||
}
|
||
|
||
export function group(scenarios: Scenario[]): Array<{ name: string; items: Scenario[] }> {
|
||
const map = new Map<string, Scenario[]>();
|
||
for (const scenario of scenarios) {
|
||
const list = map.get(scenario.file) ?? [];
|
||
list.push(scenario);
|
||
map.set(scenario.file, list);
|
||
}
|
||
|
||
const rank = (file: string) => {
|
||
const index = AREA_ORDER.indexOf(file.split('/').pop() ?? file);
|
||
return index === -1 ? AREA_ORDER.length : index;
|
||
};
|
||
|
||
return [...map.entries()]
|
||
.sort(([a], [b]) => rank(a) - rank(b) || a.localeCompare(b))
|
||
.map(([name, items]) => ({ name, items }));
|
||
}
|
||
|
||
// --- formatting ------------------------------------------------------------
|
||
|
||
export function escapeHtml(value: unknown): string {
|
||
return String(value)
|
||
.replace(/&/g, '&')
|
||
.replace(/</g, '<')
|
||
.replace(/>/g, '>')
|
||
.replace(/"/g, '"');
|
||
}
|
||
|
||
export function formatValue(value: unknown): string {
|
||
if (value === null || value === undefined) {
|
||
return '—';
|
||
}
|
||
if (typeof value === 'boolean') {
|
||
return value ? 'Yes' : 'No';
|
||
}
|
||
if (typeof value === 'object') {
|
||
return JSON.stringify(value);
|
||
}
|
||
return String(value);
|
||
}
|
||
|
||
/** "1 min 30 s" reads better in a summary than "89.8 s". */
|
||
export function friendly(ms: number): string {
|
||
if (ms < 1000) {
|
||
return `${ms} ms`;
|
||
}
|
||
const total = Math.round(ms / 1000);
|
||
const minutes = Math.floor(total / 60);
|
||
const secs = total % 60;
|
||
return minutes === 0 ? `${secs} s` : `${minutes} min ${String(secs).padStart(2, '0')} s`;
|
||
}
|
||
|
||
export function sentenceCase(value: string): string {
|
||
return value.charAt(0).toUpperCase() + value.slice(1);
|
||
}
|
||
|
||
/** seatsAvailable -> "Seats available", so data tables read as English. */
|
||
export function label(key: string): string {
|
||
const spaced = key
|
||
.replace(/([a-z0-9])([A-Z])/g, '$1 $2')
|
||
.replace(/[_-]+/g, ' ')
|
||
.toLowerCase()
|
||
.trim();
|
||
return sentenceCase(spaced);
|
||
}
|
||
|
||
export function statusWord(status: Scenario['status']): string {
|
||
switch (status) {
|
||
case 'passed':
|
||
return 'Passed';
|
||
case 'skipped':
|
||
return 'Not run';
|
||
case 'timedOut':
|
||
return 'Timed out';
|
||
default:
|
||
return 'Failed';
|
||
}
|
||
}
|
||
|
||
export function statusClass(status: Scenario['status']): string {
|
||
if (status === 'passed') {
|
||
return 'pass';
|
||
}
|
||
return status === 'skipped' ? 'skip' : 'fail';
|
||
}
|