Add AI-assisted QA test workflow: scenario cards, runner, hooks, API capture
A complete loop for defining, generating, running and gating tests: - qa/: scenario-card template + examples (API/UI/binding), the Claude generation prompt, and a QA runbook tying it together - e2e/CLAUDE.md: conventions that keep AI-generated tests consistent - run-tests.ps1: one runner for every layer (backend/api/web/ui/binding), logs per run under .testlogs/, JUnit output, evidence report, and a re-seed guard against stale demo data - .githooks/ + scripts/install-hooks.ps1: pre-commit secret scan and pre-push unit tests (shift-left before CI) - evidence report now captures each API call's request, payload and response (Authorization redacted); JUnit reporter added for CI - scripts/db.mjs: db:check freshness guard Azure DevOps pipeline deferred; the runner already emits JUnit + the evidence report for any CI to publish. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.8
parent
df9e534e42
commit
45b5b34680
@@ -0,0 +1,110 @@
|
||||
/**
|
||||
* Captures the API traffic a test produced, for the evidence report.
|
||||
*
|
||||
* A screenshot shows what the user saw; this shows what crossed the wire -- the
|
||||
* request (method, path, payload) and the response (status, body) for every
|
||||
* `/api/**` call. Together they make the report a complete record of the
|
||||
* scenario.
|
||||
*
|
||||
* Security: the Authorization header carries the Basic-auth credential (a
|
||||
* base64 of email:password), so it is redacted before anything is stored. The
|
||||
* report is meant to be shared; a credential must never ride along in it.
|
||||
*/
|
||||
import type { Response } from '@playwright/test';
|
||||
|
||||
export interface ApiCall {
|
||||
method: string;
|
||||
path: string;
|
||||
status: number;
|
||||
statusText: string;
|
||||
durationMs: number;
|
||||
requestHeaders: Record<string, string>;
|
||||
requestBody?: string;
|
||||
responseBody?: string;
|
||||
truncated?: boolean;
|
||||
}
|
||||
|
||||
// Headers that must never appear in a shareable report.
|
||||
const SENSITIVE = new Set(['authorization', 'cookie', 'set-cookie', 'proxy-authorization']);
|
||||
|
||||
// Response bodies are capped so a large flight list cannot bloat the report.
|
||||
const MAX_BODY = 12000;
|
||||
|
||||
function redactHeaders(headers: Record<string, string>): Record<string, string> {
|
||||
const out: Record<string, string> = {};
|
||||
for (const [key, value] of Object.entries(headers)) {
|
||||
out[key] = SENSITIVE.has(key.toLowerCase()) ? '[redacted]' : value;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/** Pretty-prints JSON and caps length, reporting whether it was truncated. */
|
||||
function presentBody(raw: string, contentType: string): { body: string; truncated: boolean } {
|
||||
let text = raw;
|
||||
if (/json/i.test(contentType)) {
|
||||
try {
|
||||
text = JSON.stringify(JSON.parse(raw), null, 2);
|
||||
} catch {
|
||||
// Leave non-parseable bodies as-is.
|
||||
}
|
||||
}
|
||||
if (text.length > MAX_BODY) {
|
||||
return { body: text.slice(0, MAX_BODY) + '\n… (truncated)', truncated: true };
|
||||
}
|
||||
return { body: text, truncated: false };
|
||||
}
|
||||
|
||||
/**
|
||||
* Turns one response into an ApiCall record, or null if it is not an API call.
|
||||
* The body is read here, while the response is still live, and returned via the
|
||||
* promise the caller awaits at test teardown.
|
||||
*/
|
||||
export function captureApiCall(response: Response): Promise<ApiCall | null> {
|
||||
const request = response.request();
|
||||
|
||||
let url: URL;
|
||||
try {
|
||||
url = new URL(request.url());
|
||||
} catch {
|
||||
return Promise.resolve(null);
|
||||
}
|
||||
if (!url.pathname.startsWith('/api/')) {
|
||||
return Promise.resolve(null);
|
||||
}
|
||||
|
||||
return (async () => {
|
||||
try {
|
||||
const contentType = response.headers()['content-type'] ?? '';
|
||||
let responseBody: string | undefined;
|
||||
let truncated = false;
|
||||
|
||||
if (/json|text|xml|form/i.test(contentType)) {
|
||||
const raw = await response.text().catch(() => '');
|
||||
if (raw) {
|
||||
const shown = presentBody(raw, contentType);
|
||||
responseBody = shown.body;
|
||||
truncated = shown.truncated;
|
||||
}
|
||||
}
|
||||
|
||||
const timing = request.timing();
|
||||
const durationMs = timing.responseEnd > 0 ? Math.round(timing.responseEnd) : 0;
|
||||
|
||||
return {
|
||||
method: request.method(),
|
||||
path: url.pathname + url.search,
|
||||
status: response.status(),
|
||||
statusText: response.statusText(),
|
||||
durationMs,
|
||||
requestHeaders: redactHeaders(request.headers()),
|
||||
requestBody: request.postData() ?? undefined,
|
||||
responseBody,
|
||||
truncated: truncated || undefined,
|
||||
};
|
||||
} catch {
|
||||
// A response that cannot be read (e.g. the page navigated away) is
|
||||
// skipped rather than failing the test it was only observing.
|
||||
return null;
|
||||
}
|
||||
})();
|
||||
}
|
||||
@@ -2,6 +2,7 @@ import { expect, test as base } from '@playwright/test';
|
||||
|
||||
import { Account } from './accounts';
|
||||
import { Evidence } from './evidence';
|
||||
import { ApiCall, captureApiCall } from './http-capture';
|
||||
import { BookingDetailPage, BookingsPage } from '../pages/bookings.page';
|
||||
import { CheckoutPage } from '../pages/checkout.page';
|
||||
import { DashboardPage } from '../pages/dashboard.page';
|
||||
@@ -26,6 +27,9 @@ interface Fixtures {
|
||||
|
||||
/** Console and uncaught errors seen during the test; asserted empty at the end. */
|
||||
consoleErrors: string[];
|
||||
|
||||
/** Records every /api/** request and response for the run report. */
|
||||
apiTraffic: void;
|
||||
}
|
||||
|
||||
export const test = base.extend<Fixtures>({
|
||||
@@ -75,6 +79,28 @@ export const test = base.extend<Fixtures>({
|
||||
},
|
||||
{ auto: true },
|
||||
],
|
||||
|
||||
// Records the request/response of every API call the test made, attached as
|
||||
// evidence-http for the reporter. Auto, so no test has to opt in.
|
||||
apiTraffic: [
|
||||
async ({ page }, use, testInfo) => {
|
||||
const pending: Array<Promise<ApiCall | null>> = [];
|
||||
page.on('response', (response) => {
|
||||
pending.push(captureApiCall(response));
|
||||
});
|
||||
|
||||
await use();
|
||||
|
||||
const calls = (await Promise.all(pending)).filter((c): c is ApiCall => c !== null);
|
||||
if (calls.length > 0) {
|
||||
await testInfo.attach('evidence-http', {
|
||||
body: Buffer.from(JSON.stringify(calls, null, 2), 'utf8'),
|
||||
contentType: 'application/json',
|
||||
});
|
||||
}
|
||||
},
|
||||
{ auto: true },
|
||||
],
|
||||
});
|
||||
|
||||
export { expect } from '@playwright/test';
|
||||
|
||||
Reference in New Issue
Block a user