Add AI-assisted QA test workflow: scenario cards, runner, hooks, API capture
A complete loop for defining, generating, running and gating tests: - qa/: scenario-card template + examples (API/UI/binding), the Claude generation prompt, and a QA runbook tying it together - e2e/CLAUDE.md: conventions that keep AI-generated tests consistent - run-tests.ps1: one runner for every layer (backend/api/web/ui/binding), logs per run under .testlogs/, JUnit output, evidence report, and a re-seed guard against stale demo data - .githooks/ + scripts/install-hooks.ps1: pre-commit secret scan and pre-push unit tests (shift-left before CI) - evidence report now captures each API call's request, payload and response (Authorization redacted); JUnit reporter added for CI - scripts/db.mjs: db:check freshness guard Azure DevOps pipeline deferred; the runner already emits JUnit + the evidence report for any CI to publish. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.8
parent
df9e534e42
commit
45b5b34680
@@ -21,6 +21,7 @@ import type { FullConfig, FullResult, Reporter, Suite, TestCase, TestResult } fr
|
||||
import { DATA_PREFIX, SHOT_PREFIX } from '../fixtures/evidence';
|
||||
import { renderHtml } from './render-html';
|
||||
import {
|
||||
ApiCall,
|
||||
RunMeta,
|
||||
Scenario,
|
||||
Step,
|
||||
@@ -69,6 +70,7 @@ export default class EvidenceReporter implements Reporter {
|
||||
const suite = path.slice(2, -1).join(' › ');
|
||||
|
||||
const steps = this.collectSteps(test, result);
|
||||
const apiCalls = this.collectApiCalls(result);
|
||||
|
||||
this.scenarios.push({
|
||||
project,
|
||||
@@ -79,10 +81,28 @@ export default class EvidenceReporter implements Reporter {
|
||||
expected: result.status === test.expectedStatus,
|
||||
durationMs: result.duration,
|
||||
steps,
|
||||
apiCalls,
|
||||
error: result.error?.message?.split('\n').slice(0, 4).join('\n'),
|
||||
});
|
||||
}
|
||||
|
||||
/** Reads the evidence-http attachment the apiTraffic fixture produced. */
|
||||
private collectApiCalls(result: TestResult): ApiCall[] {
|
||||
const att = result.attachments.find((a) => a.name === 'evidence-http');
|
||||
if (!att) {
|
||||
return [];
|
||||
}
|
||||
const raw = att.body ? att.body.toString('utf8') : att.path ? safeRead(att.path) : '';
|
||||
if (!raw) {
|
||||
return [];
|
||||
}
|
||||
try {
|
||||
return JSON.parse(raw) as ApiCall[];
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
/** Pairs `evidence:NN:name` screenshots with `evidence-data:NN:name` blobs. */
|
||||
private collectSteps(test: TestCase, result: TestResult): Step[] {
|
||||
const bySeq = new Map<string, Step>();
|
||||
@@ -180,6 +200,7 @@ export default class EvidenceReporter implements Reporter {
|
||||
(n, s) => n + s.steps.filter((step) => step.shot).length,
|
||||
0,
|
||||
),
|
||||
apiCalls: this.scenarios.reduce((n, s) => n + s.apiCalls.length, 0),
|
||||
};
|
||||
|
||||
mkdirSync(OUT_DIR, { recursive: true });
|
||||
@@ -206,7 +227,7 @@ export default class EvidenceReporter implements Reporter {
|
||||
'Evidence report — what passed, with data and screenshots:',
|
||||
` ${url}`,
|
||||
` ${summary.passed}/${summary.total} passed · ${summary.screenshots} screenshots · ` +
|
||||
`${summary.narrated} scenarios with recorded data`,
|
||||
`${summary.apiCalls} API calls · ${summary.narrated} scenarios with recorded data`,
|
||||
'',
|
||||
' Open it: npm run report',
|
||||
' Markdown: ' + pathToFileURL(join(OUT_DIR, 'report.md')).href,
|
||||
@@ -254,7 +275,7 @@ function renderMarkdown(summary: Summary, scenarios: Scenario[]): string {
|
||||
lines.push(`### ${mark} ${heading}`, '');
|
||||
lines.push(`*${scenario.suite ? scenario.suite + ' · ' : ''}${scenario.project} · ${friendly(scenario.durationMs)}*`, '');
|
||||
|
||||
if (scenario.steps.length === 0) {
|
||||
if (scenario.steps.length === 0 && scenario.apiCalls.length === 0) {
|
||||
lines.push('_No evidence captured._', '');
|
||||
continue;
|
||||
}
|
||||
@@ -272,10 +293,32 @@ function renderMarkdown(summary: Summary, scenarios: Scenario[]): string {
|
||||
lines.push(``, '');
|
||||
}
|
||||
}
|
||||
|
||||
if (scenario.apiCalls.length > 0) {
|
||||
lines.push(`**API calls (${scenario.apiCalls.length})**`, '');
|
||||
for (const call of scenario.apiCalls) {
|
||||
lines.push(`- \`${call.method} ${call.path}\` → **${call.status}** (${call.durationMs} ms)`);
|
||||
if (call.requestBody) {
|
||||
lines.push('', ' Request:', '', ' ```json', indent(call.requestBody), ' ```');
|
||||
}
|
||||
if (call.responseBody) {
|
||||
lines.push('', ' Response:', '', ' ```json', indent(call.responseBody), ' ```');
|
||||
}
|
||||
}
|
||||
lines.push('');
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return lines.join('\n');
|
||||
}
|
||||
|
||||
/** Indents a block two spaces so it nests under a Markdown list item. */
|
||||
function indent(text: string): string {
|
||||
return text
|
||||
.split('\n')
|
||||
.map((line) => ' ' + line)
|
||||
.join('\n');
|
||||
}
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user