Add AI-assisted QA test workflow: scenario cards, runner, hooks, API capture

A complete loop for defining, generating, running and gating tests:

- qa/: scenario-card template + examples (API/UI/binding), the Claude
  generation prompt, and a QA runbook tying it together
- e2e/CLAUDE.md: conventions that keep AI-generated tests consistent
- run-tests.ps1: one runner for every layer (backend/api/web/ui/binding),
  logs per run under .testlogs/, JUnit output, evidence report, and a
  re-seed guard against stale demo data
- .githooks/ + scripts/install-hooks.ps1: pre-commit secret scan and
  pre-push unit tests (shift-left before CI)
- evidence report now captures each API call's request, payload and
  response (Authorization redacted); JUnit reporter added for CI
- scripts/db.mjs: db:check freshness guard

Azure DevOps pipeline deferred; the runner already emits JUnit + the
evidence report for any CI to publish.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
nilesh.p
2026-10-10 13:04:46 +05:30
co-authored by Claude Opus 4.8
parent df9e534e42
commit 45b5b34680
20 changed files with 1054 additions and 11 deletions
+45 -2
View File
@@ -21,6 +21,7 @@ import type { FullConfig, FullResult, Reporter, Suite, TestCase, TestResult } fr
import { DATA_PREFIX, SHOT_PREFIX } from '../fixtures/evidence';
import { renderHtml } from './render-html';
import {
ApiCall,
RunMeta,
Scenario,
Step,
@@ -69,6 +70,7 @@ export default class EvidenceReporter implements Reporter {
const suite = path.slice(2, -1).join(' › ');
const steps = this.collectSteps(test, result);
const apiCalls = this.collectApiCalls(result);
this.scenarios.push({
project,
@@ -79,10 +81,28 @@ export default class EvidenceReporter implements Reporter {
expected: result.status === test.expectedStatus,
durationMs: result.duration,
steps,
apiCalls,
error: result.error?.message?.split('\n').slice(0, 4).join('\n'),
});
}
/** Reads the evidence-http attachment the apiTraffic fixture produced. */
private collectApiCalls(result: TestResult): ApiCall[] {
const att = result.attachments.find((a) => a.name === 'evidence-http');
if (!att) {
return [];
}
const raw = att.body ? att.body.toString('utf8') : att.path ? safeRead(att.path) : '';
if (!raw) {
return [];
}
try {
return JSON.parse(raw) as ApiCall[];
} catch {
return [];
}
}
/** Pairs `evidence:NN:name` screenshots with `evidence-data:NN:name` blobs. */
private collectSteps(test: TestCase, result: TestResult): Step[] {
const bySeq = new Map<string, Step>();
@@ -180,6 +200,7 @@ export default class EvidenceReporter implements Reporter {
(n, s) => n + s.steps.filter((step) => step.shot).length,
0,
),
apiCalls: this.scenarios.reduce((n, s) => n + s.apiCalls.length, 0),
};
mkdirSync(OUT_DIR, { recursive: true });
@@ -206,7 +227,7 @@ export default class EvidenceReporter implements Reporter {
'Evidence report — what passed, with data and screenshots:',
` ${url}`,
` ${summary.passed}/${summary.total} passed · ${summary.screenshots} screenshots · ` +
`${summary.narrated} scenarios with recorded data`,
`${summary.apiCalls} API calls · ${summary.narrated} scenarios with recorded data`,
'',
' Open it: npm run report',
' Markdown: ' + pathToFileURL(join(OUT_DIR, 'report.md')).href,
@@ -254,7 +275,7 @@ function renderMarkdown(summary: Summary, scenarios: Scenario[]): string {
lines.push(`### ${mark} ${heading}`, '');
lines.push(`*${scenario.suite ? scenario.suite + ' · ' : ''}${scenario.project} · ${friendly(scenario.durationMs)}*`, '');
if (scenario.steps.length === 0) {
if (scenario.steps.length === 0 && scenario.apiCalls.length === 0) {
lines.push('_No evidence captured._', '');
continue;
}
@@ -272,10 +293,32 @@ function renderMarkdown(summary: Summary, scenarios: Scenario[]): string {
lines.push(`![${step.name}](${step.shot})`, '');
}
}
if (scenario.apiCalls.length > 0) {
lines.push(`**API calls (${scenario.apiCalls.length})**`, '');
for (const call of scenario.apiCalls) {
lines.push(`- \`${call.method} ${call.path}\` → **${call.status}** (${call.durationMs} ms)`);
if (call.requestBody) {
lines.push('', ' Request:', '', ' ```json', indent(call.requestBody), ' ```');
}
if (call.responseBody) {
lines.push('', ' Response:', '', ' ```json', indent(call.responseBody), ' ```');
}
}
lines.push('');
}
}
}
return lines.join('\n');
}
/** Indents a block two spaces so it nests under a Markdown list item. */
function indent(text: string): string {
return text
.split('\n')
.map((line) => ' ' + line)
.join('\n');
}