Add AI-assisted QA test workflow: scenario cards, runner, hooks, API capture
A complete loop for defining, generating, running and gating tests: - qa/: scenario-card template + examples (API/UI/binding), the Claude generation prompt, and a QA runbook tying it together - e2e/CLAUDE.md: conventions that keep AI-generated tests consistent - run-tests.ps1: one runner for every layer (backend/api/web/ui/binding), logs per run under .testlogs/, JUnit output, evidence report, and a re-seed guard against stale demo data - .githooks/ + scripts/install-hooks.ps1: pre-commit secret scan and pre-push unit tests (shift-left before CI) - evidence report now captures each API call's request, payload and response (Authorization redacted); JUnit reporter added for CI - scripts/db.mjs: db:check freshness guard Azure DevOps pipeline deferred; the runner already emits JUnit + the evidence report for any CI to publish. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
nilesh.p
co-authored by
Claude Opus 4.8
Claude Opus 4.8
parent
df9e534e42
commit
45b5b34680
+252
@@ -0,0 +1,252 @@
|
||||
#requires -Version 5.1
|
||||
<#
|
||||
.SYNOPSIS
|
||||
Run the TravelHub test suite by layer, log everything, and emit CI-ready
|
||||
results and an evidence report. The same script runs locally, in a git
|
||||
hook, and in a pipeline.
|
||||
|
||||
.DESCRIPTION
|
||||
One runner for every test layer so there is a single source of truth for
|
||||
"how tests run here". It:
|
||||
- refreshes PATH from the registry (works in stale terminals)
|
||||
- guards e2e against stale demo data by re-seeding when inventory is old
|
||||
- writes a timestamped log per step under .testlogs\<run-id>\
|
||||
- emits JUnit XML (e2e) that Azure DevOps / Gitea can publish
|
||||
- regenerates the evidence report (screenshots + API request/response)
|
||||
- returns a non-zero exit code if any layer fails, so CI can gate on it
|
||||
|
||||
Layers map to the scenario taxonomy in qa\:
|
||||
backend Go unit + integration tests (repository, middleware, config)
|
||||
api Go HTTP-contract tests (handlers, middleware)
|
||||
web Angular unit tests
|
||||
ui Playwright browser journeys (UI + binding, with API capture)
|
||||
binding alias for ui -- the UI<->API seam, proven by captured traffic
|
||||
unit backend + web (no browser; fast)
|
||||
all backend + web + ui
|
||||
|
||||
.PARAMETER Layer
|
||||
Which layer to run. Default: all.
|
||||
|
||||
.PARAMETER Grep
|
||||
Only run tests whose title matches this (passed to `go test -run` and
|
||||
`playwright -g`).
|
||||
|
||||
.PARAMETER Reseed
|
||||
Force a database re-seed before running, instead of only when stale.
|
||||
|
||||
.PARAMETER NoReport
|
||||
Skip regenerating the evidence report (faster; ui/binding only).
|
||||
|
||||
.PARAMETER Open
|
||||
Open the evidence report in the browser when the run finishes.
|
||||
|
||||
.EXAMPLE
|
||||
.\run-tests.ps1 -Layer all
|
||||
.EXAMPLE
|
||||
.\run-tests.ps1 -Layer ui -Grep "oversell" -Open
|
||||
.EXAMPLE
|
||||
.\run-tests.ps1 -Layer unit # fast pre-commit gate, no browser
|
||||
#>
|
||||
[CmdletBinding()]
|
||||
param(
|
||||
[ValidateSet('all', 'unit', 'backend', 'api', 'web', 'ui', 'binding')]
|
||||
[string]$Layer = 'all',
|
||||
|
||||
[string]$Grep,
|
||||
[switch]$Reseed,
|
||||
[switch]$NoReport,
|
||||
[switch]$Open
|
||||
)
|
||||
|
||||
$ErrorActionPreference = 'Stop'
|
||||
|
||||
# --- configuration ---------------------------------------------------------
|
||||
|
||||
$Root = $PSScriptRoot
|
||||
$Backend = Join-Path $Root 'backend'
|
||||
$Frontend = Join-Path $Root 'frontend'
|
||||
$E2E = Join-Path $Root 'e2e'
|
||||
|
||||
$RunId = Get-Date -Format 'yyyyMMdd-HHmmss'
|
||||
$LogDir = Join-Path $Root ".testlogs\$RunId"
|
||||
|
||||
# --- console ---------------------------------------------------------------
|
||||
|
||||
function Write-Head($msg) { Write-Host "`n==== $msg ====" -ForegroundColor Cyan }
|
||||
function Write-Ok($msg) { Write-Host " PASS $msg" -ForegroundColor Green }
|
||||
function Write-Fail($msg) { Write-Host " FAIL $msg" -ForegroundColor Red }
|
||||
function Write-Info($msg) { Write-Host " $msg" -ForegroundColor DarkGray }
|
||||
|
||||
# --- environment -----------------------------------------------------------
|
||||
|
||||
function Sync-Path {
|
||||
$m = [Environment]::GetEnvironmentVariable('Path', 'Machine')
|
||||
$u = [Environment]::GetEnvironmentVariable('Path', 'User')
|
||||
$env:Path = ($m, $u | Where-Object { $_ }) -join ';'
|
||||
}
|
||||
|
||||
function Resolve-Tool($name, [string[]]$fallbacks) {
|
||||
$cmd = Get-Command $name -ErrorAction SilentlyContinue
|
||||
if ($cmd) { return $cmd.Source }
|
||||
foreach ($f in $fallbacks) { if (Test-Path $f) { return $f } }
|
||||
throw "$name not found on PATH."
|
||||
}
|
||||
function Get-Go { Resolve-Tool 'go.exe' @('C:\Program Files\Go\bin\go.exe') }
|
||||
function Get-Npm { Resolve-Tool 'npm.cmd' @('C:\Program Files\nodejs\npm.cmd') }
|
||||
function Get-Npx { Resolve-Tool 'npx.cmd' @('C:\Program Files\nodejs\npx.cmd') }
|
||||
function Get-Node { Resolve-Tool 'node.exe' @('C:\Program Files\nodejs\node.exe') }
|
||||
|
||||
# --- step runner -----------------------------------------------------------
|
||||
|
||||
$script:Results = @()
|
||||
|
||||
# Run one command, tee its output to a log file, and record pass/fail by exit
|
||||
# code. Never throws -- a failing step is recorded and the run continues so the
|
||||
# report covers every layer.
|
||||
function Invoke-Step {
|
||||
param(
|
||||
[string]$Name,
|
||||
[string]$WorkDir,
|
||||
[string]$Exe,
|
||||
[string[]]$CmdArgs,
|
||||
[string]$LogName
|
||||
)
|
||||
Write-Head $Name
|
||||
$log = Join-Path $LogDir $LogName
|
||||
Write-Info "log: $log"
|
||||
|
||||
$sw = [Diagnostics.Stopwatch]::StartNew()
|
||||
Push-Location $WorkDir
|
||||
# A native tool writing to stderr (e.g. a webserver startup warning) must
|
||||
# not be treated as a PowerShell error. Pass/fail is the exit code alone;
|
||||
# stderr is just logged. Continue preference keeps 2>&1 from throwing.
|
||||
$prev = $ErrorActionPreference
|
||||
$ErrorActionPreference = 'Continue'
|
||||
try {
|
||||
& $Exe @CmdArgs 2>&1 | Tee-Object -FilePath $log
|
||||
$code = $LASTEXITCODE
|
||||
} finally {
|
||||
$ErrorActionPreference = $prev
|
||||
Pop-Location
|
||||
}
|
||||
$sw.Stop()
|
||||
|
||||
$ok = ($code -eq 0)
|
||||
$script:Results += [pscustomobject]@{
|
||||
Layer = $Name; Exit = $code; Ok = $ok; Seconds = [math]::Round($sw.Elapsed.TotalSeconds, 1); Log = $log
|
||||
}
|
||||
if ($ok) { Write-Ok "$Name ($($sw.Elapsed.TotalSeconds.ToString('0.0'))s)" }
|
||||
else { Write-Fail "$Name (exit $code)" }
|
||||
}
|
||||
|
||||
# --- preconditions ---------------------------------------------------------
|
||||
|
||||
function Ensure-Inventory {
|
||||
# Only the browser layers need live, future-dated flight data.
|
||||
$node = Get-Node
|
||||
$check = Join-Path $Root 'scripts\db.mjs'
|
||||
|
||||
if ($Reseed) {
|
||||
Write-Info 'Re-seeding database (-Reseed).'
|
||||
& $node (Join-Path $Root 'scripts\db.mjs') reset | Out-Null
|
||||
return
|
||||
}
|
||||
|
||||
& $node $check check 2>&1 | Out-Null
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
Write-Info 'Demo inventory is stale or missing -- re-seeding before e2e.'
|
||||
& $node (Join-Path $Root 'scripts\db.mjs') reset | Out-Null
|
||||
} else {
|
||||
Write-Info 'Demo inventory is fresh.'
|
||||
}
|
||||
}
|
||||
|
||||
# --- layers ----------------------------------------------------------------
|
||||
|
||||
function Run-Backend {
|
||||
$go = Get-Go
|
||||
$cmdArgs = @('test', './...')
|
||||
if ($Grep) { $cmdArgs += @('-run', $Grep) }
|
||||
Invoke-Step 'backend' $Backend $go $cmdArgs 'backend.log'
|
||||
}
|
||||
|
||||
function Run-Api {
|
||||
# The HTTP contract lives in the handler + middleware packages.
|
||||
$go = Get-Go
|
||||
$cmdArgs = @('test', './internal/handlers/...', './internal/middleware/...')
|
||||
if ($Grep) { $cmdArgs += @('-run', $Grep) }
|
||||
Invoke-Step 'api' $Backend $go $cmdArgs 'api.log'
|
||||
}
|
||||
|
||||
function Run-Web {
|
||||
$npx = Get-Npx
|
||||
Invoke-Step 'web' $Frontend $npx @('ng', 'test', '--no-watch') 'web.log'
|
||||
}
|
||||
|
||||
function Run-Ui {
|
||||
param([string]$Label = 'ui')
|
||||
Ensure-Inventory
|
||||
$npx = Get-Npx
|
||||
$cmdArgs = @('playwright', 'test')
|
||||
if ($Grep) { $cmdArgs += @('-g', $Grep) }
|
||||
Invoke-Step $Label $E2E $npx $cmdArgs "$Label.log"
|
||||
}
|
||||
|
||||
# --- evidence + results collection -----------------------------------------
|
||||
|
||||
function Collect-Artifacts {
|
||||
# JUnit (for CI test publishing) and the evidence report both live under e2e.
|
||||
$junit = Join-Path $E2E 'results\junit.xml'
|
||||
if (Test-Path $junit) {
|
||||
Copy-Item $junit (Join-Path $LogDir 'junit.xml') -Force
|
||||
Write-Info "JUnit results -> $(Join-Path $LogDir 'junit.xml')"
|
||||
}
|
||||
$evidence = Join-Path $E2E 'evidence-report\index.html'
|
||||
if (Test-Path $evidence) {
|
||||
Write-Info "Evidence report -> $evidence"
|
||||
if ($Open) { Start-Process $evidence }
|
||||
}
|
||||
}
|
||||
|
||||
# --- dispatch --------------------------------------------------------------
|
||||
|
||||
Sync-Path
|
||||
New-Item -ItemType Directory -Force -Path $LogDir | Out-Null
|
||||
Write-Host "TravelHub test run $RunId (layer: $Layer)" -ForegroundColor White
|
||||
|
||||
switch ($Layer) {
|
||||
'backend' { Run-Backend }
|
||||
'api' { Run-Api }
|
||||
'web' { Run-Web }
|
||||
'ui' { Run-Ui 'ui' }
|
||||
'binding' { Run-Ui 'binding' }
|
||||
'unit' { Run-Backend; Run-Web }
|
||||
'all' { Run-Backend; Run-Web; Run-Ui 'ui' }
|
||||
}
|
||||
|
||||
if (-not $NoReport) { Collect-Artifacts }
|
||||
|
||||
# --- summary ---------------------------------------------------------------
|
||||
|
||||
Write-Head 'Summary'
|
||||
$script:Results | Format-Table Layer, @{N='Result';E={ if ($_.Ok) { 'PASS' } else { "FAIL ($($_.Exit))" } }}, Seconds -AutoSize | Out-String | Write-Host
|
||||
|
||||
# Write a small machine-readable summary alongside the logs.
|
||||
$summary = [pscustomobject]@{
|
||||
runId = $RunId
|
||||
layer = $Layer
|
||||
when = (Get-Date).ToString('o')
|
||||
results = $script:Results
|
||||
passed = @($script:Results | Where-Object Ok).Count
|
||||
failed = @($script:Results | Where-Object { -not $_.Ok }).Count
|
||||
}
|
||||
$summary | ConvertTo-Json -Depth 4 | Out-File (Join-Path $LogDir 'summary.json') -Encoding utf8
|
||||
Write-Info "Logs: $LogDir"
|
||||
|
||||
$failed = @($script:Results | Where-Object { -not $_.Ok }).Count
|
||||
if ($failed -gt 0) {
|
||||
Write-Fail "$failed layer(s) failed."
|
||||
exit 1
|
||||
}
|
||||
Write-Ok 'All layers passed.'
|
||||
exit 0
|
||||
Reference in New Issue
Block a user