new changes
This commit is contained in:
@@ -40,3 +40,6 @@ e2e/**/*.pdf
|
||||
.run/
|
||||
.testlogs/
|
||||
e2e/results/
|
||||
|
||||
# qa tool generated prompts (ephemeral)
|
||||
qa/.prompts/
|
||||
|
||||
+5
-1
@@ -38,6 +38,10 @@
|
||||
"db:reset": "node scripts/db.mjs reset",
|
||||
"db:test": "node scripts/db.mjs test-db",
|
||||
"start:api": "cd backend && go run ./cmd/api",
|
||||
"start:web": "cd frontend && npm start"
|
||||
"start:web": "cd frontend && npm start",
|
||||
"qa": "node scripts/qa-tool.mjs",
|
||||
"qa:index": "node scripts/qa-tool.mjs index",
|
||||
"qa:list": "node scripts/qa-tool.mjs list",
|
||||
"qa:ui": "node scripts/qa-ui.mjs"
|
||||
}
|
||||
}
|
||||
|
||||
+26
-1
@@ -37,7 +37,32 @@ exactly those screenshots and API calls, and the report must show them.
|
||||
|
||||
## 2. Automate it with Claude Code
|
||||
|
||||
From the repo root, use the prompt in
|
||||
**The quick way — the `qa` tool.** It assembles a *grounded* generation prompt
|
||||
(your card + the app model + the conventions) so Claude never guesses selectors,
|
||||
routes, or endpoints:
|
||||
|
||||
```bash
|
||||
npm run qa:index # build the app-model grounding index (do once / after UI changes)
|
||||
node scripts/qa-tool.mjs list # list scenario cards
|
||||
node scripts/qa-tool.mjs prompt BOOK-012 # print the grounded prompt — paste into Claude Code
|
||||
node scripts/qa-tool.mjs run BOOK-012 # run the card's tests + open the evidence report
|
||||
```
|
||||
|
||||
`npm run qa:index` reads the codebase and writes
|
||||
[`app-model.md`](app-model.md) / `app-model.json`: the routes, page objects,
|
||||
API endpoints, areas and accounts that actually exist. `prompt` embeds that plus
|
||||
the card and [`e2e/CLAUDE.md`](../e2e/CLAUDE.md) into one ready-to-paste ask, so
|
||||
the generated test is correct by construction. `run` wraps `run-tests.ps1` for
|
||||
the card's layer and opens the evidence.
|
||||
|
||||
Generation stays human-in-the-loop: the tool prepares the grounded ask and runs
|
||||
the result; you (with Claude Code) write and review the test.
|
||||
|
||||
**Prefer a screen?** `npm run qa:ui` opens a small local web UI (localhost:9500)
|
||||
over the same commands: pick a card, copy its grounded prompt, hit **Run tests**,
|
||||
and see the evidence report inline. Same engine, nicer front door.
|
||||
|
||||
**Or by hand:** use the template in
|
||||
[`generate-test.prompt.md`](generate-test.prompt.md) with your card id. Claude
|
||||
reads the card and the conventions in [`e2e/CLAUDE.md`](../e2e/CLAUDE.md) and
|
||||
generates the spec using our page objects and fixtures.
|
||||
|
||||
@@ -0,0 +1,288 @@
|
||||
{
|
||||
"generatedAt": "2026-10-10T09:15:03.203Z",
|
||||
"baseUrl": "http://localhost:4200",
|
||||
"auth": {
|
||||
"type": "basic",
|
||||
"tenantHeader": "X-Tenant",
|
||||
"note": "Credentials verified within the named tenant; take accounts from e2e/fixtures/accounts.ts, never hardcode.",
|
||||
"accounts": [
|
||||
{
|
||||
"name": "SKYLINE_AGENT",
|
||||
"email": "agent@skyline.test",
|
||||
"tenant": "skyline",
|
||||
"role": "agent"
|
||||
},
|
||||
{
|
||||
"name": "SKYLINE_ADMIN",
|
||||
"email": "admin@skyline.test",
|
||||
"tenant": "skyline",
|
||||
"role": "admin"
|
||||
},
|
||||
{
|
||||
"name": "GLOBETROTTER_AGENT",
|
||||
"email": "agent@globetrotter.test",
|
||||
"tenant": "globetrotter",
|
||||
"role": "agent"
|
||||
},
|
||||
{
|
||||
"name": "GLOBETROTTER_ADMIN",
|
||||
"email": "admin@globetrotter.test",
|
||||
"tenant": "globetrotter",
|
||||
"role": "admin"
|
||||
}
|
||||
]
|
||||
},
|
||||
"routes": [
|
||||
"/login",
|
||||
"/dashboard",
|
||||
"/search",
|
||||
"/book/:flightId",
|
||||
"/bookings",
|
||||
"/bookings/:bookingId"
|
||||
],
|
||||
"areas": [
|
||||
{
|
||||
"spec": "auth.spec.ts",
|
||||
"title": "Signing in and sessions"
|
||||
},
|
||||
{
|
||||
"spec": "search.spec.ts",
|
||||
"title": "Searching for flights"
|
||||
},
|
||||
{
|
||||
"spec": "booking.spec.ts",
|
||||
"title": "Booking and cancelling"
|
||||
},
|
||||
{
|
||||
"spec": "tenant-isolation.spec.ts",
|
||||
"title": "Keeping agencies separate"
|
||||
},
|
||||
{
|
||||
"spec": "dashboard.spec.ts",
|
||||
"title": "Agency dashboard"
|
||||
},
|
||||
{
|
||||
"spec": "simulation.spec.ts",
|
||||
"title": "Behaviour when things go wrong"
|
||||
},
|
||||
{
|
||||
"spec": "responsive.spec.ts",
|
||||
"title": "Phones and small screens"
|
||||
}
|
||||
],
|
||||
"api": [
|
||||
{
|
||||
"method": "GET",
|
||||
"path": "/api/session",
|
||||
"auth": true
|
||||
},
|
||||
{
|
||||
"method": "GET",
|
||||
"path": "/api/flights",
|
||||
"auth": true
|
||||
},
|
||||
{
|
||||
"method": "GET",
|
||||
"path": "/api/flights/{id}",
|
||||
"auth": true
|
||||
},
|
||||
{
|
||||
"method": "GET",
|
||||
"path": "/api/airports",
|
||||
"auth": true
|
||||
},
|
||||
{
|
||||
"method": "GET",
|
||||
"path": "/api/airlines",
|
||||
"auth": true
|
||||
},
|
||||
{
|
||||
"method": "GET",
|
||||
"path": "/api/bookings",
|
||||
"auth": true
|
||||
},
|
||||
{
|
||||
"method": "POST",
|
||||
"path": "/api/bookings",
|
||||
"auth": true
|
||||
},
|
||||
{
|
||||
"method": "GET",
|
||||
"path": "/api/bookings/{id}",
|
||||
"auth": true
|
||||
},
|
||||
{
|
||||
"method": "POST",
|
||||
"path": "/api/bookings/{id}/cancel",
|
||||
"auth": true
|
||||
},
|
||||
{
|
||||
"method": "GET",
|
||||
"path": "/api/dashboard",
|
||||
"auth": true
|
||||
},
|
||||
{
|
||||
"method": "GET",
|
||||
"path": "/api/health",
|
||||
"auth": false
|
||||
},
|
||||
{
|
||||
"method": "GET",
|
||||
"path": "/api/tenants",
|
||||
"auth": false
|
||||
}
|
||||
],
|
||||
"pageObjects": [
|
||||
{
|
||||
"file": "e2e/pages/bookings.page.ts",
|
||||
"classes": [
|
||||
"BookingsPage",
|
||||
"BookingDetailPage"
|
||||
],
|
||||
"locators": [
|
||||
"rows",
|
||||
"emptyState",
|
||||
"filterAll",
|
||||
"filterConfirmed",
|
||||
"filterCancelled",
|
||||
"confirmationBanner",
|
||||
"reference",
|
||||
"statusBadge",
|
||||
"travellerRows",
|
||||
"totalPaid",
|
||||
"cancelButton",
|
||||
"confirmCancelButton",
|
||||
"keepButton",
|
||||
"cancelledNotice"
|
||||
],
|
||||
"methods": [
|
||||
"goto()",
|
||||
"settled()",
|
||||
"rowFor(reference: string)",
|
||||
"open(reference: string)",
|
||||
"filterBy(status: 'all' | 'confirmed' | 'cancelled')",
|
||||
"bookingReference()",
|
||||
"cancel(expectedReference: string)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"file": "e2e/pages/checkout.page.ts",
|
||||
"classes": [
|
||||
"CheckoutPage"
|
||||
],
|
||||
"locators": [
|
||||
"travellerFieldsets",
|
||||
"addTraveller",
|
||||
"contactEmail",
|
||||
"contactPhone",
|
||||
"confirm",
|
||||
"total",
|
||||
"error",
|
||||
"seatsLeft"
|
||||
],
|
||||
"methods": [
|
||||
"waitUntilReady()",
|
||||
"totalAmount()",
|
||||
"fillTraveller(index: number, traveller: Traveller)",
|
||||
"fillTravellers(travellers: Traveller[])",
|
||||
"removeTraveller(index: number)",
|
||||
"confirmBooking()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"file": "e2e/pages/dashboard.page.ts",
|
||||
"classes": [
|
||||
"DashboardPage"
|
||||
],
|
||||
"locators": [
|
||||
"greeting",
|
||||
"tiles",
|
||||
"recentRows",
|
||||
"topRoutes"
|
||||
],
|
||||
"methods": [
|
||||
"goto()",
|
||||
"settled()",
|
||||
"tileValue(label: string)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"file": "e2e/pages/login.page.ts",
|
||||
"classes": [
|
||||
"LoginPage"
|
||||
],
|
||||
"locators": [
|
||||
"tenantSelect",
|
||||
"email",
|
||||
"password",
|
||||
"submit",
|
||||
"error",
|
||||
"demoChips"
|
||||
],
|
||||
"methods": [
|
||||
"goto()",
|
||||
"useDemoChip(account: Account)",
|
||||
"fill(account: Account, password = account.password)",
|
||||
"signIn(account: Account)",
|
||||
"signInExpectingFailure(account: Account, password: string)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"file": "e2e/pages/search.page.ts",
|
||||
"classes": [
|
||||
"FlightCard",
|
||||
"SearchPage"
|
||||
],
|
||||
"locators": [
|
||||
"root",
|
||||
"origin",
|
||||
"destination",
|
||||
"date",
|
||||
"cabin",
|
||||
"travellers",
|
||||
"searchButton",
|
||||
"resetButton",
|
||||
"sort",
|
||||
"resultSummary",
|
||||
"results",
|
||||
"emptyState",
|
||||
"nextPage",
|
||||
"prevPage"
|
||||
],
|
||||
"methods": [
|
||||
"fare()",
|
||||
"seatsLeft()",
|
||||
"flightNumber()",
|
||||
"durationMinutes()",
|
||||
"goto()",
|
||||
"card(index = 0)",
|
||||
"settled()",
|
||||
"searchRoute(from: string, to: string)",
|
||||
"setTravellers(n: number)",
|
||||
"sortBy(value: 'price' | 'duration' | 'departure')",
|
||||
"totalResults()",
|
||||
"cardForFlight(flightNumber: string)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"file": "e2e/pages/shell.page.ts",
|
||||
"classes": [
|
||||
"ShellPage"
|
||||
],
|
||||
"locators": [
|
||||
"agencyName",
|
||||
"agencySlug",
|
||||
"accountName",
|
||||
"accountMeta",
|
||||
"signOut"
|
||||
],
|
||||
"methods": [
|
||||
"goToDashboard()",
|
||||
"goToSearch()",
|
||||
"goToBookings()",
|
||||
"signOutAndWait()"
|
||||
]
|
||||
}
|
||||
],
|
||||
"conventions": "e2e/CLAUDE.md"
|
||||
}
|
||||
@@ -0,0 +1,78 @@
|
||||
# App model — grounding index for test generation
|
||||
|
||||
> Generated by `npm run qa:index` from the codebase. Do not edit by hand.
|
||||
|
||||
Base URL: `http://localhost:4200` · Auth: basic (header `X-Tenant`)
|
||||
|
||||
## Accounts
|
||||
|
||||
| const | email | tenant | role |
|
||||
| --- | --- | --- | --- |
|
||||
| `SKYLINE_AGENT` | agent@skyline.test | skyline | agent |
|
||||
| `SKYLINE_ADMIN` | admin@skyline.test | skyline | admin |
|
||||
| `GLOBETROTTER_AGENT` | agent@globetrotter.test | globetrotter | agent |
|
||||
| `GLOBETROTTER_ADMIN` | admin@globetrotter.test | globetrotter | admin |
|
||||
|
||||
_Password: see `DEMO_PASSWORD` in accounts.ts. Credentials verified within the named tenant; take accounts from e2e/fixtures/accounts.ts, never hardcode._
|
||||
|
||||
## Routes
|
||||
|
||||
- `/login`
|
||||
- `/dashboard`
|
||||
- `/search`
|
||||
- `/book/:flightId`
|
||||
- `/bookings`
|
||||
- `/bookings/:bookingId`
|
||||
|
||||
## Areas (spec file → report area)
|
||||
|
||||
- `auth.spec.ts` — Signing in and sessions
|
||||
- `search.spec.ts` — Searching for flights
|
||||
- `booking.spec.ts` — Booking and cancelling
|
||||
- `tenant-isolation.spec.ts` — Keeping agencies separate
|
||||
- `dashboard.spec.ts` — Agency dashboard
|
||||
- `simulation.spec.ts` — Behaviour when things go wrong
|
||||
- `responsive.spec.ts` — Phones and small screens
|
||||
|
||||
## API endpoints
|
||||
|
||||
| method | path | auth |
|
||||
| --- | --- | --- |
|
||||
| GET | `/api/session` | required |
|
||||
| GET | `/api/flights` | required |
|
||||
| GET | `/api/flights/{id}` | required |
|
||||
| GET | `/api/airports` | required |
|
||||
| GET | `/api/airlines` | required |
|
||||
| GET | `/api/bookings` | required |
|
||||
| POST | `/api/bookings` | required |
|
||||
| GET | `/api/bookings/{id}` | required |
|
||||
| POST | `/api/bookings/{id}/cancel` | required |
|
||||
| GET | `/api/dashboard` | required |
|
||||
| GET | `/api/health` | public |
|
||||
| GET | `/api/tenants` | public |
|
||||
|
||||
## Page objects (use these — no raw selectors in specs)
|
||||
|
||||
### `e2e/pages/bookings.page.ts` — BookingsPage, BookingDetailPage
|
||||
- locators: `rows`, `emptyState`, `filterAll`, `filterConfirmed`, `filterCancelled`, `confirmationBanner`, `reference`, `statusBadge`, `travellerRows`, `totalPaid`, `cancelButton`, `confirmCancelButton`, `keepButton`, `cancelledNotice`
|
||||
- methods: `goto()`, `settled()`, `rowFor(reference: string)`, `open(reference: string)`, `filterBy(status: 'all' | 'confirmed' | 'cancelled')`, `bookingReference()`, `cancel(expectedReference: string)`
|
||||
|
||||
### `e2e/pages/checkout.page.ts` — CheckoutPage
|
||||
- locators: `travellerFieldsets`, `addTraveller`, `contactEmail`, `contactPhone`, `confirm`, `total`, `error`, `seatsLeft`
|
||||
- methods: `waitUntilReady()`, `totalAmount()`, `fillTraveller(index: number, traveller: Traveller)`, `fillTravellers(travellers: Traveller[])`, `removeTraveller(index: number)`, `confirmBooking()`
|
||||
|
||||
### `e2e/pages/dashboard.page.ts` — DashboardPage
|
||||
- locators: `greeting`, `tiles`, `recentRows`, `topRoutes`
|
||||
- methods: `goto()`, `settled()`, `tileValue(label: string)`
|
||||
|
||||
### `e2e/pages/login.page.ts` — LoginPage
|
||||
- locators: `tenantSelect`, `email`, `password`, `submit`, `error`, `demoChips`
|
||||
- methods: `goto()`, `useDemoChip(account: Account)`, `fill(account: Account, password = account.password)`, `signIn(account: Account)`, `signInExpectingFailure(account: Account, password: string)`
|
||||
|
||||
### `e2e/pages/search.page.ts` — FlightCard, SearchPage
|
||||
- locators: `root`, `origin`, `destination`, `date`, `cabin`, `travellers`, `searchButton`, `resetButton`, `sort`, `resultSummary`, `results`, `emptyState`, `nextPage`, `prevPage`
|
||||
- methods: `fare()`, `seatsLeft()`, `flightNumber()`, `durationMinutes()`, `goto()`, `card(index = 0)`, `settled()`, `searchRoute(from: string, to: string)`, `setTravellers(n: number)`, `sortBy(value: 'price' | 'duration' | 'departure')`, `totalResults()`, `cardForFlight(flightNumber: string)`
|
||||
|
||||
### `e2e/pages/shell.page.ts` — ShellPage
|
||||
- locators: `agencyName`, `agencySlug`, `accountName`, `accountMeta`, `signOut`
|
||||
- methods: `goToDashboard()`, `goToSearch()`, `goToBookings()`, `signOutAndWait()`
|
||||
@@ -0,0 +1,311 @@
|
||||
#!/usr/bin/env node
|
||||
/**
|
||||
* QA test-generation helper (Phase 1: the natural-language door).
|
||||
*
|
||||
* The loop it supports:
|
||||
* 1. index build the app-model index — a grounded map of the app (routes,
|
||||
* page objects, API endpoints, areas, accounts) so generation does
|
||||
* not guess selectors or contracts.
|
||||
* 2. prompt assemble a complete, grounded generation prompt for a scenario
|
||||
* card (card + app model + conventions). Paste it into Claude Code
|
||||
* to generate the Playwright test.
|
||||
* 3. run run the test(s) for a card and open the evidence report.
|
||||
*
|
||||
* node scripts/qa-tool.mjs index
|
||||
* node scripts/qa-tool.mjs prompt BOOK-012
|
||||
* node scripts/qa-tool.mjs run BOOK-012
|
||||
* node scripts/qa-tool.mjs list
|
||||
*
|
||||
* Nothing here calls an AI directly — generation stays human-in-the-loop in
|
||||
* Claude Code. This tool prepares the grounded ask and runs the result.
|
||||
*/
|
||||
|
||||
import { existsSync, mkdirSync, readFileSync, readdirSync, writeFileSync } from 'node:fs';
|
||||
import { spawnSync } from 'node:child_process';
|
||||
import { dirname, join, resolve } from 'node:path';
|
||||
import { fileURLToPath, pathToFileURL } from 'node:url';
|
||||
|
||||
export const ROOT = resolve(dirname(fileURLToPath(import.meta.url)), '..');
|
||||
const rel = (p) => join(ROOT, p);
|
||||
const read = (p) => (existsSync(rel(p)) ? readFileSync(rel(p), 'utf8') : '');
|
||||
|
||||
// --- parsers: build the app model from source -------------------------------
|
||||
|
||||
function parseRoutes() {
|
||||
const src = read('frontend/src/app/app.routes.ts');
|
||||
const routes = [];
|
||||
for (const m of src.matchAll(/path:\s*'([^']*)'/g)) {
|
||||
const path = m[1];
|
||||
if (path === '' || path === '**') continue;
|
||||
routes.push('/' + path.replace(/^\/+/, ''));
|
||||
}
|
||||
return [...new Set(routes)];
|
||||
}
|
||||
|
||||
function parseAccounts() {
|
||||
const src = read('e2e/fixtures/accounts.ts');
|
||||
const accounts = [];
|
||||
const re = /export const (\w+):\s*Account\s*=\s*\{([\s\S]*?)\};/g;
|
||||
for (const m of src.matchAll(re)) {
|
||||
const body = m[2];
|
||||
const g = (k) => (body.match(new RegExp(`${k}:\\s*'([^']*)'`)) || [])[1];
|
||||
accounts.push({ name: m[1], email: g('email'), tenant: g('tenant'), role: g('role') });
|
||||
}
|
||||
return accounts;
|
||||
}
|
||||
|
||||
function parseAreas() {
|
||||
const src = read('e2e/reporters/report-model.ts');
|
||||
const areas = [];
|
||||
const re = /'([\w-]+\.spec\.ts)':\s*\{\s*title:\s*'([^']*)'/g;
|
||||
for (const m of src.matchAll(re)) areas.push({ spec: m[1], title: m[2] });
|
||||
return areas;
|
||||
}
|
||||
|
||||
function parseApi() {
|
||||
const src = read('backend/internal/httpapi/router.go');
|
||||
const api = [];
|
||||
// protected.* routes need auth; root.* (health, tenants) are public.
|
||||
for (const m of src.matchAll(/(protected|root)\.HandleFunc\("(\w+)\s+(\/api\/[^"]*)"/g)) {
|
||||
api.push({ method: m[2], path: m[3], auth: m[1] === 'protected' });
|
||||
}
|
||||
return api;
|
||||
}
|
||||
|
||||
function parsePageObjects() {
|
||||
const dir = rel('e2e/pages');
|
||||
if (!existsSync(dir)) return [];
|
||||
const objects = [];
|
||||
for (const file of readdirSync(dir).filter((f) => f.endsWith('.page.ts'))) {
|
||||
const src = readFileSync(join(dir, file), 'utf8');
|
||||
const classes = [...src.matchAll(/export class (\w+)/g)].map((m) => m[1]);
|
||||
const locators = [...src.matchAll(/readonly (\w+):\s*Locator/g)].map((m) => m[1]);
|
||||
// Public methods: skip the constructor and private helpers.
|
||||
const methods = [...src.matchAll(/^\s{2}(?:async\s+)?([a-z]\w*)\s*\(([^)]*)\)/gm)]
|
||||
.map((m) => `${m[1]}(${m[2].trim()})`)
|
||||
.filter((sig) => !sig.startsWith('constructor('));
|
||||
objects.push({ file: `e2e/pages/${file}`, classes, locators, methods });
|
||||
}
|
||||
return objects;
|
||||
}
|
||||
|
||||
export function buildModel() {
|
||||
return {
|
||||
generatedAt: new Date().toISOString(),
|
||||
baseUrl: 'http://localhost:4200',
|
||||
auth: {
|
||||
type: 'basic',
|
||||
tenantHeader: 'X-Tenant',
|
||||
note: 'Credentials verified within the named tenant; take accounts from e2e/fixtures/accounts.ts, never hardcode.',
|
||||
accounts: parseAccounts(),
|
||||
},
|
||||
routes: parseRoutes(),
|
||||
areas: parseAreas(),
|
||||
api: parseApi(),
|
||||
pageObjects: parsePageObjects(),
|
||||
conventions: 'e2e/CLAUDE.md',
|
||||
};
|
||||
}
|
||||
|
||||
// --- index: write app-model.json + app-model.md -----------------------------
|
||||
|
||||
function renderModelMd(m) {
|
||||
const L = [];
|
||||
L.push('# App model — grounding index for test generation', '');
|
||||
L.push('> Generated by `npm run qa:index` from the codebase. Do not edit by hand.', '');
|
||||
L.push(`Base URL: \`${m.baseUrl}\` · Auth: ${m.auth.type} (header \`${m.auth.tenantHeader}\`)`, '');
|
||||
|
||||
L.push('## Accounts', '', '| const | email | tenant | role |', '| --- | --- | --- | --- |');
|
||||
for (const a of m.auth.accounts) L.push(`| \`${a.name}\` | ${a.email} | ${a.tenant} | ${a.role} |`);
|
||||
L.push('', `_Password: see \`DEMO_PASSWORD\` in accounts.ts. ${m.auth.note}_`, '');
|
||||
|
||||
L.push('## Routes', '');
|
||||
for (const r of m.routes) L.push(`- \`${r}\``);
|
||||
L.push('');
|
||||
|
||||
L.push('## Areas (spec file → report area)', '');
|
||||
for (const a of m.areas) L.push(`- \`${a.spec}\` — ${a.title}`);
|
||||
L.push('');
|
||||
|
||||
L.push('## API endpoints', '', '| method | path | auth |', '| --- | --- | --- |');
|
||||
for (const e of m.api) L.push(`| ${e.method} | \`${e.path}\` | ${e.auth ? 'required' : 'public'} |`);
|
||||
L.push('');
|
||||
|
||||
L.push('## Page objects (use these — no raw selectors in specs)', '');
|
||||
for (const po of m.pageObjects) {
|
||||
L.push(`### \`${po.file}\` — ${po.classes.join(', ')}`);
|
||||
if (po.locators.length) L.push(`- locators: ${po.locators.map((x) => `\`${x}\``).join(', ')}`);
|
||||
if (po.methods.length) L.push(`- methods: ${po.methods.map((x) => `\`${x}\``).join(', ')}`);
|
||||
L.push('');
|
||||
}
|
||||
return L.join('\n');
|
||||
}
|
||||
|
||||
export function writeIndex() {
|
||||
const m = buildModel();
|
||||
writeFileSync(rel('qa/app-model.json'), JSON.stringify(m, null, 2), 'utf8');
|
||||
writeFileSync(rel('qa/app-model.md'), renderModelMd(m), 'utf8');
|
||||
return m;
|
||||
}
|
||||
|
||||
function cmdIndex() {
|
||||
const m = writeIndex();
|
||||
console.log('✓ wrote qa/app-model.json and qa/app-model.md');
|
||||
console.log(
|
||||
` ${m.routes.length} routes · ${m.api.length} endpoints · ` +
|
||||
`${m.pageObjects.length} page objects · ${m.auth.accounts.length} accounts`,
|
||||
);
|
||||
}
|
||||
|
||||
// --- scenario cards ---------------------------------------------------------
|
||||
|
||||
function cardPath(id) {
|
||||
const dir = rel('qa/scenarios');
|
||||
const hit = existsSync(dir) && readdirSync(dir).find((f) => f.toUpperCase().startsWith(id.toUpperCase()));
|
||||
return hit ? join(dir, hit) : null;
|
||||
}
|
||||
|
||||
export function readCard(id) {
|
||||
const p = cardPath(id);
|
||||
if (!p) return null;
|
||||
const text = readFileSync(p, 'utf8');
|
||||
const field = (k) => (text.match(new RegExp(`^${k}:\\s*(.+)$`, 'm')) || [])[1]?.trim();
|
||||
return {
|
||||
path: p,
|
||||
file: p.split(/[\\/]/).pop(),
|
||||
text,
|
||||
id: field('id') || id,
|
||||
layer: field('layer') || 'ui',
|
||||
title: field('title') || id,
|
||||
area: field('area') || '',
|
||||
risk: field('risk') || '',
|
||||
};
|
||||
}
|
||||
|
||||
/** All scenario cards as structured objects, sorted by filename. */
|
||||
export function listCards() {
|
||||
const dir = rel('qa/scenarios');
|
||||
const files = existsSync(dir) ? readdirSync(dir).filter((f) => f.endsWith('.yml')).sort() : [];
|
||||
return files.map((f) => readCard(f.replace(/\.yml$/, ''))).filter(Boolean);
|
||||
}
|
||||
|
||||
function cmdList() {
|
||||
const cards = listCards();
|
||||
if (!cards.length) {
|
||||
console.log('No scenario cards yet. Copy qa/scenario-template.yml to qa/scenarios/<ID>.yml');
|
||||
return;
|
||||
}
|
||||
console.log('Scenario cards:');
|
||||
for (const c of cards) console.log(` ${c.file.padEnd(34)} [${c.layer}] ${c.title}`);
|
||||
}
|
||||
|
||||
// --- prompt: assemble the grounded generation ask ---------------------------
|
||||
|
||||
/** Assemble the grounded generation prompt for a card. Pure: returns a string. */
|
||||
export function buildPrompt(id) {
|
||||
if (!existsSync(rel('qa/app-model.md'))) writeIndex();
|
||||
const card = readCard(id);
|
||||
if (!card) throw new Error(`No scenario card for "${id}" in qa/scenarios/.`);
|
||||
const conventions = read('e2e/CLAUDE.md');
|
||||
const appModel = read('qa/app-model.md');
|
||||
|
||||
return [
|
||||
`# Generate the Playwright test for scenario ${id}`,
|
||||
'',
|
||||
'You are generating an end-to-end test for the TravelHub app. Use ONLY the',
|
||||
'page objects, routes, accounts and endpoints in the app model below — do not',
|
||||
'invent selectors or URLs. Follow the conventions exactly.',
|
||||
'',
|
||||
'## Scenario card',
|
||||
'',
|
||||
'```yaml',
|
||||
card.text.trim(),
|
||||
'```',
|
||||
'',
|
||||
'## App model (grounding — what actually exists)',
|
||||
'',
|
||||
appModel.trim(),
|
||||
'',
|
||||
'## Conventions (e2e/CLAUDE.md)',
|
||||
'',
|
||||
conventions.trim(),
|
||||
'',
|
||||
'## Your task',
|
||||
'',
|
||||
`- Add the test to the spec for this card's area; reuse the \`test.describe\` block.`,
|
||||
'- Drive the UI only through the page objects above; add a method if one is missing.',
|
||||
`- For every \`then:\` clause and every \`evidence:\` item, add an \`evidence.step(name, {...})\``,
|
||||
' capturing a screenshot and the observed figures.',
|
||||
'- No raw selectors, no `waitForTimeout`, no hardcoded secrets, no weakened assertions.',
|
||||
'- Clean up any data the test creates.',
|
||||
'',
|
||||
'Then show the diff, run it, and confirm the evidence report shows the required',
|
||||
`screenshots and API request/response:`,
|
||||
'',
|
||||
` node scripts/qa-tool.mjs run ${id}`,
|
||||
'',
|
||||
'If the app behaves differently from the card, stop and report the difference —',
|
||||
'that is a finding, not something to code around.',
|
||||
].join('\n');
|
||||
}
|
||||
|
||||
function cmdPrompt(id) {
|
||||
const card = readCard(id);
|
||||
if (!card) { console.error(`No scenario card for "${id}". Run: node scripts/qa-tool.mjs list`); process.exit(1); }
|
||||
const prompt = buildPrompt(id);
|
||||
mkdirSync(rel('qa/.prompts'), { recursive: true });
|
||||
writeFileSync(rel(`qa/.prompts/${id}.prompt.md`), prompt, 'utf8');
|
||||
console.log(`✓ grounded prompt written to qa/.prompts/${id}.prompt.md`);
|
||||
console.log(' Paste it into Claude Code (or open the file) to generate the test.\n');
|
||||
console.log(prompt);
|
||||
}
|
||||
|
||||
// --- run: execute the card's layer and open evidence ------------------------
|
||||
|
||||
/** Map a card's layer to a run-tests.ps1 layer. */
|
||||
export function layerFor(card) {
|
||||
return ({ backend: 'backend', api: 'api', ui: 'ui', binding: 'binding' })[card.layer] || 'ui';
|
||||
}
|
||||
|
||||
function cmdRun(id, extra) {
|
||||
const card = readCard(id);
|
||||
if (!card) { console.error(`No scenario card for "${id}".`); process.exit(1); }
|
||||
const layer = layerFor(card);
|
||||
|
||||
// Grep on a distinctive slice of the card title so only this scenario runs.
|
||||
const grep = extra.grep || card.title.split(/\s+/).slice(0, 4).join(' ');
|
||||
|
||||
const psArgs = ['-NoProfile', '-ExecutionPolicy', 'Bypass', '-File', 'run-tests.ps1',
|
||||
'-Layer', layer, '-Grep', grep, '-Open'];
|
||||
|
||||
console.log(`Running ${id} [layer: ${layer}] matching "${grep}" ...`);
|
||||
if (process.platform !== 'win32') {
|
||||
console.log(`(non-Windows) run manually:\n pwsh ${psArgs.join(' ')}`);
|
||||
return;
|
||||
}
|
||||
const r = spawnSync('powershell', psArgs, { cwd: ROOT, stdio: 'inherit' });
|
||||
process.exit(r.status ?? 1);
|
||||
}
|
||||
|
||||
// --- dispatch (only when run directly, not when imported) -------------------
|
||||
|
||||
const runDirectly = process.argv[1] && pathToFileURL(process.argv[1]).href === import.meta.url;
|
||||
if (runDirectly) main();
|
||||
|
||||
function main() {
|
||||
const [cmd, arg, ...rest] = process.argv.slice(2);
|
||||
const extra = { grep: (rest.includes('--grep') ? rest[rest.indexOf('--grep') + 1] : null) };
|
||||
|
||||
switch (cmd) {
|
||||
case 'index': cmdIndex(); break;
|
||||
case 'list': cmdList(); break;
|
||||
case 'prompt': if (!arg) { console.error('usage: qa-tool.mjs prompt <CARD-ID>'); process.exit(1); } cmdPrompt(arg); break;
|
||||
case 'run': if (!arg) { console.error('usage: qa-tool.mjs run <CARD-ID> [--grep "..."]'); process.exit(1); } cmdRun(arg, extra); break;
|
||||
default:
|
||||
console.log('TravelHub QA tool — Phase 1 (natural-language door)\n');
|
||||
console.log(' node scripts/qa-tool.mjs index build the app-model grounding index');
|
||||
console.log(' node scripts/qa-tool.mjs list list scenario cards');
|
||||
console.log(' node scripts/qa-tool.mjs prompt <CARD> assemble the grounded generation prompt');
|
||||
console.log(' node scripts/qa-tool.mjs run <CARD> run the card\'s tests + open evidence');
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,231 @@
|
||||
#!/usr/bin/env node
|
||||
/**
|
||||
* QA tool UI — a small local web page over the Phase-1 `qa` commands.
|
||||
*
|
||||
* Pick a scenario card, see the grounded generation prompt (copy it into Claude
|
||||
* Code), run the card's tests, and view the evidence inline. It wraps the exact
|
||||
* same functions the CLI uses (scripts/qa-tool.mjs), so there is one engine.
|
||||
*
|
||||
* node scripts/qa-ui.mjs http://localhost:9500
|
||||
* node scripts/qa-ui.mjs --port 8200 --no-open
|
||||
*
|
||||
* Local dev tool: binds localhost only. The /run endpoint executes the test
|
||||
* runner, so do not expose this port beyond your machine.
|
||||
*/
|
||||
import { createServer } from 'node:http';
|
||||
import { createReadStream, existsSync, statSync } from 'node:fs';
|
||||
import { spawn } from 'node:child_process';
|
||||
import { extname, join, normalize } from 'node:path';
|
||||
import { pathToFileURL } from 'node:url';
|
||||
|
||||
import { ROOT, buildPrompt, layerFor, listCards, readCard, writeIndex } from './qa-tool.mjs';
|
||||
|
||||
const args = process.argv.slice(2);
|
||||
const PORT = args.includes('--port') ? Number(args[args.indexOf('--port') + 1]) : 9500;
|
||||
const shouldOpen = !args.includes('--no-open');
|
||||
|
||||
const EVIDENCE_DIR = join(ROOT, 'e2e', 'evidence-report');
|
||||
const TYPES = {
|
||||
'.html': 'text/html; charset=utf-8', '.css': 'text/css', '.js': 'text/javascript',
|
||||
'.json': 'application/json', '.png': 'image/png', '.md': 'text/markdown; charset=utf-8',
|
||||
};
|
||||
|
||||
function sendJson(res, code, body) {
|
||||
const s = JSON.stringify(body);
|
||||
res.writeHead(code, { 'Content-Type': 'application/json; charset=utf-8', 'Content-Length': Buffer.byteLength(s) });
|
||||
res.end(s);
|
||||
}
|
||||
|
||||
// Run run-tests.ps1 for a card, buffering output; resolve when it exits.
|
||||
function runCard(id) {
|
||||
const card = readCard(id);
|
||||
if (!card) return Promise.resolve({ ok: false, output: `No card "${id}".` });
|
||||
const layer = layerFor(card);
|
||||
const grep = card.title.split(/\s+/).slice(0, 4).join(' ');
|
||||
|
||||
return new Promise((resolve) => {
|
||||
const ps = spawn(
|
||||
'powershell',
|
||||
['-NoProfile', '-ExecutionPolicy', 'Bypass', '-File', 'run-tests.ps1',
|
||||
'-Layer', layer, '-Grep', grep],
|
||||
{ cwd: ROOT },
|
||||
);
|
||||
let out = '';
|
||||
ps.stdout.on('data', (d) => (out += d));
|
||||
ps.stderr.on('data', (d) => (out += d));
|
||||
ps.on('close', (code) => resolve({ ok: code === 0, exitCode: code, layer, grep, output: out }));
|
||||
ps.on('error', (e) => resolve({ ok: false, output: String(e) }));
|
||||
});
|
||||
}
|
||||
|
||||
const server = createServer(async (req, res) => {
|
||||
const url = new URL(req.url, `http://localhost:${PORT}`);
|
||||
const path = url.pathname;
|
||||
|
||||
try {
|
||||
if (path === '/') { res.writeHead(200, { 'Content-Type': 'text/html; charset=utf-8' }); return res.end(PAGE); }
|
||||
|
||||
if (path === '/qa/cards') return sendJson(res, 200, listCards());
|
||||
|
||||
if (path === '/qa/index') { const m = writeIndex(); return sendJson(res, 200, { ok: true, counts: {
|
||||
routes: m.routes.length, api: m.api.length, pageObjects: m.pageObjects.length, accounts: m.auth.accounts.length } }); }
|
||||
|
||||
const pm = path.match(/^\/qa\/prompt\/([\w-]+)$/);
|
||||
if (pm) {
|
||||
try { return sendJson(res, 200, { prompt: buildPrompt(pm[1]) }); }
|
||||
catch (e) { return sendJson(res, 404, { error: String(e.message || e) }); }
|
||||
}
|
||||
|
||||
const rm = path.match(/^\/qa\/run\/([\w-]+)$/);
|
||||
if (rm && req.method === 'POST') return sendJson(res, 200, await runCard(rm[1]));
|
||||
|
||||
// Serve the evidence report for the inline iframe.
|
||||
if (path.startsWith('/evidence/')) {
|
||||
const target = join(EVIDENCE_DIR, normalize(path.slice('/evidence/'.length)) || 'index.html');
|
||||
if (!target.startsWith(EVIDENCE_DIR) || !existsSync(target) || statSync(target).isDirectory()) {
|
||||
res.writeHead(404); return res.end('not found');
|
||||
}
|
||||
res.writeHead(200, { 'Content-Type': TYPES[extname(target).toLowerCase()] || 'application/octet-stream' });
|
||||
return createReadStream(target).pipe(res);
|
||||
}
|
||||
|
||||
res.writeHead(404); res.end('not found');
|
||||
} catch (e) {
|
||||
sendJson(res, 500, { error: String(e.message || e) });
|
||||
}
|
||||
});
|
||||
|
||||
server.on('error', (e) => {
|
||||
if (e.code === 'EADDRINUSE') { console.error(`Port ${PORT} in use. Try: node scripts/qa-ui.mjs --port ${PORT + 1}`); process.exit(1); }
|
||||
throw e;
|
||||
});
|
||||
|
||||
server.listen(PORT, () => {
|
||||
const url = `http://localhost:${PORT}/`;
|
||||
console.log(`\n QA tool UI: ${url}\n Ctrl+C to stop.\n`);
|
||||
if (shouldOpen) {
|
||||
const [c, a] = process.platform === 'win32' ? ['cmd', ['/c', 'start', '', url]]
|
||||
: process.platform === 'darwin' ? ['open', [url]] : ['xdg-open', [url]];
|
||||
spawn(c, a, { detached: true, stdio: 'ignore' }).unref();
|
||||
}
|
||||
});
|
||||
|
||||
// --- the page (self-contained, no external resources) -----------------------
|
||||
|
||||
const PAGE = `<!doctype html>
|
||||
<html lang="en"><head><meta charset="utf-8"/><meta name="viewport" content="width=device-width,initial-scale=1"/>
|
||||
<title>TravelHub QA tool</title>
|
||||
<style>
|
||||
:root{--ink:#111827;--muted:#6b7280;--faint:#9ca3af;--line:#e5e7eb;--bg:#f3f4f6;--card:#fff;
|
||||
--accent:#0b5cd6;--ok:#067a6e;--ok-bg:#e6f4f1;--fail:#b3243f;--fail-bg:#fdeaee;}
|
||||
*{box-sizing:border-box}
|
||||
body{margin:0;font:14px/1.55 'Inter',-apple-system,'Segoe UI',Roboto,sans-serif;color:var(--ink);background:var(--bg)}
|
||||
header{background:#0b1727;color:#fff;padding:14px 20px;display:flex;align-items:center;gap:14px}
|
||||
header h1{font-size:1.05rem;margin:0}
|
||||
header .sub{color:#9fb3cd;font-size:.8rem}
|
||||
header button{margin-left:auto}
|
||||
.wrap{display:grid;grid-template-columns:300px 1fr;gap:0;height:calc(100vh - 50px)}
|
||||
.list{border-right:1px solid var(--line);overflow:auto;background:var(--card)}
|
||||
.card{padding:12px 16px;border-bottom:1px solid var(--line);cursor:pointer}
|
||||
.card:hover{background:#f8fafc}
|
||||
.card.active{background:var(--ok-bg);border-left:3px solid var(--ok)}
|
||||
.card .id{font-weight:650;font-size:.82rem}
|
||||
.card .title{color:var(--muted);font-size:.82rem}
|
||||
.card .tag{display:inline-block;font-size:.64rem;font-weight:700;text-transform:uppercase;letter-spacing:.04em;
|
||||
padding:1px 6px;border-radius:4px;background:#eef2f7;color:var(--muted);margin-top:4px}
|
||||
.main{overflow:auto;padding:20px 24px}
|
||||
.empty{color:var(--faint);padding:40px;text-align:center}
|
||||
h2{font-size:1.1rem;margin:0 0 4px}
|
||||
.meta{color:var(--muted);font-size:.82rem;margin-bottom:14px}
|
||||
.panel{background:var(--card);border:1px solid var(--line);border-radius:8px;margin-bottom:16px}
|
||||
.panel h3{font-size:.72rem;text-transform:uppercase;letter-spacing:.06em;color:var(--faint);
|
||||
margin:0;padding:11px 14px;border-bottom:1px solid var(--line);display:flex;align-items:center;gap:10px}
|
||||
.panel .body{padding:0}
|
||||
pre{margin:0;padding:14px;background:#0f172a;color:#e2e8f0;border-radius:0 0 8px 8px;overflow:auto;
|
||||
font:12px/1.5 ui-monospace,Menlo,monospace;max-height:360px;white-space:pre-wrap}
|
||||
pre.card-yaml{background:#f8fafc;color:var(--ink);max-height:none}
|
||||
button{font:inherit;font-size:.82rem;border:1px solid #cfd9e8;background:#fff;color:var(--ink);
|
||||
border-radius:6px;padding:6px 12px;cursor:pointer}
|
||||
button:hover{border-color:var(--accent);color:var(--accent)}
|
||||
button.primary{background:var(--accent);border-color:var(--accent);color:#fff}
|
||||
button.primary:hover{background:#0a4fb8;color:#fff}
|
||||
button:disabled{opacity:.5;cursor:not-allowed}
|
||||
.copy{margin-left:auto}
|
||||
.run-status{font-weight:650;font-size:.8rem}
|
||||
.run-status.ok{color:var(--ok)} .run-status.fail{color:var(--fail)}
|
||||
iframe{width:100%;height:520px;border:1px solid var(--line);border-radius:8px;background:#fff}
|
||||
.hint{color:var(--faint);font-size:.8rem;padding:8px 14px}
|
||||
</style></head>
|
||||
<body>
|
||||
<header>
|
||||
<h1>TravelHub QA tool</h1>
|
||||
<span class="sub">scenario → grounded prompt → run → evidence</span>
|
||||
<button id="reindex">Refresh app model</button>
|
||||
</header>
|
||||
<div class="wrap">
|
||||
<div class="list" id="list"></div>
|
||||
<div class="main" id="main"><div class="empty">Select a scenario card on the left.</div></div>
|
||||
</div>
|
||||
<script>
|
||||
let cards=[], current=null;
|
||||
|
||||
async function load(){
|
||||
cards = await (await fetch('/qa/cards')).json();
|
||||
const el=document.getElementById('list');
|
||||
if(!cards.length){ el.innerHTML='<div class="empty">No scenario cards.<br>Add one in qa/scenarios/.</div>'; return; }
|
||||
el.innerHTML = cards.map(c=>
|
||||
'<div class="card" data-id="'+c.id+'"><div class="id">'+c.id+'</div>'+
|
||||
'<div class="title">'+esc(c.title)+'</div><span class="tag">'+c.layer+'</span></div>').join('');
|
||||
el.querySelectorAll('.card').forEach(d=>d.onclick=()=>select(d.dataset.id));
|
||||
}
|
||||
|
||||
function esc(s){return String(s).replace(/[&<>"]/g,m=>({'&':'&','<':'<','>':'>','"':'"'}[m]));}
|
||||
|
||||
async function select(id){
|
||||
current=id;
|
||||
document.querySelectorAll('.card').forEach(d=>d.classList.toggle('active',d.dataset.id===id));
|
||||
const c=cards.find(x=>x.id===id);
|
||||
const main=document.getElementById('main');
|
||||
main.innerHTML=
|
||||
'<h2>'+esc(c.id)+' — '+esc(c.title)+'</h2>'+
|
||||
'<div class="meta">layer <b>'+c.layer+'</b> · area '+esc(c.area||'—')+' · risk '+esc(c.risk||'—')+'</div>'+
|
||||
'<div class="panel"><h3>Scenario card</h3><div class="body"><pre class="card-yaml">'+esc(c.text)+'</pre></div></div>'+
|
||||
'<div class="panel"><h3>Grounded prompt'+
|
||||
'<button class="copy" id="copyBtn">Copy</button></h3>'+
|
||||
'<div class="hint">Paste into Claude Code to generate the Playwright test (grounded in the real app model).</div>'+
|
||||
'<div class="body"><pre id="prompt">loading…</pre></div></div>'+
|
||||
'<div class="panel"><h3>Run'+
|
||||
'<button class="primary" id="runBtn" style="margin-left:auto">Run tests</button>'+
|
||||
'<span class="run-status" id="runStatus"></span></h3>'+
|
||||
'<div class="body"><pre id="runOut" style="max-height:220px">Not run yet.</pre></div></div>'+
|
||||
'<div class="panel"><h3>Evidence report</h3><div class="body" style="padding:10px">'+
|
||||
'<iframe id="evidence" src="/evidence/index.html?t='+Date.now()+'"></iframe></div></div>';
|
||||
|
||||
const pr=await (await fetch('/qa/prompt/'+id)).json();
|
||||
document.getElementById('prompt').textContent = pr.prompt || pr.error || '(none)';
|
||||
document.getElementById('copyBtn').onclick=()=>navigator.clipboard.writeText(document.getElementById('prompt').textContent);
|
||||
document.getElementById('runBtn').onclick=()=>run(id);
|
||||
}
|
||||
|
||||
async function run(id){
|
||||
const btn=document.getElementById('runBtn'), st=document.getElementById('runStatus'), out=document.getElementById('runOut');
|
||||
btn.disabled=true; st.textContent='running…'; st.className='run-status'; out.textContent='Running — this can take ~10–30s…';
|
||||
try{
|
||||
const r=await (await fetch('/qa/run/'+id,{method:'POST'})).json();
|
||||
st.textContent = r.ok?'passed':'failed'; st.className='run-status '+(r.ok?'ok':'fail');
|
||||
out.textContent = r.output || '(no output)';
|
||||
document.getElementById('evidence').src='/evidence/index.html?t='+Date.now();
|
||||
}catch(e){ st.textContent='error'; st.className='run-status fail'; out.textContent=String(e); }
|
||||
btn.disabled=false;
|
||||
}
|
||||
|
||||
document.getElementById('reindex').onclick=async()=>{
|
||||
const r=await (await fetch('/qa/index')).json();
|
||||
alert('App model rebuilt: '+r.counts.routes+' routes, '+r.counts.api+' endpoints, '+
|
||||
r.counts.pageObjects+' page objects, '+r.counts.accounts+' accounts.');
|
||||
if(current) select(current);
|
||||
};
|
||||
|
||||
load();
|
||||
</script>
|
||||
</body></html>`;
|
||||
Reference in New Issue
Block a user