new changes
This commit is contained in:
nilesh.p
@@ -0,0 +1,311 @@
|
||||
#!/usr/bin/env node
|
||||
/**
|
||||
* QA test-generation helper (Phase 1: the natural-language door).
|
||||
*
|
||||
* The loop it supports:
|
||||
* 1. index build the app-model index — a grounded map of the app (routes,
|
||||
* page objects, API endpoints, areas, accounts) so generation does
|
||||
* not guess selectors or contracts.
|
||||
* 2. prompt assemble a complete, grounded generation prompt for a scenario
|
||||
* card (card + app model + conventions). Paste it into Claude Code
|
||||
* to generate the Playwright test.
|
||||
* 3. run run the test(s) for a card and open the evidence report.
|
||||
*
|
||||
* node scripts/qa-tool.mjs index
|
||||
* node scripts/qa-tool.mjs prompt BOOK-012
|
||||
* node scripts/qa-tool.mjs run BOOK-012
|
||||
* node scripts/qa-tool.mjs list
|
||||
*
|
||||
* Nothing here calls an AI directly — generation stays human-in-the-loop in
|
||||
* Claude Code. This tool prepares the grounded ask and runs the result.
|
||||
*/
|
||||
|
||||
import { existsSync, mkdirSync, readFileSync, readdirSync, writeFileSync } from 'node:fs';
|
||||
import { spawnSync } from 'node:child_process';
|
||||
import { dirname, join, resolve } from 'node:path';
|
||||
import { fileURLToPath, pathToFileURL } from 'node:url';
|
||||
|
||||
export const ROOT = resolve(dirname(fileURLToPath(import.meta.url)), '..');
|
||||
const rel = (p) => join(ROOT, p);
|
||||
const read = (p) => (existsSync(rel(p)) ? readFileSync(rel(p), 'utf8') : '');
|
||||
|
||||
// --- parsers: build the app model from source -------------------------------
|
||||
|
||||
function parseRoutes() {
|
||||
const src = read('frontend/src/app/app.routes.ts');
|
||||
const routes = [];
|
||||
for (const m of src.matchAll(/path:\s*'([^']*)'/g)) {
|
||||
const path = m[1];
|
||||
if (path === '' || path === '**') continue;
|
||||
routes.push('/' + path.replace(/^\/+/, ''));
|
||||
}
|
||||
return [...new Set(routes)];
|
||||
}
|
||||
|
||||
function parseAccounts() {
|
||||
const src = read('e2e/fixtures/accounts.ts');
|
||||
const accounts = [];
|
||||
const re = /export const (\w+):\s*Account\s*=\s*\{([\s\S]*?)\};/g;
|
||||
for (const m of src.matchAll(re)) {
|
||||
const body = m[2];
|
||||
const g = (k) => (body.match(new RegExp(`${k}:\\s*'([^']*)'`)) || [])[1];
|
||||
accounts.push({ name: m[1], email: g('email'), tenant: g('tenant'), role: g('role') });
|
||||
}
|
||||
return accounts;
|
||||
}
|
||||
|
||||
function parseAreas() {
|
||||
const src = read('e2e/reporters/report-model.ts');
|
||||
const areas = [];
|
||||
const re = /'([\w-]+\.spec\.ts)':\s*\{\s*title:\s*'([^']*)'/g;
|
||||
for (const m of src.matchAll(re)) areas.push({ spec: m[1], title: m[2] });
|
||||
return areas;
|
||||
}
|
||||
|
||||
function parseApi() {
|
||||
const src = read('backend/internal/httpapi/router.go');
|
||||
const api = [];
|
||||
// protected.* routes need auth; root.* (health, tenants) are public.
|
||||
for (const m of src.matchAll(/(protected|root)\.HandleFunc\("(\w+)\s+(\/api\/[^"]*)"/g)) {
|
||||
api.push({ method: m[2], path: m[3], auth: m[1] === 'protected' });
|
||||
}
|
||||
return api;
|
||||
}
|
||||
|
||||
function parsePageObjects() {
|
||||
const dir = rel('e2e/pages');
|
||||
if (!existsSync(dir)) return [];
|
||||
const objects = [];
|
||||
for (const file of readdirSync(dir).filter((f) => f.endsWith('.page.ts'))) {
|
||||
const src = readFileSync(join(dir, file), 'utf8');
|
||||
const classes = [...src.matchAll(/export class (\w+)/g)].map((m) => m[1]);
|
||||
const locators = [...src.matchAll(/readonly (\w+):\s*Locator/g)].map((m) => m[1]);
|
||||
// Public methods: skip the constructor and private helpers.
|
||||
const methods = [...src.matchAll(/^\s{2}(?:async\s+)?([a-z]\w*)\s*\(([^)]*)\)/gm)]
|
||||
.map((m) => `${m[1]}(${m[2].trim()})`)
|
||||
.filter((sig) => !sig.startsWith('constructor('));
|
||||
objects.push({ file: `e2e/pages/${file}`, classes, locators, methods });
|
||||
}
|
||||
return objects;
|
||||
}
|
||||
|
||||
export function buildModel() {
|
||||
return {
|
||||
generatedAt: new Date().toISOString(),
|
||||
baseUrl: 'http://localhost:4200',
|
||||
auth: {
|
||||
type: 'basic',
|
||||
tenantHeader: 'X-Tenant',
|
||||
note: 'Credentials verified within the named tenant; take accounts from e2e/fixtures/accounts.ts, never hardcode.',
|
||||
accounts: parseAccounts(),
|
||||
},
|
||||
routes: parseRoutes(),
|
||||
areas: parseAreas(),
|
||||
api: parseApi(),
|
||||
pageObjects: parsePageObjects(),
|
||||
conventions: 'e2e/CLAUDE.md',
|
||||
};
|
||||
}
|
||||
|
||||
// --- index: write app-model.json + app-model.md -----------------------------
|
||||
|
||||
function renderModelMd(m) {
|
||||
const L = [];
|
||||
L.push('# App model — grounding index for test generation', '');
|
||||
L.push('> Generated by `npm run qa:index` from the codebase. Do not edit by hand.', '');
|
||||
L.push(`Base URL: \`${m.baseUrl}\` · Auth: ${m.auth.type} (header \`${m.auth.tenantHeader}\`)`, '');
|
||||
|
||||
L.push('## Accounts', '', '| const | email | tenant | role |', '| --- | --- | --- | --- |');
|
||||
for (const a of m.auth.accounts) L.push(`| \`${a.name}\` | ${a.email} | ${a.tenant} | ${a.role} |`);
|
||||
L.push('', `_Password: see \`DEMO_PASSWORD\` in accounts.ts. ${m.auth.note}_`, '');
|
||||
|
||||
L.push('## Routes', '');
|
||||
for (const r of m.routes) L.push(`- \`${r}\``);
|
||||
L.push('');
|
||||
|
||||
L.push('## Areas (spec file → report area)', '');
|
||||
for (const a of m.areas) L.push(`- \`${a.spec}\` — ${a.title}`);
|
||||
L.push('');
|
||||
|
||||
L.push('## API endpoints', '', '| method | path | auth |', '| --- | --- | --- |');
|
||||
for (const e of m.api) L.push(`| ${e.method} | \`${e.path}\` | ${e.auth ? 'required' : 'public'} |`);
|
||||
L.push('');
|
||||
|
||||
L.push('## Page objects (use these — no raw selectors in specs)', '');
|
||||
for (const po of m.pageObjects) {
|
||||
L.push(`### \`${po.file}\` — ${po.classes.join(', ')}`);
|
||||
if (po.locators.length) L.push(`- locators: ${po.locators.map((x) => `\`${x}\``).join(', ')}`);
|
||||
if (po.methods.length) L.push(`- methods: ${po.methods.map((x) => `\`${x}\``).join(', ')}`);
|
||||
L.push('');
|
||||
}
|
||||
return L.join('\n');
|
||||
}
|
||||
|
||||
export function writeIndex() {
|
||||
const m = buildModel();
|
||||
writeFileSync(rel('qa/app-model.json'), JSON.stringify(m, null, 2), 'utf8');
|
||||
writeFileSync(rel('qa/app-model.md'), renderModelMd(m), 'utf8');
|
||||
return m;
|
||||
}
|
||||
|
||||
function cmdIndex() {
|
||||
const m = writeIndex();
|
||||
console.log('✓ wrote qa/app-model.json and qa/app-model.md');
|
||||
console.log(
|
||||
` ${m.routes.length} routes · ${m.api.length} endpoints · ` +
|
||||
`${m.pageObjects.length} page objects · ${m.auth.accounts.length} accounts`,
|
||||
);
|
||||
}
|
||||
|
||||
// --- scenario cards ---------------------------------------------------------
|
||||
|
||||
function cardPath(id) {
|
||||
const dir = rel('qa/scenarios');
|
||||
const hit = existsSync(dir) && readdirSync(dir).find((f) => f.toUpperCase().startsWith(id.toUpperCase()));
|
||||
return hit ? join(dir, hit) : null;
|
||||
}
|
||||
|
||||
export function readCard(id) {
|
||||
const p = cardPath(id);
|
||||
if (!p) return null;
|
||||
const text = readFileSync(p, 'utf8');
|
||||
const field = (k) => (text.match(new RegExp(`^${k}:\\s*(.+)$`, 'm')) || [])[1]?.trim();
|
||||
return {
|
||||
path: p,
|
||||
file: p.split(/[\\/]/).pop(),
|
||||
text,
|
||||
id: field('id') || id,
|
||||
layer: field('layer') || 'ui',
|
||||
title: field('title') || id,
|
||||
area: field('area') || '',
|
||||
risk: field('risk') || '',
|
||||
};
|
||||
}
|
||||
|
||||
/** All scenario cards as structured objects, sorted by filename. */
|
||||
export function listCards() {
|
||||
const dir = rel('qa/scenarios');
|
||||
const files = existsSync(dir) ? readdirSync(dir).filter((f) => f.endsWith('.yml')).sort() : [];
|
||||
return files.map((f) => readCard(f.replace(/\.yml$/, ''))).filter(Boolean);
|
||||
}
|
||||
|
||||
function cmdList() {
|
||||
const cards = listCards();
|
||||
if (!cards.length) {
|
||||
console.log('No scenario cards yet. Copy qa/scenario-template.yml to qa/scenarios/<ID>.yml');
|
||||
return;
|
||||
}
|
||||
console.log('Scenario cards:');
|
||||
for (const c of cards) console.log(` ${c.file.padEnd(34)} [${c.layer}] ${c.title}`);
|
||||
}
|
||||
|
||||
// --- prompt: assemble the grounded generation ask ---------------------------
|
||||
|
||||
/** Assemble the grounded generation prompt for a card. Pure: returns a string. */
|
||||
export function buildPrompt(id) {
|
||||
if (!existsSync(rel('qa/app-model.md'))) writeIndex();
|
||||
const card = readCard(id);
|
||||
if (!card) throw new Error(`No scenario card for "${id}" in qa/scenarios/.`);
|
||||
const conventions = read('e2e/CLAUDE.md');
|
||||
const appModel = read('qa/app-model.md');
|
||||
|
||||
return [
|
||||
`# Generate the Playwright test for scenario ${id}`,
|
||||
'',
|
||||
'You are generating an end-to-end test for the TravelHub app. Use ONLY the',
|
||||
'page objects, routes, accounts and endpoints in the app model below — do not',
|
||||
'invent selectors or URLs. Follow the conventions exactly.',
|
||||
'',
|
||||
'## Scenario card',
|
||||
'',
|
||||
'```yaml',
|
||||
card.text.trim(),
|
||||
'```',
|
||||
'',
|
||||
'## App model (grounding — what actually exists)',
|
||||
'',
|
||||
appModel.trim(),
|
||||
'',
|
||||
'## Conventions (e2e/CLAUDE.md)',
|
||||
'',
|
||||
conventions.trim(),
|
||||
'',
|
||||
'## Your task',
|
||||
'',
|
||||
`- Add the test to the spec for this card's area; reuse the \`test.describe\` block.`,
|
||||
'- Drive the UI only through the page objects above; add a method if one is missing.',
|
||||
`- For every \`then:\` clause and every \`evidence:\` item, add an \`evidence.step(name, {...})\``,
|
||||
' capturing a screenshot and the observed figures.',
|
||||
'- No raw selectors, no `waitForTimeout`, no hardcoded secrets, no weakened assertions.',
|
||||
'- Clean up any data the test creates.',
|
||||
'',
|
||||
'Then show the diff, run it, and confirm the evidence report shows the required',
|
||||
`screenshots and API request/response:`,
|
||||
'',
|
||||
` node scripts/qa-tool.mjs run ${id}`,
|
||||
'',
|
||||
'If the app behaves differently from the card, stop and report the difference —',
|
||||
'that is a finding, not something to code around.',
|
||||
].join('\n');
|
||||
}
|
||||
|
||||
function cmdPrompt(id) {
|
||||
const card = readCard(id);
|
||||
if (!card) { console.error(`No scenario card for "${id}". Run: node scripts/qa-tool.mjs list`); process.exit(1); }
|
||||
const prompt = buildPrompt(id);
|
||||
mkdirSync(rel('qa/.prompts'), { recursive: true });
|
||||
writeFileSync(rel(`qa/.prompts/${id}.prompt.md`), prompt, 'utf8');
|
||||
console.log(`✓ grounded prompt written to qa/.prompts/${id}.prompt.md`);
|
||||
console.log(' Paste it into Claude Code (or open the file) to generate the test.\n');
|
||||
console.log(prompt);
|
||||
}
|
||||
|
||||
// --- run: execute the card's layer and open evidence ------------------------
|
||||
|
||||
/** Map a card's layer to a run-tests.ps1 layer. */
|
||||
export function layerFor(card) {
|
||||
return ({ backend: 'backend', api: 'api', ui: 'ui', binding: 'binding' })[card.layer] || 'ui';
|
||||
}
|
||||
|
||||
function cmdRun(id, extra) {
|
||||
const card = readCard(id);
|
||||
if (!card) { console.error(`No scenario card for "${id}".`); process.exit(1); }
|
||||
const layer = layerFor(card);
|
||||
|
||||
// Grep on a distinctive slice of the card title so only this scenario runs.
|
||||
const grep = extra.grep || card.title.split(/\s+/).slice(0, 4).join(' ');
|
||||
|
||||
const psArgs = ['-NoProfile', '-ExecutionPolicy', 'Bypass', '-File', 'run-tests.ps1',
|
||||
'-Layer', layer, '-Grep', grep, '-Open'];
|
||||
|
||||
console.log(`Running ${id} [layer: ${layer}] matching "${grep}" ...`);
|
||||
if (process.platform !== 'win32') {
|
||||
console.log(`(non-Windows) run manually:\n pwsh ${psArgs.join(' ')}`);
|
||||
return;
|
||||
}
|
||||
const r = spawnSync('powershell', psArgs, { cwd: ROOT, stdio: 'inherit' });
|
||||
process.exit(r.status ?? 1);
|
||||
}
|
||||
|
||||
// --- dispatch (only when run directly, not when imported) -------------------
|
||||
|
||||
const runDirectly = process.argv[1] && pathToFileURL(process.argv[1]).href === import.meta.url;
|
||||
if (runDirectly) main();
|
||||
|
||||
function main() {
|
||||
const [cmd, arg, ...rest] = process.argv.slice(2);
|
||||
const extra = { grep: (rest.includes('--grep') ? rest[rest.indexOf('--grep') + 1] : null) };
|
||||
|
||||
switch (cmd) {
|
||||
case 'index': cmdIndex(); break;
|
||||
case 'list': cmdList(); break;
|
||||
case 'prompt': if (!arg) { console.error('usage: qa-tool.mjs prompt <CARD-ID>'); process.exit(1); } cmdPrompt(arg); break;
|
||||
case 'run': if (!arg) { console.error('usage: qa-tool.mjs run <CARD-ID> [--grep "..."]'); process.exit(1); } cmdRun(arg, extra); break;
|
||||
default:
|
||||
console.log('TravelHub QA tool — Phase 1 (natural-language door)\n');
|
||||
console.log(' node scripts/qa-tool.mjs index build the app-model grounding index');
|
||||
console.log(' node scripts/qa-tool.mjs list list scenario cards');
|
||||
console.log(' node scripts/qa-tool.mjs prompt <CARD> assemble the grounded generation prompt');
|
||||
console.log(' node scripts/qa-tool.mjs run <CARD> run the card\'s tests + open evidence');
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user