TS_Proto/e2e/reporters/evidence.ts

/**
 * Builds a run report from the evidence captured by passing tests.
 *
 * Playwright's own HTML report is a debugging tool: it shows what went wrong.
 * This one is the opposite -- a record of what the suite proved, with the
 * screenshots and the data each scenario observed, readable by somebody who
 * will never open a terminal.
 *
 * Output lands in `e2e/evidence-report/`:
 *   index.html   the report
 *   report.json  the same content, machine readable
 *   report.md    a text summary for pasting into a ticket or a PR
 *   shots/       the screenshots
 */
import { copyFileSync, mkdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
import { join, relative, resolve, sep } from 'node:path';
import { pathToFileURL } from 'node:url';

import type { FullConfig, FullResult, Reporter, Suite, TestCase, TestResult } from '@playwright/test/reporter';

import { DATA_PREFIX, SHOT_PREFIX } from '../fixtures/evidence';
import { renderHtml } from './render-html';
import {
  ApiCall,
  RunMeta,
  Scenario,
  Step,
  Summary,
  areaFor,
  formatValue,
  friendly,
  group,
  label,
  sentenceCase,
} from './report-model';

const OUT_DIR = resolve(__dirname, '..', 'evidence-report');
const SHOT_DIR = join(OUT_DIR, 'shots');

export default class EvidenceReporter implements Reporter {
  private readonly scenarios: Scenario[] = [];
  private startedAt = Date.now();
  private rootDir = '';
  private meta: RunMeta = { baseURL: 'unknown', projects: [], workers: 1 };

  onBegin(config: FullConfig, _suite: Suite): void {
    this.startedAt = Date.now();
    this.rootDir = config.rootDir;

    this.meta = {
      baseURL: String(config.projects[0]?.use?.baseURL ?? 'unknown'),
      projects: config.projects.map((project) => {
        const viewport = project.use?.viewport;
        const size = viewport ? `${viewport.width} x ${viewport.height}` : 'default size';
        return `${project.name} (${size})`;
      }),
      workers: config.workers,
    };

    rmSync(OUT_DIR, { recursive: true, force: true });
    mkdirSync(SHOT_DIR, { recursive: true });
  }

  onTestEnd(test: TestCase, result: TestResult): void {
    const path = test.titlePath().filter(Boolean);
    // titlePath is [project, file, ...describes, title]
    const project = path[0] ?? 'default';
    const file = relative(this.rootDir, test.location.file).split(sep).join('/');
    const title = path[path.length - 1] ?? test.title;
    const suite = path.slice(2, -1).join(' › ');

    const steps = this.collectSteps(test, result);
    const apiCalls = this.collectApiCalls(result);

    this.scenarios.push({
      project,
      file,
      suite,
      title,
      status: result.status,
      expected: result.status === test.expectedStatus,
      durationMs: result.duration,
      steps,
      apiCalls,
      error: result.error?.message?.split('\n').slice(0, 4).join('\n'),
    });
  }

  /** Reads the evidence-http attachment the apiTraffic fixture produced. */
  private collectApiCalls(result: TestResult): ApiCall[] {
    const att = result.attachments.find((a) => a.name === 'evidence-http');
    if (!att) {
      return [];
    }
    const raw = att.body ? att.body.toString('utf8') : att.path ? safeRead(att.path) : '';
    if (!raw) {
      return [];
    }
    try {
      return JSON.parse(raw) as ApiCall[];
    } catch {
      return [];
    }
  }

  /** Pairs `evidence:NN:name` screenshots with `evidence-data:NN:name` blobs. */
  private collectSteps(test: TestCase, result: TestResult): Step[] {
    const bySeq = new Map<string, Step>();

    for (const attachment of result.attachments) {
      const match = /^(evidence|evidence-data):(\d+):(.*)$/.exec(attachment.name);
      if (!match) {
        continue;
      }
      const [, kind, seq, name] = match;

      let step = bySeq.get(seq);
      if (!step) {
        step = { seq, name };
        bySeq.set(seq, step);
      }

      if (kind === SHOT_PREFIX) {
        // An attachment made with `body` carries the bytes in memory; one made
        // with `path` points at a file on disk. Both shapes occur.
        const safe = `${slug(test.id)}-${seq}.png`;
        try {
          if (attachment.body) {
            writeFileSync(join(SHOT_DIR, safe), attachment.body);
            step.shot = `shots/${safe}`;
          } else if (attachment.path) {
            copyFileSync(attachment.path, join(SHOT_DIR, safe));
            step.shot = `shots/${safe}`;
          }
        } catch {
          // A screenshot that could not be saved should not sink the report.
        }
      }

      if (kind === DATA_PREFIX) {
        const raw = attachment.body
          ? attachment.body.toString('utf8')
          : attachment.path
            ? safeRead(attachment.path)
            : '';
        if (raw) {
          try {
            step.facts = JSON.parse(raw) as Record<string, unknown>;
          } catch {
            step.facts = { raw };
          }
        }
      }
    }

    const steps = [...bySeq.values()].sort((a, b) => a.seq.localeCompare(b.seq));
    if (steps.length > 0) {
      return steps;
    }

    // Not every test is instrumented. Fall back to Playwright's own
    // end-of-test screenshot so the report still shows what the run looked
    // like, just without the narrated data.
    const auto = result.attachments.find(
      (a) => a.name === 'screenshot' && a.contentType === 'image/png',
    );
    if (!auto) {
      return [];
    }

    const safe = `${slug(test.id)}-final.png`;
    try {
      if (auto.body) {
        writeFileSync(join(SHOT_DIR, safe), auto.body);
      } else if (auto.path) {
        copyFileSync(auto.path, join(SHOT_DIR, safe));
      } else {
        return [];
      }
    } catch {
      return [];
    }

    return [{ seq: '01', name: 'Final state', shot: `shots/${safe}` }];
  }

  async onEnd(result: FullResult): Promise<void> {
    const durationMs = Date.now() - this.startedAt;

    const summary = {
      generatedAt: new Date().toISOString(),
      outcome: result.status,
      durationMs,
      total: this.scenarios.length,
      passed: this.scenarios.filter((s) => s.status === 'passed').length,
      failed: this.scenarios.filter((s) => s.status === 'failed' || s.status === 'timedOut').length,
      skipped: this.scenarios.filter((s) => s.status === 'skipped').length,
      narrated: this.scenarios.filter((s) => s.steps.some((step) => step.facts)).length,
      screenshots: this.scenarios.reduce(
        (n, s) => n + s.steps.filter((step) => step.shot).length,
        0,
      ),
      apiCalls: this.scenarios.reduce((n, s) => n + s.apiCalls.length, 0),
    };

    mkdirSync(OUT_DIR, { recursive: true });
    writeFileSync(
      join(OUT_DIR, 'report.json'),
      JSON.stringify({ summary, scenarios: this.scenarios }, null, 2),
      'utf8',
    );
    writeFileSync(join(OUT_DIR, 'report.md'), renderMarkdown(summary, this.scenarios), 'utf8');
    writeFileSync(
      join(OUT_DIR, 'index.html'),
      renderHtml(summary, this.scenarios, this.meta),
      'utf8',
    );

    // An absolute file:// URL, because a relative path is ambiguous: the suite
    // runs with its cwd in e2e/, but people launch it from the repo root.
    const file = join(OUT_DIR, 'index.html');
    const url = pathToFileURL(file).href;

    console.log(
      [
        '',
        'Evidence report — what passed, with data and screenshots:',
        `  ${url}`,
        `  ${summary.passed}/${summary.total} passed · ${summary.screenshots} screenshots · ` +
          `${summary.apiCalls} API calls · ${summary.narrated} scenarios with recorded data`,
        '',
        '  Open it:  npm run report',
        '  Markdown: ' + pathToFileURL(join(OUT_DIR, 'report.md')).href,
        '',
        "  (`npx playwright show-report` opens Playwright's own report, which is a",
        '   different artifact — it is for debugging failures.)',
      ].join('\n'),
    );
  }
}

// --- helpers ---------------------------------------------------------------

function slug(value: string): string {
  return value.replace(/[^a-zA-Z0-9]+/g, '-').slice(0, 60);
}

function safeRead(path: string): string {
  try {
    return readFileSync(path, 'utf8');
  } catch {
    return '';
  }
}




// --- markdown --------------------------------------------------------------

function renderMarkdown(summary: Summary, scenarios: Scenario[]): string {
  const lines: string[] = [
    '# TravelHub end-to-end run report',
    '',
    `Generated ${summary.generatedAt} · ${summary.passed}/${summary.total} passed · ` +
      `${friendly(summary.durationMs)}`,
    '',
  ];

  for (const file of group(scenarios)) {
    lines.push(`## ${areaFor(file.name).title}`, '', `_${areaFor(file.name).blurb}_`, '');
    for (const scenario of file.items) {
      const mark = scenario.status === 'passed' ? '✅' : scenario.status === 'skipped' ? '⏭️' : '❌';
      const heading = sentenceCase(scenario.title);
      lines.push(`### ${mark} ${heading}`, '');
      lines.push(`*${scenario.suite ? scenario.suite + ' · ' : ''}${scenario.project} · ${friendly(scenario.durationMs)}*`, '');

      if (scenario.steps.length === 0 && scenario.apiCalls.length === 0) {
        lines.push('_No evidence captured._', '');
        continue;
      }

      for (const step of scenario.steps) {
        lines.push(`**${step.seq}. ${step.name}**`, '');
        if (step.facts) {
          lines.push('| Field | Value |', '| ----- | ----- |');
          for (const [key, value] of Object.entries(step.facts)) {
            lines.push(`| ${label(key)} | ${formatValue(value)} |`);
          }
          lines.push('');
        }
        if (step.shot) {
          lines.push(`![${step.name}](${step.shot})`, '');
        }
      }

      if (scenario.apiCalls.length > 0) {
        lines.push(`**API calls (${scenario.apiCalls.length})**`, '');
        for (const call of scenario.apiCalls) {
          lines.push(`- \`${call.method} ${call.path}\` → **${call.status}** (${call.durationMs} ms)`);
          if (call.requestBody) {
            lines.push('', '  Request:', '', '  ```json', indent(call.requestBody), '  ```');
          }
          if (call.responseBody) {
            lines.push('', '  Response:', '', '  ```json', indent(call.responseBody), '  ```');
          }
        }
        lines.push('');
      }
    }
  }

  return lines.join('\n');
}

/** Indents a block two spaces so it nests under a Markdown list item. */
function indent(text: string): string {
  return text
    .split('\n')
    .map((line) => '  ' + line)
    .join('\n');
}