Automate 54/60 manual QA rows as Playwright tests on MySQL 8 CI, with coverage matrix

.github/workflows/e2e.yml

@@ -0,0 +1,81 @@
+name: e2e
+
+on:
+  pull_request:
+  push:
+    branches: [main, master, demo/qa-live-before]
+  workflow_dispatch:
+
+jobs:
+  playwright:
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    services:
+      mysql:
+        image: mirror.gcr.io/library/mysql:8.0
+        env:
+          MYSQL_ROOT_PASSWORD: Test@123
+          MYSQL_DATABASE: bankappdb
+        ports:
+          - 3306:3306
+        options: >-
+          --health-cmd="mysqladmin ping -h 127.0.0.1 -uroot -pTest@123"
+          --health-interval=5s --health-timeout=5s --health-retries=30
+    env:
+      MVNW_REPOURL: https://maven-central.storage-download.googleapis.com/maven2
+      SPRING_DATASOURCE_URL: jdbc:mysql://127.0.0.1:3306/bankappdb?useSSL=false&allowPublicKeyRetrieval=true&serverTimezone=UTC
+      BASE_URL: http://localhost:8080
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-java@v4
+        with:
+          distribution: temurin
+          java-version: '17'
+          cache: maven
+
+      - uses: actions/setup-node@v4
+        with:
+          node-version: '20'
+          cache: npm
+          cache-dependency-path: e2e/package-lock.json
+
+      - name: Coverage matrix accounts for all 60 plan rows
+        run: node e2e/scripts/check-coverage.mjs
+
+      - name: Build app
+        run: ./mvnw -B -s demo/maven-settings.xml -DskipTests package
+
+      - name: Start app against MySQL 8
+        run: |
+          mysql_version=$(docker exec ${{ job.services.mysql.id }} mysql -uroot -pTest@123 -N -e 'select version()' 2>/dev/null)
+          echo "MySQL server version: $mysql_version"
+          [[ "$mysql_version" == 8.* ]]
+          setsid nohup java -jar target/bankapp-0.0.1-SNAPSHOT.jar > app.log 2>&1 < /dev/null &
+          for i in $(seq 1 120); do
+            curl -sf -o /dev/null "$BASE_URL/login" && { echo "app up after ${i}s"; exit 0; }
+            sleep 1
+          done
+          tail -n 80 app.log
+          exit 1
+
+      - name: Install Playwright
+        working-directory: e2e
+        run: |
+          npm ci
+          npx playwright install --with-deps chromium
+
+      - name: Run Playwright tests
+        working-directory: e2e
+        run: npx playwright test
+
+      - name: Upload report
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: playwright-report
+          path: |
+            e2e/playwright-report
+            e2e/test-results
+            app.log
+          retention-days: 14

e2e/.gitignore

@@ -0,0 +1,3 @@
+node_modules/
+playwright-report/
+test-results/

e2e/README.md

@@ -0,0 +1,12 @@
+# Playwright regression suite
+
+Automates `qa/manual-test-plan.csv`. `qa/coverage-matrix.csv` maps every plan row to its test or to the reason it stays manual; `scripts/check-coverage.mjs` fails CI if any row is missing, duplicated, or out of sync with the tests.
+
+```bash
+demo/start-demo.sh            # app on :8080 against MySQL 8
+cd e2e && npm ci && npx playwright install chromium
+npx playwright test           # BASE_URL overrides http://localhost:8080
+node scripts/check-coverage.mjs
+```
+
+Each test registers its own unique customers, so tests run in parallel against a shared database. CI: `.github/workflows/e2e.yml` (MySQL 8 service container).

e2e/package.json

@@ -0,0 +1,13 @@
+{
+  "name": "bankapp-e2e",
+  "private": true,
+  "scripts": {
+    "test": "playwright test",
+    "coverage:check": "node scripts/check-coverage.mjs"
+  },
+  "devDependencies": {
+    "@playwright/test": "1.52.0",
+    "@types/node": "^20.0.0",
+    "typescript": "^5.0.0"
+  }
+}

e2e/playwright.config.ts

@@ -0,0 +1,18 @@
+import { defineConfig, devices } from '@playwright/test';
+
+export default defineConfig({
+  testDir: './tests',
+  fullyParallel: true,
+  forbidOnly: !!process.env.CI,
+  retries: process.env.CI ? 1 : 0,
+  workers: process.env.CI ? 2 : 4,
+  reporter: process.env.CI
+    ? [['list'], ['html', { open: 'never' }], ['github']]
+    : [['list'], ['html', { open: 'never' }]],
+  use: {
+    baseURL: process.env.BASE_URL ?? 'http://localhost:8080',
+    trace: 'retain-on-failure',
+    screenshot: 'only-on-failure',
+  },
+  projects: [{ name: 'chromium', use: { ...devices['Desktop Chrome'] } }],
+});

e2e/scripts/check-coverage.mjs

@@ -0,0 +1,82 @@
+#!/usr/bin/env node
+// Fails unless qa/coverage-matrix.csv accounts for every row of qa/manual-test-plan.csv exactly once,
+// every "automated" row has a matching Playwright test, and every "manual" row has a reason and no test.
+import { readFileSync, readdirSync } from 'node:fs';
+import { dirname, join } from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+const root = join(dirname(fileURLToPath(import.meta.url)), '..', '..');
+
+function parseCsv(text) {
+  const rows = [];
+  let row = [], field = '', quoted = false;
+  for (let i = 0; i < text.length; i++) {
+    const c = text[i];
+    if (quoted) {
+      if (c === '"' && text[i + 1] === '"') { field += '"'; i++; }
+      else if (c === '"') quoted = false;
+      else field += c;
+    } else if (c === '"') quoted = true;
+    else if (c === ',') { row.push(field); field = ''; }
+    else if (c === '\n' || c === '\r') {
+      if (c === '\r' && text[i + 1] === '\n') i++;
+      row.push(field); field = '';
+      if (row.some((f) => f !== '')) rows.push(row);
+      row = [];
+    } else field += c;
+  }
+  if (field !== '' || row.length) { row.push(field); rows.push(row); }
+  const [header, ...body] = rows;
+  return body.map((r) => Object.fromEntries(header.map((h, i) => [h, r[i] ?? ''])));
+}
+
+const plan = parseCsv(readFileSync(join(root, 'qa/manual-test-plan.csv'), 'utf8'));
+const matrix = parseCsv(readFileSync(join(root, 'qa/coverage-matrix.csv'), 'utf8'));
+
+const testsDir = join(root, 'e2e/tests');
+const testTitles = new Map();
+for (const file of readdirSync(testsDir).filter((f) => f.endsWith('.spec.ts'))) {
+  const src = readFileSync(join(testsDir, file), 'utf8');
+  for (const m of src.matchAll(/\btest(?:\.\w+)?\(\s*'(MT-\d{3}) ([^']*)'/g)) {
+    const id = m[1];
+    if (!testTitles.has(id)) testTitles.set(id, []);
+    testTitles.get(id).push({ file: `e2e/tests/${file}`, title: `${id} ${m[2]}` });
+  }
+}
+
+const errors = [];
+const planIds = plan.map((r) => r.id);
+const matrixById = new Map();
+for (const row of matrix) {
+  if (matrixById.has(row.id)) errors.push(`${row.id}: listed more than once in the coverage matrix`);
+  matrixById.set(row.id, row);
+}
+
+for (const p of plan) {
+  const m = matrixById.get(p.id);
+  if (!m) { errors.push(`${p.id}: missing from the coverage matrix`); continue; }
+  if (m.title !== p.title) errors.push(`${p.id}: matrix title "${m.title}" does not match plan "${p.title}"`);
+  const tests = testTitles.get(p.id) ?? [];
+  if (m.status === 'automated') {
+    if (tests.length !== 1) errors.push(`${p.id}: marked automated but has ${tests.length} Playwright tests`);
+    else {
+      const t = tests[0];
+      if (m.automated_test !== `${t.file} :: ${t.title}`) errors.push(`${p.id}: automated_test "${m.automated_test}" should be "${t.file} :: ${t.title}"`);
+      if (t.title !== `${p.id} ${p.title}`) errors.push(`${p.id}: test title "${t.title}" does not match plan title`);
+    }
+  } else if (m.status === 'manual') {
+    if (!m.notes.trim()) errors.push(`${p.id}: manual row needs a reason in notes`);
+    if (tests.length) errors.push(`${p.id}: marked manual but a Playwright test exists`);
+  } else errors.push(`${p.id}: unknown status "${m.status}"`);
+}
+for (const id of matrixById.keys()) if (!planIds.includes(id)) errors.push(`${id}: in the coverage matrix but not in the plan`);
+for (const id of testTitles.keys()) if (!planIds.includes(id)) errors.push(`${id}: Playwright test has no plan row`);
+
+const automated = matrix.filter((r) => r.status === 'automated').length;
+const manual = matrix.filter((r) => r.status === 'manual');
+console.log(`Plan rows: ${plan.length}. Automated: ${automated}. Manual: ${manual.length} (${manual.map((r) => r.id).join(', ')}).`);
+if (errors.length) {
+  console.error(`Coverage matrix check FAILED:\n- ${errors.join('\n- ')}`);
+  process.exit(1);
+}
+console.log('Coverage matrix check passed: every plan row is accounted for exactly once.');