test: automate 54/60 manual QA rows with Playwright on MySQL 8 + coverage matrix

.github/workflows/e2e.yml

@@ -0,0 +1,90 @@
+name: e2e (Playwright on MySQL 8)
+
+on:
+  pull_request:
+  push:
+    branches: [demo/qa-live-before]
+  workflow_dispatch:
+
+jobs:
+  playwright:
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    services:
+      mysql:
+        image: mirror.gcr.io/library/mysql:8.0
+        env:
+          MYSQL_ROOT_PASSWORD: Test@123
+          MYSQL_DATABASE: bankappdb
+        ports:
+          - 3306:3306
+        options: >-
+          --health-cmd="mysqladmin ping -h 127.0.0.1 -uroot -pTest@123"
+          --health-interval=5s
+          --health-timeout=5s
+          --health-retries=30
+    env:
+      MVNW_REPOURL: https://maven-central.storage-download.googleapis.com/maven2
+      SPRING_DATASOURCE_URL: jdbc:mysql://127.0.0.1:3306/bankappdb?useSSL=false&allowPublicKeyRetrieval=true&serverTimezone=UTC
+      BASE_URL: http://localhost:8080
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-java@v4
+        with:
+          distribution: temurin
+          java-version: '17'
+          cache: maven
+
+      - uses: actions/setup-node@v4
+        with:
+          node-version: '20'
+          cache: npm
+          cache-dependency-path: e2e/package-lock.json
+
+      - name: Coverage matrix accounts for every manual row
+        run: node e2e/scripts/check-coverage.mjs
+
+      - name: Confirm MySQL 8
+        run: |
+          version="$(docker exec ${{ job.services.mysql.id }} mysql -uroot -pTest@123 -N -e 'select version()' 2>/dev/null)"
+          echo "MySQL server version: $version"
+          [[ "$version" == 8.* ]]
+
+      - name: Build app
+        run: ./mvnw -B -s demo/maven-settings.xml -DskipTests package
+
+      - name: Start app
+        run: |
+          setsid nohup java -jar target/bankapp-0.0.1-SNAPSHOT.jar > app.log 2>&1 < /dev/null &
+          for _ in $(seq 1 120); do
+            curl -sf -o /dev/null "$BASE_URL/login" && exit 0
+            sleep 1
+          done
+          tail -n 100 app.log
+          exit 1
+
+      - name: Install Playwright
+        working-directory: e2e
+        run: |
+          npm ci
+          npx playwright install --with-deps chromium
+
+      - name: Typecheck tests
+        working-directory: e2e
+        run: npx tsc --noEmit
+
+      - name: Run Playwright suite
+        working-directory: e2e
+        run: npx playwright test
+
+      - name: Upload Playwright report
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: playwright-report
+          path: |
+            e2e/playwright-report/
+            e2e/test-results/
+            app.log
+          retention-days: 14

.gitignore

@@ -32,3 +32,8 @@ build/
 
 ### VS Code ###
 .vscode/
+
+### Playwright (e2e/) ###
+e2e/node_modules/
+e2e/test-results/
+e2e/playwright-report/

e2e/package.json

@@ -0,0 +1,15 @@
+{
+  "name": "bankapp-e2e",
+  "private": true,
+  "description": "Playwright automation of qa/manual-test-plan.csv",
+  "scripts": {
+    "test": "playwright test",
+    "typecheck": "tsc --noEmit",
+    "coverage:check": "node scripts/check-coverage.mjs"
+  },
+  "devDependencies": {
+    "@playwright/test": "1.52.0",
+    "@types/node": "^20.0.0",
+    "typescript": "^5.0.0"
+  }
+}

e2e/playwright.config.ts

@@ -0,0 +1,22 @@
+import { defineConfig, devices } from '@playwright/test';
+
+export default defineConfig({
+  testDir: './tests',
+  fullyParallel: true,
+  forbidOnly: !!process.env.CI,
+  retries: 0,
+  workers: process.env.CI ? 2 : undefined,
+  timeout: 60_000,
+  expect: { timeout: 10_000 },
+  reporter: [
+    ['list'],
+    ['html', { open: 'never', outputFolder: 'playwright-report' }],
+    ['junit', { outputFile: 'test-results/junit.xml' }],
+  ],
+  use: {
+    baseURL: process.env.BASE_URL ?? 'http://localhost:8080',
+    trace: 'retain-on-failure',
+    screenshot: 'only-on-failure',
+  },
+  projects: [{ name: 'chromium', use: { ...devices['Desktop Chrome'] } }],
+});

e2e/scripts/check-coverage.mjs

@@ -0,0 +1,76 @@
+#!/usr/bin/env node
+// Fails unless qa/coverage-matrix.md accounts for every row of qa/manual-test-plan.csv exactly once,
+// every "Automated" row has exactly one Playwright test titled with its ID, and every
+// "Not automated" row has a reason and no test.
+import { readFileSync, readdirSync } from 'node:fs';
+import { dirname, join } from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+const root = join(dirname(fileURLToPath(import.meta.url)), '..', '..');
+const errors = [];
+
+function parseCsv(text) {
+  const rows = [];
+  let row = [], field = '', quoted = false;
+  for (let i = 0; i < text.length; i++) {
+    const c = text[i];
+    if (quoted) {
+      if (c === '"' && text[i + 1] === '"') { field += '"'; i++; }
+      else if (c === '"') quoted = false;
+      else field += c;
+    } else if (c === '"') quoted = true;
+    else if (c === ',') { row.push(field); field = ''; }
+    else if (c === '\n' || c === '\r') {
+      if (c === '\r' && text[i + 1] === '\n') i++;
+      row.push(field); rows.push(row); row = []; field = '';
+    } else field += c;
+  }
+  if (field || row.length) { row.push(field); rows.push(row); }
+  return rows.filter((r) => r.some((f) => f.trim()));
+}
+
+const [header, ...planRows] = parseCsv(readFileSync(join(root, 'qa/manual-test-plan.csv'), 'utf8'));
+const idCol = header.indexOf('id');
+const planIds = planRows.map((r) => r[idCol]);
+if (new Set(planIds).size !== planIds.length) errors.push('qa/manual-test-plan.csv has duplicate IDs');
+
+const matrix = new Map();
+for (const line of readFileSync(join(root, 'qa/coverage-matrix.md'), 'utf8').split('\n')) {
+  const cells = line.split('|').map((c) => c.trim());
+  if (!/^MT-\d{3}$/.test(cells[1] ?? '')) continue;
+  const [, id, , , , status, test, notes] = cells;
+  if (matrix.has(id)) errors.push(`${id} appears more than once in the coverage matrix`);
+  matrix.set(id, { status, test, notes });
+}
+
+const testsById = new Map();
+const testsDir = join(root, 'e2e/tests');
+for (const file of readdirSync(testsDir).filter((f) => f.endsWith('.spec.ts'))) {
+  const source = readFileSync(join(testsDir, file), 'utf8');
+  for (const m of source.matchAll(/\btest(?:\.\w+)?\(\s*['"`](MT-\d{3})\b/g)) {
+    testsById.set(m[1], [...(testsById.get(m[1]) ?? []), file]);
+  }
+}
+
+for (const id of planIds) {
+  const entry = matrix.get(id);
+  const tests = testsById.get(id) ?? [];
+  if (!entry) { errors.push(`${id} is in the plan but missing from the coverage matrix`); continue; }
+  if (entry.status === 'Automated') {
+    if (tests.length !== 1) errors.push(`${id} is marked Automated but has ${tests.length} Playwright tests`);
+    else if (!entry.test.includes(tests[0])) errors.push(`${id} matrix points at "${entry.test}" but the test is in ${tests[0]}`);
+  } else if (entry.status === 'Not automated') {
+    if (tests.length) errors.push(`${id} is marked Not automated but has a test in ${tests.join(', ')}`);
+    if (!entry.notes) errors.push(`${id} is marked Not automated without a reason`);
+  } else errors.push(`${id} has unknown status "${entry.status}"`);
+}
+for (const id of [...matrix.keys(), ...testsById.keys()]) {
+  if (!planIds.includes(id)) errors.push(`${id} is not in qa/manual-test-plan.csv`);
+}
+
+const automated = planIds.filter((id) => matrix.get(id)?.status === 'Automated').length;
+if (errors.length) {
+  console.error(`Coverage matrix check FAILED:\n- ${[...new Set(errors)].join('\n- ')}`);
+  process.exit(1);
+}
+console.log(`Coverage matrix OK: ${planIds.length} plan rows, ${automated} automated, ${planIds.length - automated} not automated (with reasons).`);