test: automate 54/60 manual regression rows (Playwright on MySQL 8) + coverage matrix

.github/workflows/e2e.yml

@@ -0,0 +1,99 @@
+name: E2E regression (Playwright + MySQL 8)
+
+on:
+  push:
+    branches: [demo/qa-live-before, main, master]
+  pull_request:
+  workflow_dispatch:
+
+concurrency:
+  group: e2e-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  e2e:
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    services:
+      mysql:
+        # mirror.gcr.io avoids Docker Hub rate limits (see AGENTS.md).
+        image: mirror.gcr.io/library/mysql:8.0
+        env:
+          MYSQL_ROOT_PASSWORD: Test@123
+          MYSQL_DATABASE: bankappdb
+        ports:
+          - 3306:3306
+        options: >-
+          --health-cmd="mysqladmin ping -h 127.0.0.1 -uroot -pTest@123"
+          --health-interval=5s --health-timeout=5s --health-retries=30
+    env:
+      MVNW_REPOURL: https://maven-central.storage-download.googleapis.com/maven2
+      SPRING_DATASOURCE_URL: jdbc:mysql://127.0.0.1:3306/bankappdb?useSSL=false&allowPublicKeyRetrieval=true&serverTimezone=UTC
+      SPRING_DATASOURCE_USERNAME: root
+      SPRING_DATASOURCE_PASSWORD: Test@123
+      BASE_URL: http://localhost:8080
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-java@v4
+        with:
+          distribution: temurin
+          java-version: '17'
+          cache: maven
+
+      - uses: actions/setup-node@v4
+        with:
+          node-version: '20'
+          cache: npm
+          cache-dependency-path: e2e/package-lock.json
+
+      - name: Install e2e dependencies
+        working-directory: e2e
+        run: npm ci
+
+      - name: Check every plan row is accounted for
+        working-directory: e2e
+        run: |
+          npm run typecheck
+          npm run matrix:check
+
+      - name: Build app (GCS Maven mirror)
+        run: ./mvnw -B -s demo/maven-settings.xml -DskipTests package
+
+      - name: Start app against MySQL 8
+        run: |
+          mysql_version=$(docker exec "${{ job.services.mysql.id }}" mysql -uroot -pTest@123 -N -e 'select version()' 2>/dev/null)
+          echo "MySQL server version: $mysql_version"
+          [[ "$mysql_version" == 8.* ]]
+          setsid nohup java -jar target/bankapp-0.0.1-SNAPSHOT.jar > app.log 2>&1 < /dev/null &
+          for i in $(seq 1 120); do
+            curl -sf -o /dev/null "$BASE_URL/login" && { echo "app up after ${i}s"; exit 0; }
+            sleep 1
+          done
+          echo "app did not serve /login within 120s"; tail -n 80 app.log; exit 1
+
+      - name: Install Playwright Chromium
+        working-directory: e2e
+        run: npx playwright install --with-deps chromium
+
+      - name: Run Playwright suite
+        working-directory: e2e
+        run: npx playwright test
+
+      - name: Publish coverage matrix
+        if: always()
+        working-directory: e2e
+        run: npm run matrix
+
+      - name: Upload report, matrix and app log
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: e2e-report
+          path: |
+            qa/coverage-matrix.md
+            e2e/playwright-report/
+            e2e/results.json
+            e2e/test-results/
+            app.log
+          if-no-files-found: warn

e2e/.gitignore

@@ -0,0 +1,4 @@
+node_modules/
+test-results/
+playwright-report/
+results.json

e2e/README.md

@@ -0,0 +1,29 @@
+# Playwright regression suite
+
+Automates [`qa/manual-test-plan.csv`](../qa/manual-test-plan.csv). Each test title starts with its plan ID (`MT-0xx`), one test per row.
+Rows that cannot be automated are listed with a reason in [`qa/not-automated.json`](../qa/not-automated.json).
+[`qa/coverage-matrix.md`](../qa/coverage-matrix.md) accounts for all 60 rows and is regenerated on every CI run
+(job summary + `e2e-report` artifact).
+
+## Run locally
+
+```bash
+demo/start-demo.sh            # MySQL 8 in Docker + app on :8080
+cd e2e
+npm ci
+npx playwright install chromium
+npx playwright test           # BASE_URL defaults to http://localhost:8080
+npm run matrix                # rewrite ../qa/coverage-matrix.md with these results
+```
+
+`npm run matrix:check` asks Playwright for the test list (`--list`) and fails if a plan row is neither automated nor listed in `qa/not-automated.json`, is in both, has two tests, or its test is `test.skip` / `test.fixme` / `test.fail`. With results (`npm run matrix`) it also fails on tests skipped at runtime.
+
+Tests register their own unique users, so they run in parallel and do not need a clean database.
+The Deposit/Withdraw/Transfer panels use Bootstrap from public CDNs, so the browser needs internet access.
+
+## Adding or changing a row
+
+1. Add the row to `qa/manual-test-plan.csv`.
+2. Either add `test('MT-0xx <title>', ...)` to the matching spec, or add an entry with a reason to `qa/not-automated.json`.
+   Do not make up an expected result the plan does not state.
+3. `npm run matrix:check`.

e2e/package.json

@@ -0,0 +1,16 @@
+{
+  "name": "bankapp-e2e",
+  "private": true,
+  "description": "Playwright automation of qa/manual-test-plan.csv",
+  "scripts": {
+    "test": "playwright test",
+    "matrix": "node scripts/coverage-matrix.mjs",
+    "matrix:check": "node scripts/coverage-matrix.mjs --check",
+    "typecheck": "tsc --noEmit"
+  },
+  "devDependencies": {
+    "@playwright/test": "1.52.0",
+    "@types/node": "^20.0.0",
+    "typescript": "^5.0.0"
+  }
+}

e2e/playwright.config.ts

@@ -0,0 +1,22 @@
+import { defineConfig, devices } from '@playwright/test';
+
+export default defineConfig({
+  testDir: './tests',
+  fullyParallel: true,
+  forbidOnly: !!process.env.CI,
+  retries: 0,
+  workers: process.env.CI ? 2 : 4,
+  timeout: 30_000,
+  expect: { timeout: 5_000 },
+  reporter: [
+    ['list'],
+    ['html', { open: 'never', outputFolder: 'playwright-report' }],
+    ['json', { outputFile: 'results.json' }],
+  ],
+  use: {
+    baseURL: process.env.BASE_URL ?? 'http://localhost:8080',
+    trace: 'retain-on-failure',
+    screenshot: 'only-on-failure',
+  },
+  projects: [{ name: 'chromium', use: { ...devices['Desktop Chrome'] } }],
+});