From 2e5b39c3fe8c0f4531841f5b7fc2d25d55d82dbf Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Fri, 23 Oct 2026 02:54:37 +0000 Subject: [PATCH 1/3] test: automate 54 of 60 manual regression rows with Playwright, CI on MySQL 8, coverage matrix Co-Authored-By: Garret Andreine --- .github/workflows/e2e.yml | 99 ++++++++++++++++++ e2e/.gitignore | 4 + e2e/README.md | 29 ++++++ e2e/package-lock.json | 109 ++++++++++++++++++++ e2e/package.json | 16 +++ e2e/playwright.config.ts | 22 ++++ e2e/scripts/coverage-matrix.mjs | 171 +++++++++++++++++++++++++++++++ e2e/tests/authentication.spec.ts | 108 +++++++++++++++++++ e2e/tests/cross-cutting.spec.ts | 44 ++++++++ e2e/tests/dashboard.spec.ts | 47 +++++++++ e2e/tests/deposit.spec.ts | 62 +++++++++++ e2e/tests/fixtures.ts | 105 +++++++++++++++++++ e2e/tests/transactions.spec.ts | 54 ++++++++++ e2e/tests/transfer.spec.ts | 128 +++++++++++++++++++++++ e2e/tests/withdraw.spec.ts | 64 ++++++++++++ e2e/tsconfig.json | 12 +++ qa/coverage-matrix.md | 73 +++++++++++++ qa/not-automated.json | 32 ++++++ 18 files changed, 1179 insertions(+) create mode 100644 .github/workflows/e2e.yml create mode 100644 e2e/.gitignore create mode 100644 e2e/README.md create mode 100644 e2e/package-lock.json create mode 100644 e2e/package.json create mode 100644 e2e/playwright.config.ts create mode 100644 e2e/scripts/coverage-matrix.mjs create mode 100644 e2e/tests/authentication.spec.ts create mode 100644 e2e/tests/cross-cutting.spec.ts create mode 100644 e2e/tests/dashboard.spec.ts create mode 100644 e2e/tests/deposit.spec.ts create mode 100644 e2e/tests/fixtures.ts create mode 100644 e2e/tests/transactions.spec.ts create mode 100644 e2e/tests/transfer.spec.ts create mode 100644 e2e/tests/withdraw.spec.ts create mode 100644 e2e/tsconfig.json create mode 100644 qa/coverage-matrix.md create mode 100644 qa/not-automated.json diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml new file mode 100644 index 00000000..ba9a46c4 --- /dev/null +++ b/.github/workflows/e2e.yml @@ -0,0 +1,99 @@ +name: E2E regression (Playwright + MySQL 8) + +on: + push: + branches: [demo/qa-live-before, main, master] + pull_request: + workflow_dispatch: + +concurrency: + group: e2e-${{ github.ref }} + cancel-in-progress: true + +jobs: + e2e: + runs-on: ubuntu-latest + timeout-minutes: 30 + services: + mysql: + # mirror.gcr.io avoids Docker Hub rate limits (see AGENTS.md). + image: mirror.gcr.io/library/mysql:8.0 + env: + MYSQL_ROOT_PASSWORD: Test@123 + MYSQL_DATABASE: bankappdb + ports: + - 3306:3306 + options: >- + --health-cmd="mysqladmin ping -h 127.0.0.1 -uroot -pTest@123" + --health-interval=5s --health-timeout=5s --health-retries=30 + env: + MVNW_REPOURL: https://maven-central.storage-download.googleapis.com/maven2 + SPRING_DATASOURCE_URL: jdbc:mysql://127.0.0.1:3306/bankappdb?useSSL=false&allowPublicKeyRetrieval=true&serverTimezone=UTC + SPRING_DATASOURCE_USERNAME: root + SPRING_DATASOURCE_PASSWORD: Test@123 + BASE_URL: http://localhost:8080 + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-java@v4 + with: + distribution: temurin + java-version: '17' + cache: maven + + - uses: actions/setup-node@v4 + with: + node-version: '20' + cache: npm + cache-dependency-path: e2e/package-lock.json + + - name: Install e2e dependencies + working-directory: e2e + run: npm ci + + - name: Check every plan row is accounted for + working-directory: e2e + run: | + npm run typecheck + npm run matrix:check + + - name: Build app (GCS Maven mirror) + run: ./mvnw -B -s demo/maven-settings.xml -DskipTests package + + - name: Start app against MySQL 8 + run: | + mysql_version=$(docker exec "${{ job.services.mysql.id }}" mysql -uroot -pTest@123 -N -e 'select version()' 2>/dev/null) + echo "MySQL server version: $mysql_version" + [[ "$mysql_version" == 8.* ]] + setsid nohup java -jar target/bankapp-0.0.1-SNAPSHOT.jar > app.log 2>&1 < /dev/null & + for i in $(seq 1 120); do + curl -sf -o /dev/null "$BASE_URL/login" && { echo "app up after ${i}s"; exit 0; } + sleep 1 + done + echo "app did not serve /login within 120s"; tail -n 80 app.log; exit 1 + + - name: Install Playwright Chromium + working-directory: e2e + run: npx playwright install --with-deps chromium + + - name: Run Playwright suite + working-directory: e2e + run: npx playwright test + + - name: Publish coverage matrix + if: always() + working-directory: e2e + run: npm run matrix + + - name: Upload report, matrix and app log + if: always() + uses: actions/upload-artifact@v4 + with: + name: e2e-report + path: | + qa/coverage-matrix.md + e2e/playwright-report/ + e2e/results.json + e2e/test-results/ + app.log + if-no-files-found: warn diff --git a/e2e/.gitignore b/e2e/.gitignore new file mode 100644 index 00000000..e2a17b28 --- /dev/null +++ b/e2e/.gitignore @@ -0,0 +1,4 @@ +node_modules/ +test-results/ +playwright-report/ +results.json diff --git a/e2e/README.md b/e2e/README.md new file mode 100644 index 00000000..caa29f21 --- /dev/null +++ b/e2e/README.md @@ -0,0 +1,29 @@ +# Playwright regression suite + +Automates [`qa/manual-test-plan.csv`](../qa/manual-test-plan.csv). Each test title starts with its plan ID (`MT-0xx`), one test per row. +Rows that cannot be automated are listed with a reason in [`qa/not-automated.json`](../qa/not-automated.json). +[`qa/coverage-matrix.md`](../qa/coverage-matrix.md) accounts for all 60 rows and is regenerated on every CI run +(job summary + `e2e-report` artifact). + +## Run locally + +```bash +demo/start-demo.sh # MySQL 8 in Docker + app on :8080 +cd e2e +npm ci +npx playwright install chromium +npx playwright test # BASE_URL defaults to http://localhost:8080 +npm run matrix # rewrite ../qa/coverage-matrix.md with these results +``` + +`npm run matrix:check` fails if a plan row is neither automated nor listed in `qa/not-automated.json`, is in both, or has two tests. + +Tests register their own unique users, so they run in parallel and do not need a clean database. +The Deposit/Withdraw/Transfer panels use Bootstrap from public CDNs, so the browser needs internet access. + +## Adding or changing a row + +1. Add the row to `qa/manual-test-plan.csv`. +2. Either add `test('MT-0xx ', ...)` to the matching spec, or add an entry with a reason to `qa/not-automated.json`. + Do not make up an expected result the plan does not state. +3. `npm run matrix:check`. diff --git a/e2e/package-lock.json b/e2e/package-lock.json new file mode 100644 index 00000000..e43f214a --- /dev/null +++ b/e2e/package-lock.json @@ -0,0 +1,109 @@ +{ + "name": "bankapp-e2e", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "bankapp-e2e", + "devDependencies": { + "@playwright/test": "1.52.0", + "@types/node": "^20.0.0", + "typescript": "^5.0.0" + } + }, + "node_modules/@playwright/test": { + "version": "1.52.0", + "resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.52.0.tgz", + "integrity": "sha512-uh6W7sb55hl7D6vsAeA+V2p5JnlAqzhqFyF0VcJkKZXkgnFcVG9PziERRHQfPLfNGx1C292a4JqbWzhR8L4R1g==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "playwright": "1.52.0" + }, + "bin": { + "playwright": "cli.js" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@types/node": { + "version": "20.19.43", + "resolved": "https://registry.npmjs.org/@types/node/-/node-20.19.43.tgz", + "integrity": "sha512-6oYBAi5ikg4Pl+kGsoYtawUMBT2zZMCvPNF7pVLnHZfd1zf38DRiWn/gT01RYCdUqkv7Fhr+C9ot4/tb+2sVvA==", + "dev": true, + "license": "MIT", + "dependencies": { + "undici-types": "~6.21.0" + } + }, + "node_modules/fsevents": { + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz", + "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, + "node_modules/playwright": { + "version": "1.52.0", + "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.52.0.tgz", + "integrity": "sha512-JAwMNMBlxJ2oD1kce4KPtMkDeKGHQstdpFPcPH3maElAXon/QZeTvtsfXmTMRyO9TslfoYOXkSsvao2nE1ilTw==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "playwright-core": "1.52.0" + }, + "bin": { + "playwright": "cli.js" + }, + "engines": { + "node": ">=18" + }, + "optionalDependencies": { + "fsevents": "2.3.2" + } + }, + "node_modules/playwright-core": { + "version": "1.52.0", + "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.52.0.tgz", + "integrity": "sha512-l2osTgLXSMeuLZOML9qYODUQoPPnUsKsb5/P6LJ2e6uPKXUdPK5WYhN4z03G+YNbWmGDY4YENauNu4ZKczreHg==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "playwright-core": "cli.js" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/typescript": { + "version": "5.9.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz", + "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=14.17" + } + }, + "node_modules/undici-types": { + "version": "6.21.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz", + "integrity": "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==", + "dev": true, + "license": "MIT" + } + } +} diff --git a/e2e/package.json b/e2e/package.json new file mode 100644 index 00000000..a821560a --- /dev/null +++ b/e2e/package.json @@ -0,0 +1,16 @@ +{ + "name": "bankapp-e2e", + "private": true, + "description": "Playwright automation of qa/manual-test-plan.csv", + "scripts": { + "test": "playwright test", + "matrix": "node scripts/coverage-matrix.mjs", + "matrix:check": "node scripts/coverage-matrix.mjs --check", + "typecheck": "tsc --noEmit" + }, + "devDependencies": { + "@playwright/test": "1.52.0", + "@types/node": "^20.0.0", + "typescript": "^5.0.0" + } +} diff --git a/e2e/playwright.config.ts b/e2e/playwright.config.ts new file mode 100644 index 00000000..6b38f307 --- /dev/null +++ b/e2e/playwright.config.ts @@ -0,0 +1,22 @@ +import { defineConfig, devices } from '@playwright/test'; + +export default defineConfig({ + testDir: './tests', + fullyParallel: true, + forbidOnly: !!process.env.CI, + retries: 0, + workers: process.env.CI ? 2 : 4, + timeout: 30_000, + expect: { timeout: 5_000 }, + reporter: [ + ['list'], + ['html', { open: 'never', outputFolder: 'playwright-report' }], + ['json', { outputFile: 'results.json' }], + ], + use: { + baseURL: process.env.BASE_URL ?? 'http://localhost:8080', + trace: 'retain-on-failure', + screenshot: 'only-on-failure', + }, + projects: [{ name: 'chromium', use: { ...devices['Desktop Chrome'] } }], +}); diff --git a/e2e/scripts/coverage-matrix.mjs b/e2e/scripts/coverage-matrix.mjs new file mode 100644 index 00000000..5b8b5c62 --- /dev/null +++ b/e2e/scripts/coverage-matrix.mjs @@ -0,0 +1,171 @@ +#!/usr/bin/env node +// Builds qa/coverage-matrix.md from qa/manual-test-plan.csv, the Playwright specs, qa/not-automated.json +// and (when present) e2e/results.json. Exits 1 if any plan row is unaccounted for or accounted twice. +// Usage: node scripts/coverage-matrix.mjs [--check] [--results results.json] [--out ../qa/coverage-matrix.md] +import { readFileSync, readdirSync, writeFileSync, existsSync, appendFileSync } from 'node:fs'; +import { dirname, join, relative, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const e2eDir = resolve(dirname(fileURLToPath(import.meta.url)), '..'); +const repoDir = resolve(e2eDir, '..'); +const args = process.argv.slice(2); +const opt = (name, fallback) => { + const i = args.indexOf(name); + return i >= 0 ? args[i + 1] : fallback; +}; +const checkOnly = args.includes('--check'); +const resultsPath = resolve(e2eDir, opt('--results', 'results.json')); +const outPath = resolve(e2eDir, opt('--out', '../qa/coverage-matrix.md')); + +function parseCsv(text) { + const rows = []; + let row = [], field = '', quoted = false; + for (let i = 0; i < text.length; i++) { + const c = text[i]; + if (quoted) { + if (c === '"' && text[i + 1] === '"') { field += '"'; i++; } + else if (c === '"') quoted = false; + else field += c; + } else if (c === '"') quoted = true; + else if (c === ',') { row.push(field); field = ''; } + else if (c === '\n' || c === '\r') { + if (c === '\r' && text[i + 1] === '\n') i++; + row.push(field); field = ''; + if (row.some((f) => f !== '')) rows.push(row); + row = []; + } else field += c; + } + row.push(field); + if (row.some((f) => f !== '')) rows.push(row); + const [header, ...body] = rows; + return body.map((r) => Object.fromEntries(header.map((h, i) => [h, r[i] ?? '']))); +} + +const plan = parseCsv(readFileSync(join(repoDir, 'qa/manual-test-plan.csv'), 'utf8')); +const notAutomated = JSON.parse(readFileSync(join(repoDir, 'qa/not-automated.json'), 'utf8')); +const errors = []; + +const planIds = new Set(); +for (const row of plan) { + if (planIds.has(row.id)) errors.push(`${row.id} appears twice in the plan`); + planIds.add(row.id); +} + +// Static scan: every test('MT-xxx ...') in the specs. +const automated = new Map(); +const testsDir = join(e2eDir, 'tests'); +for (const file of readdirSync(testsDir).filter((f) => f.endsWith('.spec.ts')).sort()) { + const src = readFileSync(join(testsDir, file), 'utf8'); + for (const m of src.matchAll(/\btest(?:\.\w+)?\(\s*['"`](MT-\d{3})\b([^'"`]*)['"`]/g)) { + const [, id, rest] = m; + const line = src.slice(0, m.index).split('\n').length; + const spec = `e2e/tests/${file}:${line}`; + if (automated.has(id)) errors.push(`${id} has more than one test (${automated.get(id).spec}, ${spec})`); + automated.set(id, { spec, title: `${id}${rest}` }); + } +} + +const excluded = new Map(); +for (const entry of notAutomated) { + if (excluded.has(entry.id)) errors.push(`${entry.id} is listed twice in qa/not-automated.json`); + if (!entry.reason?.trim()) errors.push(`${entry.id} in qa/not-automated.json has no reason`); + excluded.set(entry.id, entry); +} + +for (const id of [...automated.keys(), ...excluded.keys()]) { + if (!planIds.has(id)) errors.push(`${id} is referenced but is not in qa/manual-test-plan.csv`); +} +for (const { id } of plan) { + const a = automated.has(id), e = excluded.has(id); + if (a && e) errors.push(`${id} is both automated and listed as not automated`); + if (!a && !e) errors.push(`${id} is neither automated nor listed in qa/not-automated.json`); +} + +// Results from Playwright's JSON reporter, keyed by MT id. +const results = new Map(); +const haveResults = !checkOnly && existsSync(resultsPath); +if (haveResults) { + const report = JSON.parse(readFileSync(resultsPath, 'utf8')); + const walk = (suite) => { + for (const spec of suite.specs ?? []) { + const id = spec.title.match(/^MT-\d{3}/)?.[0]; + if (!id) continue; + const statuses = spec.tests.map((t) => t.status); + const status = statuses.includes('unexpected') ? 'failed' + : statuses.includes('flaky') ? 'flaky' + : statuses.every((s) => s === 'skipped') ? 'skipped' : 'passed'; + results.set(id, status); + } + for (const child of suite.suites ?? []) walk(child); + }; + for (const suite of report.suites ?? []) walk(suite); + for (const id of automated.keys()) { + if (!results.has(id)) errors.push(`${id} has a test but no result in ${relative(repoDir, resultsPath)}`); + } +} + +const badge = { passed: 'PASS', failed: 'FAIL', flaky: 'FLAKY', skipped: 'SKIPPED' }; +const esc = (s) => String(s).replace(/\|/g, '\\|').replace(/\n/g, ' '); +const counts = { automated: 0, notAutomated: 0, passed: 0, failed: 0, other: 0 }; +const lines = []; +for (const row of plan) { + const a = automated.get(row.id); + const e = excluded.get(row.id); + let coverage, evidence, result; + if (a) { + counts.automated++; + coverage = 'Automated'; + evidence = `\`${a.spec}\``; + const r = results.get(row.id); + result = haveResults ? badge[r] ?? 'NO RESULT' : 'not run'; + if (r === 'passed') counts.passed++; + else if (r === 'failed') counts.failed++; + else if (haveResults) counts.other++; + } else if (e) { + counts.notAutomated++; + coverage = `Not automated (${e.category})`; + evidence = e.reason; + result = 'manual'; + } else { + coverage = 'UNACCOUNTED'; + evidence = ''; + result = ''; + } + lines.push(`| ${row.id} | ${esc(row.area)} | ${esc(row.title)} | ${row.priority} | ${coverage} | ${result} | ${esc(evidence)} |`); +} + +const generatedFrom = haveResults + ? `Playwright results from \`${relative(repoDir, resultsPath)}\` (${new Date().toISOString()})` + : 'static scan only (no Playwright results)'; +const md = [ + '# Manual regression plan: automation coverage matrix', + '', + 'Generated by `e2e/scripts/coverage-matrix.mjs` from `qa/manual-test-plan.csv`, `e2e/tests/*.spec.ts` and `qa/not-automated.json`. Do not edit by hand.', + '', + `Source: ${generatedFrom}.`, + '', + `- Plan rows: **${plan.length}**`, + `- Automated (Playwright, Chromium, against MySQL 8): **${counts.automated}**` + + (haveResults ? ` (passed ${counts.passed}, failed ${counts.failed}${counts.other ? `, other ${counts.other}` : ''})` : ''), + `- Not automated, stays manual: **${counts.notAutomated}**`, + `- Unaccounted: **${plan.length - counts.automated - counts.notAutomated}**`, + '', + '| ID | Area | Title | Priority | Coverage | Result | Test / reason not automated |', + '|---|---|---|---|---|---|---|', + ...lines, + '', +].join('\n'); + +if (errors.length) { + console.error('Coverage accounting errors:\n' + errors.map((e) => ` - ${e}`).join('\n')); +} +if (!checkOnly) { + writeFileSync(outPath, md); + console.log(`Wrote ${relative(repoDir, outPath)}`); + if (process.env.GITHUB_STEP_SUMMARY) appendFileSync(process.env.GITHUB_STEP_SUMMARY, md); +} +console.log( + `${plan.length} rows: ${counts.automated} automated, ${counts.notAutomated} not automated` + + (haveResults ? `; results: ${counts.passed} passed, ${counts.failed} failed` : ''), +); +process.exit(errors.length ? 1 : 0); diff --git a/e2e/tests/authentication.spec.ts b/e2e/tests/authentication.spec.ts new file mode 100644 index 00000000..53a5cd1e --- /dev/null +++ b/e2e/tests/authentication.spec.ts @@ -0,0 +1,108 @@ +import { test, expect, login, logout, newCustomer, registerViaApi, uniqueUsername, PASSWORD } from './fixtures'; + +const loginError = 'Invalid username or password.'; + +test.describe('Authentication', () => { + test('MT-001 Login page shows username and password fields', async ({ page }) => { + await page.goto('/login'); + await expect(page.locator('input[name="username"]')).toBeVisible(); + await expect(page.locator('input[name="password"]')).toBeVisible(); + await expect(page.locator('input[name="password"]')).toHaveAttribute('type', 'password'); + await expect(page.getByRole('button', { name: 'Login' })).toBeVisible(); + }); + + test('MT-002 Register link on login page opens registration', async ({ page }) => { + await page.goto('/login'); + await page.getByRole('link', { name: 'Register here' }).click(); + await expect(page).toHaveURL(/\/register$/); + await expect(page.getByRole('heading', { name: 'Register a New Account' })).toBeVisible(); + }); + + test('MT-003 Login link on register page returns to login', async ({ page }) => { + await page.goto('/register'); + await page.getByRole('link', { name: 'Login here' }).click(); + await expect(page).toHaveURL(/\/login$/); + await expect(page.getByRole('heading', { name: 'Login' })).toBeVisible(); + }); + + test('MT-004 New customer can register', async ({ page }) => { + await page.goto('/register'); + await page.locator('input[name="username"]').fill(uniqueUsername('reg')); + await page.locator('input[name="password"]').fill(PASSWORD); + await page.getByRole('button', { name: 'Register' }).click(); + await expect(page).toHaveURL(/\/login$/); + await expect(page.getByRole('heading', { name: 'Login' })).toBeVisible(); + }); + + test('MT-005 Duplicate username is rejected', async ({ page }) => { + const username = uniqueUsername('dup'); + await registerViaApi(page, username); + await page.goto('/register'); + await page.locator('input[name="username"]').fill(username); + await page.locator('input[name="password"]').fill('Other0ne!'); + await page.getByRole('button', { name: 'Register' }).click(); + await expect(page.getByText('User already present.')).toBeVisible(); + await expect(page.getByRole('heading', { name: 'Register a New Account' })).toBeVisible(); + }); + + test('MT-006 Valid login opens dashboard', async ({ page }) => { + const username = uniqueUsername('login'); + await registerViaApi(page, username); + await login(page, username); + await expect(page).toHaveURL(/\/dashboard$/); + await expect(page.getByRole('heading', { level: 2 })).toHaveText(`Welcome, ${username}`); + }); + + test('MT-007 Wrong password is rejected', async ({ page }) => { + const username = uniqueUsername('badpw'); + await registerViaApi(page, username); + await login(page, username, 'WrongPass1!'); + await expect(page).toHaveURL(/\/login\?error/); + await expect(page.getByText(loginError)).toBeVisible(); + }); + + test('MT-008 Unknown username gets the same error', async ({ page }) => { + const known = uniqueUsername('known'); + await registerViaApi(page, known); + await login(page, known, 'WrongPass1!'); + const knownAlert = (await page.locator('.alert-danger').innerText()).trim(); + + await login(page, uniqueUsername('ghost'), PASSWORD); + await expect(page).toHaveURL(/\/login\?error/); + await expect(page.locator('.alert-danger')).toHaveText(knownAlert); + expect(knownAlert).toBe(loginError); + }); + + test('MT-009 Password is case sensitive', async ({ page }) => { + const username = uniqueUsername('case'); + await registerViaApi(page, username); + await login(page, username, PASSWORD.toUpperCase()); + await expect(page).toHaveURL(/\/login\?error/); + await login(page, username, PASSWORD.toLowerCase()); + await expect(page).toHaveURL(/\/login\?error/); + }); + + test('MT-010 Dashboard requires login', async ({ page }) => { + await page.goto('/dashboard'); + await expect(page).toHaveURL(/\/login$/); + }); + + test('MT-011 Transaction history requires login', async ({ page }) => { + await page.goto('/transactions'); + await expect(page).toHaveURL(/\/login$/); + }); + + test('MT-012 Logout returns to login', async ({ page }) => { + await newCustomer(page); + await page.getByRole('link', { name: 'Logout' }).click(); + await expect(page).toHaveURL(/\/login\?logout$/); + await expect(page.getByRole('heading', { name: 'Login' })).toBeVisible(); + }); + + test('MT-013 Session ends after logout', async ({ page }) => { + await newCustomer(page); + await logout(page); + await page.goto('/dashboard'); + await expect(page).toHaveURL(/\/login$/); + }); +}); diff --git a/e2e/tests/cross-cutting.spec.ts b/e2e/tests/cross-cutting.spec.ts new file mode 100644 index 00000000..85b79e93 --- /dev/null +++ b/e2e/tests/cross-cutting.spec.ts @@ -0,0 +1,44 @@ +import { test, expect, deposit, expectBalance, login, logout, newCustomer, registerViaApi, sessionFor, uniqueUsername } from './fixtures'; + +const BANK_NAME = 'Wells Fargo'; + +test.describe('Cross-cutting', () => { + test('MT-056 Customers only see their own account', async ({ page, browser, baseURL }) => { + const alice = await newCustomer(page, 'alice'); + await deposit(page, '100'); + const bob = uniqueUsername('bob'); + await registerViaApi(page, bob); + const bobPage = await sessionFor(browser, baseURL, bob); + await deposit(bobPage, '25'); + + await page.goto('/dashboard'); + await expect(page.getByRole('heading', { level: 2 })).toHaveText(`Welcome, ${alice}`); + await expectBalance(page, '100.00'); + await expect(page.getByText(bob)).toHaveCount(0); + + await bobPage.goto('/dashboard'); + await expect(bobPage.getByRole('heading', { level: 2 })).toHaveText(`Welcome, ${bob}`); + await expectBalance(bobPage, '25.00'); + await expect(bobPage.getByText(alice)).toHaveCount(0); + }); + + test('MT-057 Balance survives logout', async ({ page }) => { + const username = await newCustomer(page); + await deposit(page, '100'); + await logout(page); + await login(page, username); + await expect(page).toHaveURL(/\/dashboard$/); + await expectBalance(page, '100.00'); + }); + + test('MT-058 Bank name shows on every page tab', async ({ page }) => { + await page.goto('/login'); + await expect(page).toHaveTitle(new RegExp(BANK_NAME)); + await page.goto('/register'); + await expect(page).toHaveTitle(new RegExp(BANK_NAME)); + await newCustomer(page); + await expect(page).toHaveTitle(new RegExp(BANK_NAME)); + await page.goto('/transactions'); + await expect(page).toHaveTitle(new RegExp(BANK_NAME)); + }); +}); diff --git a/e2e/tests/dashboard.spec.ts b/e2e/tests/dashboard.spec.ts new file mode 100644 index 00000000..8c3e9655 --- /dev/null +++ b/e2e/tests/dashboard.spec.ts @@ -0,0 +1,47 @@ +import { test, expect, expectBalance, newCustomer, openPanel } from './fixtures'; + +test.describe('Dashboard', () => { + test('MT-014 New customer starts at zero', async ({ page }) => { + await newCustomer(page); + await expectBalance(page, '0.00'); + }); + + test('MT-015 Welcome message shows the customer', async ({ page }) => { + const username = await newCustomer(page); + await expect(page.getByRole('heading', { level: 2 })).toHaveText(`Welcome, ${username}`); + }); + + test('MT-016 Account number is shown', async ({ page }) => { + await newCustomer(page); + await expect(page.getByRole('heading', { name: 'Account Details' })).toBeVisible(); + const number = page.locator('p', { hasText: 'Account Number:' }).locator('span'); + await expect(number).toHaveText(/^\d+$/); + }); + + test('MT-017 Navigation links are present', async ({ page }) => { + await newCustomer(page); + const nav = page.locator('nav'); + for (const name of ['Dashboard', 'Transactions', 'Logout']) { + await expect(nav.getByRole('link', { name, exact: true })).toBeVisible(); + } + }); + + test('MT-018 Deposit panel opens', async ({ page }) => { + await newCustomer(page); + const form = await openPanel(page, 'deposit'); + await expect(form.locator('input[name="amount"]')).toBeVisible(); + }); + + test('MT-019 Withdraw panel opens', async ({ page }) => { + await newCustomer(page); + const form = await openPanel(page, 'withdraw'); + await expect(form.locator('input[name="amount"]')).toBeVisible(); + }); + + test('MT-020 Transfer panel opens', async ({ page }) => { + await newCustomer(page); + const form = await openPanel(page, 'transfer'); + await expect(form.locator('input[name="toUsername"]')).toBeVisible(); + await expect(form.locator('input[name="amount"]')).toBeVisible(); + }); +}); diff --git a/e2e/tests/deposit.spec.ts b/e2e/tests/deposit.spec.ts new file mode 100644 index 00000000..e9ec336b --- /dev/null +++ b/e2e/tests/deposit.spec.ts @@ -0,0 +1,62 @@ +import { test, expect, deposit, expectBalance, history, newCustomer, openPanel, trackPosts } from './fixtures'; + +test.describe('Deposit', () => { + test('MT-022 Deposit increases balance', async ({ page }) => { + await newCustomer(page); + await deposit(page, '100'); + await expectBalance(page, '100.00'); + }); + + test('MT-023 Deposits add up', async ({ page }) => { + await newCustomer(page); + await deposit(page, '100'); + await deposit(page, '250'); + await expectBalance(page, '350.00'); + }); + + test('MT-024 Deposit appears in history', async ({ page }) => { + await newCustomer(page); + await deposit(page, '100'); + const rows = await history(page); + expect(rows.map(({ type, amount }) => ({ type, amount }))).toEqual([{ type: 'Deposit', amount: '100.00' }]); + }); + + test('MT-025 Empty deposit is blocked', async ({ page }) => { + await newCustomer(page); + const posts = trackPosts(page, '/deposit'); + const form = await openPanel(page, 'deposit'); + await form.getByRole('button', { name: 'Submit' }).click(); + expect(await form.locator('input[name="amount"]').evaluate((el: HTMLInputElement) => el.validity.valueMissing)).toBe(true); + await expect(page).toHaveURL(/\/dashboard$/); + expect(posts).toEqual([]); + await page.reload(); + await expectBalance(page, '0.00'); + }); + + test('MT-026 Deposit field accepts whole dollars only', async ({ page }) => { + await newCustomer(page); + const posts = trackPosts(page, '/deposit'); + const form = await openPanel(page, 'deposit'); + const amount = form.locator('input[name="amount"]'); + await amount.fill('10.50'); + await form.getByRole('button', { name: 'Submit' }).click(); + expect(await amount.evaluate((el: HTMLInputElement) => el.validity.stepMismatch)).toBe(true); + expect(posts).toEqual([]); + await page.reload(); + await expectBalance(page, '0.00'); + }); + + test('MT-027 Large deposit is handled', async ({ page }) => { + await newCustomer(page); + await deposit(page, '1000000'); + await expectBalance(page, '1000000.00'); + }); + + test('MT-028 Letters cannot be typed into amount', async ({ page }) => { + await newCustomer(page); + const form = await openPanel(page, 'deposit'); + const amount = form.locator('input[name="amount"]'); + await amount.pressSequentially('abc'); + await expect(amount).toHaveValue(''); + }); +}); diff --git a/e2e/tests/fixtures.ts b/e2e/tests/fixtures.ts new file mode 100644 index 00000000..829caa6c --- /dev/null +++ b/e2e/tests/fixtures.ts @@ -0,0 +1,105 @@ +import { test as base, expect, type Browser, type Page } from '@playwright/test'; + +export const PASSWORD = 'Passw0rd!'; + +let counter = 0; +export function uniqueUsername(prefix = 'qa'): string { + counter += 1; + return `${prefix}_${Date.now().toString(36)}_${process.pid}_${counter}`; +} + +export async function registerViaApi(page: Page, username: string, password = PASSWORD): Promise<void> { + const res = await page.request.post('/register', { + form: { username, password }, + maxRedirects: 0, + }); + expect(res.status(), `register ${username}`).toBe(302); +} + +export async function login(page: Page, username: string, password = PASSWORD): Promise<void> { + await page.goto('/login'); + await page.locator('input[name="username"]').fill(username); + await page.locator('input[name="password"]').fill(password); + await page.getByRole('button', { name: 'Login' }).click(); +} + +export async function logout(page: Page): Promise<void> { + await page.getByRole('link', { name: 'Logout' }).click(); + await expect(page).toHaveURL(/\/login\?logout/); +} + +export async function newCustomer(page: Page, prefix = 'qa'): Promise<string> { + const username = uniqueUsername(prefix); + await registerViaApi(page, username); + await login(page, username); + await expect(page).toHaveURL(/\/dashboard$/); + return username; +} + +/** Opens a fresh browser context (separate cookies) logged in as `username`. */ +export async function sessionFor(browser: Browser, baseURL: string | undefined, username: string): Promise<Page> { + const context = await browser.newContext({ baseURL }); + const page = await context.newPage(); + await login(page, username); + await expect(page).toHaveURL(/\/dashboard$/); + return page; +} + +export const balance = (page: Page) => page.getByRole('heading', { level: 3, name: /^Current Balance:/ }); + +export async function expectBalance(page: Page, amount: string): Promise<void> { + await expect(balance(page)).toHaveText(`Current Balance: $${amount}`); +} + +export const errorMessage = (page: Page) => page.locator('p.error'); + +const panelButton = { deposit: 'Deposit', withdraw: 'Withdraw', transfer: 'Transfer Money' } as const; +const panelId = { deposit: '#depositForm', withdraw: '#withdrawForm', transfer: '#transferForm' } as const; +export type Panel = keyof typeof panelId; + +export async function openPanel(page: Page, panel: Panel) { + await page.getByRole('button', { name: panelButton[panel], exact: true }).click(); + const form = page.locator(`${panelId[panel]} form`); + await expect(form).toBeVisible(); + return form; +} + +async function submitAmount(page: Page, panel: Panel, amount: string, toUsername?: string): Promise<void> { + await page.goto('/dashboard'); + const form = await openPanel(page, panel); + if (toUsername !== undefined) await form.locator('input[name="toUsername"]').fill(toUsername); + await form.locator('input[name="amount"]').fill(amount); + await form.getByRole('button', { name: 'Submit' }).click(); + await page.waitForLoadState('domcontentloaded'); +} + +export const deposit = (page: Page, amount: string) => submitAmount(page, 'deposit', amount); +export const withdraw = (page: Page, amount: string) => submitAmount(page, 'withdraw', amount); +export const transfer = (page: Page, toUsername: string, amount: string) => + submitAmount(page, 'transfer', amount, toUsername); + +export type HistoryRow = { id: string; type: string; amount: string; date: string }; + +export async function history(page: Page): Promise<HistoryRow[]> { + await page.goto('/transactions'); + await expect(page.getByRole('heading', { name: 'Transaction History' })).toBeVisible(); + const rows = page.locator('table tbody tr'); + const out: HistoryRow[] = []; + for (const row of await rows.all()) { + const cells = (await row.locator('td').allInnerTexts()).map((c) => c.trim()); + out.push({ id: cells[0], type: cells[1], amount: cells[2], date: cells[3] }); + } + return out; +} + +/** Records every POST the page sends to `path`, so a test can assert a form was never submitted. */ +export function trackPosts(page: Page, path: string): string[] { + const seen: string[] = []; + page.on('request', (req) => { + if (req.method() === 'POST' && new URL(req.url()).pathname === path) seen.push(req.url()); + }); + return seen; +} + +export const test = base; +export { expect }; diff --git a/e2e/tests/transactions.spec.ts b/e2e/tests/transactions.spec.ts new file mode 100644 index 00000000..2414b81b --- /dev/null +++ b/e2e/tests/transactions.spec.ts @@ -0,0 +1,54 @@ +import { test, expect, deposit, history, newCustomer, registerViaApi, transfer, uniqueUsername, withdraw } from './fixtures'; + +const DATE_TIME = /^\d{4}-\d{2}-\d{2}[T ]\d{2}:\d{2}(:\d{2}(\.\d+)?)?$/; + +test.describe('Transactions', () => { + test('MT-049 New customer has empty history', async ({ page }) => { + await newCustomer(page); + await page.goto('/transactions'); + await expect(page.locator('table thead th')).toHaveCount(4); + await expect(page.locator('table tbody tr')).toHaveCount(0); + }); + + test('MT-050 History columns are correct', async ({ page }) => { + await newCustomer(page); + await page.goto('/transactions'); + await expect(page.locator('table thead th')).toHaveText(['ID', 'Type', 'Amount', 'Date']); + }); + + test('MT-051 All transaction types are listed', async ({ page }) => { + const payee = uniqueUsername('payee'); + await registerViaApi(page, payee); + await newCustomer(page); + await deposit(page, '100'); + await withdraw(page, '10'); + await transfer(page, payee, '20'); + const types = (await history(page)).map((r) => r.type); + expect(types).toEqual(expect.arrayContaining(['Deposit', 'Withdrawal', `Transfer Out to ${payee}`])); + expect(types).toHaveLength(3); + }); + + test('MT-052 Back to Dashboard link works', async ({ page }) => { + await newCustomer(page); + await page.goto('/transactions'); + await page.getByRole('link', { name: 'Back to Dashboard' }).click(); + await expect(page).toHaveURL(/\/dashboard$/); + await expect(page.getByRole('heading', { name: /^Welcome, / })).toBeVisible(); + }); + + test('MT-053 Transactions link works from dashboard', async ({ page }) => { + await newCustomer(page); + await page.locator('nav').getByRole('link', { name: 'Transactions', exact: true }).click(); + await expect(page).toHaveURL(/\/transactions$/); + await expect(page.getByRole('heading', { name: 'Transaction History' })).toBeVisible(); + }); + + test('MT-054 Every row has a date', async ({ page }) => { + await newCustomer(page); + await deposit(page, '100'); + await withdraw(page, '10'); + const rows = await history(page); + expect(rows).toHaveLength(2); + for (const row of rows) expect(row.date).toMatch(DATE_TIME); + }); +}); diff --git a/e2e/tests/transfer.spec.ts b/e2e/tests/transfer.spec.ts new file mode 100644 index 00000000..5888edc2 --- /dev/null +++ b/e2e/tests/transfer.spec.ts @@ -0,0 +1,128 @@ +import { + test, expect, deposit, errorMessage, expectBalance, history, newCustomer, openPanel, + registerViaApi, sessionFor, trackPosts, transfer, uniqueUsername, +} from './fixtures'; +import type { Page } from '@playwright/test'; + +/** Registers a recipient (no login) and returns its username. */ +async function recipient(page: Page): Promise<string> { + const username = uniqueUsername('payee'); + await registerViaApi(page, username); + return username; +} + +test.describe('Transfer', () => { + test('MT-036 Transfer debits the sender', async ({ page }) => { + const payee = await recipient(page); + await newCustomer(page, 'sender'); + await deposit(page, '100'); + await transfer(page, payee, '30'); + await expect(errorMessage(page)).toHaveCount(0); + await expectBalance(page, '70.00'); + }); + + // The plan says "After MT-036"; each test repeats MT-036's steps so it can run on its own. + test('MT-037 Transfer credits the recipient', async ({ page, browser, baseURL }) => { + const payee = await recipient(page); + await newCustomer(page, 'sender'); + await deposit(page, '100'); + await transfer(page, payee, '30'); + const payeePage = await sessionFor(browser, baseURL, payee); + await expectBalance(payeePage, '30.00'); + }); + + test('MT-038 Sender history shows the transfer', async ({ page }) => { + const payee = await recipient(page); + await newCustomer(page, 'sender'); + await deposit(page, '100'); + await transfer(page, payee, '30'); + const rows = await history(page); + expect(rows.filter((r) => r.type.startsWith('Transfer')).map(({ type, amount }) => ({ type, amount }))).toEqual([ + { type: `Transfer Out to ${payee}`, amount: '30.00' }, + ]); + }); + + test('MT-039 Recipient history shows the transfer', async ({ page, browser, baseURL }) => { + const payee = await recipient(page); + const sender = await newCustomer(page, 'sender'); + await deposit(page, '100'); + await transfer(page, payee, '30'); + const rows = await history(await sessionFor(browser, baseURL, payee)); + expect(rows.map(({ type, amount }) => ({ type, amount }))).toEqual([ + { type: `Transfer In from ${sender}`, amount: '30.00' }, + ]); + }); + + test('MT-040 Customer can transfer the full balance', async ({ page, browser, baseURL }) => { + const payee = await recipient(page); + await newCustomer(page, 'sender'); + await deposit(page, '100'); + await transfer(page, payee, '100'); + await expect(errorMessage(page)).toHaveCount(0); + await expectBalance(page, '0.00'); + await expectBalance(await sessionFor(browser, baseURL, payee), '100.00'); + }); + + test('MT-041 Transfer above balance is rejected', async ({ page, browser, baseURL }) => { + const payee = await recipient(page); + await newCustomer(page, 'sender'); + await deposit(page, '100'); + await transfer(page, payee, '150'); + await expect(errorMessage(page)).toHaveText('Insufficient funds'); + await page.goto('/dashboard'); + await expectBalance(page, '100.00'); + await expectBalance(await sessionFor(browser, baseURL, payee), '0.00'); + }); + + test('MT-042 Transfer to unknown customer is rejected', async ({ page }) => { + await newCustomer(page, 'sender'); + await deposit(page, '100'); + await transfer(page, uniqueUsername('ghost'), '10'); + await expect(errorMessage(page)).toHaveText('Recipient account not found'); + await page.goto('/dashboard'); + await expectBalance(page, '100.00'); + }); + + test('MT-043 Recipient is required', async ({ page }) => { + await newCustomer(page, 'sender'); + const posts = trackPosts(page, '/transfer'); + const form = await openPanel(page, 'transfer'); + await form.locator('input[name="amount"]').fill('10'); + await form.getByRole('button', { name: 'Submit' }).click(); + expect(await form.locator('input[name="toUsername"]').evaluate((el: HTMLInputElement) => el.validity.valueMissing)).toBe(true); + await expect(page).toHaveURL(/\/dashboard$/); + expect(posts).toEqual([]); + }); + + test('MT-044 Amount is required', async ({ page }) => { + const payee = await recipient(page); + await newCustomer(page, 'sender'); + const posts = trackPosts(page, '/transfer'); + const form = await openPanel(page, 'transfer'); + await form.locator('input[name="toUsername"]').fill(payee); + await form.getByRole('button', { name: 'Submit' }).click(); + expect(await form.locator('input[name="amount"]').evaluate((el: HTMLInputElement) => el.validity.valueMissing)).toBe(true); + await expect(page).toHaveURL(/\/dashboard$/); + expect(posts).toEqual([]); + }); + + test('MT-045 Multiple transfers add up', async ({ page, browser, baseURL }) => { + const payee = await recipient(page); + await newCustomer(page, 'sender'); + await deposit(page, '100'); + await transfer(page, payee, '20'); + await transfer(page, payee, '30'); + await expectBalance(page, '50.00'); + await expectBalance(await sessionFor(browser, baseURL, payee), '50.00'); + }); + + test('MT-046 Rejected transfer leaves no history', async ({ page }) => { + await newCustomer(page, 'sender'); + await deposit(page, '100'); + await transfer(page, uniqueUsername('ghost'), '10'); + await expect(errorMessage(page)).toHaveText('Recipient account not found'); + const rows = await history(page); + expect(rows.filter((r) => r.type.startsWith('Transfer'))).toEqual([]); + }); +}); + diff --git a/e2e/tests/withdraw.spec.ts b/e2e/tests/withdraw.spec.ts new file mode 100644 index 00000000..1c0a1081 --- /dev/null +++ b/e2e/tests/withdraw.spec.ts @@ -0,0 +1,64 @@ +import { test, expect, deposit, errorMessage, expectBalance, history, newCustomer, openPanel, trackPosts, withdraw } from './fixtures'; + +test.describe('Withdraw', () => { + test('MT-029 Withdrawal decreases balance', async ({ page }) => { + await newCustomer(page); + await deposit(page, '100'); + await withdraw(page, '40'); + await expectBalance(page, '60.00'); + }); + + test('MT-030 Customer can withdraw the full balance', async ({ page }) => { + await newCustomer(page); + await deposit(page, '100'); + await withdraw(page, '100'); + await expect(errorMessage(page)).toHaveCount(0); + await expectBalance(page, '0.00'); + }); + + test('MT-031 Overdraw is rejected', async ({ page }) => { + await newCustomer(page); + await deposit(page, '100'); + await withdraw(page, '500'); + await expect(errorMessage(page)).toHaveText('Insufficient funds'); + await expectBalance(page, '100.00'); + await page.goto('/dashboard'); + await expectBalance(page, '100.00'); + }); + + test('MT-032 Withdrawal from empty account is rejected', async ({ page }) => { + await newCustomer(page); + await withdraw(page, '1'); + await expect(errorMessage(page)).toHaveText('Insufficient funds'); + await page.goto('/dashboard'); + await expectBalance(page, '0.00'); + }); + + test('MT-033 Withdrawal appears in history', async ({ page }) => { + await newCustomer(page); + await deposit(page, '100'); + await withdraw(page, '40'); + const rows = await history(page); + expect(rows.filter((r) => r.type === 'Withdrawal').map((r) => r.amount)).toEqual(['40.00']); + }); + + test('MT-034 Empty withdrawal is blocked', async ({ page }) => { + await newCustomer(page); + await deposit(page, '100'); + const posts = trackPosts(page, '/withdraw'); + const form = await openPanel(page, 'withdraw'); + await form.getByRole('button', { name: 'Submit' }).click(); + expect(await form.locator('input[name="amount"]').evaluate((el: HTMLInputElement) => el.validity.valueMissing)).toBe(true); + await expect(page).toHaveURL(/\/dashboard$/); + expect(posts).toEqual([]); + }); + + test('MT-035 Rejected withdrawal leaves no history', async ({ page }) => { + await newCustomer(page); + await deposit(page, '100'); + await withdraw(page, '500'); + await expect(errorMessage(page)).toHaveText('Insufficient funds'); + const rows = await history(page); + expect(rows.filter((r) => r.type === 'Withdrawal')).toEqual([]); + }); +}); diff --git a/e2e/tsconfig.json b/e2e/tsconfig.json new file mode 100644 index 00000000..42e30ec4 --- /dev/null +++ b/e2e/tsconfig.json @@ -0,0 +1,12 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "commonjs", + "strict": true, + "esModuleInterop": true, + "skipLibCheck": true, + "noEmit": true, + "types": ["node"] + }, + "include": ["playwright.config.ts", "tests/**/*.ts"] +} diff --git a/qa/coverage-matrix.md b/qa/coverage-matrix.md new file mode 100644 index 00000000..8e596fd8 --- /dev/null +++ b/qa/coverage-matrix.md @@ -0,0 +1,73 @@ +# Manual regression plan: automation coverage matrix + +Generated by `e2e/scripts/coverage-matrix.mjs` from `qa/manual-test-plan.csv`, `e2e/tests/*.spec.ts` and `qa/not-automated.json`. Do not edit by hand. + +Source: Playwright results from `e2e/results.json` (2026-10-23T02:54:31.202Z). + +- Plan rows: **60** +- Automated (Playwright, Chromium, against MySQL 8): **54** (passed 54, failed 0) +- Not automated, stays manual: **6** +- Unaccounted: **0** + +| ID | Area | Title | Priority | Coverage | Result | Test / reason not automated | +|---|---|---|---|---|---|---| +| MT-001 | Authentication | Login page shows username and password fields | High | Automated | PASS | `e2e/tests/authentication.spec.ts:6` | +| MT-002 | Authentication | Register link on login page opens registration | Medium | Automated | PASS | `e2e/tests/authentication.spec.ts:14` | +| MT-003 | Authentication | Login link on register page returns to login | Low | Automated | PASS | `e2e/tests/authentication.spec.ts:21` | +| MT-004 | Authentication | New customer can register | High | Automated | PASS | `e2e/tests/authentication.spec.ts:28` | +| MT-005 | Authentication | Duplicate username is rejected | High | Automated | PASS | `e2e/tests/authentication.spec.ts:37` | +| MT-006 | Authentication | Valid login opens dashboard | High | Automated | PASS | `e2e/tests/authentication.spec.ts:48` | +| MT-007 | Authentication | Wrong password is rejected | High | Automated | PASS | `e2e/tests/authentication.spec.ts:56` | +| MT-008 | Authentication | Unknown username gets the same error | High | Automated | PASS | `e2e/tests/authentication.spec.ts:64` | +| MT-009 | Authentication | Password is case sensitive | Medium | Automated | PASS | `e2e/tests/authentication.spec.ts:76` | +| MT-010 | Authentication | Dashboard requires login | High | Automated | PASS | `e2e/tests/authentication.spec.ts:85` | +| MT-011 | Authentication | Transaction history requires login | High | Automated | PASS | `e2e/tests/authentication.spec.ts:90` | +| MT-012 | Authentication | Logout returns to login | High | Automated | PASS | `e2e/tests/authentication.spec.ts:95` | +| MT-013 | Authentication | Session ends after logout | High | Automated | PASS | `e2e/tests/authentication.spec.ts:102` | +| MT-014 | Dashboard | New customer starts at zero | High | Automated | PASS | `e2e/tests/dashboard.spec.ts:4` | +| MT-015 | Dashboard | Welcome message shows the customer | Low | Automated | PASS | `e2e/tests/dashboard.spec.ts:9` | +| MT-016 | Dashboard | Account number is shown | Medium | Automated | PASS | `e2e/tests/dashboard.spec.ts:14` | +| MT-017 | Dashboard | Navigation links are present | Low | Automated | PASS | `e2e/tests/dashboard.spec.ts:21` | +| MT-018 | Dashboard | Deposit panel opens | Medium | Automated | PASS | `e2e/tests/dashboard.spec.ts:29` | +| MT-019 | Dashboard | Withdraw panel opens | Medium | Automated | PASS | `e2e/tests/dashboard.spec.ts:35` | +| MT-020 | Dashboard | Transfer panel opens | Medium | Automated | PASS | `e2e/tests/dashboard.spec.ts:41` | +| MT-021 | Dashboard | Brand colours and logo match the brand guide | Low | Not automated (Human sign-off) | manual | The expected result is a brand-team sign-off, which is a human approval, not an observable app state. The brand guide is also not in the repo, so there is nothing to compare against. Stays manual; once a baseline is approved it could back a screenshot-comparison test. | +| MT-022 | Deposit | Deposit increases balance | High | Automated | PASS | `e2e/tests/deposit.spec.ts:4` | +| MT-023 | Deposit | Deposits add up | High | Automated | PASS | `e2e/tests/deposit.spec.ts:10` | +| MT-024 | Deposit | Deposit appears in history | High | Automated | PASS | `e2e/tests/deposit.spec.ts:17` | +| MT-025 | Deposit | Empty deposit is blocked | Medium | Automated | PASS | `e2e/tests/deposit.spec.ts:24` | +| MT-026 | Deposit | Deposit field accepts whole dollars only | Medium | Automated | PASS | `e2e/tests/deposit.spec.ts:36` | +| MT-027 | Deposit | Large deposit is handled | Medium | Automated | PASS | `e2e/tests/deposit.spec.ts:49` | +| MT-028 | Deposit | Letters cannot be typed into amount | Low | Automated | PASS | `e2e/tests/deposit.spec.ts:55` | +| MT-029 | Withdraw | Withdrawal decreases balance | High | Automated | PASS | `e2e/tests/withdraw.spec.ts:4` | +| MT-030 | Withdraw | Customer can withdraw the full balance | High | Automated | PASS | `e2e/tests/withdraw.spec.ts:11` | +| MT-031 | Withdraw | Overdraw is rejected | High | Automated | PASS | `e2e/tests/withdraw.spec.ts:19` | +| MT-032 | Withdraw | Withdrawal from empty account is rejected | High | Automated | PASS | `e2e/tests/withdraw.spec.ts:29` | +| MT-033 | Withdraw | Withdrawal appears in history | High | Automated | PASS | `e2e/tests/withdraw.spec.ts:37` | +| MT-034 | Withdraw | Empty withdrawal is blocked | Medium | Automated | PASS | `e2e/tests/withdraw.spec.ts:45` | +| MT-035 | Withdraw | Rejected withdrawal leaves no history | High | Automated | PASS | `e2e/tests/withdraw.spec.ts:56` | +| MT-036 | Transfer | Transfer debits the sender | High | Automated | PASS | `e2e/tests/transfer.spec.ts:15` | +| MT-037 | Transfer | Transfer credits the recipient | High | Automated | PASS | `e2e/tests/transfer.spec.ts:25` | +| MT-038 | Transfer | Sender history shows the transfer | High | Automated | PASS | `e2e/tests/transfer.spec.ts:34` | +| MT-039 | Transfer | Recipient history shows the transfer | High | Automated | PASS | `e2e/tests/transfer.spec.ts:45` | +| MT-040 | Transfer | Customer can transfer the full balance | High | Automated | PASS | `e2e/tests/transfer.spec.ts:56` | +| MT-041 | Transfer | Transfer above balance is rejected | High | Automated | PASS | `e2e/tests/transfer.spec.ts:66` | +| MT-042 | Transfer | Transfer to unknown customer is rejected | High | Automated | PASS | `e2e/tests/transfer.spec.ts:77` | +| MT-043 | Transfer | Recipient is required | Medium | Automated | PASS | `e2e/tests/transfer.spec.ts:86` | +| MT-044 | Transfer | Amount is required | Medium | Automated | PASS | `e2e/tests/transfer.spec.ts:97` | +| MT-045 | Transfer | Multiple transfers add up | High | Automated | PASS | `e2e/tests/transfer.spec.ts:109` | +| MT-046 | Transfer | Rejected transfer leaves no history | High | Automated | PASS | `e2e/tests/transfer.spec.ts:119` | +| MT-047 | Transfer | Negative transfer amount | High | Not automated (Expected result not testable) | manual | 'System handles it appropriately' does not define a pass condition, so no assertion was invented. Needs a product decision (for example: reject with message X, balances unchanged, no history row). Observed on this build: a -50 transfer is accepted, the sender balance goes UP by 50, the recipient goes DOWN by 50, and the sender history shows 'Transfer Out to <recipient>' for -50.00. | +| MT-048 | Transfer | Customer receives a transfer confirmation by SMS or email | Medium | Not automated (Feature or external system missing) | manual | Needs real SMS and email delivery to a customer's phone and inbox. The app has no notification feature (no mail or SMS dependency or code in the repo), so there is nothing to automate, and running this row by hand today would fail. | +| MT-049 | Transactions | New customer has empty history | Medium | Automated | PASS | `e2e/tests/transactions.spec.ts:6` | +| MT-050 | Transactions | History columns are correct | Low | Automated | PASS | `e2e/tests/transactions.spec.ts:13` | +| MT-051 | Transactions | All transaction types are listed | High | Automated | PASS | `e2e/tests/transactions.spec.ts:19` | +| MT-052 | Transactions | Back to Dashboard link works | Low | Automated | PASS | `e2e/tests/transactions.spec.ts:31` | +| MT-053 | Transactions | Transactions link works from dashboard | Low | Automated | PASS | `e2e/tests/transactions.spec.ts:39` | +| MT-054 | Transactions | Every row has a date | Medium | Automated | PASS | `e2e/tests/transactions.spec.ts:46` | +| MT-055 | Transactions | Printed history is legible on the branch printer | Low | Not automated (Physical hardware) | manual | Needs a physical branch printer, and 'legible' is a human judgment. Print-media CSS emulation would not prove what the branch printer outputs. | +| MT-056 | Cross-cutting | Customers only see their own account | High | Automated | PASS | `e2e/tests/cross-cutting.spec.ts:6` | +| MT-057 | Cross-cutting | Balance survives logout | High | Automated | PASS | `e2e/tests/cross-cutting.spec.ts:25` | +| MT-058 | Cross-cutting | Bank name shows on every page tab | Low | Automated | PASS | `e2e/tests/cross-cutting.spec.ts:34` | +| MT-059 | Cross-cutting | Screen reader reads balance and errors | Medium | Not automated (Assistive technology / human judgment) | manual | Needs JAWS on Windows, and 'announced clearly' is a human judgment. Playwright on Linux CI cannot drive JAWS. An automated accessibility scan (e.g. axe) could be added as a supplement, but it would not satisfy this row. | +| MT-060 | Cross-cutting | First-time customer completes a transfer unaided | Low | Not automated (Moderated usability study) | manual | Needs a moderated session with a real first-time customer. A script completing a transfer says nothing about whether a person can do it unaided in under 2 minutes. | diff --git a/qa/not-automated.json b/qa/not-automated.json new file mode 100644 index 00000000..21a206b9 --- /dev/null +++ b/qa/not-automated.json @@ -0,0 +1,32 @@ +[ + { + "id": "MT-021", + "category": "Human sign-off", + "reason": "The expected result is a brand-team sign-off, which is a human approval, not an observable app state. The brand guide is also not in the repo, so there is nothing to compare against. Stays manual; once a baseline is approved it could back a screenshot-comparison test." + }, + { + "id": "MT-047", + "category": "Expected result not testable", + "reason": "'System handles it appropriately' does not define a pass condition, so no assertion was invented. Needs a product decision (for example: reject with message X, balances unchanged, no history row). Observed on this build: a -50 transfer is accepted, the sender balance goes UP by 50, the recipient goes DOWN by 50, and the sender history shows 'Transfer Out to <recipient>' for -50.00." + }, + { + "id": "MT-048", + "category": "Feature or external system missing", + "reason": "Needs real SMS and email delivery to a customer's phone and inbox. The app has no notification feature (no mail or SMS dependency or code in the repo), so there is nothing to automate, and running this row by hand today would fail." + }, + { + "id": "MT-055", + "category": "Physical hardware", + "reason": "Needs a physical branch printer, and 'legible' is a human judgment. Print-media CSS emulation would not prove what the branch printer outputs." + }, + { + "id": "MT-059", + "category": "Assistive technology / human judgment", + "reason": "Needs JAWS on Windows, and 'announced clearly' is a human judgment. Playwright on Linux CI cannot drive JAWS. An automated accessibility scan (e.g. axe) could be added as a supplement, but it would not satisfy this row." + }, + { + "id": "MT-060", + "category": "Moderated usability study", + "reason": "Needs a moderated session with a real first-time customer. A script completing a transfer says nothing about whether a person can do it unaided in under 2 minutes." + } +] From 604bf8dc1949914b9648d4aa10cd1999d42975f8 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Fri, 23 Oct 2026 03:07:49 +0000 Subject: [PATCH 2/3] test: coverage check rejects skipped, fixme and test.fail rows (Playwright --list instead of source regex) Co-Authored-By: Garret Andreine <garret.andreine@cognition.ai> --- e2e/README.md | 2 +- e2e/scripts/coverage-matrix.mjs | 61 +++++++++++++++++++++++++-------- qa/coverage-matrix.md | 2 +- 3 files changed, 48 insertions(+), 17 deletions(-) diff --git a/e2e/README.md b/e2e/README.md index caa29f21..4a19bf1c 100644 --- a/e2e/README.md +++ b/e2e/README.md @@ -16,7 +16,7 @@ npx playwright test # BASE_URL defaults to http://localhost:8080 npm run matrix # rewrite ../qa/coverage-matrix.md with these results ``` -`npm run matrix:check` fails if a plan row is neither automated nor listed in `qa/not-automated.json`, is in both, or has two tests. +`npm run matrix:check` asks Playwright for the test list (`--list`) and fails if a plan row is neither automated nor listed in `qa/not-automated.json`, is in both, has two tests, or its test is `test.skip` / `test.fixme` / `test.fail`. With results (`npm run matrix`) it also fails on tests skipped at runtime. Tests register their own unique users, so they run in parallel and do not need a clean database. The Deposit/Withdraw/Transfer panels use Bootstrap from public CDNs, so the browser needs internet access. diff --git a/e2e/scripts/coverage-matrix.mjs b/e2e/scripts/coverage-matrix.mjs index 5b8b5c62..45a4741e 100644 --- a/e2e/scripts/coverage-matrix.mjs +++ b/e2e/scripts/coverage-matrix.mjs @@ -1,10 +1,12 @@ #!/usr/bin/env node // Builds qa/coverage-matrix.md from qa/manual-test-plan.csv, the Playwright specs, qa/not-automated.json -// and (when present) e2e/results.json. Exits 1 if any plan row is unaccounted for or accounted twice. +// and (when present) e2e/results.json. Exits 1 if any plan row is unaccounted for, accounted twice, +// or covered only by a skipped/fixme test. // Usage: node scripts/coverage-matrix.mjs [--check] [--results results.json] [--out ../qa/coverage-matrix.md] -import { readFileSync, readdirSync, writeFileSync, existsSync, appendFileSync } from 'node:fs'; +import { readFileSync, writeFileSync, existsSync, appendFileSync } from 'node:fs'; import { dirname, join, relative, resolve } from 'node:path'; import { fileURLToPath } from 'node:url'; +import { execFileSync } from 'node:child_process'; const e2eDir = resolve(dirname(fileURLToPath(import.meta.url)), '..'); const repoDir = resolve(e2eDir, '..'); @@ -51,18 +53,42 @@ for (const row of plan) { planIds.add(row.id); } -// Static scan: every test('MT-xxx ...') in the specs. +// Ask Playwright which tests exist (commented-out code never counts) and whether each is expected to run. +// A row only counts as automated if its test is expected to run and pass: test.skip / test.fixme / test.fail are +// rejected here, and runtime skips are rejected when results are supplied. +function listTests() { + const cli = join(e2eDir, 'node_modules', '@playwright', 'test', 'cli.js'); + const out = execFileSync(process.execPath, [cli, 'test', '--list', '--reporter=json'], { + cwd: e2eDir, + encoding: 'utf8', + maxBuffer: 64 * 1024 * 1024, + }); + const report = JSON.parse(out); + for (const e of report.errors ?? []) errors.push(`playwright --list: ${e.message ?? JSON.stringify(e)}`); + const specs = []; + const walk = (suite) => { + for (const spec of suite.specs ?? []) specs.push(spec); + for (const child of suite.suites ?? []) walk(child); + }; + for (const suite of report.suites ?? []) walk(suite); + return specs; +} + +const DISABLING = new Set(['skip', 'fixme', 'fail']); const automated = new Map(); -const testsDir = join(e2eDir, 'tests'); -for (const file of readdirSync(testsDir).filter((f) => f.endsWith('.spec.ts')).sort()) { - const src = readFileSync(join(testsDir, file), 'utf8'); - for (const m of src.matchAll(/\btest(?:\.\w+)?\(\s*['"`](MT-\d{3})\b([^'"`]*)['"`]/g)) { - const [, id, rest] = m; - const line = src.slice(0, m.index).split('\n').length; - const spec = `e2e/tests/${file}:${line}`; - if (automated.has(id)) errors.push(`${id} has more than one test (${automated.get(id).spec}, ${spec})`); - automated.set(id, { spec, title: `${id}${rest}` }); +for (const spec of listTests()) { + const id = spec.title.match(/^MT-\d{3}\b/)?.[0]; + if (!id) continue; + const where = `e2e/tests/${spec.file}:${spec.line}`; + if (automated.has(id)) errors.push(`${id} has more than one test (${automated.get(id).spec}, ${where})`); + const disabled = spec.tests.filter( + (t) => t.expectedStatus !== 'passed' || t.annotations.some((a) => DISABLING.has(a.type)), + ); + if (disabled.length) { + const how = [...new Set(disabled.flatMap((t) => t.annotations.map((a) => a.type)).concat(disabled.map((t) => t.expectedStatus)))]; + errors.push(`${id} test is disabled (${how.join(', ')}) at ${where}; a disabled test is not coverage, list the row in qa/not-automated.json instead`); } + automated.set(id, { spec: where, title: spec.title }); } const excluded = new Map(); @@ -91,7 +117,9 @@ if (haveResults) { const id = spec.title.match(/^MT-\d{3}/)?.[0]; if (!id) continue; const statuses = spec.tests.map((t) => t.status); - const status = statuses.includes('unexpected') ? 'failed' + const inverted = spec.tests.some((t) => t.expectedStatus !== 'passed' && t.expectedStatus !== 'skipped'); + const status = inverted ? 'inverted' + : statuses.includes('unexpected') ? 'failed' : statuses.includes('flaky') ? 'flaky' : statuses.every((s) => s === 'skipped') ? 'skipped' : 'passed'; results.set(id, status); @@ -100,11 +128,14 @@ if (haveResults) { }; for (const suite of report.suites ?? []) walk(suite); for (const id of automated.keys()) { - if (!results.has(id)) errors.push(`${id} has a test but no result in ${relative(repoDir, resultsPath)}`); + const r = results.get(id); + if (!r) errors.push(`${id} has a test but no result in ${relative(repoDir, resultsPath)}`); + else if (r === 'skipped') errors.push(`${id} was skipped at runtime; a skipped test is not coverage`); + else if (r === 'inverted') errors.push(`${id} is marked test.fail, so a failing check reports as expected; that is not coverage`); } } -const badge = { passed: 'PASS', failed: 'FAIL', flaky: 'FLAKY', skipped: 'SKIPPED' }; +const badge = { passed: 'PASS', failed: 'FAIL', flaky: 'FLAKY', skipped: 'SKIPPED', inverted: 'EXPECTED-FAIL' }; const esc = (s) => String(s).replace(/\|/g, '\\|').replace(/\n/g, ' '); const counts = { automated: 0, notAutomated: 0, passed: 0, failed: 0, other: 0 }; const lines = []; diff --git a/qa/coverage-matrix.md b/qa/coverage-matrix.md index 8e596fd8..9c060328 100644 --- a/qa/coverage-matrix.md +++ b/qa/coverage-matrix.md @@ -2,7 +2,7 @@ Generated by `e2e/scripts/coverage-matrix.mjs` from `qa/manual-test-plan.csv`, `e2e/tests/*.spec.ts` and `qa/not-automated.json`. Do not edit by hand. -Source: Playwright results from `e2e/results.json` (2026-10-23T02:54:31.202Z). +Source: Playwright results from `e2e/results.json` (2026-10-23T03:07:41.918Z). - Plan rows: **60** - Automated (Playwright, Chromium, against MySQL 8): **54** (passed 54, failed 0) From 14935d3d4fcf3f9a9f6c314523248e881ad179c9 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Fri, 23 Oct 2026 03:20:53 +0000 Subject: [PATCH 3/3] test: wait for Bootstrap collapse before opening dashboard panels (MT-020/MT-025 CI race) Co-Authored-By: Garret Andreine <garret.andreine@cognition.ai> --- e2e/tests/fixtures.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/e2e/tests/fixtures.ts b/e2e/tests/fixtures.ts index 829caa6c..c61266f8 100644 --- a/e2e/tests/fixtures.ts +++ b/e2e/tests/fixtures.ts @@ -58,6 +58,9 @@ const panelId = { deposit: '#depositForm', withdraw: '#withdrawForm', transfer: export type Panel = keyof typeof panelId; export async function openPanel(page: Page, panel: Panel) { + // The panels are Bootstrap collapses loaded from a CDN at the end of <body>; a click before the plugin is + // wired up does nothing, so wait for it (a login redirect can resolve toHaveURL before the scripts load). + await page.waitForFunction(() => typeof (window as any).jQuery?.fn?.collapse === 'function'); await page.getByRole('button', { name: panelButton[panel], exact: true }).click(); const form = page.locator(`${panelId[panel]} form`); await expect(form).toBeVisible();