From b8885fc352886948949fc68482150aa33a9b7f0b Mon Sep 17 00:00:00 2001 From: Pasit Sangprachathanarak Date: Mon, 27 Jul 2026 00:47:07 +0800 Subject: [PATCH 1/2] Add Playwright browser tests to CI pipeline --- .github/workflows/main.yml | 8 +++ Dockerfile | 2 + cmstestsuite/browser_tests/__init__.py | 19 ++++++++ cmstestsuite/browser_tests/conftest.py | 49 +++++++++++++++++++ cmstestsuite/browser_tests/runservice.sh | 62 ++++++++++++++++++++++++ cmstestsuite/browser_tests/test_aws.py | 28 +++++++++++ cmstestsuite/browser_tests/test_cws.py | 28 +++++++++++ docker/_cms-test-internal.sh | 9 +++- pyproject.toml | 1 + 9 files changed, 205 insertions(+), 1 deletion(-) create mode 100644 cmstestsuite/browser_tests/__init__.py create mode 100644 cmstestsuite/browser_tests/conftest.py create mode 100644 cmstestsuite/browser_tests/runservice.sh create mode 100644 cmstestsuite/browser_tests/test_aws.py create mode 100644 cmstestsuite/browser_tests/test_cws.py diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 1699261dbc..e514abd365 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -44,6 +44,14 @@ jobs: flags: unittests token: ${{ secrets.CODECOV_TOKEN }} + - name: Upload browser test results to Codecov + uses: codecov/codecov-action@v4 + if: ${{ !cancelled() }} + with: + files: ./codecov/browsertests.xml + flags: browsertests + token: ${{ secrets.CODECOV_TOKEN }} + - name: Upload functional test coverage results to Codecov uses: codecov/codecov-action@v4 if: ${{ !cancelled() }} diff --git a/Dockerfile b/Dockerfile index 9a2827e783..38848448f9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -87,6 +87,8 @@ COPY --chown=cmsuser:cmsuser . /home/cmsuser/src RUN --mount=type=cache,target=/home/cmsuser/.cache/pip,uid=2000 ./install.py cms --devel +RUN playwright install --with-deps chromium + RUN < +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +"""Browser-based tests for CWS and AWS.""" diff --git a/cmstestsuite/browser_tests/conftest.py b/cmstestsuite/browser_tests/conftest.py new file mode 100644 index 0000000000..fbf075b0d5 --- /dev/null +++ b/cmstestsuite/browser_tests/conftest.py @@ -0,0 +1,49 @@ +#!/usr/bin/env python3 + +# Contest Management System - http://cms-dev.github.io/ +# Copyright © 2026 Pasit Sangprachathanarak +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +import logging +import os +import sys +import pytest + +logger = logging.getLogger(__name__) + + +def pytest_configure(config): + """Initialize CONFIG dictionary before tests run.""" + from cmstestsuite import CONFIG + + CONFIG["CONFIG_PATH"] = os.path.join(sys.prefix, "etc/cms.toml") + + # Allow override via CMS_CONFIG env + if "CMS_CONFIG" in os.environ: + CONFIG["CONFIG_PATH"] = os.environ["CMS_CONFIG"] + + +@pytest.fixture(scope="session") +def browser_context_args(browser_context_args): + """Configure browser context for tests. + + Runs headless by default (for CI), but can be overridden + with pytest --headed flag for local debugging. + """ + return { + **browser_context_args, + "viewport": {"width": 1280, "height": 720}, + "ignore_https_errors": True, + } diff --git a/cmstestsuite/browser_tests/runservice.sh b/cmstestsuite/browser_tests/runservice.sh new file mode 100644 index 0000000000..ef1c1880c5 --- /dev/null +++ b/cmstestsuite/browser_tests/runservice.sh @@ -0,0 +1,62 @@ +#!/usr/bin/env bash + +set -e + +echo "Creating test contest..." +CONTEST_ID=$(python3 << 'EOF' +from cms.db import Contest, SessionGen + +with SessionGen() as session: + contest = Contest( + name="test_contest", + description="Browser test contest", + ) + session.add(contest) + session.commit() + print(contest.id) +EOF +) + +echo "Created contest with ID: $CONTEST_ID" + +# Start minimal services for browser tests +echo "Starting LogService..." +cmsLogService 0 & +LOG_PID=$! +sleep 2 + +echo "Starting AdminWebServer..." +cmsAdminWebServer 0 & +AWS_PID=$! + +echo "Starting ContestWebServer for contest $CONTEST_ID..." +cmsContestWebServer -c $CONTEST_ID 0 & +CWS_PID=$! + +# Wait for services to be ready +echo "Waiting for services to be ready..." +for i in {1..30}; do + if curl -s http://localhost:8889 > /dev/null 2>&1 && \ + curl -s http://localhost:8888 > /dev/null 2>&1; then + echo "Services are ready!" + break + fi + if [ $i -eq 30 ]; then + echo "Services failed to start within 30 seconds" + kill $CWS_PID $AWS_PID $LOG_PID 2>/dev/null || true + exit 1 + fi + sleep 1 +done + +# Run the browser tests +echo "Running browser tests..." +pytest cmstestsuite/browser_tests/ "$@" +TEST_EXIT=$? + +# Cleanup +echo "Stopping services..." +kill $CWS_PID $AWS_PID $LOG_PID 2>/dev/null || true +wait 2>/dev/null || true + +exit $TEST_EXIT diff --git a/cmstestsuite/browser_tests/test_aws.py b/cmstestsuite/browser_tests/test_aws.py new file mode 100644 index 0000000000..9e4132bd37 --- /dev/null +++ b/cmstestsuite/browser_tests/test_aws.py @@ -0,0 +1,28 @@ +#!/usr/bin/env python3 + +# Contest Management System - http://cms-dev.github.io/ +# Copyright © 2026 Pasit Sangprachathanarak +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +import pytest + + +def test_aws_responds(page): + """Test that AWS responds on port 8889.""" + # Navigate to AWS + page.goto("http://localhost:8889") + + # Just verify page loads (200 status) + assert page.url.startswith("http://localhost:8889") diff --git a/cmstestsuite/browser_tests/test_cws.py b/cmstestsuite/browser_tests/test_cws.py new file mode 100644 index 0000000000..e26b4408e9 --- /dev/null +++ b/cmstestsuite/browser_tests/test_cws.py @@ -0,0 +1,28 @@ +#!/usr/bin/env python3 + +# Contest Management System - http://cms-dev.github.io/ +# Copyright © 2026 Pasit Sangprachathanarak +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +import pytest + + +def test_cws_responds(page): + """Test that CWS responds on port 8888.""" + # Navigate to CWS + page.goto("http://localhost:8888") + + # Just verify page loads (200 status) + assert page.url.startswith("http://localhost:8888") diff --git a/docker/_cms-test-internal.sh b/docker/_cms-test-internal.sh index 6608e5ef47..44e13a26c4 100755 --- a/docker/_cms-test-internal.sh +++ b/docker/_cms-test-internal.sh @@ -13,6 +13,13 @@ dropdb --host=testdb --username=postgres cmsdbfortesting createdb --host=testdb --username=postgres cmsdbfortesting cmsInitDB +bash cmstestsuite/browser_tests/runservice.sh --junitxml=codecov/browsertests.xml +BROWSER=$? + +dropdb --host=testdb --username=postgres cmsdbfortesting +createdb --host=testdb --username=postgres cmsdbfortesting +cmsInitDB + cmsRunFunctionalTests -v --coverage codecov/functionaltests.xml FUNC=$? @@ -20,7 +27,7 @@ FUNC=$? # the CI as long as the functional tests are passing. Ideally we should get rid # of `cmsRunFunctionalTests` and make those tests work with pytest so they can # be auto-discovered and run in a single command. -if [ $UNIT -ne 0 ] || [ $FUNC -ne 0 ] +if [ $UNIT -ne 0 ] || [ $BROWSER -ne 0 ] || [ $FUNC -ne 0 ] then exit 1 else diff --git a/pyproject.toml b/pyproject.toml index 7cabb1ba1e..b0f03b85fa 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -57,6 +57,7 @@ devel = [ "coverage>=4.5,<7.10", "pytest", "pytest-cov", + "pytest-playwright", # Only for building documentation # XXX: The version of Sphinx needed to build our documentation From 94f7845901c841d851ab158413c21e647d6b0bfd Mon Sep 17 00:00:00 2001 From: Pasit Sangprachathanarak Date: Mon, 27 Jul 2026 15:14:40 +0800 Subject: [PATCH 2/2] Fix --- cmstestsuite/browser_tests/runservice.sh | 93 +++++++++++++++++------- docker/_cms-test-internal.sh | 6 -- 2 files changed, 68 insertions(+), 31 deletions(-) diff --git a/cmstestsuite/browser_tests/runservice.sh b/cmstestsuite/browser_tests/runservice.sh index ef1c1880c5..a4411d7fe2 100644 --- a/cmstestsuite/browser_tests/runservice.sh +++ b/cmstestsuite/browser_tests/runservice.sh @@ -2,47 +2,90 @@ set -e -echo "Creating test contest..." -CONTEST_ID=$(python3 << 'EOF' -from cms.db import Contest, SessionGen - -with SessionGen() as session: - contest = Contest( - name="test_contest", - description="Browser test contest", - ) - session.add(contest) - session.commit() - print(contest.id) -EOF -) - -echo "Created contest with ID: $CONTEST_ID" - -# Start minimal services for browser tests +# Start LogService first (needed by AdminWebServer) echo "Starting LogService..." cmsLogService 0 & LOG_PID=$! sleep 2 +# Start AdminWebServer (needed to create contest properly) echo "Starting AdminWebServer..." cmsAdminWebServer 0 & AWS_PID=$! +# Wait for AdminWebServer to be ready +echo "Waiting for AdminWebServer..." +for i in {1..20}; do + if curl -s http://localhost:8889 > /dev/null 2>&1; then + break + fi + if [ $i -eq 20 ]; then + echo "AdminWebServer failed to start" + kill $AWS_PID $LOG_PID 2>/dev/null || true + exit 1 + fi + sleep 1 +done + +# Create admin and contest via FunctionalTestFramework (creates proper main_group) +echo "Setting up admin and contest..." +CONTEST_ID=$(python3 << 'EOF' +import datetime +import os +import sys + +from cms import TOKEN_MODE_FINITE +from cmscommon.datetime import get_system_timezone +from cmstestsuite import CONFIG +from cmstestsuite.functionaltestframework import FunctionalTestFramework + +CONFIG["CONFIG_PATH"] = os.path.join(sys.prefix, "etc/cms.toml") +if "CMS_CONFIG" in os.environ: + CONFIG["CONFIG_PATH"] = os.environ["CMS_CONFIG"] + +framework = FunctionalTestFramework() +framework.initialize_aws() + +start_time = datetime.datetime.utcnow() +stop_time = start_time + datetime.timedelta(hours=2) + +contest_id, _ = framework.add_contest( + name="test_contest", + description="Browser test contest", + languages=["C++17 / g++"], + allow_password_authentication="checked", + start=start_time.strftime("%Y-%m-%d %H:%M:%S.%f"), + stop=stop_time.strftime("%Y-%m-%d %H:%M:%S.%f"), + timezone=get_system_timezone(), + allow_user_tests="checked", + token_mode=TOKEN_MODE_FINITE, + token_max_number="100", + token_min_interval="0", + token_gen_initial="100", + token_gen_number="0", + token_gen_interval="1", + token_gen_max="100", +) +print(contest_id) +EOF +) + +echo "Created contest with ID: $CONTEST_ID" + +# Start ContestWebServer with the proper contest echo "Starting ContestWebServer for contest $CONTEST_ID..." cmsContestWebServer -c $CONTEST_ID 0 & CWS_PID=$! -# Wait for services to be ready -echo "Waiting for services to be ready..." -for i in {1..30}; do - if curl -s http://localhost:8889 > /dev/null 2>&1 && \ - curl -s http://localhost:8888 > /dev/null 2>&1; then +# Wait for ContestWebServer to be ready +echo "Waiting for ContestWebServer..." +for i in {1..20}; do + if curl -s http://localhost:8888 > /dev/null 2>&1; then echo "Services are ready!" break fi - if [ $i -eq 30 ]; then - echo "Services failed to start within 30 seconds" + if [ $i -eq 20 ]; then + echo "ContestWebServer failed to start" kill $CWS_PID $AWS_PID $LOG_PID 2>/dev/null || true exit 1 fi diff --git a/docker/_cms-test-internal.sh b/docker/_cms-test-internal.sh index 44e13a26c4..c2cdb89979 100755 --- a/docker/_cms-test-internal.sh +++ b/docker/_cms-test-internal.sh @@ -9,17 +9,11 @@ cmsInitDB pytest --cov . --cov-report xml:codecov/unittests.xml --junitxml=codecov/junit.xml -o junit_family=legacy UNIT=$? -dropdb --host=testdb --username=postgres cmsdbfortesting -createdb --host=testdb --username=postgres cmsdbfortesting cmsInitDB bash cmstestsuite/browser_tests/runservice.sh --junitxml=codecov/browsertests.xml BROWSER=$? -dropdb --host=testdb --username=postgres cmsdbfortesting -createdb --host=testdb --username=postgres cmsdbfortesting -cmsInitDB - cmsRunFunctionalTests -v --coverage codecov/functionaltests.xml FUNC=$?