From 77d55fdffb44617f62e4542f57a7b4bfb3607eaa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edouard=20Choini=C3=A8re?= <27212526+echoix@users.noreply.github.com> Date: Mon, 20 May 2024 17:38:13 +0000 Subject: [PATCH 01/21] GridModule: Change max process to fixture --- pyproject.toml | 3 + .../modules/tests/grass_pygrass_grid_test.py | 61 ++++++++++++++----- 2 files changed, 49 insertions(+), 15 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index c99b85ab4dc..14acedf663b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -21,6 +21,9 @@ addopts = """ --ignore='raster/r.category/test_rcategory_doctest.txt' """ timeout = 300 +markers = [ + "serial", +] [tool.bandit] diff --git a/python/grass/pygrass/modules/tests/grass_pygrass_grid_test.py b/python/grass/pygrass/modules/tests/grass_pygrass_grid_test.py index c2ec828908f..3779c426ead 100644 --- a/python/grass/pygrass/modules/tests/grass_pygrass_grid_test.py +++ b/python/grass/pygrass/modules/tests/grass_pygrass_grid_test.py @@ -6,9 +6,23 @@ import grass.script as gs +@pytest.fixture(scope="module") +def max_processes() -> int: + """Get max useful number of parallel processes to run""" + import os + if "PYTEST_XDIST_WORKER_COUNT" in os.environ: + n_workers = int(os.environ["PYTEST_XDIST_WORKER_COUNT"]) + allowed_logical_cpus = len(os.sched_getaffinity(0)) + return min(allowed_logical_cpus // n_workers, 4) + return min(multiprocessing.cpu_count(), 4) -def max_processes(): +def max_processes_result(): """Get max useful number of parallel processes to run""" + import os + if "PYTEST_XDIST_WORKER_COUNT" in os.environ: + n_workers = int(os.environ["PYTEST_XDIST_WORKER_COUNT"]) + allowed_logical_cpus = len(os.sched_getaffinity(0)) + return min(allowed_logical_cpus // n_workers, 4) return min(multiprocessing.cpu_count(), 4) @@ -21,8 +35,9 @@ def run_in_subprocess(function): process.start() process.join() - -@pytest.mark.parametrize("processes", list(range(1, max_processes() + 1)) + [None]) +# @pytest.mark.parametrize("processes", list(range(1, max_processes() + 1)) + [None]) +@pytest.mark.serial +@pytest.mark.parametrize("processes", list(range(1, max_processes_result() + 1)) + [None]) def test_processes(tmp_path, processes): """Check that running with multiple processes works""" location = "test" @@ -61,7 +76,8 @@ def run_grid_module(): @pytest.mark.parametrize("width", [5, 10, 50]) # None does not work. @pytest.mark.parametrize("height", [5, 10, 50]) -def test_tiling_schemes(tmp_path, width, height): +# def test_tiling_schemes(tmp_path, width, height): +def test_tiling_schemes(tmp_path, width, height, max_processes): """Check that different shapes of tiles work""" location = "test" gs.core._create_location_xy(tmp_path, location) # pylint: disable=protected-access @@ -81,7 +97,8 @@ def run_grid_module(): width=width, height=height, overlap=2, - processes=max_processes(), + processes=max_processes, + # processes=max_processes(), elevation=surface, slope="slope", aspect="aspect", @@ -95,7 +112,8 @@ def run_grid_module(): @pytest.mark.parametrize("overlap", [0, 1, 2, 5]) -def test_overlaps(tmp_path, overlap): +# def test_overlaps(tmp_path, overlap): +def test_overlaps(tmp_path, overlap, max_processes): """Check that overlap accepts different values""" location = "test" gs.core._create_location_xy(tmp_path, location) # pylint: disable=protected-access @@ -114,7 +132,8 @@ def run_grid_module(): width=10, height=5, overlap=overlap, - processes=max_processes(), + # processes=max_processes(), + processes=max_processes, elevation=surface, slope="slope", aspect="aspect", @@ -129,7 +148,8 @@ def run_grid_module(): @pytest.mark.parametrize("clean", [True, False]) @pytest.mark.parametrize("surface", ["surface", "non_exist_surface"]) -def test_cleans(tmp_path, clean, surface): +# def test_cleans(tmp_path, clean, surface): +def test_cleans(tmp_path, clean, surface, max_processes): """Check that temporary mapsets are cleaned when appropriate""" location = "test" mapset_prefix = "abc" @@ -149,7 +169,8 @@ def run_grid_module(): width=10, height=5, overlap=0, - processes=max_processes(), + # processes=max_processes(), + processes=max_processes, elevation=surface, slope="slope", aspect="aspect", @@ -174,7 +195,8 @@ def run_grid_module(): @pytest.mark.parametrize("patch_backend", [None, "r.patch", "RasterRow"]) -def test_patching_backend(tmp_path, patch_backend): +# def test_patching_backend(tmp_path, patch_backend): +def test_patching_backend(tmp_path, patch_backend, max_processes): """Check patching backend works""" location = "test" gs.core._create_location_xy(tmp_path, location) # pylint: disable=protected-access @@ -199,7 +221,8 @@ def run_grid_module(): height=5, overlap=0, patch_backend=patch_backend, - processes=max_processes(), + # processes=max_processes(), + processes=max_processes, input=points, output="output", type="point", @@ -214,12 +237,20 @@ def run_grid_module(): assert abs(mean - mean_ref) < 0.0001 +# @pytest.mark.parametrize( +# "width, height, processes", +# [ +# (None, None, max_processes()), +# (10, None, max_processes()), +# (None, 5, max_processes()), +# ], +# ) @pytest.mark.parametrize( "width, height, processes", [ - (None, None, max_processes()), - (10, None, max_processes()), - (None, 5, max_processes()), + (None, None, max_processes_result()), + (10, None, max_processes_result()), + (None, 5, max_processes_result()), ], ) def test_tiling(tmp_path, width, height, processes): @@ -254,7 +285,7 @@ def run_grid_module(): info = gs.raster_info("slope") assert info["min"] > 0 - +@pytest.mark.serial @pytest.mark.parametrize( "processes, backend", [ From 7dc94fcd5cd93f46e03cb41552c14d09da2ccb78 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edouard=20Choini=C3=A8re?= <27212526+echoix@users.noreply.github.com> Date: Mon, 20 May 2024 18:04:45 +0000 Subject: [PATCH 02/21] pytest: Run serial tests separately Tests that already use multiple processes can use the "serial" marker to be ran without other tests running --- .github/workflows/pytest.yml | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 24e25052b70..2f97fb5c46c 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -77,7 +77,17 @@ jobs: run: | export PYTHONPATH=`grass --config python_path`:$PYTHONPATH export LD_LIBRARY_PATH=$HOME/install/grass84/lib:$LD_LIBRARY_PATH - pytest --verbose --color=yes --durations=0 --durations-min=0.5 --numprocesses auto -ra . + pytest --verbose --color=yes --durations=0 --durations-min=0.5 --numprocesses auto -ra . -m 'not serial' + - name: Run pytest (serial-only test) + run: | + export PYTHONPATH=`grass --config python_path`:$PYTHONPATH + export LD_LIBRARY_PATH=$HOME/install/grass84/lib:$LD_LIBRARY_PATH + pytest --verbose --color=yes --durations=0 --durations-min=0.5 --numprocesses 1 -ra . -m 'serial' + - name: Run pytest (serial-only test) + run: | + export PYTHONPATH=`grass --config python_path`:$PYTHONPATH + export LD_LIBRARY_PATH=$HOME/install/grass84/lib:$LD_LIBRARY_PATH + pytest --verbose --color=yes --durations=0 --durations-min=0.5 -ra . -m 'serial' - name: Print installed versions if: always() From 912f484e7cecad8f459d1f7c86a989f03fcf1247 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edouard=20Choini=C3=A8re?= <27212526+echoix@users.noreply.github.com> Date: Mon, 20 May 2024 20:19:06 +0000 Subject: [PATCH 03/21] pytest: .env and vscode configuration --- .github/workflows/.env | 2 ++ .vscode/settings.json | 9 +++++++++ 2 files changed, 11 insertions(+) create mode 100644 .github/workflows/.env create mode 100644 .vscode/settings.json diff --git a/.github/workflows/.env b/.github/workflows/.env new file mode 100644 index 00000000000..b654d017478 --- /dev/null +++ b/.github/workflows/.env @@ -0,0 +1,2 @@ +PYTHONPATH=/home/gitpod/install/grass84/etc/python:/workspace/grass +PATH=/home/gitpod/install/bin \ No newline at end of file diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 00000000000..4d17ec0cf6e --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,9 @@ +{ + "python.envFile": "${workspaceFolder}/.env", + "python.testing.pytestArgs": [ + ".", + "--continue-on-collection-errors" + ], + "python.testing.unittestEnabled": false, + "python.testing.pytestEnabled": true +} \ No newline at end of file From fdd3d414dc26c5751c2f3e2155e7ca6a8b2aa77d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edouard=20Choini=C3=A8re?= <27212526+echoix@users.noreply.github.com> Date: Mon, 20 May 2024 20:19:39 +0000 Subject: [PATCH 04/21] pytest: pyproject.toml --continue-on-collection-errors --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index 14acedf663b..b9b3fb4fb7d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -19,6 +19,7 @@ addopts = """ --ignore-glob='*/grass/pygrass/tests/*' --doctest-glob='*doctest*.txt' --ignore='raster/r.category/test_rcategory_doctest.txt' + --continue-on-collection-errors """ timeout = 300 markers = [ From 419810292b73c7a46fa964a3e46726c1657dbf46 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edouard=20Choini=C3=A8re?= <27212526+echoix@users.noreply.github.com> Date: Mon, 20 May 2024 20:19:54 +0000 Subject: [PATCH 05/21] pytest: pyproject.toml slow marker --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index b9b3fb4fb7d..5231aa27aac 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -23,6 +23,7 @@ addopts = """ """ timeout = 300 markers = [ + "slow: marks tests as slow (deselect with '-m \"not slow\"')", "serial", ] From 2cb9fa7312c22ec0bde21db77aca4d3568aae518 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edouard=20Choini=C3=A8re?= <27212526+echoix@users.noreply.github.com> Date: Mon, 20 May 2024 21:02:07 +0000 Subject: [PATCH 06/21] pytest: .env comment --- .github/workflows/.env | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/.github/workflows/.env b/.github/workflows/.env index b654d017478..8475da64fba 100644 --- a/.github/workflows/.env +++ b/.github/workflows/.env @@ -1,2 +1,7 @@ -PYTHONPATH=/home/gitpod/install/grass84/etc/python:/workspace/grass -PATH=/home/gitpod/install/bin \ No newline at end of file +# # PYTHONPATH=/home/gitpod/install/grass84/etc/python:/workspace/grass +# # PYTHONPATH=/home/gitpod/install/grass84/etc/python:/workspace/grass/python +# PYTHONPATH=/home/gitpod/install/grass84/etc/python +# # PATH=/home/gitpod/install/bin:$PATH +# PATH=/home/gitpod/install/bin:/home/gitpod/.sdkman/candidates/maven/current/bin:/home/gitpod/.sdkman/candidates/java/current/bin:/home/gitpod/.sdkman/candidates/gradle/current/bin:/workspace/.cargo/bin:/home/gitpod/.rvm/gems/ruby-3.2.4/bin:/home/gitpod/.rvm/gems/ruby-3.2.4@global/bin:/home/gitpod/.rvm/rubies/ruby-3.2.4/bin:/home/gitpod/.pyenv/shims:/workspace/go/bin:/home/gitpod/.nix-profile/bin:/ide/bin/remote-cli:/home/gitpod/go/bin:/home/gitpod/go-packages/bin:/home/gitpod/.nvm/versions/node/v20.13.1/bin:/home/gitpod/.yarn/bin:/home/gitpod/.pnpm:/home/gitpod/.pyenv/bin:/home/gitpod/.rvm/bin:/home/gitpod/.cargo/bin:/home/linuxbrew/.linuxbrew/bin:/home/linuxbrew/.linuxbrew/sbin/:/home/gitpod/.local/bin:/usr/games:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/gitpod/.nvm/versions/node/v20.13.1/bin:/home/gitpod/.rvm/bin +# LD_LIBRARY_PATH=/home/gitpod/install/grass84/lib +# # LD_LIBRARY_PATH=$HOME/install/grass84/lib \ No newline at end of file From 3c5958013533137a7cb5af15941f746ffe0a026e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edouard=20Choini=C3=A8re?= <27212526+echoix@users.noreply.github.com> Date: Mon, 20 May 2024 21:18:48 +0000 Subject: [PATCH 07/21] changes to vscode pytest settings --- .vscode/settings.json | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/.vscode/settings.json b/.vscode/settings.json index 4d17ec0cf6e..df00dfd987a 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,9 +1,16 @@ { - "python.envFile": "${workspaceFolder}/.env", - "python.testing.pytestArgs": [ - ".", - "--continue-on-collection-errors" - ], - "python.testing.unittestEnabled": false, - "python.testing.pytestEnabled": true -} \ No newline at end of file + "python.envFile": "${workspaceFolder}/.env", + "python.testing.pytestArgs": [ + "--continue-on-collection-errors", + ".", + "--override-ini", + "pythonpath=/home/gitpod/install/grass84/etc/python", + "--numprocesses", + "auto", + "--color=yes", + "--durations=0", + "--durations-min=0.5" + ], + "python.testing.unittestEnabled": false, + "python.testing.pytestEnabled": true +} From 5582b31a4625059f36483e6e38a42e23bfcb84bf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edouard=20Choini=C3=A8re?= <27212526+echoix@users.noreply.github.com> Date: Mon, 20 May 2024 21:19:04 +0000 Subject: [PATCH 08/21] changes to vscode pytest settings --- .vscode/settings.json | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.vscode/settings.json b/.vscode/settings.json index df00dfd987a..161c69b78c9 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,6 +1,7 @@ { "python.envFile": "${workspaceFolder}/.env", "python.testing.pytestArgs": [ + "-ra", "--continue-on-collection-errors", ".", "--override-ini", @@ -9,7 +10,8 @@ "auto", "--color=yes", "--durations=0", - "--durations-min=0.5" + "--durations-min=0.5", + "--verbose" ], "python.testing.unittestEnabled": false, "python.testing.pytestEnabled": true From 6742ecdd9e0836de02b28d547fd5414500cfd820 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edouard=20Choini=C3=A8re?= <27212526+echoix@users.noreply.github.com> Date: Mon, 20 May 2024 21:19:46 +0000 Subject: [PATCH 09/21] move .env --- .github/workflows/.env => .env | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/.env => .env (100%) diff --git a/.github/workflows/.env b/.env similarity index 100% rename from .github/workflows/.env rename to .env From 79589e1be13f96ad295d1099616c4037ddd139aa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edouard=20Choini=C3=A8re?= <27212526+echoix@users.noreply.github.com> Date: Tue, 21 May 2024 01:39:10 +0000 Subject: [PATCH 10/21] collect pytest for gunittest --- pyproject.toml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 5231aa27aac..669ed77de9f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,16 +11,18 @@ extend-exclude = ''' [tool.pytest.ini_options] minversion = "6.0" -python_files = "*/tests/*_test.py" +#python_files = "*/tests/*_test.py" addopts = """ --ignore-glob='dist.*' --ignore-glob='bin.*' --ignore-glob='*/tests/data/*' - --ignore-glob='*/grass/pygrass/tests/*' + --ignore-glob='*/gunittest/testsuite/data/*' + --ignore-glob='*/tests/*_test.py' --doctest-glob='*doctest*.txt' --ignore='raster/r.category/test_rcategory_doctest.txt' --continue-on-collection-errors """ +# --ignore-glob='*/grass/pygrass/tests/*' timeout = 300 markers = [ "slow: marks tests as slow (deselect with '-m \"not slow\"')", From 1aa4f1504d85f22c1ae28d8be83b0f83c50d6f69 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edouard=20Choini=C3=A8re?= <27212526+echoix@users.noreply.github.com> Date: Tue, 21 May 2024 01:39:43 +0000 Subject: [PATCH 11/21] change pytest vscode settings --- .vscode/settings.json | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.vscode/settings.json b/.vscode/settings.json index 161c69b78c9..df00dfd987a 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,7 +1,6 @@ { "python.envFile": "${workspaceFolder}/.env", "python.testing.pytestArgs": [ - "-ra", "--continue-on-collection-errors", ".", "--override-ini", @@ -10,8 +9,7 @@ "auto", "--color=yes", "--durations=0", - "--durations-min=0.5", - "--verbose" + "--durations-min=0.5" ], "python.testing.unittestEnabled": false, "python.testing.pytestEnabled": true From 459c56331e98bca5ec7de06f7407fa56e9be357d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edouard=20Choini=C3=A8re?= <27212526+echoix@users.noreply.github.com> Date: Wed, 22 May 2024 20:20:00 -0400 Subject: [PATCH 12/21] pytest: use importlib import mode --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index 669ed77de9f..2cfa8193326 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -21,6 +21,7 @@ addopts = """ --doctest-glob='*doctest*.txt' --ignore='raster/r.category/test_rcategory_doctest.txt' --continue-on-collection-errors + --import-mode=importlib """ # --ignore-glob='*/grass/pygrass/tests/*' timeout = 300 From 4e3d89ea6a410728e51a137ffe85aef6c5c319b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edouard=20Choini=C3=A8re?= <27212526+echoix@users.noreply.github.com> Date: Wed, 22 May 2024 21:03:49 -0400 Subject: [PATCH 13/21] Pytest: call inside grass --exec --- .github/workflows/pytest.yml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 2f97fb5c46c..87dd17edbc7 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -73,6 +73,18 @@ jobs: - name: Test executing of the grass command run: .github/workflows/test_simple.sh + - run: | + grass --tmp-location XY --exec \ + g.download.location url=https://grass.osgeo.org/sampledata/north_carolina/nc_spm_full_v2alpha2.tar.gz path=$HOME + + grass --tmp-location XY --exec \ + pytest -m grass.gunittest.main \ + --grassdata $HOME --location nc_spm_full_v2alpha2 --location-type nc \ + --min-success 100 $@ + continue-on-error: true + - run: | + grass --tmp-location XY --exec pytest + continue-on-error: true - name: Run pytest run: | export PYTHONPATH=`grass --config python_path`:$PYTHONPATH From 8858812c67dbc81a4be797476ef70fb9abfcdcd8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edouard=20Choini=C3=A8re?= <27212526+echoix@users.noreply.github.com> Date: Wed, 22 May 2024 21:38:39 -0400 Subject: [PATCH 14/21] Update pytest.yml console style count --- .github/workflows/pytest.yml | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 87dd17edbc7..4d48c0aacaa 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -74,16 +74,7 @@ jobs: run: .github/workflows/test_simple.sh - run: | - grass --tmp-location XY --exec \ - g.download.location url=https://grass.osgeo.org/sampledata/north_carolina/nc_spm_full_v2alpha2.tar.gz path=$HOME - - grass --tmp-location XY --exec \ - pytest -m grass.gunittest.main \ - --grassdata $HOME --location nc_spm_full_v2alpha2 --location-type nc \ - --min-success 100 $@ - continue-on-error: true - - run: | - grass --tmp-location XY --exec pytest + grass --tmp-location XY --exec pytest --verbose --color=yes --durations=0 --durations-min=0.005 --numprocesses auto -ra . -m 'not serial' --tb=line -o console_output_style=count continue-on-error: true - name: Run pytest run: | From a27126d03ab676d1a9ecab0585374bef80321597 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edouard=20Choini=C3=A8re?= <27212526+echoix@users.noreply.github.com> Date: Wed, 22 May 2024 22:08:35 -0400 Subject: [PATCH 15/21] Download test data in pytest.yml --- .github/workflows/pytest.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 4d48c0aacaa..cb6bfa70a94 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -74,6 +74,8 @@ jobs: run: .github/workflows/test_simple.sh - run: | + grass --tmp-location XY --exec \ + g.download.location url=https://grass.osgeo.org/sampledata/north_carolina/nc_spm_full_v2alpha2.tar.gz path=$HOME grass --tmp-location XY --exec pytest --verbose --color=yes --durations=0 --durations-min=0.005 --numprocesses auto -ra . -m 'not serial' --tb=line -o console_output_style=count continue-on-error: true - name: Run pytest From 75bf19364c2067de261d16219f09c00ef45c0891 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edouard=20Choini=C3=A8re?= <27212526+echoix@users.noreply.github.com> Date: Wed, 22 May 2024 22:33:01 -0400 Subject: [PATCH 16/21] Update pytest.yml try changing directory to home --- .github/workflows/pytest.yml | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index cb6bfa70a94..ec9e16e857e 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -19,11 +19,13 @@ jobs: matrix: os: - ubuntu-22.04 + - ubuntu-24.04 python-version: - '3.8' - '3.10' - '3.12' - fail-fast: true + - '3.13' + fail-fast: false #true runs-on: ${{ matrix.os }} env: @@ -76,8 +78,14 @@ jobs: - run: | grass --tmp-location XY --exec \ g.download.location url=https://grass.osgeo.org/sampledata/north_carolina/nc_spm_full_v2alpha2.tar.gz path=$HOME + cd $HOME grass --tmp-location XY --exec pytest --verbose --color=yes --durations=0 --durations-min=0.005 --numprocesses auto -ra . -m 'not serial' --tb=line -o console_output_style=count continue-on-error: true + + - run: | + cd $HOME + grass --tmp-location XY --exec pytest --verbose --color=yes --durations=0 --durations-min=0.005 -ra . -m 'not serial' --tb=line -o console_output_style=count + continue-on-error: true - name: Run pytest run: | export PYTHONPATH=`grass --config python_path`:$PYTHONPATH From d7870df4ab934a48419f67a49a1e92b538676617 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edouard=20Choini=C3=A8re?= <27212526+echoix@users.noreply.github.com> Date: Wed, 22 May 2024 23:11:50 -0400 Subject: [PATCH 17/21] Update pytest.yml randomly, -o faulthandler_timeout=45 --- .github/workflows/pytest.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index ec9e16e857e..c4a7070bae3 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -19,12 +19,12 @@ jobs: matrix: os: - ubuntu-22.04 - - ubuntu-24.04 + #- ubuntu-24.04 python-version: - '3.8' - '3.10' - '3.12' - - '3.13' + # - '3.13' fail-fast: false #true runs-on: ${{ matrix.os }} @@ -55,7 +55,7 @@ jobs: python -m pip install --upgrade pip pip install -r .github/workflows/python_requirements.txt pip install -r .github/workflows/optional_requirements.txt - pip install pytest pytest-timeout pytest-github-actions-annotate-failures pytest-xdist + pip install pytest pytest-timeout pytest-github-actions-annotate-failures pytest-xdist pytest-randomly - name: Create installation directory run: | @@ -79,12 +79,12 @@ jobs: grass --tmp-location XY --exec \ g.download.location url=https://grass.osgeo.org/sampledata/north_carolina/nc_spm_full_v2alpha2.tar.gz path=$HOME cd $HOME - grass --tmp-location XY --exec pytest --verbose --color=yes --durations=0 --durations-min=0.005 --numprocesses auto -ra . -m 'not serial' --tb=line -o console_output_style=count + grass --tmp-location XY --exec pytest --verbose --color=yes --durations=0 --durations-min=0.005 --numprocesses auto -ra . -m 'not serial' --tb=line -o console_output_style=count -o faulthandler_timeout=45 continue-on-error: true - run: | cd $HOME - grass --tmp-location XY --exec pytest --verbose --color=yes --durations=0 --durations-min=0.005 -ra . -m 'not serial' --tb=line -o console_output_style=count + grass --tmp-location XY --exec pytest --verbose --color=yes --durations=0 --durations-min=0.005 -ra . -m 'not serial' --tb=line -o console_output_style=count -o faulthandler_timeout=45 continue-on-error: true - name: Run pytest run: | From 1a9b32badbbbcc2472a58e67443df4df9d95393f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edouard=20Choini=C3=A8re?= <27212526+echoix@users.noreply.github.com> Date: Wed, 22 May 2024 23:56:58 -0400 Subject: [PATCH 18/21] Update pyproject.toml ignore gunittest/testsuite/data --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index 2cfa8193326..1c9bb30f7e2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -22,6 +22,7 @@ addopts = """ --ignore='raster/r.category/test_rcategory_doctest.txt' --continue-on-collection-errors --import-mode=importlib + --ignore=gunittest/testsuite/data/ """ # --ignore-glob='*/grass/pygrass/tests/*' timeout = 300 From 7f15fe92f88f7930374374533a2c5276db57cc7a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edouard=20Choini=C3=A8re?= <27212526+echoix@users.noreply.github.com> Date: Wed, 22 May 2024 23:58:37 -0400 Subject: [PATCH 19/21] Update pytest.yml --- .github/workflows/pytest.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index c4a7070bae3..9237caa95a5 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -78,10 +78,11 @@ jobs: - run: | grass --tmp-location XY --exec \ g.download.location url=https://grass.osgeo.org/sampledata/north_carolina/nc_spm_full_v2alpha2.tar.gz path=$HOME + continue-on-error: true + - run: | cd $HOME grass --tmp-location XY --exec pytest --verbose --color=yes --durations=0 --durations-min=0.005 --numprocesses auto -ra . -m 'not serial' --tb=line -o console_output_style=count -o faulthandler_timeout=45 continue-on-error: true - - run: | cd $HOME grass --tmp-location XY --exec pytest --verbose --color=yes --durations=0 --durations-min=0.005 -ra . -m 'not serial' --tb=line -o console_output_style=count -o faulthandler_timeout=45 From 4a1861d6c20d9c08f178c1c30fb6f1c3231d4905 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edouard=20Choini=C3=A8re?= <27212526+echoix@users.noreply.github.com> Date: Sat, 12 Sep 2026 19:18:42 -0400 Subject: [PATCH 20/21] Update grass_pygrass_grid_test.py Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- python/grass/pygrass/modules/tests/grass_pygrass_grid_test.py | 1 + 1 file changed, 1 insertion(+) diff --git a/python/grass/pygrass/modules/tests/grass_pygrass_grid_test.py b/python/grass/pygrass/modules/tests/grass_pygrass_grid_test.py index c14eb147e47..55eecc64e8b 100644 --- a/python/grass/pygrass/modules/tests/grass_pygrass_grid_test.py +++ b/python/grass/pygrass/modules/tests/grass_pygrass_grid_test.py @@ -14,6 +14,7 @@ strict=True, ) + @pytest.fixture(scope="module") def max_processes() -> int: """Get max useful number of parallel processes to run""" From 1665ee589cdec34b6a851a1055c402f706f949de Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edouard=20Choini=C3=A8re?= <27212526+echoix@users.noreply.github.com> Date: Sat, 12 Sep 2026 19:22:10 -0400 Subject: [PATCH 21/21] Apply batched suggestions from code review Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- .env | 2 +- .vscode/settings.json | 28 +++++++++---------- .../modules/tests/grass_pygrass_grid_test.py | 7 ++++- 3 files changed, 21 insertions(+), 16 deletions(-) diff --git a/.env b/.env index 8475da64fba..017a9b0c447 100644 --- a/.env +++ b/.env @@ -4,4 +4,4 @@ # # PATH=/home/gitpod/install/bin:$PATH # PATH=/home/gitpod/install/bin:/home/gitpod/.sdkman/candidates/maven/current/bin:/home/gitpod/.sdkman/candidates/java/current/bin:/home/gitpod/.sdkman/candidates/gradle/current/bin:/workspace/.cargo/bin:/home/gitpod/.rvm/gems/ruby-3.2.4/bin:/home/gitpod/.rvm/gems/ruby-3.2.4@global/bin:/home/gitpod/.rvm/rubies/ruby-3.2.4/bin:/home/gitpod/.pyenv/shims:/workspace/go/bin:/home/gitpod/.nix-profile/bin:/ide/bin/remote-cli:/home/gitpod/go/bin:/home/gitpod/go-packages/bin:/home/gitpod/.nvm/versions/node/v20.13.1/bin:/home/gitpod/.yarn/bin:/home/gitpod/.pnpm:/home/gitpod/.pyenv/bin:/home/gitpod/.rvm/bin:/home/gitpod/.cargo/bin:/home/linuxbrew/.linuxbrew/bin:/home/linuxbrew/.linuxbrew/sbin/:/home/gitpod/.local/bin:/usr/games:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/gitpod/.nvm/versions/node/v20.13.1/bin:/home/gitpod/.rvm/bin # LD_LIBRARY_PATH=/home/gitpod/install/grass84/lib -# # LD_LIBRARY_PATH=$HOME/install/grass84/lib \ No newline at end of file +# # LD_LIBRARY_PATH=$HOME/install/grass84/lib diff --git a/.vscode/settings.json b/.vscode/settings.json index df00dfd987a..b634547927f 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,16 +1,16 @@ { - "python.envFile": "${workspaceFolder}/.env", - "python.testing.pytestArgs": [ - "--continue-on-collection-errors", - ".", - "--override-ini", - "pythonpath=/home/gitpod/install/grass84/etc/python", - "--numprocesses", - "auto", - "--color=yes", - "--durations=0", - "--durations-min=0.5" - ], - "python.testing.unittestEnabled": false, - "python.testing.pytestEnabled": true + "python.envFile": "${workspaceFolder}/.env", + "python.testing.pytestArgs": [ + "--continue-on-collection-errors", + ".", + "--override-ini", + "pythonpath=/home/gitpod/install/grass84/etc/python", + "--numprocesses", + "auto", + "--color=yes", + "--durations=0", + "--durations-min=0.5" + ], + "python.testing.unittestEnabled": false, + "python.testing.pytestEnabled": true } diff --git a/python/grass/pygrass/modules/tests/grass_pygrass_grid_test.py b/python/grass/pygrass/modules/tests/grass_pygrass_grid_test.py index 55eecc64e8b..6659b0330df 100644 --- a/python/grass/pygrass/modules/tests/grass_pygrass_grid_test.py +++ b/python/grass/pygrass/modules/tests/grass_pygrass_grid_test.py @@ -19,15 +19,18 @@ def max_processes() -> int: """Get max useful number of parallel processes to run""" import os + if "PYTEST_XDIST_WORKER_COUNT" in os.environ: n_workers = int(os.environ["PYTEST_XDIST_WORKER_COUNT"]) allowed_logical_cpus = len(os.sched_getaffinity(0)) return min(allowed_logical_cpus // n_workers, 4) return min(multiprocessing.cpu_count(), 4) + def max_processes_result(): """Get max useful number of parallel processes to run""" import os + if "PYTEST_XDIST_WORKER_COUNT" in os.environ: n_workers = int(os.environ["PYTEST_XDIST_WORKER_COUNT"]) allowed_logical_cpus = len(os.sched_getaffinity(0)) @@ -47,7 +50,9 @@ def run_in_subprocess(function): @xfail_mp_spawn @pytest.mark.needs_solo_run -@pytest.mark.parametrize("processes", list(range(1, max_processes_result() + 1)) + [None]) +@pytest.mark.parametrize( + "processes", list(range(1, max_processes_result() + 1)) + [None] +) def test_processes(tmp_path, processes): """Check that running with multiple processes works""" project = tmp_path / "test"