diff --git a/.env b/.env new file mode 100644 index 00000000000..017a9b0c447 --- /dev/null +++ b/.env @@ -0,0 +1,7 @@ +# # PYTHONPATH=/home/gitpod/install/grass84/etc/python:/workspace/grass +# # PYTHONPATH=/home/gitpod/install/grass84/etc/python:/workspace/grass/python +# PYTHONPATH=/home/gitpod/install/grass84/etc/python +# # PATH=/home/gitpod/install/bin:$PATH +# PATH=/home/gitpod/install/bin:/home/gitpod/.sdkman/candidates/maven/current/bin:/home/gitpod/.sdkman/candidates/java/current/bin:/home/gitpod/.sdkman/candidates/gradle/current/bin:/workspace/.cargo/bin:/home/gitpod/.rvm/gems/ruby-3.2.4/bin:/home/gitpod/.rvm/gems/ruby-3.2.4@global/bin:/home/gitpod/.rvm/rubies/ruby-3.2.4/bin:/home/gitpod/.pyenv/shims:/workspace/go/bin:/home/gitpod/.nix-profile/bin:/ide/bin/remote-cli:/home/gitpod/go/bin:/home/gitpod/go-packages/bin:/home/gitpod/.nvm/versions/node/v20.13.1/bin:/home/gitpod/.yarn/bin:/home/gitpod/.pnpm:/home/gitpod/.pyenv/bin:/home/gitpod/.rvm/bin:/home/gitpod/.cargo/bin:/home/linuxbrew/.linuxbrew/bin:/home/linuxbrew/.linuxbrew/sbin/:/home/gitpod/.local/bin:/usr/games:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/gitpod/.nvm/versions/node/v20.13.1/bin:/home/gitpod/.rvm/bin +# LD_LIBRARY_PATH=/home/gitpod/install/grass84/lib +# # LD_LIBRARY_PATH=$HOME/install/grass84/lib diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index bfd8d77b693..f42b3684c6c 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -63,7 +63,8 @@ jobs: pip install --break-system-packages -r .github/workflows/optional_requirements.txt pip install --break-system-packages GDAL=="$(gdal-config --version)" pip install --break-system-packages pytest pytest-timeout \ - pytest-github-actions-annotate-failures pytest-xdist pytest-cov + pytest-github-actions-annotate-failures pytest-xdist pytest-cov \ + pytest-randomly - name: Create installation directory run: | diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 00000000000..b634547927f --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,16 @@ +{ + "python.envFile": "${workspaceFolder}/.env", + "python.testing.pytestArgs": [ + "--continue-on-collection-errors", + ".", + "--override-ini", + "pythonpath=/home/gitpod/install/grass84/etc/python", + "--numprocesses", + "auto", + "--color=yes", + "--durations=0", + "--durations-min=0.5" + ], + "python.testing.unittestEnabled": false, + "python.testing.pytestEnabled": true +} diff --git a/pyproject.toml b/pyproject.toml index 1d275ba8afe..d7ab82f0cce 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -526,10 +526,14 @@ addopts = """ --ignore-glob='dist.*' --ignore-glob='bin.*' --ignore-glob='*/tests/data/*' + --ignore-glob='*/gunittest/testsuite/data/*' + --ignore-glob='*/tests/*_test.py' --ignore-glob='*/testsuite/data/**' - --ignore-glob='*/grass/pygrass/tests/*' --doctest-glob='*doctest*.txt' --ignore='raster/r.category/test_rcategory_doctest.txt' + --continue-on-collection-errors + --import-mode=importlib + --ignore=gunittest/testsuite/data/ """ markers = [ "slow: marks tests as slow (deselect with '-m \"not slow\"')", diff --git a/python/grass/pygrass/modules/tests/grass_pygrass_grid_test.py b/python/grass/pygrass/modules/tests/grass_pygrass_grid_test.py index 5f49689688f..6659b0330df 100644 --- a/python/grass/pygrass/modules/tests/grass_pygrass_grid_test.py +++ b/python/grass/pygrass/modules/tests/grass_pygrass_grid_test.py @@ -15,8 +15,26 @@ ) -def max_processes(): +@pytest.fixture(scope="module") +def max_processes() -> int: """Get max useful number of parallel processes to run""" + import os + + if "PYTEST_XDIST_WORKER_COUNT" in os.environ: + n_workers = int(os.environ["PYTEST_XDIST_WORKER_COUNT"]) + allowed_logical_cpus = len(os.sched_getaffinity(0)) + return min(allowed_logical_cpus // n_workers, 4) + return min(multiprocessing.cpu_count(), 4) + + +def max_processes_result(): + """Get max useful number of parallel processes to run""" + import os + + if "PYTEST_XDIST_WORKER_COUNT" in os.environ: + n_workers = int(os.environ["PYTEST_XDIST_WORKER_COUNT"]) + allowed_logical_cpus = len(os.sched_getaffinity(0)) + return min(allowed_logical_cpus // n_workers, 4) return min(multiprocessing.cpu_count(), 4) @@ -32,7 +50,9 @@ def run_in_subprocess(function): @xfail_mp_spawn @pytest.mark.needs_solo_run -@pytest.mark.parametrize("processes", list(range(1, max_processes() + 1)) + [None]) +@pytest.mark.parametrize( + "processes", list(range(1, max_processes_result() + 1)) + [None] +) def test_processes(tmp_path, processes): """Check that running with multiple processes works""" project = tmp_path / "test" @@ -68,7 +88,8 @@ def run_grid_module(): @xfail_mp_spawn @pytest.mark.parametrize("width", [5, 10, 50]) # None does not work. @pytest.mark.parametrize("height", [5, 10, 50]) -def test_tiling_schemes(tmp_path, width, height): +# def test_tiling_schemes(tmp_path, width, height): +def test_tiling_schemes(tmp_path, width, height, max_processes): """Check that different shapes of tiles work""" project = tmp_path / "test" gs.create_project(project) @@ -84,7 +105,8 @@ def run_grid_module(): width=width, height=height, overlap=2, - processes=max_processes(), + processes=max_processes, + # processes=max_processes(), elevation=surface, slope="slope", aspect="aspect", @@ -99,7 +121,8 @@ def run_grid_module(): @xfail_mp_spawn @pytest.mark.parametrize("overlap", [0, 1, 2, 5]) -def test_overlaps(tmp_path, overlap): +# def test_overlaps(tmp_path, overlap): +def test_overlaps(tmp_path, overlap, max_processes): """Check that overlap accepts different values""" project = tmp_path / "test" gs.create_project(project) @@ -114,7 +137,8 @@ def run_grid_module(): width=10, height=5, overlap=overlap, - processes=max_processes(), + # processes=max_processes(), + processes=max_processes, elevation=surface, slope="slope", aspect="aspect", @@ -130,7 +154,8 @@ def run_grid_module(): @xfail_mp_spawn @pytest.mark.parametrize("clean", [True, False]) @pytest.mark.parametrize("surface", ["surface", "non_exist_surface"]) -def test_cleans(tmp_path, clean, surface): +# def test_cleans(tmp_path, clean, surface): +def test_cleans(tmp_path, clean, surface, max_processes): """Check that temporary mapsets are cleaned when appropriate""" project = tmp_path / "test" mapset_prefix = "abc" @@ -146,7 +171,8 @@ def run_grid_module(): width=10, height=5, overlap=0, - processes=max_processes(), + # processes=max_processes(), + processes=max_processes, elevation=surface, slope="slope", aspect="aspect", @@ -171,7 +197,8 @@ def run_grid_module(): @xfail_mp_spawn @pytest.mark.parametrize("patch_backend", [None, "r.patch", "RasterRow"]) -def test_patching_backend(tmp_path, patch_backend): +# def test_patching_backend(tmp_path, patch_backend): +def test_patching_backend(tmp_path, patch_backend, max_processes): """Check patching backend works""" project = tmp_path / "test" gs.create_project(project) @@ -192,7 +219,8 @@ def run_grid_module(): height=5, overlap=0, patch_backend=patch_backend, - processes=max_processes(), + # processes=max_processes(), + processes=max_processes, input=points, output="output", type="point", @@ -211,9 +239,9 @@ def run_grid_module(): @pytest.mark.parametrize( ("width", "height", "processes"), [ - (None, None, max_processes()), - (10, None, max_processes()), - (None, 5, max_processes()), + (None, None, max_processes_result()), + (10, None, max_processes_result()), + (None, 5, max_processes_result()), ], ) def test_tiling(tmp_path, width, height, processes):