diff --git a/.github/workflows/test_thorough.bat b/.github/workflows/test_thorough.bat index d24276b48a1..e61e58aec7a 100644 --- a/.github/workflows/test_thorough.bat +++ b/.github/workflows/test_thorough.bat @@ -2,4 +2,4 @@ set grass=%1 set python=%2 call %grass% --tmp-project XY --exec g.download.project url=https://grass.osgeo.org/sampledata/north_carolina/nc_spm_full_v2beta1.tar.gz path=%USERPROFILE% -call %grass% --tmp-project XY --exec %python% -m grass.gunittest.main --grassdata %USERPROFILE% --location nc_spm_full_v2beta1 --location-type nc --min-success 97.3 --config .github\workflows\osgeo4w_gunittest.cfg +call %grass% --tmp-project XY --exec %python% -m grass.gunittest.main --grassdata %USERPROFILE% --location nc_spm_full_v2beta1 --location-type nc --min-success 97.3 --random-order --config .github\workflows\osgeo4w_gunittest.cfg diff --git a/.github/workflows/test_thorough.sh b/.github/workflows/test_thorough.sh index 31b225dc26e..dfc121d3c78 100755 --- a/.github/workflows/test_thorough.sh +++ b/.github/workflows/test_thorough.sh @@ -11,4 +11,4 @@ grass --tmp-project XY --exec \ grass --tmp-project XY --exec \ python3 -m grass.gunittest.main \ --grassdata $HOME --location nc_spm_full_v2beta1 --location-type nc \ - --min-success 100 $@ + --min-success 100 --random-order $@ diff --git a/python/grass/docs/src/gunittest_running_tests.rst b/python/grass/docs/src/gunittest_running_tests.rst index 4ccccbcf523..8272474a67b 100644 --- a/python/grass/docs/src/gunittest_running_tests.rst +++ b/python/grass/docs/src/gunittest_running_tests.rst @@ -83,6 +83,32 @@ to run the test and ``grass.gunittest.main`` returning zero return code even if some tests fail, use ``--min-success 0`` +Running test files in random order +---------------------------------- + +By default, test files run in the same order every time. A fixed order can +hide a test which passes only because of what an earlier test did, +for example a file an earlier test left in a shared directory. To run the +test files in a random order, use:: + + python -m grass.gunittest.main ... --random-order + +The seed of the random order is printed at the start and stored in the +report. To run the test files again in the same order, pass this seed:: + + python -m grass.gunittest.main ... --randomly-seed 1234 + +To reuse the seed of the previous run, which is read from the report in the +output directory, use ``--randomly-seed last``. + +The same seed gives the same order only for the same set of test files. +Regardless of the order the test files run in, the reports list them in the +usual order, so that reports from different runs can be compared. +Tests inside a test file still run in their usual order. +The ``--randomly-seed`` option and the printed seed follow the +*pytest-randomly* plugin, but the order is random only when requested. + + Excluding test files from testing --------------------------------- diff --git a/python/grass/gunittest/invoker.py b/python/grass/gunittest/invoker.py index 7805902c7bd..cbe60f0eee4 100644 --- a/python/grass/gunittest/invoker.py +++ b/python/grass/gunittest/invoker.py @@ -12,6 +12,7 @@ import collections import os from pathlib import Path +import random import shutil import subprocess import sys @@ -140,7 +141,6 @@ def _run_test_module( self, module, results_dir: StrPath, gisdbase, location, timeout: float | None ) -> None: """Run one test file.""" - self.testsuite_dirs[module.tested_dir].append(module.name) cwd: str = os.path.join(results_dir, module.tested_dir, module.name) data_dir: str = os.path.join(module.file_dir, "data") if Path(data_dir).exists(): @@ -284,7 +284,13 @@ def try_decode(data, encodings): shutil.rmtree(mapset_dir, ignore_errors=True) def run_in_location( - self, gisdbase, location, location_type, results_dir: StrPath, exclude + self, + gisdbase, + location, + location_type, + results_dir: StrPath, + exclude, + seed: int | None = None, ) -> GrassTestFilesMultiReporter: """Run tests in a given location @@ -292,6 +298,11 @@ def run_in_location( i.e., a file-oriented reporter as opposed to testsuite-oriented one. Use only the attributes related to the summary, such as file_pass_per, not to one file as these will simply contain the last executed file. + + :param seed: if not None, test files run in a random order + generated from this seed (the same seed gives the same order + for the same set of test files); reports still list the test + files in discovery order """ if os.path.abspath(results_dir) == os.path.abspath(self.start_dir): msg = ( @@ -299,6 +310,9 @@ def run_in_location( " as discovery start directory" ) raise RuntimeError(msg) + info = {"location": location, "location_type": location_type} + if seed is not None: + info["random_seed"] = seed self.reporter = GrassTestFilesMultiReporter( reporters=[ GrassTestFilesTextReporter(stream=sys.stderr), @@ -306,9 +320,7 @@ def run_in_location( file_anonymizer=self._file_anonymizer, main_page_name="testfiles.html", ), - GrassTestFilesKeyValueReporter( - info={"location": location, "location_type": location_type} - ), + GrassTestFilesKeyValueReporter(info=info), ] ) self.testsuite_dirs = collections.defaultdict( @@ -326,6 +338,10 @@ def run_in_location( import_modules=False, exclude=exclude, ) + for module in modules: + self.testsuite_dirs[module.tested_dir].append(module.name) + if seed is not None: + random.Random(seed).shuffle(modules) self.reporter.start(results_dir) for module in modules: @@ -340,16 +356,23 @@ def run_in_location( # TODO: move this to some (new?) reporter # TODO: add basic summary of linked files so that the page is not empty + if seed is None: + order_text = "" + else: + order_text = f"

Test files ran in random order with seed {seed}.

" Path(os.path.join(results_dir, "index.html")).write_text( "" "

Tests for <{location}>" " using <{type}> type tests

" + "{order_text}" "