diff --git a/.github/workflows/osgeo4w.yml b/.github/workflows/osgeo4w.yml index 96b5a6bb9aa..583bb0ac9b9 100644 --- a/.github/workflows/osgeo4w.yml +++ b/.github/workflows/osgeo4w.yml @@ -185,6 +185,7 @@ jobs: run: .github/workflows/test_thorough.bat '%O4WROOT%\opt\grass\grass86.bat' '%O4WROOT%\bin\python3' env: O4WROOT: ${{ steps.osgeo4w.outputs.root }} + GRASS_GUNITTEST_RANDOM_ORDER: "1" - name: Make HTML test report available if: ${{ always() }} diff --git a/python/grass/gunittest/loader.py b/python/grass/gunittest/loader.py index f536ab6a83d..3eebe5af748 100644 --- a/python/grass/gunittest/loader.py +++ b/python/grass/gunittest/loader.py @@ -11,6 +11,7 @@ import fnmatch import os +import random import re import unittest from pathlib import PurePath @@ -95,12 +96,25 @@ def discover_modules( :returns: a list of GrassTestPythonModule objects + Discovery order is alphabetical (both directories and files) unless the + ``GRASS_GUNITTEST_RANDOM_ORDER`` environment variable is set, in which + case both are shuffled instead. Since ``main.py`` runs the whole + discovered suite in a single process, a fixed order means whichever + test happens to run first in a given testsuite directory never changes, + which can hide an order-dependent bug (e.g. one that depends on some + state, such as a native library search path, established by an earlier + test) behind whatever the alphabetically-first test happens to do. + .. todo:: Implement import_modules. """ + random_order = bool(os.environ.get("GRASS_GUNITTEST_RANDOM_ORDER")) modules = [] for root, dirs, unused_files in os.walk(start_dir, topdown=True): - dirs.sort() + if random_order: + random.shuffle(dirs) + else: + dirs.sort() for dir_pattern in skip_dirs: to_skip = fnmatch.filter(dirs, dir_pattern) for skip in to_skip: @@ -118,7 +132,10 @@ def discover_modules( files = [f for f in files if re.match(file_regexp, f)] if exclude: files = fnmatch_exclude_with_base(files, full, exclude) - files = sorted(files) + if random_order: + random.shuffle(files) + else: + files = sorted(files) # get test/module name without .py # expecting all files to end with .py # this will not work for invoking bat files but it works fine