From 0a2c2e410faae72bbcbe9418006b92bf83e955b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edouard=20Choini=C3=A8re?= <27212526+echoix@users.noreply.github.com> Date: Sat, 26 Sep 2026 11:46:11 -0400 Subject: [PATCH 1/4] CI: Capture Windows loader snaps to find the exact unresolved symbol (not for merge) An exhaustive, cross-checked symbol-level graph covering grass_gis.8.6.dll's entire dependency closure -- gdal313.dll, curl, OpenSSL (libssl/libcrypto), libjpeg-turbo, expat, PROJ, sqlite3, libxml2, brotli, zstd, msvcrt2019, and most MSYS2-provided libraries -- comes back completely clean: every symbol each of these needs from any of the others is present. Rather than keep guessing which single binary might have an unresolved symbol, this asks Windows' own loader directly. Enables Loader Snaps (verbose OS-level trace of every module load and symbol resolution step) for python.exe via the Image File Execution Options GlobalFlag, captures it with Sysinternals DebugView, and runs `python -m pytest` (which reliably crashes with "Windows fatal exception: code 0xc0000139") underneath it. Also dumps the real libgomp-1.dll/libsystre-0.dll exports directly from the CI environment, completing the dependency graph check (only spot-checked locally against non-identical builds so far). Temporary, for CI investigation only. Co-Authored-By: Claude Sonnet 5 --- .github/workflows/osgeo4w.yml | 72 +++++++++++++++++++++++++++++++++++ 1 file changed, 72 insertions(+) diff --git a/.github/workflows/osgeo4w.yml b/.github/workflows/osgeo4w.yml index 96b5a6bb9aa..49084822a56 100644 --- a/.github/workflows/osgeo4w.yml +++ b/.github/workflows/osgeo4w.yml @@ -133,6 +133,78 @@ jobs: pytest-randomly # Obfuscated usage of GitHub Actions features, Windows CMD shell limits analysis shell: cmd /D /E:ON /V:OFF /S /C "CALL C:/OSGeo4W/OSGeo4W.bat "{0}"" # zizmor: ignore[misfeature] + + - name: "[CI diagnostic, not for merge] Dump libgomp-1.dll / libsystre-0.dll exports" + # Completes an exhaustive symbol-level cross-check of grass_gis.8.6.dll's + # entire non-system dependency closure (gdal313.dll, curl, openssl, + # libjpeg-turbo, expat, PROJ, sqlite3, libxml2, brotli, zstd all + # already checked clean offline). These two MSYS2-provided libraries + # could only be spot-checked locally against a non-identical build, + # so dump the real ones here. + shell: pwsh + run: | + $ErrorActionPreference = 'Stop' + $vswhere = "${env:ProgramFiles(x86)}\Microsoft Visual Studio\Installer\vswhere.exe" + $dumpbin = & $vswhere -latest -products * -find '**\dumpbin.exe' | Select-Object -First 1 + foreach ($dll in @('libgomp-1.dll', 'libsystre-0.dll')) { + $path = "D:\msys64\mingw64\bin\$dll" + if (Test-Path $path) { + Write-Output "::group::$dll exports" + & $dumpbin /exports $path + Write-Output '::endgroup::' + } else { + Write-Output "::warning::$path not found" + } + } + + - name: "[CI diagnostic, not for merge] Capture Windows loader snaps around the crash" + # Every native dependency in this investigation has been checked + # clean via manual dumpbin export/import diffing (GDAL, curl, + # OpenSSL, libjpeg-turbo, expat, PROJ, sqlite3, libxml2, brotli, + # zstd, msvcrt2019, and most MSYS2-provided libraries). Rather + # than continue guessing which binary might have an unresolved + # symbol, this asks Windows' own loader directly: enables Loader + # Snaps (verbose OS-level trace of every module load and symbol + # resolution step) for python.exe via the Image File Execution + # Options GlobalFlag, captures it with Sysinternals DebugView, + # and runs `python -m pytest` (which reliably crashes with + # "Windows fatal exception: code 0xc0000139") underneath it. + shell: pwsh + run: | + $ErrorActionPreference = 'Stop' + + Write-Output '::group::Enable loader snaps for python.exe' + $ifeoPath = 'HKLM:\SOFTWARE\Microsoft\Windows NT\CurrentVersion\Image File Execution Options\python.exe' + New-Item -Path $ifeoPath -Force | Out-Null + New-ItemProperty -Path $ifeoPath -Name 'GlobalFlag' -PropertyType DWord -Value 0x2 -Force | Out-Null + Write-Output '::endgroup::' + + Write-Output '::group::Download DebugView' + Invoke-WebRequest -Uri 'https://download.sysinternals.com/files/DebugView.zip' -OutFile "$env:RUNNER_TEMP\DebugView.zip" + Expand-Archive -Path "$env:RUNNER_TEMP\DebugView.zip" -DestinationPath "$env:RUNNER_TEMP\DebugView" + Write-Output '::endgroup::' + + $logfile = "$env:RUNNER_TEMP\loadersnaps.log" + $dbgview = Start-Process -FilePath "$env:RUNNER_TEMP\DebugView\Dbgview64.exe" -ArgumentList @('/accepteula', '/l', $logfile, '/min') -PassThru + Start-Sleep -Seconds 5 + + Write-Output '::group::Run python -m pytest under loader snaps' + $cmd = 'call "C:\OSGeo4W\opt\grass\etc\env.bat" && set PYTHONPATH=%GISBASE%\etc\python;%PYTHONPATH% && path %GISBASE%\lib;%GISBASE%\bin;%PATH% && python -m pytest -v imagery/i.gensig/testsuite/test_i_gensig.py lib/arraystats/tests' + & "C:\OSGeo4W\OSGeo4W.bat" cmd /c $cmd + Write-Output '::endgroup::' + + Start-Sleep -Seconds 5 + Stop-Process -Id $dbgview.Id -Force -ErrorAction SilentlyContinue + + Write-Output '::group::Loader snaps log (filtered)' + if (Test-Path $logfile) { + Get-Content $logfile | Select-String -Pattern 'python|grass|proj_9|gdal313|not found|procedure' | Select-Object -First 500 + } else { + Write-Output '::error::No loader snaps log was produced' + } + Write-Output '::endgroup::' + continue-on-error: true + - name: Run pytest with multiple workers in parallel run: | call %OSGEO4W_ROOT%\opt\grass\etc\env.bat From 245ece21136199776c6f9326dbcfcccce2342c83 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edouard=20Choini=C3=A8re?= <27212526+echoix@users.noreply.github.com> Date: Sat, 26 Sep 2026 12:10:47 -0400 Subject: [PATCH 2/4] CI: Fix loader snaps capture (split into separate steps) (not for merge) The previous version wrapped the actual `python -m pytest` invocation in a PowerShell string passed through `& "OSGeo4W.bat" cmd /c $cmd`, which corrupted the huge %PATH% value during the nested cmd.exe re-invocation ("\NSIS\ was unexpected at this time"), so python never actually ran inside the loader-snaps capture window at all -- the step produced no log and errored out before testing anything. This splits the diagnostic into three steps: start DebugView/enable loader snaps (PowerShell), run python -m pytest using the exact same proven cmd/OSGeo4W.bat wrapper every other pytest step in this workflow already uses, then stop DebugView and dump the trace (PowerShell). Confirmed via a previous run of this PR: every native dependency in grass_gis.8.6.dll's dependency graph (GDAL, curl, OpenSSL, libjpeg-turbo, expat, PROJ, sqlite3, libxml2, brotli, zstd, msvcrt2019, and now all 6 MSYS2-provided libraries pulled directly from the CI environment) is symbol-level clean, so this is the last remaining way to directly identify the actual failure without guessing further. Temporary, for CI investigation only. Co-Authored-By: Claude Sonnet 5 --- .github/workflows/osgeo4w.yml | 52 ++++++++++++++++++++++------------- 1 file changed, 33 insertions(+), 19 deletions(-) diff --git a/.github/workflows/osgeo4w.yml b/.github/workflows/osgeo4w.yml index 49084822a56..e668c63e79c 100644 --- a/.github/workflows/osgeo4w.yml +++ b/.github/workflows/osgeo4w.yml @@ -157,53 +157,67 @@ jobs: } } - - name: "[CI diagnostic, not for merge] Capture Windows loader snaps around the crash" + - name: "[CI diagnostic, not for merge] Start loader snaps capture" # Every native dependency in this investigation has been checked # clean via manual dumpbin export/import diffing (GDAL, curl, # OpenSSL, libjpeg-turbo, expat, PROJ, sqlite3, libxml2, brotli, - # zstd, msvcrt2019, and most MSYS2-provided libraries). Rather + # zstd, msvcrt2019, and all MSYS2-provided libraries). Rather # than continue guessing which binary might have an unresolved # symbol, this asks Windows' own loader directly: enables Loader # Snaps (verbose OS-level trace of every module load and symbol # resolution step) for python.exe via the Image File Execution - # Options GlobalFlag, captures it with Sysinternals DebugView, - # and runs `python -m pytest` (which reliably crashes with - # "Windows fatal exception: code 0xc0000139") underneath it. + # Options GlobalFlag, captured with Sysinternals DebugView. Kept + # in its own step (rather than wrapping the actual pytest + # invocation in PowerShell string interpolation) because nesting + # OSGeo4W.bat's own cmd /c re-invocation inside a PowerShell + # -ArgumentList corrupts the huge %PATH% value. shell: pwsh run: | $ErrorActionPreference = 'Stop' - - Write-Output '::group::Enable loader snaps for python.exe' $ifeoPath = 'HKLM:\SOFTWARE\Microsoft\Windows NT\CurrentVersion\Image File Execution Options\python.exe' New-Item -Path $ifeoPath -Force | Out-Null New-ItemProperty -Path $ifeoPath -Name 'GlobalFlag' -PropertyType DWord -Value 0x2 -Force | Out-Null - Write-Output '::endgroup::' - Write-Output '::group::Download DebugView' Invoke-WebRequest -Uri 'https://download.sysinternals.com/files/DebugView.zip' -OutFile "$env:RUNNER_TEMP\DebugView.zip" Expand-Archive -Path "$env:RUNNER_TEMP\DebugView.zip" -DestinationPath "$env:RUNNER_TEMP\DebugView" - Write-Output '::endgroup::' $logfile = "$env:RUNNER_TEMP\loadersnaps.log" $dbgview = Start-Process -FilePath "$env:RUNNER_TEMP\DebugView\Dbgview64.exe" -ArgumentList @('/accepteula', '/l', $logfile, '/min') -PassThru + Set-Content -Path "$env:RUNNER_TEMP\dbgview.pid" -Value $dbgview.Id Start-Sleep -Seconds 5 + Write-Output "Started DebugView (pid $($dbgview.Id)), logging to $logfile" - Write-Output '::group::Run python -m pytest under loader snaps' - $cmd = 'call "C:\OSGeo4W\opt\grass\etc\env.bat" && set PYTHONPATH=%GISBASE%\etc\python;%PYTHONPATH% && path %GISBASE%\lib;%GISBASE%\bin;%PATH% && python -m pytest -v imagery/i.gensig/testsuite/test_i_gensig.py lib/arraystats/tests' - & "C:\OSGeo4W\OSGeo4W.bat" cmd /c $cmd - Write-Output '::endgroup::' + - name: "[CI diagnostic, not for merge] Run python -m pytest under loader snaps" + run: | + call C:\OSGeo4W\opt\grass\etc\env.bat + set PYTHONPATH=%GISBASE%\etc\python;%PYTHONPATH% + path %GISBASE%\lib;%GISBASE%\bin;%PATH% + python -m pytest -v imagery/i.gensig/testsuite/test_i_gensig.py lib/arraystats/tests + # Obfuscated usage of GitHub Actions features, Windows CMD shell limits analysis + shell: cmd /D /E:ON /V:OFF /S /C "CALL C:/OSGeo4W/OSGeo4W.bat "{0}"" # zizmor: ignore[misfeature] + continue-on-error: true + - name: "[CI diagnostic, not for merge] Stop loader snaps capture and dump the trace" + if: always() + shell: pwsh + run: | Start-Sleep -Seconds 5 - Stop-Process -Id $dbgview.Id -Force -ErrorAction SilentlyContinue + $pidFile = "$env:RUNNER_TEMP\dbgview.pid" + if (Test-Path $pidFile) { + Stop-Process -Id (Get-Content $pidFile) -Force -ErrorAction SilentlyContinue + } - Write-Output '::group::Loader snaps log (filtered)' + $logfile = "$env:RUNNER_TEMP\loadersnaps.log" if (Test-Path $logfile) { - Get-Content $logfile | Select-String -Pattern 'python|grass|proj_9|gdal313|not found|procedure' | Select-Object -First 500 + Write-Output "::group::Full loader snaps log line count" + (Get-Content $logfile).Count + Write-Output '::endgroup::' + Write-Output '::group::Loader snaps log (filtered for python/grass/not found/procedure)' + Get-Content $logfile | Select-String -Pattern 'python|grass_|proj_9|gdal313|not found|procedure|LDR' | Select-Object -First 800 + Write-Output '::endgroup::' } else { Write-Output '::error::No loader snaps log was produced' } - Write-Output '::endgroup::' - continue-on-error: true - name: Run pytest with multiple workers in parallel run: | From 299545cc3f905af831cd6c873a2bf6cd999aca3d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edouard=20Choini=C3=A8re?= <27212526+echoix@users.noreply.github.com> Date: Sat, 26 Sep 2026 12:34:19 -0400 Subject: [PATCH 3/4] CI: Enable kernel-mode capture in DebugView for loader snaps (not for merge) The previous run's crash reproduced correctly under the capture window, but the loader snaps log came back empty. Windows Loader Snaps are emitted through the kernel-mode DbgPrint path, not Win32 OutputDebugString, and DebugView only listens to DbgPrint when its /k (capture kernel) switch is passed. Co-Authored-By: Claude Sonnet 5 --- .github/workflows/osgeo4w.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/osgeo4w.yml b/.github/workflows/osgeo4w.yml index e668c63e79c..cef64cd5c7f 100644 --- a/.github/workflows/osgeo4w.yml +++ b/.github/workflows/osgeo4w.yml @@ -182,7 +182,10 @@ jobs: Expand-Archive -Path "$env:RUNNER_TEMP\DebugView.zip" -DestinationPath "$env:RUNNER_TEMP\DebugView" $logfile = "$env:RUNNER_TEMP\loadersnaps.log" - $dbgview = Start-Process -FilePath "$env:RUNNER_TEMP\DebugView\Dbgview64.exe" -ArgumentList @('/accepteula', '/l', $logfile, '/min') -PassThru + # /k enables kernel-mode DbgPrint capture; loader snaps are emitted + # through that path, not Win32 OutputDebugString, so the previous + # run (without /k) produced an empty log despite the crash firing. + $dbgview = Start-Process -FilePath "$env:RUNNER_TEMP\DebugView\Dbgview64.exe" -ArgumentList @('/accepteula', '/k', '/l', $logfile, '/min') -PassThru Set-Content -Path "$env:RUNNER_TEMP\dbgview.pid" -Value $dbgview.Id Start-Sleep -Seconds 5 Write-Output "Started DebugView (pid $($dbgview.Id)), logging to $logfile" From e3e2651f7a4bae1d29dc912356984977a3f4a31b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edouard=20Choini=C3=A8re?= <27212526+echoix@users.noreply.github.com> Date: Sat, 26 Sep 2026 13:14:56 -0400 Subject: [PATCH 4/4] CI: Switch loader snaps capture from DebugView to cdb.exe (not for merge) DebugView never produced a log file at all, with or without kernel capture enabled, most likely because GitHub Actions runs steps in a non-interactive Session 0 where its GUI message pump cannot initialize. cdb.exe (the console-mode debugger from the Windows SDK Debugging Tools) is itself a debugger attached directly to the python.exe process, which is the OS-level sink loader snaps are meant to write to, and needs no desktop session at all. Co-Authored-By: Claude Sonnet 5 --- .github/workflows/osgeo4w.yml | 65 ++++++++++++++++++----------------- 1 file changed, 34 insertions(+), 31 deletions(-) diff --git a/.github/workflows/osgeo4w.yml b/.github/workflows/osgeo4w.yml index cef64cd5c7f..7b5854f2a44 100644 --- a/.github/workflows/osgeo4w.yml +++ b/.github/workflows/osgeo4w.yml @@ -157,7 +157,7 @@ jobs: } } - - name: "[CI diagnostic, not for merge] Start loader snaps capture" + - name: "[CI diagnostic, not for merge] Enable loader snaps and locate cdb.exe" # Every native dependency in this investigation has been checked # clean via manual dumpbin export/import diffing (GDAL, curl, # OpenSSL, libjpeg-turbo, expat, PROJ, sqlite3, libxml2, brotli, @@ -166,11 +166,13 @@ jobs: # symbol, this asks Windows' own loader directly: enables Loader # Snaps (verbose OS-level trace of every module load and symbol # resolution step) for python.exe via the Image File Execution - # Options GlobalFlag, captured with Sysinternals DebugView. Kept - # in its own step (rather than wrapping the actual pytest - # invocation in PowerShell string interpolation) because nesting - # OSGeo4W.bat's own cmd /c re-invocation inside a PowerShell - # -ArgumentList corrupts the huge %PATH% value. + # Options GlobalFlag. DebugView (a GUI app) was tried first but + # never produced a log file at all, with or without its /k switch, + # most likely because GitHub Actions runs steps in a non-interactive + # Session 0 where its window/message pump cannot initialize. cdb.exe + # (console-mode debugger, part of the Windows SDK Debugging Tools) + # is a debugger itself, which is the OS-level sink loader snaps are + # designed to write to, and needs no desktop session. shell: pwsh run: | $ErrorActionPreference = 'Stop' @@ -178,48 +180,49 @@ jobs: New-Item -Path $ifeoPath -Force | Out-Null New-ItemProperty -Path $ifeoPath -Name 'GlobalFlag' -PropertyType DWord -Value 0x2 -Force | Out-Null - Invoke-WebRequest -Uri 'https://download.sysinternals.com/files/DebugView.zip' -OutFile "$env:RUNNER_TEMP\DebugView.zip" - Expand-Archive -Path "$env:RUNNER_TEMP\DebugView.zip" -DestinationPath "$env:RUNNER_TEMP\DebugView" - - $logfile = "$env:RUNNER_TEMP\loadersnaps.log" - # /k enables kernel-mode DbgPrint capture; loader snaps are emitted - # through that path, not Win32 OutputDebugString, so the previous - # run (without /k) produced an empty log despite the crash firing. - $dbgview = Start-Process -FilePath "$env:RUNNER_TEMP\DebugView\Dbgview64.exe" -ArgumentList @('/accepteula', '/k', '/l', $logfile, '/min') -PassThru - Set-Content -Path "$env:RUNNER_TEMP\dbgview.pid" -Value $dbgview.Id - Start-Sleep -Seconds 5 - Write-Output "Started DebugView (pid $($dbgview.Id)), logging to $logfile" + $candidates = @( + "${env:ProgramFiles(x86)}\Windows Kits\10\Debuggers\x64\cdb.exe", + "${env:ProgramFiles}\Windows Kits\10\Debuggers\x64\cdb.exe" + ) + $cdb = $candidates | Where-Object { Test-Path $_ } | Select-Object -First 1 + if (-not $cdb) { + Write-Output "cdb.exe not preinstalled, installing Debugging Tools for Windows via choco" + choco install windbg -y --no-progress + $cdb = Get-ChildItem -Path "${env:ProgramFiles(x86)}\Windows Kits\10\Debuggers\x64\cdb.exe" -ErrorAction SilentlyContinue | + Select-Object -First 1 -ExpandProperty FullName + } + if (-not $cdb) { + throw "cdb.exe could not be found or installed" + } + Write-Output "Using cdb at $cdb" + Set-Content -Path "$env:RUNNER_TEMP\cdb_path.txt" -Value $cdb -NoNewline - - name: "[CI diagnostic, not for merge] Run python -m pytest under loader snaps" + - name: "[CI diagnostic, not for merge] Run python -m pytest under cdb (loader snaps)" run: | call C:\OSGeo4W\opt\grass\etc\env.bat set PYTHONPATH=%GISBASE%\etc\python;%PYTHONPATH% path %GISBASE%\lib;%GISBASE%\bin;%PATH% - python -m pytest -v imagery/i.gensig/testsuite/test_i_gensig.py lib/arraystats/tests + set /p CDB_PATH=<"%RUNNER_TEMP%\cdb_path.txt" + "%CDB_PATH%" -g -G -c "g;q" python -m pytest -v imagery/i.gensig/testsuite/test_i_gensig.py lib/arraystats/tests > "%RUNNER_TEMP%\cdb_output.log" 2>&1 + type "%RUNNER_TEMP%\cdb_output.log" # Obfuscated usage of GitHub Actions features, Windows CMD shell limits analysis shell: cmd /D /E:ON /V:OFF /S /C "CALL C:/OSGeo4W/OSGeo4W.bat "{0}"" # zizmor: ignore[misfeature] continue-on-error: true - - name: "[CI diagnostic, not for merge] Stop loader snaps capture and dump the trace" + - name: "[CI diagnostic, not for merge] Dump cdb loader snaps output" if: always() shell: pwsh run: | - Start-Sleep -Seconds 5 - $pidFile = "$env:RUNNER_TEMP\dbgview.pid" - if (Test-Path $pidFile) { - Stop-Process -Id (Get-Content $pidFile) -Force -ErrorAction SilentlyContinue - } - - $logfile = "$env:RUNNER_TEMP\loadersnaps.log" + $logfile = "$env:RUNNER_TEMP\cdb_output.log" if (Test-Path $logfile) { - Write-Output "::group::Full loader snaps log line count" + Write-Output "::group::Full cdb output line count" (Get-Content $logfile).Count Write-Output '::endgroup::' - Write-Output '::group::Loader snaps log (filtered for python/grass/not found/procedure)' - Get-Content $logfile | Select-String -Pattern 'python|grass_|proj_9|gdal313|not found|procedure|LDR' | Select-Object -First 800 + Write-Output '::group::cdb output (filtered for LDR/not found/procedure/ModLoad/python/grass_/Exception)' + Get-Content $logfile | Select-String -Pattern 'LDR|not found|procedure|ModLoad|python|grass_|proj_9|gdal313|Exception' | Select-Object -First 1000 Write-Output '::endgroup::' } else { - Write-Output '::error::No loader snaps log was produced' + Write-Output '::error::No cdb output log was produced' } - name: Run pytest with multiple workers in parallel