From ed27e36bb3fa3fc0d3e2e9a8127bc0b3173ed380 Mon Sep 17 00:00:00 2001 From: ninsbl Date: Wed, 5 Jan 2022 22:55:16 +0100 Subject: [PATCH 1/4] move from #1873 --- .github/workflows/build_osgeo4w.sh | 135 ++-------------------------- .github/workflows/osgeo4w.yml | 22 +++-- .github/workflows/test_simple.bat | 5 ++ .github/workflows/test_simple.sh | 5 ++ .github/workflows/test_thorough.bat | 2 +- 5 files changed, 32 insertions(+), 137 deletions(-) diff --git a/.github/workflows/build_osgeo4w.sh b/.github/workflows/build_osgeo4w.sh index 09eb271d94b..32fb82ea4d1 100755 --- a/.github/workflows/build_osgeo4w.sh +++ b/.github/workflows/build_osgeo4w.sh @@ -1,145 +1,22 @@ #!/bin/sh # -# Usage: build_osgeo4w.sh [-p] [path] +# Usage: build_osgeo4w.sh [path] # # By default, the script will look for the source code in the current directory # and create bin.x86_64-w64-mingw32\grass$ver.bat (run this batch file to start # GRASS GIS) and dist.x86_64-w64-mingw32\etc\env.bat. # -# -p optionally install GRASS GIS to C:\OSGeo4W64\opt\grass (run -# C:\OSGeo4W64\opt\grass\grass$ver.bat) and create an unzippable package -# grass$ver-x86_64-w64-mingw32-osgeo4w64-$date.zip -# # path specify a path to the source code # # stop on errors set -e -if [ "$1" = "-p" ]; then - package=1 - shift -else - package=0 -fi - test -d "$1" && cd "$1" -osgeo4w_path=/c/OSGeo4W64 -arch=x86_64-w64-mingw32 -src=$(pwd) - -# compile - -export PATH=/mingw64/bin:/c/OSGeo4W64/bin:/usr/bin -export PROJ_LIB=$osgeo4w_path/share/proj - -OSGEO4W_ROOT_MSYS=$osgeo4w_path \ - ./configure \ - --host=$arch \ - --with-includes=$osgeo4w_path/include \ - --with-libs="$osgeo4w_path/lib $osgeo4w_path/bin" \ - --with-nls \ - --with-freetype-includes=$osgeo4w_path/include/freetype2 \ - --with-bzlib \ - --with-geos=$src/mswindows/osgeo4w/geos-config \ - --with-netcdf=$osgeo4w_path/bin/nc-config \ - --with-gdal=$src/mswindows/osgeo4w/gdal-config \ - --with-liblas=$src/mswindows/osgeo4w/liblas-config \ - --with-opengl=windows - -make - -# install - -bin=bin.$arch -dist=dist.$arch -ver=$(sed -n '/^INST_DIR[ \t]*=/{s/^.*grass//; p}' include/Make/Platform.make) - -rm -f $dist/grass$ver.tmp $dist/etc/fontcap - -# create batch files - -src_esc=$(echo $src | sed 's#^/\([a-z]\)#\1:#; s#/#\\\\\\\\#g') -dist_esc="$src_esc\\\\$dist" - -( - sed 's/^\(set GISBASE=\).*/\1'$dist_esc'/' \ - mswindows/osgeo4w/env.bat.tmpl - cat <$dist/etc/env.bat -unix2dos $dist/etc/env.bat - -( - sed -e 's/^\(call "\)%~dp0\(.*\)$/\1C:\\OSGeo4W64\\bin\2/' \ - -e 's/^\(call "\).*\(\\etc\\env\.bat"\)$/\1'$dist_esc'\2/' \ - -e 's/^\(.* "\)%GISBASE%\\etc\(\\grass.*\)$/\1%GISBASE%\\..\\'$bin'\2/' \ - -e 's/@POSTFIX@/'$ver'/g' \ - mswindows/osgeo4w/grass.bat.tmpl -) >$bin/grass$ver.bat -unix2dos $bin/grass$ver.bat - -test $package -eq 0 && exit - -# package - -opt_path=$osgeo4w_path/opt -grass_path=$opt_path/grass - -mkdir -p $opt_path -cp -a $dist $grass_path -# have a versionless and versioned startup script -cp -a $bin/grass.py $bin/grass$ver.py -cp -a $bin/grass$ver.py $grass_path/etc -cp -a $(ldd $dist/lib/*.dll | awk '/mingw64/{print $3}' | - sort -u | grep -v 'lib\(crypto\|ssl\)') $grass_path/lib - -( - sed -e 's/^\(set GISBASE=\).*/\1%OSGEO4W_ROOT%\\opt\\grass/' \ - mswindows/osgeo4w/env.bat.tmpl - cat <$grass_path/etc/env.bat -unix2dos $grass_path/etc/env.bat - -( - sed -e 's/^\(call "%~dp0\)\(.*\)$/\1\\..\\..\\bin\2/' \ - -e 's/^\(call "%OSGEO4W_ROOT%\\\).*\(\\etc\\env\.bat"\)$/\1opt\\grass\2/' \ - -e 's/@POSTFIX@/'$ver'/g' \ - mswindows/osgeo4w/grass.bat.tmpl -) >$grass_path/grass$ver.bat -unix2dos $grass_path/grass$ver.bat - -exit - -# don't package for GitHub workflow; unnecessary - -osgeo4w_basename=$(basename $osgeo4w_path) -date=$(date +%Y%m%d) -zip=$src/grass$ver-$arch-osgeo4w64-$date.zip +export OSGEO4W_ROOT_MSYS=/c/OSGeo4W +export SRC=$(pwd) +export UNITTEST=1 -cd $osgeo4w_path/.. -zip -r $zip $osgeo4w_basename -x "$osgeo4w_basename/var/*" "*/__pycache__/*" +# Build according to OSGeo4W recipe +${SRC}/mswindows/osgeo4w/build_osgeo4W.sh diff --git a/.github/workflows/osgeo4w.yml b/.github/workflows/osgeo4w.yml index 0afd5cc575e..cf34ed24f72 100644 --- a/.github/workflows/osgeo4w.yml +++ b/.github/workflows/osgeo4w.yml @@ -21,17 +21,25 @@ jobs: git config --global core.autocrlf false git config --global core.eol lf - uses: actions/checkout@v2 + - name: Install OSGeo4W run: | - $exe = 'osgeo4w-setup-x86_64-v1.exe' - $url = 'http://download.osgeo.org/osgeo4w/' + $exe + $exe = 'osgeo4w-setup.exe' + $url = 'http://download.osgeo.org/osgeo4w/v2/' + $exe (New-Object System.Net.WebClient).DownloadFile($url, $exe) - Start-Process ('.\'+$exe) -ArgumentList '-A -g -k -q -s http://download.osgeo.org/x86_64 -P cairo,fftw,freetype-devel,gdal-ecw,gdal-mrsid,liblas-devel,libxdr,msys,pdcurses,python3-numpy,python3-pywin32,python3-wx,regex-devel,wxpython,zstd-devel' -Wait + Start-Process ('.\'+$exe) -ArgumentList '-A -g -k -q -s http://download.osgeo.org/x86_64 -P proj-devel,gdal-devel,geos-devel,libtiff-devel,libpng-devel,pdal-devel,netcdf-devel,cairo-devel,fftw,freetype-devel,gdal-ecw,gdal-mrsid,liblas-devel,libxdr,libpq-devel,pdcurses,python3-matplotlib,python3-numpy,python3-ply,python3-pywin32,python3-six,python3-wxpython,regex-devel,wxwidgets-devel,zstd-devel' -Wait + - name: Install MSYS2 packages - run: C:\msys64\usr\bin\pacman.exe --noconfirm -S tar libintl make bison diffutils git dos2unix zip mingw-w64-x86_64-toolchain mingw-w64-x86_64-cairo mingw-w64-x86_64-python3-six + run: C:\msys64\usr\bin\pacman.exe --noconfirm -S tar libintl make bison flex diffutils git dos2unix zip mingw-w64-x86_64-toolchain mingw-w64-x86_64-cairo mingw-w64-x86_64-fftw mingw-w64-x86_64-lapack mingw-w64-x86_64-pkg-config mingw-w64-x86_64-gcc mingw-w64-x86_64-ccache mingw-w64-x86_64-zlib mingw-w64-x86_64-libiconv mingw-w64-x86_64-bzip2 mingw-w64-x86_64-gettext mingw-w64-x86_64-libsystre mingw-w64-x86_64-libtre-git mingw-w64-x86_64-libwinpthread-git mingw-w64-x86_64-libpng mingw-w64-x86_64-pcre mingw-w64-x86_64-python3-six + - name: Compile GRASS GIS - run: C:\msys64\usr\bin\bash.exe -l (''+(Get-Location)+'\.github\workflows\build_osgeo4w.sh') -p (Get-Location) + run: C:\msys64\usr\bin\bash.exe -l (''+(Get-Location)+'\.github\workflows\build_osgeo4w.sh') (Get-Location) + - name: Test executing of the grass command - run: .github/workflows/test_simple.bat 'C:\OSGeo4W64\opt\grass\grass80.bat' + run: .github/workflows/test_simple.bat 'C:\OSGeo4W\opt\grass\grass80.bat' + + - name: Test executing of the grass command in bash + run: C:\msys64\usr\bin\bash.exe .github/workflows/test_simple.sh + - name: Run tests - run: .github/workflows/test_thorough.bat 'C:\OSGeo4W64\opt\grass\grass80.bat' + run: .github/workflows/test_thorough.bat 'C:\OSGeo4W\opt\grass\grass80.bat' 'C:\OSGeo4W\bin\python3' diff --git a/.github/workflows/test_simple.bat b/.github/workflows/test_simple.bat index 952c91f60bf..394421ceddb 100644 --- a/.github/workflows/test_simple.bat +++ b/.github/workflows/test_simple.bat @@ -1,3 +1,8 @@ set grass=%1 +rem Test execution of binary command call %grass% --tmp-location EPSG:4326 --exec g.region res=0.1 -p +rem Test if batch-wrapper-scripts without extension are found +call %grass% --tmp-location EPSG:4326 --exec C:/Windows/System32/where.exe t.create +rem Test if python-scripts can be called +call %grass% --tmp-location EPSG:4326 --exec t.create --help diff --git a/.github/workflows/test_simple.sh b/.github/workflows/test_simple.sh index be1ffd5132e..43a19a3d457 100755 --- a/.github/workflows/test_simple.sh +++ b/.github/workflows/test_simple.sh @@ -3,4 +3,9 @@ # fail on non-zero return code from a subprocess set -e +# Test execution of binary command grass --tmp-location EPSG:4326 --exec g.region res=0.1 -p +# Test if python modules without extension are found +grass --tmp-location EPSG:4326 --exec which t.create +# Test if python modules can be called +grass --tmp-location EPSG:4326 --exec t.create --help diff --git a/.github/workflows/test_thorough.bat b/.github/workflows/test_thorough.bat index e8816aa3ca9..fea1a66b9c1 100644 --- a/.github/workflows/test_thorough.bat +++ b/.github/workflows/test_thorough.bat @@ -1,5 +1,5 @@ set grass=%1 -set python=C:\OSGeo4W64\bin\python3 +set python=%2 call %grass% --tmp-location XY --exec g.download.location url=https://grass.osgeo.org/sampledata/north_carolina/nc_spm_full_v2alpha2.tar.gz path=%USERPROFILE% call %grass% --tmp-location XY --exec %python% -m grass.gunittest.main --grassdata %USERPROFILE% --location nc_spm_full_v2alpha2 --location-type nc --min-success 60 From f7eb74d887e67ef688c743198794350e6bc351ea Mon Sep 17 00:00:00 2001 From: ninsbl Date: Wed, 5 Jan 2022 23:01:52 +0100 Subject: [PATCH 2/4] move more from #1873 --- mswindows/osgeo4w/build_osgeo4w.sh | 193 +++++++++++++++++++++++++++++ 1 file changed, 193 insertions(+) create mode 100644 mswindows/osgeo4w/build_osgeo4w.sh diff --git a/mswindows/osgeo4w/build_osgeo4w.sh b/mswindows/osgeo4w/build_osgeo4w.sh new file mode 100644 index 00000000000..9480381bdf0 --- /dev/null +++ b/mswindows/osgeo4w/build_osgeo4w.sh @@ -0,0 +1,193 @@ +#!/bin/sh + +# +# Usage: build_osgeo4w.sh +# +# The following environment variables are supposed to be passed to the build script +# - SRC: the directory where the grass source code lives +# - OSGEO4W_ROOT_MSYS: the root directory of OSGeo4W +# - UNITTEST: If this variable is defined, addition files for unittests are created +# +# By default, the script will look for the source code in the current directory +# and create bin.x86_64-w64-mingw32\grass$ver.bat (run this batch file to start +# GRASS GIS) and dist.x86_64-w64-mingw32\etc\env.bat. +# +# -p optionally install GRASS GIS to C:\OSGeo4W\opt\grass (run +# C:\OSGeo4W64\opt\grass\grass$ver.bat) and create an unzippable package +# grass$ver-x86_64-w64-mingw32-osgeo4w64-$date.zip +# +# path specify a path to the source code +# + +# stop on errors +set -e + + +# compile +export PATH=${OSGEO4W_ROOT_MSYS}/bin:/usr/bin:/mingw64/bin +export C_INCLUDE_PATH=".:${OSGEO4W_ROOT_MSYS}/include:${SRC}/dist.${ARCH}/include:/c/msys64/mingw64/include" +export PYTHONHOME=${OSGEO4W_ROOT_MSYS}/apps/Python39 +export ARCH=x86_64-w64-mingw32 + +./configure \ + --host=${ARCH} \ + --with-libs="${OSGEO4W_ROOT_MSYS}/lib ${OSGEO4W_ROOT_MSYS}/bin" \ + --with-includes=${OSGEO4W_ROOT_MSYS}/include \ + --libexecdir=${OSGEO4W_ROOT_MSYS}/bin \ + --prefix=${OSGEO4W_ROOT_MSYS}/apps/grass \ + --bindir=${OSGEO4W_ROOT_MSYS}/bin \ + --includedir=${OSGEO4W_ROOT_MSYS}/include \ + --without-x \ + --with-cxx \ + --enable-shared \ + --enable-largefile \ + --with-openmp \ + --with-fftw \ + --with-nls \ + --with-readline \ + --with-wxwidgets \ + --with-blas \ + --with-lapack-includes=/mingw64/include/lapack \ + --with-freetype \ + --with-freetype-includes=/mingw64/include/freetype2 \ + --with-proj-share=${OSGEO4W_ROOT_MSYS}/share/proj \ + --with-proj-includes=${OSGEO4W_ROOT_MSYS}/include \ + --with-proj-libs=${OSGEO4W_ROOT_MSYS}/lib \ + --with-postgres \ + --with-postgres-includes=${OSGEO4W_ROOT_MSYS}/include \ + --with-postgres-libs=${OSGEO4W_ROOT_MSYS}/lib \ + --with-gdal=${SRC}/mswindows/osgeo4w/gdal-config \ + --with-geos=${SRC}/mswindows/osgeo4w/geos-config \ + --with-sqlite \ + --with-sqlite-includes=${OSGEO4W_ROOT_MSYS}/include \ + --with-sqlite-libs=${OSGEO4W_ROOT_MSYS}/lib \ + --with-regex \ + --with-nls \ + --with-zstd \ + --with-odbc \ + --with-cairo \ + --with-cairo-includes=${SRC}/include \ + --with-cairo-ldflags="-L${SRC}/mswindows/osgeo4w/lib -lcairo -lfontconfig" \ + --with-opengl=windows \ + --with-bzlib \ + --with-liblas=${SRC}/mswindows/osgeo4w/liblas-config \ + --with-netcdf=${OSGEO4W_ROOT_MSYS}/bin/nc-config + +make + +# install + +bin=bin.${ARCH} +dist=dist.${ARCH} +ver=$(sed -n '/^INST_DIR[ \t]*=/{s/^.*grass//; p}' include/Make/Platform.make) + +rm -f $dist/grass$ver.tmp $dist/etc/fontcap + +if [ "$UNITTEST" ]; then + # Add executables for bash scripts in unittests + bash_bin=bash_bin + mkdir "${SRC}/dist.${ARCH}/$bash_bin" + + for f in ${SRC}/dist.${ARCH}/scripts/*.py + do + bash_exe=$(echo $f | sed -e "s/\/scripts\//\/${bash_bin}\//;s/\.py$//") + cp "$f" "$bash_exe" + # dos2unix "$bash_exe" + chmod ugo+x "$bash_exe" + done + bash_exe_path=$(echo "${SRC}/dist.${ARCH}/$bash_bin" | sed -e "s/^\/c/\;c:/;s/^\/d/d:/") + export PATH="$PATH:${SRC}/dist.${ARCH}/$bash_bin" +fi + +# create batch files +src_esc=$(echo ${SRC} | sed 's#^/\([a-z]\)#\1:#; s#/#\\\\\\\\#g') +dist_esc="$src_esc\\\\$dist" + +( + sed 's/^\(set GISBASE=\).*/\1'$dist_esc'/' \ + mswindows/osgeo4w/env.bat.tmpl + cat <$dist/etc/env.bat +unix2dos $dist/etc/env.bat + +( + sed -e 's/^\(call "\)%~dp0\(.*\)$/\1C:\\OSGeo4W\\bin\2/' \ + -e 's/^\(call "\).*\(\\etc\\env\.bat"\)$/\1'$dist_esc'\2/' \ + -e 's/^\(.* "\)%GISBASE%\\etc\(\\grass.*\)$/\1%GISBASE%\\..\\'$bin'\2/' \ + -e 's/@POSTFIX@/'$ver'/g' \ + mswindows/osgeo4w/grass.bat.tmpl +) >$bin/grass$ver.bat +unix2dos $bin/grass$ver.bat + +opt_path=${OSGEO4W_ROOT_MSYS}/opt +grass_path=$opt_path/grass + +if [ "$UNITTEST" ]; then + msys_path=";C:/msys64/usr/bin;C:/msys64/mingw64/bin" +fi + +mkdir -p $opt_path +cp -a $dist $grass_path +# have a versionless and versioned startup script +cp -a $bin/grass.py $bin/grass$ver.py +cp -a $bin/grass$ver.py $grass_path/etc +cp -a $(ldd $dist/lib/*.dll | awk '/mingw64/{print $3}' | + sort -u | grep -v 'lib\(crypto\|ssl\)') $grass_path/lib + +( + sed -e 's/^\(set GISBASE=\).*/\1%OSGEO4W_ROOT%\\opt\\grass/' \ + mswindows/osgeo4w/env.bat.tmpl + cat <$grass_path/etc/env.bat +unix2dos $grass_path/etc/env.bat + +cat $grass_path/etc/env.bat + +( + sed -e 's/^\(call "%~dp0\)\(.*\)$/\1\\..\\..\\bin\2/' \ + -e 's/^\(call "%OSGEO4W_ROOT%\\\).*\(\\etc\\env\.bat"\)$/\1opt\\grass\2/' \ + -e 's/@POSTFIX@/'$ver'/g' \ + mswindows/osgeo4w/grass.bat.tmpl +) >$grass_path/grass$ver.bat +unix2dos $grass_path/grass$ver.bat + +if [ "$UNITTEST" ]; then + cp $grass_path/grass$ver.bat $grass_path/grass.bat + cp -a $bin/grass.py ${SRC}/dist.${ARCH}/$bash_bin/grass + dos2unix ${SRC}/dist.${ARCH}/$bash_bin/grass + chmod ugo+x ${SRC}/dist.${ARCH}/$bash_bin/grass + # Set path for bash if not called from OSGeo shell + arch_esc=$(sed 's/[\/\*\.]/\\&/g' <<<"${ARCH}") + src_esc=$(sed 's/[\/\*\.]/\\&/g' <<<"${SRC}") + osgeo4w_esc=$(sed 's/[\/\*\.]/\\&/g' <<<"${OSGEO4W_ROOT_MSYS}") + pythonhome_esc=$(sed 's/[\/\*\.]/\\&/g' <<<"${PYTHONHOME}") + sed -i "5s/^/export PATH=\"${osgeo4w_esc}\/bin:\/usr\/bin:\/mingw64\/bin:$src_esc\/dist\.$arch_esc\/bin:$src_esc\/dist\.$arch_esc\/$bash_bin\"\nexport PYTHONHOME=\"${pythonhome_esc}\"/" ${SRC}/.github/workflows/test_simple.sh + printf "export PATH=\"${OSGEO4W_ROOT_MSYS}/bin:/usr/bin:/mingw64/bin:${SRC}/dist.${ARCH}/bin:${SRC}/dist.${ARCH}/$bash_bin\"\nexport PYTHONHOME=\"${PYTHONHOME}\"\nexport PYTHONUTF8=1" > $HOME/.bash_profile + printf "export PATH=\"${OSGEO4W_ROOT_MSYS}/bin:/usr/bin:/mingw64/bin:${SRC}/dist.${ARCH}/bin:${SRC}/dist.${ARCH}/$bash_bin\"\nexport PYTHONHOME=\"${PYTHONHOME}\"\nexport PYTHONUTF8=1" > $HOME/.profile + printf "export PATH=\"${OSGEO4W_ROOT_MSYS}/bin:/usr/bin:/mingw64/bin:${SRC}/dist.${ARCH}/bin:${SRC}/dist.${ARCH}/$bash_bin\"\nexport PYTHONHOME=\"${PYTHONHOME}\"\nexport PYTHONUTF8=1" > $HOME/.bashrc +fi From bb5be94f51866426a3cd792ad055021d5f28cdd3 Mon Sep 17 00:00:00 2001 From: ninsbl Date: Thu, 6 Jan 2022 20:20:19 +0100 Subject: [PATCH 3/4] print versions --- .github/workflows/osgeo4w.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/osgeo4w.yml b/.github/workflows/osgeo4w.yml index cf34ed24f72..1b40f144f06 100644 --- a/.github/workflows/osgeo4w.yml +++ b/.github/workflows/osgeo4w.yml @@ -43,3 +43,6 @@ jobs: - name: Run tests run: .github/workflows/test_thorough.bat 'C:\OSGeo4W\opt\grass\grass80.bat' 'C:\OSGeo4W\bin\python3' + + - name: Print installed versions + run: C:\msys64\usr\bin\bash.exe .github/workflows/print_versions.sh From d884c1e513b957f77a0f37feabcc0dae24610f94 Mon Sep 17 00:00:00 2001 From: ninsbl Date: Thu, 6 Jan 2022 22:28:37 +0100 Subject: [PATCH 4/4] Revert "print versions" This reverts commit bb5be94f51866426a3cd792ad055021d5f28cdd3. --- .github/workflows/osgeo4w.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/osgeo4w.yml b/.github/workflows/osgeo4w.yml index 1b40f144f06..cf34ed24f72 100644 --- a/.github/workflows/osgeo4w.yml +++ b/.github/workflows/osgeo4w.yml @@ -43,6 +43,3 @@ jobs: - name: Run tests run: .github/workflows/test_thorough.bat 'C:\OSGeo4W\opt\grass\grass80.bat' 'C:\OSGeo4W\bin\python3' - - - name: Print installed versions - run: C:\msys64\usr\bin\bash.exe .github/workflows/print_versions.sh