cronjobs: various fixes and cleanup

After merging of OSGeo#896 and cronjob deployment on grass.osgeo.org various issues came up which are addressed in this PR. Important: - fix incorrect branch in `cron_grass_new_current_build_binaries.sh` - generate programmer's manual only in `cron_grass_new_current_build_binaries.sh` Improvements: - fail early rather than useless continuation (`set -e`) - compile with 2 cores (`make -j2`) - fix copying of `INSTALL` and `REQUIREMENTS.html` to server download directory as being renamed to `INSTALL.md` and `REQUIREMENTS.md` - also copy `CITATION.cff` to server download directory Misc: - no longer generate gettext `.pot` file for transifex since we migrated to https://weblate.osgeo.org/projects/grass-gis/ This PR potentially also addresses OSGeo#814 (can @ninsbl please verify what's yet missing)?
neteler · May 29, 2023 · f9d0b4f · f9d0b4f
1 parent 0065b08
commit f9d0b4f
Show file tree

Hide file tree

Showing 8 changed files with 117 additions and 76 deletions.
diff --git a/utils/cronjobs_osgeo_lxd/cron_grass_legacy_build_binaries.sh b/utils/cronjobs_osgeo_lxd/cron_grass_legacy_build_binaries.sh
@@ -41,6 +41,10 @@ VERSION=$GMAJOR$GMINOR
 GVERSION=$GMAJOR
 
 ###################
+# fail early
+set -e
+
+# compiler optimization
 CFLAGSSTRING='-O2'
 CFLAGSSTRING='-Werror-implicit-function-declaration -fno-common'
 LDFLAGSSTRING='-s'
@@ -61,7 +65,7 @@ MYBIN=$MAINDIR/binaries
 
 ############################## nothing to change below:
 
-MYMAKE="nice make LD_LIBRARY_PATH=$MYBIN/lib:/usr/lib:/usr/local/lib"
+MYMAKE="nice make -j2 LD_LIBRARY_PATH=$MYBIN/lib:/usr/lib:/usr/local/lib"
 
 # catch CTRL-C and other breaks:
 trap "echo 'user break.' ; exit" 2 3 9 15
@@ -109,6 +113,8 @@ CFLAGS=$CFLAGSSTRING LDFLAGS=$LDFLAGSSTRING ./configure \
 mkdir -p $TARGETDIR
 cd $GRASSBUILDDIR/
 
+# clean up
+touch include/Make/Platform.make
 $MYMAKE distclean > /dev/null 2>&1
 
 # cleanup leftover garbage

diff --git a/utils/cronjobs_osgeo_lxd/cron_grass_legacy_src_snapshot.sh b/utils/cronjobs_osgeo_lxd/cron_grass_legacy_src_snapshot.sh
@@ -21,6 +21,9 @@ GVERSION=$GMAJOR.$GMINOR.git
 DOTVERSION=$GMAJOR.$GMINOR
 GSHORTGVERSION=$GMAJOR$GMINOR
 
+# fail early
+set -e
+
 ###################
 # where to find the GRASS sources (git clone):
 SOURCE=$MAINDIR/src/
@@ -32,7 +35,7 @@ PACKAGENAME=grass-${GVERSION}_
 
 ############################## nothing to change below:
 
-MYMAKE="nice make"
+MYMAKE="nice make -j2"
 TAR=tar
 
 # catch CTRL-C and other breaks:
@@ -51,7 +54,8 @@ mkdir -p $TARGETDIR
 cd $SOURCE/$BRANCH/
 date
 
-#clean up
+# clean up
+touch include/Make/Platform.make
 $MYMAKE distclean > /dev/null 2>&1
 
 # cleanup leftover garbage

diff --git a/utils/cronjobs_osgeo_lxd/cron_grass_new_current_build_binaries.sh b/utils/cronjobs_osgeo_lxd/cron_grass_new_current_build_binaries.sh
@@ -1,6 +1,6 @@
 #!/bin/sh
 
-# script to build GRASS GIS new current binaries + addons from the `main`
+# script to build GRASS GIS new current binaries + addons + progman from the `releasebranch_8_3` binaries
 # (c) GPL 2+ Markus Neteler <neteler@osgeo.org>
 # 2022-2023
 #
@@ -17,15 +17,15 @@
 # - generates the user 8 HTML manuals
 # - injects DuckDuckGo search field
 
-# Preparations, on OSGeo server (neteler@grasslxd):
-# - Install PROJ: http://trac.osgeo.org/proj/ incl Datum shift grids
-# sh conf_proj4.sh
-# - Install GDAL: http://trac.osgeo.org/gdal/wiki/DownloadSource
-# sh conf_gdal.sh
+# Preparations, on server:
+# - Install PROJ incl Datum shift grids
+# - Install GDAL:
 # - Install apt-get install texlive-latex-extra python3-sphinxcontrib.apidoc
 # - Clone source from github:
 # mkdir -p ~/src ; cd ~/src
-# git clone https://github.com/OSGeo/grass.git main
+# git clone https://github.com/OSGeo/grass.git releasebranch_8_3
+# cd releasebranch_8_3
+# git checkout releasebranch_8_3
 # - Prepare target directories:
 # cd /var/www/code_and_data/
 # mkdir grass83
@@ -43,6 +43,10 @@ VERSION=$GMAJOR$GMINOR
 GVERSION=$GMAJOR
 
 ###################
+# fail early
+set -e
+
+# compiler optimization
 CFLAGSSTRING='-O2'
 CFLAGSSTRING='-Werror-implicit-function-declaration -fno-common'
 LDFLAGSSTRING='-s'
@@ -52,20 +56,20 @@ LDFLAGSSTRING='-s'
 MAINDIR=/home/neteler
 # where to find the GRASS sources (git clone):
 SOURCE=$MAINDIR/src/
-BRANCH=main
+BRANCH=releasebranch_${GMAJOR}_$GMINOR
 GRASSBUILDDIR=$SOURCE/$BRANCH
 TARGETMAIN=/var/www/code_and_data
 TARGETDIR=$TARGETMAIN/grass${VERSION}/binary/linux/snapshot
 TARGETHTMLDIR=$TARGETMAIN/grass${VERSION}/manuals/
 
-# not built for dev version
-## TARGETPROGMAN=$TARGETMAIN/programming${GVERSION}
+# only built for new stable (not built for dev version or old stable)
+TARGETPROGMAN=$TARGETMAIN/programming${GVERSION}
 
 MYBIN=$MAINDIR/binaries
 
 ############################## nothing to change below:
 
-MYMAKE="nice make LD_LIBRARY_PATH=$MYBIN/lib:/usr/lib:/usr/local/lib"
+MYMAKE="nice make -j2 LD_LIBRARY_PATH=$MYBIN/lib:/usr/lib:/usr/local/lib"
 
 # catch CTRL-C and other breaks:
 trap "echo 'user break.' ; exit" 2 3 9 15
@@ -85,7 +89,7 @@ configure_grass()
 # cleanup
 rm -f config_$GMAJOR.$GMINOR.git_log.txt
 
-# reset i18N POT files
+# reset i18N POT files to git, just to be sure
 git checkout locale/templates/*.pot
 
 CFLAGS=$CFLAGSSTRING LDFLAGS=$LDFLAGSSTRING ./configure \
@@ -118,6 +122,8 @@ CFLAGS=$CFLAGSSTRING LDFLAGS=$LDFLAGSSTRING ./configure \
 mkdir -p $TARGETDIR
 cd $GRASSBUILDDIR/
 
+# clean up
+touch include/Make/Platform.make
 $MYMAKE distclean > /dev/null 2>&1
 
 # cleanup leftover garbage
@@ -174,7 +180,7 @@ echo "Copied pygrass progman to http://grass.osgeo.org/grass${VERSION}/manuals/l
 echo "Injecting DuckDuckGo search field into manual main page..."
 (cd $TARGETHTMLDIR/ ; sed -i -e "s+</table>+</table><\!\-\- injected in cron_grass8_relbranch_build_binaries.sh \-\-> <center><iframe src=\"https://duckduckgo.com/search.html?site=grass.osgeo.org%26prefill=Search%20manual%20pages%20at%20DuckDuckGo\" style=\"overflow:hidden;margin:0;padding:0;width:410px;height:40px;\" frameborder=\"0\"></iframe></center>+g" index.html)
 
-cp -p AUTHORS CHANGES CITING COPYING GPL.TXT INSTALL REQUIREMENTS.html $TARGETDIR/
+cp -p AUTHORS CHANGES CITING CITATION.cff COPYING GPL.TXT INSTALL.md REQUIREMENTS.md $TARGETDIR/
 
 # clean wxGUI sphinx manual etc
 (cd $GRASSBUILDDIR/ ; $MYMAKE cleansphinx)
@@ -206,11 +212,12 @@ cd $GRASSBUILDDIR/
 ## bug in doxygen
 #(cd $TARGETPROGMAN/ ; ln -s index.html main.html)
 
-##### copy i18N POT files, needed for https://www.transifex.com/grass-gis/
-(cd locale ;
-mkdir -p $TARGETDIR/transifex/
-cp templates/*.pot $TARGETDIR/transifex/
-)
+##### copy i18N POT files, originally needed for https://www.transifex.com/grass-gis/
+### note: from G82+ onwards the gettext POT files are managed in git and OSGeo Weblate
+#(cd locale ;
+#mkdir -p $TARGETDIR/transifex/
+#cp templates/*.pot $TARGETDIR/transifex/
+#)
 
 ##### generate i18N stats for HTML page path (WebSVN):
 ## Structure: grasslibs_ar.po 144 translated messages 326 fuzzy translations 463 untranslated messages.
@@ -335,7 +342,7 @@ echo "Finished GRASS $VERSION $ARCH compilation."
 echo "Written to: $TARGETDIR"
 echo "Copied HTML ${GVERSION} manual to https://grass.osgeo.org/grass${VERSION}/manuals/"
 echo "Copied pygrass progman ${GVERSION} to https://grass.osgeo.org/grass${VERSION}/manuals/libpython/"
-#echo "Copied HTML ${GVERSION} progman to https://grass.osgeo.org/programming${GVERSION}"
+echo "Copied HTML ${GVERSION} progman to https://grass.osgeo.org/programming${GVERSION}"
 echo "Copied Addons ${GVERSION} to https://grass.osgeo.org/grass${VERSION}/manuals/addons/"
 
 exit 0
diff --git a/utils/cronjobs_osgeo_lxd/cron_grass_new_current_src_snapshot.sh b/utils/cronjobs_osgeo_lxd/cron_grass_new_current_src_snapshot.sh
@@ -18,6 +18,9 @@ GVERSION=$GMAJOR.$GMINOR.git
 DOTVERSION=$GMAJOR.$GMINOR
 GSHORTGVERSION=$GMAJOR$GMINOR
 
+# fail early
+set -e
+
 ###################
 # where to find the GRASS sources (git clone):
 SOURCE=$MAINDIR/src/
@@ -29,7 +32,7 @@ PACKAGENAME=grass-${GVERSION}_
 
 ############################## nothing to change below:
 
-MYMAKE="nice make"
+MYMAKE="nice make -j2"
 TAR=tar
 
 # catch CTRL-C and other breaks:
@@ -48,7 +51,8 @@ mkdir -p $TARGETDIR
 cd $SOURCE/$BRANCH/
 date
 
-#clean up
+# clean up
+touch include/Make/Platform.make
 $MYMAKE distclean > /dev/null 2>&1
 
 # cleanup leftover garbage
@@ -91,7 +95,7 @@ rm -f $TARGETDIR/ChangeLog.gz
 
 #publish the new one:
 cd $BRANCH/
-cp -p ChangeLog AUTHORS CHANGES CITING COPYING GPL.TXT INSTALL REQUIREMENTS.html $TARGETDIR
+cp -p ChangeLog AUTHORS CHANGES CITING CITATION.cff COPYING GPL.TXT INSTALL.md REQUIREMENTS.md $TARGETDIR
 
 cd ..
 gzip $TARGETDIR/ChangeLog

diff --git a/utils/cronjobs_osgeo_lxd/cron_grass_old_current_build_binaries.sh b/utils/cronjobs_osgeo_lxd/cron_grass_old_current_build_binaries.sh
@@ -43,6 +43,10 @@ VERSION=$GMAJOR$GMINOR
 GVERSION=$GMAJOR
 
 ###################
+# fail early
+set -e
+
+# compiler optimization
 CFLAGSSTRING='-O2'
 CFLAGSSTRING='-Werror-implicit-function-declaration -fno-common'
 LDFLAGSSTRING='-s'
@@ -57,13 +61,14 @@ GRASSBUILDDIR=$SOURCE/$BRANCH
 TARGETMAIN=/var/www/code_and_data
 TARGETDIR=$TARGETMAIN/grass${VERSION}/binary/linux/snapshot
 TARGETHTMLDIR=$TARGETMAIN/grass${VERSION}/manuals/
-TARGETPROGMAN=$TARGETMAIN/programming${GVERSION}
+# not built for dev version or old stable
+## TARGETPROGMAN=$TARGETMAIN/programming${GVERSION}
 
 MYBIN=$MAINDIR/binaries
 
 ############################## nothing to change below:
 
-MYMAKE="nice make LD_LIBRARY_PATH=$MYBIN/lib:/usr/lib:/usr/local/lib"
+MYMAKE="nice make -j2 LD_LIBRARY_PATH=$MYBIN/lib:/usr/lib:/usr/local/lib"
 
 # catch CTRL-C and other breaks:
 trap "echo 'user break.' ; exit" 2 3 9 15
@@ -81,7 +86,7 @@ configure_grass()
 # --with-mysql --with-mysql-includes=/usr/include/mysql --with-mysql-libs=/usr/lib/mysql \
 
 # cleanup
-rm -rf config_$GMAJOR.$GMINOR.git_log.txt
+rm -f config_$GMAJOR.$GMINOR.git_log.txt
 
 # reset i18N POT files to git, just to be sure
 git checkout locale/templates/*.pot
@@ -116,6 +121,8 @@ CFLAGS=$CFLAGSSTRING LDFLAGS=$LDFLAGSSTRING ./configure \
 mkdir -p $TARGETDIR
 cd $GRASSBUILDDIR/
 
+# clean up
+touch include/Make/Platform.make
 $MYMAKE distclean > /dev/null 2>&1
 
 # cleanup leftover garbage
@@ -172,7 +179,7 @@ echo "Copied pygrass progman to http://grass.osgeo.org/grass${VERSION}/manuals/l
 echo "Injecting DuckDuckGo search field into manual main page..."
 (cd $TARGETHTMLDIR/ ; sed -i -e "s+</table>+</table><\!\-\- injected in cron_grass8_relbranch_build_binaries.sh \-\-> <center><iframe src=\"https://duckduckgo.com/search.html?site=grass.osgeo.org%26prefill=Search%20manual%20pages%20at%20DuckDuckGo\" style=\"overflow:hidden;margin:0;padding:0;width:410px;height:40px;\" frameborder=\"0\"></iframe></center>+g" index.html)
 
-cp -p AUTHORS CHANGES CITING COPYING GPL.TXT INSTALL REQUIREMENTS.html $TARGETDIR/
+cp -p AUTHORS CHANGES CITING CITATION.cff COPYING GPL.TXT INSTALL.md REQUIREMENTS.html $TARGETDIR/
 
 # clean wxGUI sphinx manual etc
 (cd $GRASSBUILDDIR/ ; $MYMAKE cleansphinx)
@@ -185,31 +192,31 @@ $MYMAKE htmldocs-single || (echo "$0 htmldocs-single: an error occurred" ; exit
 
 cd $GRASSBUILDDIR/
 
-# clean old TARGETPROGMAN stuff from last run
-if [ -z "$TARGETPROGMAN" ] ; then
- echo "\$TARGETPROGMAN undefined, error!"
- exit 1
-fi
-mkdir -p $TARGETPROGMAN
-rm -f $TARGETPROGMAN/*.*
-
-# copy over doxygen manual
-cp -r html/* $TARGETPROGMAN/
-
-echo "Copied HTML progman to https://grass.osgeo.org/programming${GVERSION}"
-# fix permissions
-chgrp -R grass $TARGETPROGMAN/*
-chmod -R a+r,g+w $TARGETPROGMAN/
-chmod -R a+r,g+w $TARGETPROGMAN/*
-# bug in doxygen
-(cd $TARGETPROGMAN/ ; ln -s index.html main.html)
-
-##### generate i18N POT files, needed for https://www.transifex.com/grass-gis/
-# from G82+ onwards the gettext POT files are managed in git
-(cd locale ;
-mkdir -p $TARGETDIR/transifex/
-cp templates/*.pot $TARGETDIR/transifex/
-)
+## clean old TARGETPROGMAN stuff from last run
+#if [ -z "$TARGETPROGMAN" ] ; then
+# echo "\$TARGETPROGMAN undefined, error!"
+# exit 1
+#fi
+#mkdir -p $TARGETPROGMAN
+#rm -f $TARGETPROGMAN/*.*
+#
+## copy over doxygen manual
+#cp -r html/* $TARGETPROGMAN/
+#
+#echo "Copied HTML progman to https://grass.osgeo.org/programming${GVERSION}"
+## fix permissions
+#chgrp -R grass $TARGETPROGMAN/*
+#chmod -R a+r,g+w $TARGETPROGMAN/
+#chmod -R a+r,g+w $TARGETPROGMAN/*
+## bug in doxygen
+#(cd $TARGETPROGMAN/ ; ln -s index.html main.html)
+
+##### copy i18N POT files, originally needed for https://www.transifex.com/grass-gis/
+### note: from G82+ onwards the gettext POT files are managed in git and OSGeo Weblate
+#(cd locale ;
+#mkdir -p $TARGETDIR/transifex/
+#cp templates/*.pot $TARGETDIR/transifex/
+#)
 
 ##### generate i18N stats for HTML page path (WebSVN):
 ## Structure: grasslibs_ar.po 144 translated messages 326 fuzzy translations 463 untranslated messages.
@@ -334,7 +341,7 @@ echo "Finished GRASS $VERSION $ARCH compilation."
 echo "Written to: $TARGETDIR"
 echo "Copied HTML ${GVERSION} manual to https://grass.osgeo.org/grass${VERSION}/manuals/"
 echo "Copied pygrass progman ${GVERSION} to https://grass.osgeo.org/grass${VERSION}/manuals/libpython/"
-echo "Copied HTML ${GVERSION} progman to https://grass.osgeo.org/programming${GVERSION}"
+#echo "Copied HTML ${GVERSION} progman to https://grass.osgeo.org/programming${GVERSION}"
 echo "Copied Addons ${GVERSION} to https://grass.osgeo.org/grass${VERSION}/manuals/addons/"
 
 exit 0
diff --git a/utils/cronjobs_osgeo_lxd/cron_grass_old_current_src_snapshot.sh b/utils/cronjobs_osgeo_lxd/cron_grass_old_current_src_snapshot.sh
@@ -18,6 +18,9 @@ GVERSION=$GMAJOR.$GMINOR.git
 DOTVERSION=$GMAJOR.$GMINOR
 GSHORTGVERSION=$GMAJOR$GMINOR
 
+# fail early
+set -e
+
 ###################
 # where to find the GRASS sources (git clone):
 SOURCE=$MAINDIR/src/
@@ -29,7 +32,7 @@ PACKAGENAME=grass-${GVERSION}_
 
 ############################## nothing to change below:
 
-MYMAKE="nice make"
+MYMAKE="nice make -j2"
 TAR=tar
 
 # catch CTRL-C and other breaks:
@@ -48,7 +51,8 @@ mkdir -p $TARGETDIR
 cd $SOURCE/$BRANCH/
 date
 
-#clean up
+# clean up
+touch include/Make/Platform.make
 $MYMAKE distclean > /dev/null 2>&1
 
 # cleanup leftover garbage
@@ -91,7 +95,7 @@ rm -f $TARGETDIR/ChangeLog.gz
 
 #publish the new one:
 cd $BRANCH/
-cp -p ChangeLog AUTHORS CHANGES CITING COPYING GPL.TXT INSTALL REQUIREMENTS.html $TARGETDIR
+cp -p ChangeLog AUTHORS CHANGES CITING CITATION.cff COPYING GPL.TXT INSTALL.md REQUIREMENTS.html $TARGETDIR
 
 cd ..
 gzip $TARGETDIR/ChangeLog