-
Notifications
You must be signed in to change notification settings - Fork 148
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Remove hardcoded versions from release cronjobs #814
base: grass8
Are you sure you want to change the base?
Changes from all commits
7c5f6c8
3285253
601049b
14d26d6
5b62141
841a7f3
f6984cc
aea462e
a4cd3aa
7903d54
634a909
159719f
917965e
298cc95
0548c78
1d917db
23540d0
70e1198
b8529b5
3624dad
f20d06b
4de73f9
f5ebfee
e094f3d
df03226
3d5041f
f358413
2de533a
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,322 @@ | ||
#!/bin/sh | ||
|
||
# script to build GRASS 7.x binaries + addons from the `releasebranch_x_y` binaries | ||
# (c) GPL 2+ Markus Neteler <neteler@osgeo.org> | ||
# 2008-2023 | ||
# | ||
# GRASS GIS github, https://github.com/OSGeo/grass | ||
# | ||
## prep, on neteler@grasslxd:$ | ||
# mkdir -p ~/src | ||
# cd ~/src | ||
# # G76 G78 | ||
# for i in 6 8 ; do git clone https://github.com/OSGeo/grass.git releasebranch_7_$i ; done | ||
# for i in 6 8 ; do (cd releasebranch_7_$i ; git checkout releasebranch_7_$i ) ; done | ||
# | ||
################################################################### | ||
# how it works: | ||
# - it updates locally the GRASS source code from github server | ||
# - configures, compiles | ||
# - packages the binaries | ||
# - generated the install scripts | ||
# - generates the pyGRASS 7 HTML manual | ||
# - generates the user 7 HTML manuals | ||
# - injects DuckDuckGo search field | ||
# - injects G8 new version box | ||
|
||
# Preparations: | ||
# - Install PROJ: http://trac.osgeo.org/proj/ incl Datum shift grids | ||
# sh conf_proj4.sh | ||
# - Install GDAL: http://trac.osgeo.org/gdal/wiki/DownloadSource | ||
# sh conf_gdal.sh | ||
# - Install apt-get install texlive-latex-extra python3-sphinxcontrib.apidoc | ||
# - Clone source from github | ||
################################# | ||
PATH=/home/neteler/binaries/bin:/usr/bin:/bin:/usr/X11R6/bin:/usr/local/bin | ||
|
||
CURRENT_BRANCH=`curl https://api.github.com/repos/osgeo/grass/branches | grep release | grep '"name":' | cut -f4 -d'"' | sort -V | tail -n 1` | ||
CURRENT_VERSION=`curl https://raw.githubusercontent.com/osgeo/grass/$CURRENT_BRANCH/include/VERSION` | ||
CURRENT_MAJOR=`echo "$CURRENT_VERSION" | sed -n '1{p;q}'` | ||
GMAJOR=`expr $CURRENT_MAJOR - 1` | ||
|
||
BRANCH=`curl https://api.github.com/repos/osgeo/grass/branches | grep releasebranch_$GMAJOR | grep '"name":' | cut -f4 -d'"' | sort -V | tail -n 1` | ||
|
||
MAIN_VERSION=`curl https://raw.githubusercontent.com/osgeo/grass/$BRANCH/include/VERSION` | ||
|
||
GMINOR=`echo "$MAIN_VERSION" | sed -n '2{p;q}'` | ||
GPATCH=`echo "$MAIN_VERSION" | sed -n '3{p;q}' | sed 's/[^0-9]*//g'` | ||
Comment on lines
+44
to
+47
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I don't understand, minor is taken from the file while major is taken from the branch and used for the branch again. Why not getting the right branch and then the version numbers from there? There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Is this all just to keep the two scripts in sync? I think the "current" thing is really relevant for the legacy, but here where the code can be simpler. Maybe I just needs to code comments to clarify. |
||
|
||
DOTVERSION=$GMAJOR.$GMINOR | ||
VERSION=$GMAJOR$GMINOR | ||
GVERSION=$GMAJOR | ||
|
||
################### | ||
CFLAGSSTRING='-O2' | ||
CFLAGSSTRING='-Werror-implicit-function-declaration -fno-common' | ||
LDFLAGSSTRING='-s' | ||
|
||
#define several paths if required: | ||
|
||
MAINDIR=/home/neteler | ||
# where to find the GRASS sources (git clone): | ||
SOURCE=$MAINDIR/src/ | ||
BRANCH=releasebranch_${GMAJOR}_$GMINOR | ||
GRASSBUILDDIR=$SOURCE/$BRANCH | ||
TARGETMAIN=/var/www/code_and_data | ||
TARGETDIR=$TARGETMAIN/grass${VERSION}/binary/linux/snapshot | ||
TARGETHTMLDIR=$TARGETMAIN/grass${VERSION}/manuals/ | ||
# programmer's manual is only built from the relbranch_8_0 | ||
|
||
MYBIN=$MAINDIR/binaries | ||
|
||
############################## nothing to change below: | ||
|
||
MYMAKE="nice make LD_LIBRARY_PATH=$MYBIN/lib:/usr/lib:/usr/local/lib" | ||
|
||
# catch CTRL-C and other breaks: | ||
trap "echo 'user break.' ; exit" 2 3 9 15 | ||
|
||
halt_on_error() | ||
{ | ||
echo "ERROR: $1" | ||
exit 1 | ||
} | ||
|
||
configure_grass() | ||
{ | ||
|
||
# which package? | ||
# --with-mysql --with-mysql-includes=/usr/include/mysql --with-mysql-libs=/usr/lib/mysql \ | ||
|
||
|
||
CFLAGS=$CFLAGSSTRING LDFLAGS=$LDFLAGSSTRING ./configure \ | ||
--with-cxx \ | ||
--with-sqlite \ | ||
--with-postgres \ | ||
--with-geos \ | ||
--with-odbc \ | ||
--with-cairo --with-cairo-ldflags=-lfontconfig \ | ||
--with-proj-share=/usr/share/proj \ | ||
--with-postgres --with-postgres-includes=/usr/include/postgresql \ | ||
--with-freetype --with-freetype-includes=/usr/include/freetype2 \ | ||
--with-netcdf \ | ||
--with-pdal \ | ||
--with-fftw \ | ||
--with-nls \ | ||
--with-blas --with-blas-includes=/usr/include/atlas/ \ | ||
--with-lapack --with-lapack-includes=/usr/include/atlas/ \ | ||
--with-zstd \ | ||
--with-liblas \ | ||
2>&1 | tee config_$DOTVERSION.git_log.txt | ||
|
||
if [ $? -ne 0 ] ; then | ||
halt_on_error "configure doesn't work." | ||
fi | ||
} | ||
|
||
######## update from git: | ||
|
||
mkdir -p $TARGETDIR | ||
cd $GRASSBUILDDIR/ | ||
|
||
$MYMAKE distclean > /dev/null 2>&1 | ||
|
||
# cleanup leftover garbage | ||
git status | grep '.rst' | xargs rm -f | ||
rm -rf lib/python/docs/_build/ lib/python/docs/_templates/layout.html | ||
rm -f config_${DOTVERSION}.git_log.txt ChangeLog | ||
|
||
# be sure to be on branch | ||
git checkout $BRANCH | ||
|
||
echo "git update..." | ||
git fetch --all --prune && git checkout $BRANCH && git pull --rebase || halt_on_error "git update error!" | ||
git status | ||
|
||
# for the contributors list in CMS | ||
cp -f *.csv $TARGETMAIN/uploads/grass/ | ||
|
||
#configure | ||
echo "configuring" | ||
configure_grass || (echo "$0: an error occured" ; exit 1) | ||
pwd | ||
ARCH=`cat include/Make/Platform.make | grep ^ARCH | cut -d'=' -f2 | xargs` | ||
|
||
######## now GRASS is prepared #################### | ||
|
||
#### next compile GRASS: | ||
$MYMAKE | ||
|
||
|
||
echo "GRASS $VERSION compilation done" | ||
|
||
# now GRASS is prepared ############################################ | ||
|
||
#### create module overview (https://trac.osgeo.org/grass/ticket/1203) | ||
#sh utils/module_synopsis.sh | ||
|
||
#### generate developer stuff: pygrass docs + gunittest docs | ||
# generate pyGRASS sphinx manual (in docs/html/libpython/) | ||
# including source code | ||
$MYMAKE sphinxdoclib | ||
|
||
## | ||
echo "Copy over the manual + pygrass HTML pages:" | ||
mkdir -p $TARGETHTMLDIR | ||
# don't destroy the addons | ||
\mv $TARGETHTMLDIR/addons /tmp | ||
rm -f $TARGETHTMLDIR/*.* | ||
(cd $TARGETHTMLDIR ; rm -rf barscales colortables icons northarrows) | ||
\mv /tmp/addons $TARGETHTMLDIR | ||
|
||
cp -rp dist.$ARCH/docs/html/* $TARGETHTMLDIR/ | ||
echo "Copied pygrass progman to http://grass.osgeo.org/grass${VERSION}/manuals/libpython/" | ||
|
||
echo "Injecting DuckDuckGo search field into manual main page..." | ||
(cd $TARGETHTMLDIR/ ; sed -i -e "s+</table>+</table><\!\-\- injected in cron_grass7_relbranch_build_binaries.sh \-\-> <center><iframe src=\"https://duckduckgo.com/search.html?site=grass.osgeo.org%26prefill=Search%20manual%20pages%20at%20DuckDuckGo\" style=\"overflow:hidden;margin:0;padding:0;width:410px;height:40px;\" frameborder=\"0\"></iframe></center>+g" index.html) | ||
|
||
cp -p AUTHORS CHANGES CITING COPYING GPL.TXT INSTALL REQUIREMENTS.html $TARGETDIR/ | ||
|
||
# inject G8.x new version hint in a red box: | ||
(cd $TARGETHTMLDIR/ ; for myfile in `ls *.html` ; do sed -i -e "s:<hr class=\"header\">:<hr class=\"header\"><p style=\"border\:3px; border-style\:solid; border-color\:#BC1818; padding\: 1em;\">Note\: This document is for an older version of GRASS GIS that will be discontinued soon. You should upgrade, and read the <a href=\"../../grass-stable/manuals/$myfile\">current manual page</a>.</p>:g" $myfile ; done) | ||
# also for Python | ||
(cd $TARGETHTMLDIR/libpython/ ; for myfile in `ls *.html` ; do sed -i -e "s:<hr class=\"header\">:<hr class=\"header\"><p style=\"border\:3px; border-style\:solid; border-color\:#FF2121; padding\: 1em;\">Note\: This document is for an older version of GRASS GIS that will be discontinued soon. You should upgrade, and read the <a href=\"../../../grass-stable/manuals/libpython/$myfile\">current Python manual page</a>.</p>:g" $myfile ; done) | ||
|
||
# SEO: inject canonical link in all (old) manual pages to point to latest stable (avoid duplicate content SEO punishment) | ||
# see https://developers.google.com/search/docs/crawling-indexing/consolidate-duplicate-urls | ||
(cd $TARGETHTMLDIR/ ; for myfile in `ls *.html` ; do sed -i -e "s:</head>:<link rel=\"canonical\" href=\"https\://grass.osgeo.org/grass-stable/manuals/$myfile\">\n</head>:g" $myfile ; done) | ||
(cd $TARGETHTMLDIR/addons/ ; for myfile in `ls *.html` ; do sed -i -e "s:</head>:<link rel=\"canonical\" href=\"https\://grass.osgeo.org/grass-stable/manuals/addons/$myfile\">\n</head>:g" $myfile ; done) | ||
(cd $TARGETHTMLDIR/libpython/ ; for myfile in `ls *.html` ; do sed -i -e "s:</head>:<link rel=\"canonical\" href=\"https\://grass.osgeo.org/grass-stable/manuals/libpython/$myfile\">\n</head>:g" $myfile ; done) | ||
|
||
# clean wxGUI sphinx manual etc | ||
(cd $GRASSBUILDDIR/ ; $MYMAKE cleansphinx) | ||
|
||
##### generate i18N POT files, needed for https://www.transifex.com/grass-gis/ | ||
(cd locale ; | ||
$MYMAKE pot | ||
mkdir -p $TARGETDIR/transifex/ | ||
cp templates/*.pot $TARGETDIR/transifex/ | ||
) | ||
|
||
##### generate i18N stats for HTML page path (WebSVN): | ||
## Structure: grasslibs_ar.po 144 translated messages 326 fuzzy translations 463 untranslated messages. | ||
cd $GRASSBUILDDIR | ||
(cd locale/ ; | ||
touch po/*.po ; | ||
$MYMAKE mo > out.i18n 2>&1 ; | ||
# libs, mods, wx | ||
# hack: | ||
cat out.i18n | tr '\n' ' ' | sed -e 's+ msgfmt+\ | ||
msgfmt+g' | tr -s ' ' ' ' | cut -d' ' -f5,6- | sed 's+\,++g' | sed 's+^po/++g' | grep -v 'done for' | sort > $TARGETDIR/i18n_stats.txt | ||
rm -f out.i18n | ||
) | ||
|
||
cd $TARGETDIR/ | ||
cat i18n_stats.txt | grep mod > i18n_stats_mods.txt | ||
cat i18n_stats.txt | grep lib > i18n_stats_libs.txt | ||
cat i18n_stats.txt | grep wxpy > i18n_stats_wxpy.txt | ||
cd $GRASSBUILDDIR | ||
|
||
# package the package | ||
$MYMAKE bindist | ||
if [ $? -ne 0 ] ; then | ||
halt_on_error "make bindist." | ||
fi | ||
|
||
#report system: | ||
echo "System: | ||
$ARCH, compiled with:" > grass-$DOTVERSION\_$ARCH\_bin.txt | ||
## echo "Including precompiled $GDALVERSION library for r.in.gdal" >> grass-$DOTVERSION\_$ARCH\_bin.txt | ||
gcc -v 2>&1 | grep -v Reading >> grass-$DOTVERSION\_$ARCH\_bin.txt | ||
|
||
# clean old version off | ||
rm -f $TARGETDIR/grass-$DOTVERSION\_$ARCH\_bin.txt | ||
rm -f $TARGETDIR/grass-$DOTVERSION*.tar.gz | ||
rm -f $TARGETDIR/grass-$DOTVERSION*install.sh | ||
|
||
############################################ | ||
echo "Copy new binary version into web space:" | ||
cp -p grass-$DOTVERSION\_$ARCH\_bin.txt grass-$DOTVERSION*.tar.gz grass-$DOTVERSION*install.sh $TARGETDIR | ||
rm -f grass-$DOTVERSION\_$ARCH\_bin.txt grass-$DOTVERSION*.tar.gz grass-$DOTVERSION*install.sh | ||
|
||
# generate manual ZIP package | ||
(cd $TARGETHTMLDIR/.. ; rm -f $TARGETHTMLDIR/*html_manual.zip ; zip -r /tmp/grass-${DOTVERSION}_html_manual.zip manuals/) | ||
mv /tmp/grass-${DOTVERSION}_html_manual.zip $TARGETHTMLDIR/ | ||
|
||
# error log etc | ||
cp -f config_$DOTVERSION.git_log.txt $TARGETDIR | ||
cp -f error.log $TARGETDIR | ||
chmod -R a+r,g+w $TARGETDIR 2> /dev/null | ||
chmod -R a+r,g+w $TARGETHTMLDIR 2> /dev/null | ||
|
||
echo "Written to: $TARGETDIR" | ||
|
||
############################################ | ||
# compile addons | ||
|
||
# update addon repo (addon repo has been cloned twice on the server to | ||
# separate grass7 and grass8 addon compilation) | ||
(cd ~/src/grass$GMAJOR-addons/; git checkout grass$GMAJOR; git pull origin grass$GMAJOR) | ||
# compile addons | ||
cd $GRASSBUILDDIR | ||
sh ~/cronjobs/compile_addons_git.sh $GMAJOR \ | ||
$GMINOR \ | ||
~/src/grass$GMAJOR-addons/src/ \ | ||
~/src/$BRANCH/dist.$ARCH/ \ | ||
~/.grass$GMAJOR/addons \ | ||
~/src/$BRANCH/bin.$ARCH/grass$VERSION \ | ||
1 | ||
mkdir -p $TARGETHTMLDIR/addons/ | ||
# copy indvidual addon html files into one target dir if compiled addon | ||
# has own dir e.g. ~/.grass7/addons/db.join/ with bin/ docs/ etc/ scripts/ | ||
# subdir | ||
for dir in `find ~/.grass$GMAJOR/addons -maxdepth 1 -type d`; do | ||
if [ -d $dir/docs/html ] ; then | ||
if [ "$(ls -A $dir/docs/html/)" ]; then | ||
for f in $dir/docs/html/*; do | ||
cp $f $TARGETHTMLDIR/addons/ | ||
done | ||
fi | ||
fi | ||
done | ||
|
||
sh ~/cronjobs/grass-addons-index.sh $GMAJOR $GMINOR $GPATCH $TARGETHTMLDIR/addons/ | ||
chmod -R a+r,g+w $TARGETHTMLDIR 2> /dev/null | ||
# inject G8.x new version hint in a red box: into index.html and all addon manual pages | ||
(cd $TARGETHTMLDIR/addons/ ; sed -i -e "s: Addons Manual pages</h2>: Addons Manual pages</h2><p style=\"border\:3px; border-style\:solid; border-color\:#BC1818; padding\: 1em;\">Note\: This addon documentation is for an older version of GRASS GIS that will be discontinued soon. You should upgrade your GRASS GIS installation, and read the <a href=\"../../../grass-stable/manuals/addons/index.html\">current addon manual page</a>.</p>:g" index.html) | ||
(cd $TARGETHTMLDIR/addons/ ; for myfile in `ls *.html` ; do sed -i -e "s:<hr class=\"header\">:<hr class=\"header\"><p style=\"border\:3px; border-style\:solid; border-color\:#BC1818; padding\: 1em;\">Note\: This addon document is for an older version of GRASS GIS that will be discontinued soon. You should upgrade your GRASS GIS installation, and read the <a href=\"../../../grass-stable/manuals/addons/$myfile\">current addon manual page</a>.</p>:g" $myfile ; done) | ||
|
||
|
||
# cp logs from ~/.grass$GMAJOR/addons/logs/ | ||
mkdir -p $TARGETMAIN/addons/grass$GMAJOR/logs/ | ||
cp -p ~/.grass$GMAJOR/addons/logs/* $TARGETMAIN/addons/grass$GMAJOR/logs/ | ||
|
||
# generate addons module.xml file (required for g.extension module) | ||
~/src/$BRANCH/bin.$ARCH/grass$VERSION --tmp-location EPSG:4326 --exec ~/cronjobs/build-xml.py --build ~/.grass$GMAJOR/addons | ||
cp ~/.grass$GMAJOR/addons/modules.xml $TARGETMAIN/addons/grass$GMAJOR/modules.xml | ||
|
||
# regenerate keywords.html file with addons modules keywords | ||
export ARCH | ||
export ARCH_DISTDIR=$GRASSBUILDDIR/dist.$ARCH | ||
export GISBASE=$ARCH_DISTDIR | ||
export VERSION_NUMBER=$DOTVERSION | ||
python3 $GRASSBUILDDIR/man/build_keywords.py $TARGETMAIN/grass$GMAJOR$GMINOR/manuals/ $TARGETMAIN/grass$GMAJOR$GMINOR/manuals/addons/ | ||
unset ARCH ARCH_DISTDIR GISBASE VERSION_NUMBER | ||
|
||
############################################ | ||
# create sitemaps to expand the hugo sitemap | ||
|
||
python3 $HOME/src/grass$GMAJOR-addons/utils/create_manuals_sitemap.py --dir=/var/www/code_and_data/grass$GMAJOR$GMINOR/manuals/ --url=https://grass.osgeo.org/grass$GMAJOR$GMINOR/manuals/ -o | ||
python3 $HOME/src/grass$GMAJOR-addons/utils/create_manuals_sitemap.py --dir=/var/www/code_and_data/grass$GMAJOR$GMINOR/manuals/addons/ --url=https://grass.osgeo.org/grass$GMAJOR$GMINOR/manuals/addons/ -o | ||
|
||
############################################ | ||
# cleanup | ||
cd $GRASSBUILDDIR | ||
$MYMAKE distclean > /dev/null || (echo "$0: an error occured" ; exit 1) | ||
|
||
echo "Finished GRASS $VERSION $ARCH compilation." | ||
echo "Written to: $TARGETDIR" | ||
echo "Copied HTML ${GVERSION} manual to https://grass.osgeo.org/grass${VERSION}/manuals/" | ||
echo "Copied pygrass progman ${GVERSION} to https://grass.osgeo.org/grass${VERSION}/manuals/libpython/" | ||
|
||
exit 0 |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
This will always result in the same thing as CURRENT_BRANCH since GMAJOR is taken from CURRENT_BRANCH, no?