From e16f799d6f7218b414acc6cccfa788bf03838cb7 Mon Sep 17 00:00:00 2001 From: dkazanc Date: Mon, 4 Mar 2019 13:26:56 +0000 Subject: [PATCH] savu2.3.2_install --- install/2_3_2_install/__init__.py | 22 + .../2_3_2_install/conda-recipes/__init__.py | 22 + .../conda-recipes/astra/astra.pth | 2 + .../conda-recipes/astra/build.sh | 22 + .../conda-recipes/astra/meta.yaml | 25 + .../2_3_2_install/conda-recipes/h5py/build.sh | 24 + .../conda-recipes/h5py/meta.yaml | 25 + .../2_3_2_install/conda-recipes/hdf5/build.sh | 17 + .../conda-recipes/hdf5/meta.yaml | 20 + .../conda-recipes/xdesign/build.sh | 4 + .../conda-recipes/xdesign/meta.yaml | 24 + .../conda-recipes/xraylib/build.sh | 7 + .../conda-recipes/xraylib/meta.yaml | 26 + .../2_3_2_install/savu_v2.3.2/environment.yml | 30 ++ .../2_3_2_install/savu_v2.3.2/savu/build.sh | 4 + .../2_3_2_install/savu_v2.3.2/savu/meta.yaml | 23 + .../savu_v2.3.2/savu_installer.sh | 448 ++++++++++++++++++ .../savu_v2.3.2/savu_master/build.sh | 4 + .../savu_v2.3.2/savu_master/meta.yaml | 24 + install/2_3_2_install/savu_v2.3.2/version.txt | 2 + .../savu_v2.3.2/versions_file.txt | 7 + .../2_3_2_install/tests/local_mpi_cpu_test.sh | 41 ++ .../2_3_2_install/tests/local_mpi_gpu_test.sh | 41 ++ install/2_3_2_install/tests/mpi_cpu_test.sh | 24 + install/2_3_2_install/tests/mpi_gpu_test.sh | 24 + install/2_3_2_install/tests/test_setup.sh | 7 + 26 files changed, 919 insertions(+) create mode 100644 install/2_3_2_install/__init__.py create mode 100644 install/2_3_2_install/conda-recipes/__init__.py create mode 100644 install/2_3_2_install/conda-recipes/astra/astra.pth create mode 100644 install/2_3_2_install/conda-recipes/astra/build.sh create mode 100644 install/2_3_2_install/conda-recipes/astra/meta.yaml create mode 100644 install/2_3_2_install/conda-recipes/h5py/build.sh create mode 100644 install/2_3_2_install/conda-recipes/h5py/meta.yaml create mode 100644 install/2_3_2_install/conda-recipes/hdf5/build.sh create mode 100644 install/2_3_2_install/conda-recipes/hdf5/meta.yaml create mode 100644 install/2_3_2_install/conda-recipes/xdesign/build.sh create mode 100644 install/2_3_2_install/conda-recipes/xdesign/meta.yaml create mode 100644 install/2_3_2_install/conda-recipes/xraylib/build.sh create mode 100644 install/2_3_2_install/conda-recipes/xraylib/meta.yaml create mode 100644 install/2_3_2_install/savu_v2.3.2/environment.yml create mode 100644 install/2_3_2_install/savu_v2.3.2/savu/build.sh create mode 100644 install/2_3_2_install/savu_v2.3.2/savu/meta.yaml create mode 100644 install/2_3_2_install/savu_v2.3.2/savu_installer.sh create mode 100644 install/2_3_2_install/savu_v2.3.2/savu_master/build.sh create mode 100644 install/2_3_2_install/savu_v2.3.2/savu_master/meta.yaml create mode 100644 install/2_3_2_install/savu_v2.3.2/version.txt create mode 100644 install/2_3_2_install/savu_v2.3.2/versions_file.txt create mode 100644 install/2_3_2_install/tests/local_mpi_cpu_test.sh create mode 100644 install/2_3_2_install/tests/local_mpi_gpu_test.sh create mode 100644 install/2_3_2_install/tests/mpi_cpu_test.sh create mode 100644 install/2_3_2_install/tests/mpi_gpu_test.sh create mode 100644 install/2_3_2_install/tests/test_setup.sh diff --git a/install/2_3_2_install/__init__.py b/install/2_3_2_install/__init__.py new file mode 100644 index 000000000..05ec9bfd3 --- /dev/null +++ b/install/2_3_2_install/__init__.py @@ -0,0 +1,22 @@ +# Copyright 2014 Diamond Light Source Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +All the plugin architecture for Savu is contained here + + +.. moduleauthor:: Mark Basham + +""" + diff --git a/install/2_3_2_install/conda-recipes/__init__.py b/install/2_3_2_install/conda-recipes/__init__.py new file mode 100644 index 000000000..05ec9bfd3 --- /dev/null +++ b/install/2_3_2_install/conda-recipes/__init__.py @@ -0,0 +1,22 @@ +# Copyright 2014 Diamond Light Source Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +All the plugin architecture for Savu is contained here + + +.. moduleauthor:: Mark Basham + +""" + diff --git a/install/2_3_2_install/conda-recipes/astra/astra.pth b/install/2_3_2_install/conda-recipes/astra/astra.pth new file mode 100644 index 000000000..45c4c6127 --- /dev/null +++ b/install/2_3_2_install/conda-recipes/astra/astra.pth @@ -0,0 +1,2 @@ +./astra/python + diff --git a/install/2_3_2_install/conda-recipes/astra/build.sh b/install/2_3_2_install/conda-recipes/astra/build.sh new file mode 100644 index 000000000..acd72a6d6 --- /dev/null +++ b/install/2_3_2_install/conda-recipes/astra/build.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +nvcc=`command -v nvcc` +cuda=${nvcc%/bin/nvcc} + +savu_path=`command -v savu` +ana_path=${savu_path%/savu} +prefix=${ana_path%/bin}/lib/python2.7/site-packages/astra +export PATH=$ana_path:$PATH + +cd build/linux + +if [ "$cuda" ]; then + ./configure --with-cuda=$cuda --with-python --prefix=$prefix +else + echo "cuda has not been found." + ./configure --with-python --prefix=$prefix +fi + +make -j 4 +make install + diff --git a/install/2_3_2_install/conda-recipes/astra/meta.yaml b/install/2_3_2_install/conda-recipes/astra/meta.yaml new file mode 100644 index 000000000..4a04c0bc4 --- /dev/null +++ b/install/2_3_2_install/conda-recipes/astra/meta.yaml @@ -0,0 +1,25 @@ +{% set version = "1.8" %} + +package: + name: astra-toolbox + version: 1.8 + +source: + fn: astra-{{ version }}.tar.bz2 + url: http://www.astra-toolbox.com/files/astra-{{ version }}/astra-{{ version }}.tar.bz2 + sha256: 44602e0eb0789a557622eae959fc0a067d81396bcb79ec315cb39b59d52322e7 + +requirements: + build: + - python + - numpy + - cython + +build: + number: 2 + +about: + home: http://www.astra-toolbox.com + license: GPLv3 + summary: 'The ASTRA Toolbox is a Python toolbox of high-performance GPU primitives for 2D and 3D tomography.' + diff --git a/install/2_3_2_install/conda-recipes/h5py/build.sh b/install/2_3_2_install/conda-recipes/h5py/build.sh new file mode 100644 index 000000000..055293a33 --- /dev/null +++ b/install/2_3_2_install/conda-recipes/h5py/build.sh @@ -0,0 +1,24 @@ +#!/bin/bash + +# set compiler wrapper +mpicc=$(command -v mpicc) +mpi=`dirname $mpicc` + +# set anaconda distribution +ana_path=$(command -v savu) +ana_path=${ana_path%/bin/savu} +hdf5_version=1.10.1 +hdf5_build_no=1 + +export LD_LIBRARY_PATH=$mpi/lib:$mpi/include:$ana_path/lib:$LD_LIBRARY_PATH +export LD_RUN_PATH=$LD_LIBRARY_PATH +export PYTHONPATH=$PYTHONPATH:$ana_path/lib/python2.7/site-packages +export PATH=$PATH:$ana_path/bin/ +export CC=$mpicc + +$PYTHON setup.py configure --hdf5=$ana_path +$PYTHON setup.py configure --hdf5-version=$hdf5_version +$PYTHON setup.py configure --mpi +$PYTHON setup.py build +$PYTHON setup.py install + diff --git a/install/2_3_2_install/conda-recipes/h5py/meta.yaml b/install/2_3_2_install/conda-recipes/h5py/meta.yaml new file mode 100644 index 000000000..0778446ae --- /dev/null +++ b/install/2_3_2_install/conda-recipes/h5py/meta.yaml @@ -0,0 +1,25 @@ +{% set version = "2.7.1" %} # change this + +package: + name: h5py + version: {{ version }} + +source: + fn: h5py-{{ version }}.tar.gz + url: https://github.com/h5py/h5py/archive/{{ version }}.tar.gz +requirements: + build: + - python + - numpy + - hdf5 + - cython + - pkgconfig + - six + +build: + number: 1 + +about: + home: http://www.h5py.org/ + license: New BSD (http://opensource.org/licenses/bsd-license.php) + diff --git a/install/2_3_2_install/conda-recipes/hdf5/build.sh b/install/2_3_2_install/conda-recipes/hdf5/build.sh new file mode 100644 index 000000000..add607820 --- /dev/null +++ b/install/2_3_2_install/conda-recipes/hdf5/build.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +# set compiler wrapper +mpicc=$(command -v mpicc) +mpi=${mpicc%/bin/mpicc} +export LD_LIBRARY_PATH=$mpi:$mpi/lib:$mpi/include:$LD_LIBRARY_PATH +export LD_RUN_PATH=$LD_LIBRARY_PATH + +# check anaconda distribution +ana_path=$(command -v savu) + +CC=$mpicc ./configure --with-zlib --enable-parallel --enable-shared --prefix=$PREFIX +make +make install + +rm -rf $PREFIX/share/hdf5_examples + diff --git a/install/2_3_2_install/conda-recipes/hdf5/meta.yaml b/install/2_3_2_install/conda-recipes/hdf5/meta.yaml new file mode 100644 index 000000000..a12ea27a8 --- /dev/null +++ b/install/2_3_2_install/conda-recipes/hdf5/meta.yaml @@ -0,0 +1,20 @@ +{% set version = "1.10.1" %} +{% set release = "hdf5-1.10" %} +{% set patch = "hdf5-1.10.1" %} + +package: + name: hdf5 + version: {{ version }} + +source: + fn: {{ patch }}.tar.bz2 + url: https://www.hdfgroup.org/ftp/HDF5/releases/{{ release }}/{{ patch }}/src/{{ patch }}.tar.bz2 + md5: d89893c05ee7ea8611b51bb39450d64e + +build: + number: 1 + +about: + home: http://www.hdfgroup.org/HDF5/ + license: BSD-style (http://www.hdfgroup.org/ftp/HDF5/current/src/unpacked/COPYING) + diff --git a/install/2_3_2_install/conda-recipes/xdesign/build.sh b/install/2_3_2_install/conda-recipes/xdesign/build.sh new file mode 100644 index 000000000..b161f631b --- /dev/null +++ b/install/2_3_2_install/conda-recipes/xdesign/build.sh @@ -0,0 +1,4 @@ +#!/bin/bash + +$PYTHON setup.py install + diff --git a/install/2_3_2_install/conda-recipes/xdesign/meta.yaml b/install/2_3_2_install/conda-recipes/xdesign/meta.yaml new file mode 100644 index 000000000..68693b427 --- /dev/null +++ b/install/2_3_2_install/conda-recipes/xdesign/meta.yaml @@ -0,0 +1,24 @@ +package: + name: xdesign + version: '0.3.0' + +source: + git_url: https://github.com/tomography/xdesign.git + +build: + number: 0 + +requirements: + build: + - python + - setuptools + +about: + home: http://xdesign.readthedocs.io + license: BSD-3 + summary: 'Benchmarking and optimization tools for tomography.' + +# See +# http://docs.continuum.io/conda/build.html for +# more information about meta.yaml + diff --git a/install/2_3_2_install/conda-recipes/xraylib/build.sh b/install/2_3_2_install/conda-recipes/xraylib/build.sh new file mode 100644 index 000000000..8f0acd316 --- /dev/null +++ b/install/2_3_2_install/conda-recipes/xraylib/build.sh @@ -0,0 +1,7 @@ + +./configure --enable-python --disable-perl --disable-java \ + --disable-fortran2003 --disable-lua --prefix=$PREFIX +make +make check +make install + diff --git a/install/2_3_2_install/conda-recipes/xraylib/meta.yaml b/install/2_3_2_install/conda-recipes/xraylib/meta.yaml new file mode 100644 index 000000000..f924056e2 --- /dev/null +++ b/install/2_3_2_install/conda-recipes/xraylib/meta.yaml @@ -0,0 +1,26 @@ +{% set version = "3.3.0" %} + +package: + name: xraylib + version: {{ version }} + +source: + fn: xraylib-{{ version }}.tar.gz + url: https://xraylib.tomschoonjans.eu/xraylib-{{ version }}.tar.gz + sha256: a22a73b8d90eb752b034bab1a4cf6abdd81b8c7dc5020bcb22132d2ee7aacd42 + +build: + number: 0 + +requirements: + build: + - python + - setuptools + - numpy + - scipy >=0.7.0 + - cython + - swig + - autoconf + - libtool + - automake + diff --git a/install/2_3_2_install/savu_v2.3.2/environment.yml b/install/2_3_2_install/savu_v2.3.2/environment.yml new file mode 100644 index 000000000..79d658ca9 --- /dev/null +++ b/install/2_3_2_install/savu_v2.3.2/environment.yml @@ -0,0 +1,30 @@ +name: root +channels: +- defaults +dependencies: +- scipy==1.1.0 +- colorama==0.3.9 +- setuptools==39.2.0 +- jinja2==2.10 +- numpy==1.14.5 +- pandas==0.23.3 +- mock==2.0.0 +- PyWavelets==0.5.2 +- scikit-image==0.14.0 +- scikit-learn==0.19.1 +- pytest==3.6.2 +- spyder==3.3.0 +- ipython==5.7.0 +- cython==0.28.3 +- boost==1.67.0 +- pip: + - fabio==0.4.0 + - pyfftw==0.10.4 + - nvidia-ml-py==7.352.0 + - PeakUtils==1.1.1 + - pymca==5.3.1 + - tifffile==0.4.0 + - pyreadline==2.1 + - gnureadline==6.3.8 + - sphinx_rtd_theme==0.4.1 + diff --git a/install/2_3_2_install/savu_v2.3.2/savu/build.sh b/install/2_3_2_install/savu_v2.3.2/savu/build.sh new file mode 100644 index 000000000..b6fbaff2b --- /dev/null +++ b/install/2_3_2_install/savu_v2.3.2/savu/build.sh @@ -0,0 +1,4 @@ +#!/bin/bash + +$PYTHON setup.py install --facility $FACILITY # Python command to install the script. + diff --git a/install/2_3_2_install/savu_v2.3.2/savu/meta.yaml b/install/2_3_2_install/savu_v2.3.2/savu/meta.yaml new file mode 100644 index 000000000..7fa123735 --- /dev/null +++ b/install/2_3_2_install/savu_v2.3.2/savu/meta.yaml @@ -0,0 +1,23 @@ +{% set version = "2.3.2" %} + +package: + name: savu + version: {{ version }} + +source: + url: https://github.com/DiamondLightSource/Savu/archive/v{{ version }}.tar.gz + +requirements: + build: + - python + +build: + preserve_egg_dir: True + script_env: + - FACILITY + +about: + home: https://github.com/diamondlightsource/Savu + license: Apache V2 and GPL V3 + license_file: LICENCE.txt + diff --git a/install/2_3_2_install/savu_v2.3.2/savu_installer.sh b/install/2_3_2_install/savu_v2.3.2/savu_installer.sh new file mode 100644 index 000000000..e7e6ecf9d --- /dev/null +++ b/install/2_3_2_install/savu_v2.3.2/savu_installer.sh @@ -0,0 +1,448 @@ +#!/bin/bash -ex + +# change to 'latest' for the latest version +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +versions_file=$DIR/versions_file.txt + +# error log to screen and file +log_temp=`mktemp -d` +error_log=$log_temp/savu_error_log.txt +exec 2> >(tee -ia $error_log) + +oldprompt=$PS1 +newprompt=">>> " +export PS1=$newprompt + +for sig in INT TERM EXIT; do + trap "export PS1=$oldprompt; [[ $sig == EXIT ]] || kill -$sig $$" $sig +done + +PREFIX="${PREFIX:-$HOME}" + +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +savu_version=`cat $DIR/version.txt` + +# function for parsing optional arguments +function arg_parse () +{ + flag=$1 + return=$2 + while [[ $# -gt 2 ]] ; do + if [ $3 == $flag ] ; then + eval "$return"=$4 + fi + shift + done +} + +# function for parsing flags without arguments +function flag_parse () +{ + flag=$1 + return=$2 + while [[ $# -gt 2 ]] ; do + if [ $3 == $flag ] ; then + eval "$return"=true + fi + shift + done +} + +# Set the install PREFIX +flag_parse "--PREFIX" prefix "$@" +if [ $prefix ] ; then + PREFIX=true +fi + +# Set the test flag to true if test only +flag_parse "--tests_only" test_flag "$@" +if [ $test_flag ] ; then + test_flag=true +fi + +# Set the prompts flag to false if no prompts are required +flag_parse "--no_prompts" prompts "$@" +if [ $prompts ] ; then + prompts=false +else + prompts=true +fi + +# set the intermediate folder +arg_parse "-f" facility "$@" +if [ ! $facility ] ; then + facility=dls # change this default? +fi + +export FACILITY=$facility + +# set the intermediate folder +arg_parse "-c" conda_folder "$@" +if [ ! $conda_folder ] ; then + conda_folder=Savu_$savu_version +fi + +# set the intermediate folder +arg_parse "-s" savu_recipe "$@" +if [ ! $savu_recipe ] ; then + savu_recipe=savu +elif [ $savu_recipe = 'master' ] ; then + savu_recipe=savu_master +else + echo "Unknown Savu installation version." +fi + +#=========================library checking============================== + +if [ $test_flag ] ; then + echo -e "\n=============================================================" + echo -e " ......Thank you for running the Savu tests......\n" + echo -e "Performing a library check..." +else + echo -e "\n=============================================================" + echo -e " ......Thank you for running the Savu installer......\n" + echo -e "Performing a library check..." + echo -e "\nNB: An MPI implementation is required to build Savu." + echo -e "fftw is required to build Savu." + echo -e "Cuda is desirable for a full range of plugins." + echo -e "\n=============================================================" +fi + +# set compiler wrapper +MPICC=$(command -v mpicc) +if ! [ "$MPICC" ]; then + echo "ERROR: I require mpicc but I can't find it. Check /path/to/mpi_implementation/bin is in your PATH" + exit 1 +else + echo "Using mpicc: " $MPICC +fi + +# check for fftw +CFLAGS="" +LDFLAGS="" +# If no FFTWHOME provided search LD_LIBRARY_PATH for the static library +if [ -z ${FFTWHOME+x} ]; then + IFS=: + file_base=libfftw?.so + for p in ${LD_LIBRARY_PATH}; do + file_path=${p}/$file_base + if [ "x$p" != "x" -a -e $file_path ]; then + FFTWHOME=${file_path%/lib/libfftw?.so} + break + fi + done +fi +# If a valid FFTWHOME was found or provided then use it +if [ ${FFTWHOME} ]; then + CFLAGS="$FFTWHOME/include" + LDFLAGS="$FFTWHOME/lib" +fi + +if [ "$CFLAGS" ]; then + echo "Using fftw: " $FFTWHOME +else + echo "fftw has not been found." +fi + +# check for cuda +nvcc=`command -v nvcc` +CUDAHOME=${nvcc%/bin/nvcc} +if [ "$CUDAHOME" ]; then + echo "Using cuda: " $CUDAHOME +else + echo "cuda has not been found." +fi + +if [ $test_flag ] && [ $prompts = true ] ; then + + PYTHONHOME=`command -v conda` + PYTHONHOME=${PYTHONHOME%conda} + if [ ! $PYTHONHOME ] ; then + echo -e "No conda environment found in PATH. Try:" + echo -e " >>> source /savu_setup.sh" + echo -e "Aborting the tests." + exit 1 + fi + + echo -e "=============================================================\n" + while true ; do + read -n 1 -p "Are you happy to proceed with the tests? (y/n): " input + if [ "$input" = "y" ]; then + echo -e "\nProceeding with the tests." + break + elif [ "$input" = "n" ]; then + echo -e "\nAborting the tests." + exit 0 + else + echo -e "\nYour input was unknown.\n" + fi + done +elif [ $prompts = true ] ; then + echo -e "=============================================================\n" + while true ; do + read -n 1 -p "Are you happy to proceed with the installation? (y/n): " input + if [ "$input" = "y" ]; then + echo -e "\nYour input was yes" + break + elif [ "$input" = "n" ]; then + echo -e "\nInstallation process terminated." + exit 0 + else + echo -e "\nYour input was unknown.\n" + fi + done + +#=====================installing other packages========================== + + echo -e "\nInstalling Savu in" $PREFIX + read -p ">>> Press ENTER to continue or input a different path: " input + + if [ "$input" != "" ]; then + PREFIX=$input + fi + + while true; do + if [ -d "$PREFIX" ]; then + PREFIX=$PREFIX/$conda_folder/ + break + fi + echo "The path" $PREFIX "is not recognised" + read -p ">>> Please input a different installation path: " input + PREFIX=$input + done + + if [ -d "$PREFIX" ]; then + echo + while true ; do + read -n 1 -p "The folder $PREFIX already exists. Continue? [y/n]" input + if [ "$input" = "y" ]; then + echo -e "\nStarting the installation........" + break + elif [ "$input" = "n" ]; then + echo -e "\nInstallation process terminated." + exit 0 + else + echo -e "\nYour input was unknown.\n\n" + fi + done + else + # create the folder + mkdir -p $PREFIX + fi +else + if [ ! -d "$PREFIX" ] ; then + mkdir -p $PREFIX + fi +fi + +if [ ! $test_flag ] ; then + + echo -e "\nThank you! Installing Savu into" $PREFIX"\n" + + unset IFS + string=`awk '/^miniconda/' $versions_file` + miniconda_version=`echo $string | cut -d " " -f 2` + wget https://repo.continuum.io/miniconda/Miniconda2-$miniconda_version-Linux-x86_64.sh -O $PREFIX/miniconda.sh; + bash $PREFIX/miniconda.sh -b -p $PREFIX/miniconda + PYTHONHOME=$PREFIX/miniconda/bin + export PATH="$PYTHONHOME:$PATH" + + conda install -y -q conda-build conda-env + conda install -y -q conda-verify + conda env update -n root -f $DIR/environment.yml + + echo "Building Savu..." + conda build $DIR/$savu_recipe + savubuild=`conda build $DIR/$savu_recipe --output` + echo "Installing Savu..." + conda install -y -q --use-local $savubuild + + path=$(python -c "import savu; import os; print os.path.abspath(savu.__file__)") + savu_path=${path%/savu/__init__.pyc} + + # get the savu version + install_path=$(python -c "import savu; import savu.version as sv; print sv.__install__") + recipes=$savu_path/$install_path/conda-recipes + + launcher_path=`command -v savu_launcher.sh` + launcher_path=${launcher_path%/savu_launcher.sh} + if [ "$facility" ]; then + cp $savu_path/system_files/$facility/mpi/savu_launcher.sh $launcher_path + cp $savu_path/system_files/$facility/mpi/savu_mpijob.sh $launcher_path + fi + + + + #----------------------------------------------------------------- +# echo "Building astra toolbox..." +# conda build $recipes/astra +# astrabuild=`conda build $recipes/astra --output` + + echo "Installing astra toolbox..." +# conda install -y -q --use-local $astrabuild + +# site_path=$(python -c "import site; print site.getsitepackages()[0]") +# cp $recipes/astra/astra.pth $site_path +# astra_lib_path=$site_path/astra/lib + + conda install -y -q -c astra-toolbox/label/dev astra-toolbox + #----------------------------------------------------------------- + + #----------------------------------------------------------------- + echo "Building xraylib..." + conda build $recipes/xraylib + xraylibbuild=`conda build $recipes/xraylib --output` + + echo "Installing xraylib..." + conda install -y -q --use-local $xraylibbuild --no-deps + #----------------------------------------------------------------- + + #----------------------------------------------------------------- + echo "Installing tomopy..." + # these packages were missing in copied environment + string=`awk '/^tomopy/' $versions_file` + tomopy_version=`echo $string | cut -d " " -f 2` + string=`awk '/^dxchange/' $versions_file` + dxchange_version=`echo $string | cut -d " " -f 2` + conda install -y -q -c dgursoy tomopy==$tomopy_version --no-deps + conda install -y -q -c dgursoy dxchange==$dxchange_version --no-deps + #----------------------------------------------------------------- + + #----------------------------------------------------------------- + echo "Installing ccpi-reconstruction" + string=`awk '/^ccpi/' $versions_file` + ccpi_version=`echo $string | cut -d " " -f 2` + conda install -y -q -c ccpi ccpi-reconstruction==$ccpi_version -c conda-forge --no-deps + conda install -y -q -c ccpi cil_reconstruction==$ccpi_version -c conda-forge --no-deps + #----------------------------------------------------------------- + + #----------------------------------------------------------------- + echo "Installing pyfai" + string=`awk '/^pyfai/' $versions_file` + pyfai_version=`echo $string | cut -d " " -f 2` + conda install -y -q -c conda-forge pyfai==$pyfai_version --no-deps + #----------------------------------------------------------------- + + #----------------------------------------------------------------- + echo "Installing mpi4py..." + pip uninstall -y -q mpi4py || true + string=`awk '/^mpi4py/' $versions_file` + mpi4py_version=`echo $string | cut -d " " -f 2` + env MPICC=$MPICC pip install mpi4py==$mpi4py_version + #----------------------------------------------------------------- + + #----------------------------------------------------------------- + echo "Building hdf5..." + conda uninstall -y -q hdf5 || true + conda build $recipes/hdf5 + hdf5build=`conda build $recipes/hdf5 --output` + + echo "Installing hdf5..." + conda install -y -q --use-local $hdf5build + #----------------------------------------------------------------- + + #----------------------------------------------------------------- + echo "Building h5py..." + conda uninstall -y -q h5py || true + conda build $recipes/h5py --no-test + h5pybuild=`conda build $recipes/h5py --output` + + echo "Installing h5py..." + conda install -y -q --use-local $h5pybuild + #----------------------------------------------------------------- + + + echo -e "\n\t***************************************************" + echo -e "\t Package installation complete" + echo -e "\t Check $error_log for errors" + echo -e "\t***************************************************\n" + +fi + +if [ ! $test_flag ] ; then + if [ $prompts = true ] ; then + while true; do + read -n 1 -p "Would you like to run the tests? (y/n): " input + if [ "$input" = "y" ]; then + echo -e "\nYour input was yes" + test_flag=true + break + elif [ "$input" = "n" ]; then + echo -e "Aborting test run..." + echo -e "To run the tests later type: " + echo -e " >>> bash savu_v2.1/savu_installer.sh --tests_only" + exit 0 + else + echo -e "\nYour input was unknown.\n" + fi + done + fi + + setup_script=$PREFIX/'savu_setup.sh' + echo -e "\nCreating a Savu setup script" $setup_script + ( [ -e "$setup_script" ] || touch "$setup_script" ) && [ ! -w "$setup_script" ] && echo cannot write to $setup_script && exit 1 + MPIHOME="$(dirname "$(dirname $MPICC)")" + echo '#!bin/bash' > $setup_script + echo ""export PATH=$MPIHOME/bin:'$PATH'"" >> $setup_script + echo ""export LD_LIBRARY_PATH=$MPIHOME/lib:'$LD_LIBRARY_PATH'"" >> $setup_script + echo ""export PYTHONUSERSITE True"" >> $setup_script + echo ""export PATH=$PYTHONHOME:'$PATH'"" >> $setup_script + echo ""export LD_LIBRARY_PATH=$PYTHONHOME/lib:'$LD_LIBRARY_PATH'"" >> $setup_script + echo ""export LD_LIBRARY_PATH=$astra_lib_path:'$LD_LIBRARY_PATH'"" >> $setup_script + if [ "$CUDAHOME" ]; then + echo ""export PATH=$CUDAHOME/bin:'$PATH'"" >> $setup_script + echo ""export LD_LIBRARY_PATH=$CUDAHOME/lib64:'$LD_LIBRARY_PATH'"" >> $setup_script + fi + if [ "$FFTWHOME" ]; then + echo ""export FFTWDIR=$FFTWHOME"" >> $setup_script + echo ""export LD_LIBRARY_PATH=$FFTWHOME/lib:'$LD_LIBRARY_PATH'"" >> $setup_script + fi + + source $setup_script +fi + +if [ $test_flag ] ; then + + nGPUs=$(nvidia-smi -L | wc -l) + + echo -e "\n***** Testing Savu setup *****\n" + savu_quick_tests + + echo -e "\n*****Running Savu single-threaded local tests *****\n" + savu_full_tests + + echo -e "\n************** Single-threaded local tests complete ******************\n" + + test_dir=`mktemp -d` + tmp_dir=`mktemp -d` + tmpfile=$tmp_dir/temp_output.txt + touch $tmpfile + +fi + +if [ ! $test_flag ] ; then + + launcher_path=`command -v savu_launcher.sh` + mpijob_path=`command -v savu_mpijob.sh` + echo -e "\n\n===============================IMPORTANT NOTICES================================" + echo -e "If you are installing Savu for cluster use, you will need to update the savu " + echo -e "launcher scripts:" + echo -e "\n$launcher_path" + echo -e "$mpijob_path\n" + echo -e "\n\nTo run Savu type 'source $savu_setup' to set relevant paths every time you" + echo -e "open a new terminal. Alternatively, if you are using the Modules system, see" + echo -e "$DIR/module_template for an example module file." + echo -e "================================================================================\n" + + echo -e "*************** SAVU INSTALLATION COMPLETE! ******************\n" + echo -e " ......Thank you for running the Savu installer......\n" + echo -e "=============================================================\n" +else + echo -e "\n\n*************** SAVU TESTS COMPLETE! ******************\n" + echo -e " ......Thank you for running the Savu tests......\n" + echo -e " Please check $tmpfile for errors\n" + echo -e "=======================================================\n" +fi + +exit 0 + diff --git a/install/2_3_2_install/savu_v2.3.2/savu_master/build.sh b/install/2_3_2_install/savu_v2.3.2/savu_master/build.sh new file mode 100644 index 000000000..b6fbaff2b --- /dev/null +++ b/install/2_3_2_install/savu_v2.3.2/savu_master/build.sh @@ -0,0 +1,4 @@ +#!/bin/bash + +$PYTHON setup.py install --facility $FACILITY # Python command to install the script. + diff --git a/install/2_3_2_install/savu_v2.3.2/savu_master/meta.yaml b/install/2_3_2_install/savu_v2.3.2/savu_master/meta.yaml new file mode 100644 index 000000000..8f2bca48f --- /dev/null +++ b/install/2_3_2_install/savu_v2.3.2/savu_master/meta.yaml @@ -0,0 +1,24 @@ +{% set repo = "master" %} + +package: + name: savu + version: "master" + +source: + url: https://github.com/DiamondLightSource/Savu/archive/v2.3.2.tar.gz + +requirements: + build: + - python + +build: + preserve_egg_dir: True + number: 0 + script_env: + - FACILITY + +about: + home: https://github.com/diamondlightsource/Savu + license: Apache V2 and GPL V3 + license_file: LICENCE.txt + diff --git a/install/2_3_2_install/savu_v2.3.2/version.txt b/install/2_3_2_install/savu_v2.3.2/version.txt new file mode 100644 index 000000000..abb84c2c7 --- /dev/null +++ b/install/2_3_2_install/savu_v2.3.2/version.txt @@ -0,0 +1,2 @@ +2.3.2 + diff --git a/install/2_3_2_install/savu_v2.3.2/versions_file.txt b/install/2_3_2_install/savu_v2.3.2/versions_file.txt new file mode 100644 index 000000000..116a12c1a --- /dev/null +++ b/install/2_3_2_install/savu_v2.3.2/versions_file.txt @@ -0,0 +1,7 @@ +miniconda 4.5.4 # change this to 'latest' for latest version +mpi4py 3.0.0 +tomopy 1.0.1 +dxchange 0.1.1 +ccpi 0.10.0 +pyfai 0.15.0 + diff --git a/install/2_3_2_install/tests/local_mpi_cpu_test.sh b/install/2_3_2_install/tests/local_mpi_cpu_test.sh new file mode 100644 index 000000000..586af6a0b --- /dev/null +++ b/install/2_3_2_install/tests/local_mpi_cpu_test.sh @@ -0,0 +1,41 @@ +#!/bin/bash + +# function for parsing optional arguments +function arg_parse () +{ + flag=$1 + return=$2 + while [[ $# -gt 1 ]] ; do + if [ $1 == $flag ] ; then + eval "$return"=$2 + fi + shift + done +} + +output=$1 +if ! [ "$output" ]; then + echo "Please pass the path to the output folder." + exit 1 +fi + +# set the TESTDATA environment variable +. test_setup.sh + +echo "Running the mpi cpu test..." + + +arg_parse "-r" redirect "$@" +if [ $redirect ] ; then + exec 4<&2 + exec 5<&1 + exec &> $redirect +fi + +savu_mpijob_local.sh $TESTDATA/data/24737.nxs $TESTDATA/test_process_lists/mpi_cpu_test.nxs $output + +if [ $redirect ] ; then + exec 2<&4 + exec 1<&5 +fi + diff --git a/install/2_3_2_install/tests/local_mpi_gpu_test.sh b/install/2_3_2_install/tests/local_mpi_gpu_test.sh new file mode 100644 index 000000000..44cf9bbfd --- /dev/null +++ b/install/2_3_2_install/tests/local_mpi_gpu_test.sh @@ -0,0 +1,41 @@ +#!/bin/bash + +# function for parsing optional arguments +function arg_parse () +{ + flag=$1 + return=$2 + while [[ $# -gt 1 ]] ; do + if [ $1 == $flag ] ; then + eval "$return"=$2 + fi + shift + done +} + +output=$1 +if ! [ "$output" ]; then + echo "Please pass the path to the output folder." + exit 1 +fi + +# set the TESTDATA environment variable +. test_setup.sh + +echo "Running the mpi gpu test..." + + +arg_parse "-r" redirect "$@" +if [ $redirect ] ; then + exec 4<&2 + exec 5<&1 + exec &> $redirect +fi + +savu_mpijob_local.sh $TESTDATA/data/24737.nxs $TESTDATA/test_process_lists/mpi_gpu_test.nxs $output + +if [ $redirect ] ; then + exec 2<&4 + exec 1<&5 +fi + diff --git a/install/2_3_2_install/tests/mpi_cpu_test.sh b/install/2_3_2_install/tests/mpi_cpu_test.sh new file mode 100644 index 000000000..105afebcd --- /dev/null +++ b/install/2_3_2_install/tests/mpi_cpu_test.sh @@ -0,0 +1,24 @@ +#!/bin/bash + +output=$1 +if ! [ "$output" ]; then + echo "Please pass the path to the output folder." + exit 1 +fi + +# get the loaded module version of savu +IN=$LOADEDMODULES +while IFS=':' read -ra ADDR; do + for i in "${ADDR[@]}"; do + if [[ $i == *"savu"* ]]; then + version=`echo "$i" | awk -F/ '{print $NF}'` + fi + done +done <<< "$IN" + +# set the TESTDATA environment variable +. test_setup.sh + +echo "Running the mpi cpu test..." +savu_launcher.sh $version $TESTDATA/data/24737.nxs $TESTDATA/test_process_lists/mpi_cpu_test.nxs $output + diff --git a/install/2_3_2_install/tests/mpi_gpu_test.sh b/install/2_3_2_install/tests/mpi_gpu_test.sh new file mode 100644 index 000000000..fa07b4653 --- /dev/null +++ b/install/2_3_2_install/tests/mpi_gpu_test.sh @@ -0,0 +1,24 @@ +#!/bin/bash + +output=$1 +if ! [ "$output" ]; then + echo "Please pass the path to the output folder." + exit 1 +fi + +# get the loaded module version of savu +IN=$LOADEDMODULES +while IFS=':' read -ra ADDR; do + for i in "${ADDR[@]}"; do + if [[ $i == *"savu"* ]]; then + version=`echo "$i" | awk -F/ '{print $NF}'` + fi + done +done <<< "$IN" + +# set the TESTDATA environment variable +. test_setup.sh + +echo "Running the mpi gpu test..." +savu_launcher.sh $version $TESTDATA/data/24737.nxs $TESTDATA/test_process_lists/mpi_gpu_test.nxs $output + diff --git a/install/2_3_2_install/tests/test_setup.sh b/install/2_3_2_install/tests/test_setup.sh new file mode 100644 index 000000000..90a35b203 --- /dev/null +++ b/install/2_3_2_install/tests/test_setup.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +savu_path=$(python -c "import savu; import os; print os.path.abspath(os.path.dirname(savu.__file__))") +test_path=$savu_path'/../test_data' +export TESTDATA=$test_path +echo '$TESTDATA points to the Savu test data folder at' $TESTDATA +