Skip to content

Building DGSWEM HPX software stack with VTune on Rostam

Jump to bottom
Zach Byerly edited this page Feb 1, 2017 · 4 revisions

Rostam is the Ste||ar group @ LSU's research cluster (replacement for Hermione).

Clone HPX into home directory, use the following build script:

#!/bin/bash

module purge
module load gcc/6.3.0
module load cmake/3.6.2
module load hwloc/1.11.3
module load boost/1.63.0-gcc6.3.0
module load openmpi/1.10.5-gcc6.3.0 

BUILD_TYPE="RelWithDebInfo"

SCRIPTPATH=$( cd $(dirname $0) ; pwd -P )
BUILD_PATH=$HOME/build/hpx_build_${BUILD_TYPE}
INSTALL_PATH=$HOME/local_install_${BUILD_TYPE}
REPO_PATH=$HOME/hpx

NUM_BUILDCORES=20

if [ "$1" = "clean" ]; then
    rm -rf $BUILD_PATH 
    exit 0
fi

if [ ! -d "$BUILD_PATH" ]; then
    echo "Creating build path..."
    mkdir -p "$BUILD_PATH"
    cd "$BUILD_PATH"

    ############################################
    # MODIFY CMAKE FLAGS AS YOU WISH
    #

    MALLOC="jemalloc"
    IDLE_RATES="true"
    CMAKE_FLAGS="-DCMAKE_BUILD_TYPE=${BUILD_TYPE} \
-DCMAKE_INSTALL_PREFIX=$INSTALL_PATH \
-DHPX_WITH_PARCELPORT_MPI=true \
-DHPX_WITH_MALLOC=$MALLOC \
-DHPX_WITH_THREAD_IDLE_RATES=${IDLE_RATES} \
-DHPX_WITH_ITTNOTIFY=On \
-DAMPLIFIER_ROOT=/opt/eb/software/VTune/2017_update1/vtune_amplifier_xe "


    CMD="cmake ${CMAKE_FLAGS} $REPO_PATH"
    echo "CMD = $CMD"
    $CMD

    #
    ############################################


    # cleanup on failure
    rc=$?
    if [[ $rc != 0 ]] ; then
        cd "$SCRIPTPATH"
        rm -rf "$BUILD_PATH"
        exit $rc
    fi

fi

# the actual build command
cd "$BUILD_PATH"
nice make "$@" -k -j$NUM_BUILDCORES || exit 1
make install

Build and install Scotch (partitioner) with this script:

#!/bin/bash

module load gcc/6.3.0

INSTALL_DIR="$1"
if [ ! -d $INSTALL_DIR ]; then
    echo "Install directory doesn't exist"
    exit 1
fi

mkdir -p scotch/build

cd scotch
wget http://gforge.inria.fr/frs/download.php/file/34099/scotch_6.0.3.tar.gz
tar -xzvf scotch_6.0.3.tar.gz
cd scotch_6.0.3/src
cp Make.inc/Makefile.inc.x86-64_pc_linux2 Makefile.inc

make prefix=$HOME/local_install_RelWithDebInfo
make prefix=$HOME/local_install_RelWithDebInfo install

Clone https://github.com/gentryx/libgeodecomp into home directory, and build with the following script:

#!/bin/bash

module purge
module load gcc/6.3.0
module load cmake/3.6.2
module load hwloc/1.11.3
module load boost/1.63.0-gcc6.3.0
module load openmpi/1.10.5-gcc6.3.0 

BUILD_TYPE="RelWithDebInfo"

SCRIPTPATH=$( cd $(dirname $0) ; pwd -P )
BUILD_PATH=$HOME/build/lgd_build_${BUILD_TYPE}
INSTALL_PATH=$HOME/local_install_${BUILD_TYPE}
REPO_PATH=$HOME/libgeodecomp
#HPX_ROOT=$HOME/hpx_install_release_rostam

CXX_COMPILER=/opt/mn/gcc/6.3.0/bin/g++
C_COMPILER=/opt/mn/gcc/6.3.0/bin/gcc

NUM_BUILDCORES=20

if [ "$1" = "clean" ]; then
    rm -rf $BUILD_PATH
    exit 0
fi

if [ ! -d "$BUILD_PATH" ]; then
    mkdir -p "$BUILD_PATH"
    cd "$BUILD_PATH"

    ############################################
    # MODIFY CMAKE FLAGS AS YOU WISH
    #

#    echo "HPX_ROOT = ${HPX_ROOT}"
#    echo "HPX_DIR = ${HPX_ROOT}/share/hpx/cmake"

    cmake \
	-DCMAKE_BUILD_TYPE=${BUILD_TYPE} \
        -DCMAKE_INSTALL_PREFIX=${INSTALL_PATH}    \
	-DCMAKE_CXX_COMPILER=${CXX_COMPILER} \
	-DCMAKE_C_COMPILER=${C_COMPILER} \
	-DWITH_HPX=true                           \
	-DWITH_CUDA=false \
	-DWITH_SCOTCH=true \
	-DCMAKE_PREFIX_PATH=/home/zbyerly/local_install_${BUILD_TYPE}/ \
	-DHPX_WITH_THREAD_IDLE_RATES=true \
	-DSCOTCH_DIR=/usr/lib \
        $REPO_PATH

    #
    ############################################
#	-DCMAKE_CXX_COMPILER=/opt/eb/software/GCCcore/4.9.3/bin/c++ \
#	-DSCOTCH_DIR=${INSTALL_PATH} \

    # cleanup on failure
    rc=$?
    if [[ $rc != 0 ]] ; then
        cd "$SCRIPTPATH"
        rm -rf "$BUILD_PATH"
        exit $rc
    fi

fi

# the actual build command
cd "$BUILD_PATH"
nice make "$@" -k -j$NUM_BUILDCORES || exit 1
make install

Clone dgswem-hpx: git clone -b dgswem-hpx git@github.com:foci/dgswem.git dgswem-hpx

Build the application (run from inside application directory):

#!/bin/bash                                                                                                                                                                       

BUILD_TYPE="RelWithDebInfo"
BUILD_DIR="build_${BUILD_TYPE}"

REPO_PATH=".."

# Machine specific                                                                                                                                                                

CXX_COMPILER="/opt/mn/gcc/6.3.0/bin/g++"
C_COMPILER="/opt/mn/gcc/6.3.0/bin/gcc"
PREFIX_PATH="$HOME/local_install_${BUILD_TYPE}"
CMAKE="/opt/eb/software/cmake/3.6.2/bin/cmake"

# Choose between using GNU and Intel Fortran compilers
#Fortran_COMPILER="/opt/mn/gcc/6.2.0/bin/gfortran"                                                                                                                                
Fortran_COMPILER="/opt/eb/software/ifort/2013.5.192/bin/intel64/ifort"


echo "Build type: ${BUILD_TYPE}"
echo "Build directory: ${BUILD_DIR}"
echo "repo path = ${REPO_PATH}"

mkdir ${BUILD_DIR} || echo "${BUILD_DIR} already exists, continuing"
cd ${BUILD_DIR} || {
    echo "could not cd into build directory ${BUILD_DIR}, quitting."
    exit
}

$CMAKE \
    -DCMAKE_PREFIX_PATH=${PREFIX_PATH} \
    -DCMAKE_CXX_COMPILER=${CXX_COMPILER} \
    -DCMAKE_C_COMPILER=${C_COMPILER} \
    -DCMAKE_Fortran_COMPILER=${Fortran_COMPILER} \
    -DCMAKE_BUILD_TYPE=${BUILD_TYPE} \
    -DPROFILE="true"
    ${REPO_PATH}

Run command:

on a single node, after you've started a session:

mpirun amplxe-cl -r 16 -collect hotspots -knob enable-user-tasks=true ~/dgswem-hpx/build_RelWithDebInfo_intel/src/dgswem_lgd_dataflow --n_timesteps=2000 --hpx:ini=hpx.stacks.small_size=0x20000 --hpx:ini=hpx.use_itt_notify=1

Links: Intel compiler flags

mpirun amplxe-cl -r 16_submeshes -collect hotspots -knob enable-user-tasks=true /home/zbyerly/dgswem-hpx/build_RelWithDebInfo_intel/src/dgswem_lgd_dataflow --n_timesteps=2000 --hpx:print-counter=/threads/time/cumulative --hpx:print-counter=/threads/idle-rate --hpx:print-counter=/threads/time/cumulative-overhead --hpx:print-counter=/threadqueue/length --chunksize=100 --hpx:ini=hpx.stacks.small_size=0x20000 --hpx:ini=hpx.use_itt_notify=1 --hpx:print-counter-interval=10

Clone this wiki locally