test globus

.github/workflows/frontier/install.sh

@@ -0,0 +1,48 @@
+# #!/bin/bash
+
+branch=$1
+
+cd /lustre/orion/phy122/scratch/castia5/globus-compute/omega_h-test
+
+module load rocm
+module load craype-accel-amd-gfx90a
+module load cray-mpich
+export CRAYPE_LINK_TYPE=dynamic
+export MPICH_GPU_SUPPORT_ENABLED=1
+
+# # Kokkos
+# rm kokkos -rf
+# rm build-kokkos -rf
+# git clone -b 4.1.00 git@github.com:Kokkos/kokkos.git
+# cmake -S kokkos -B build-kokkos \
+#  -DCMAKE_BUILD_TYPE=RelWithDebInfo \
+#  -DCMAKE_CXX_COMPILER=CC \
+#  -DCMAKE_CXX_EXTENSIONS=OFF \
+#  -DKokkos_ENABLE_TESTS=OFF \
+#  -DKokkos_ENABLE_EXAMPLES=OFF \
+#  -DKokkos_ENABLE_HIP=ON \
+#  -DKokkos_ARCH_VEGA90A=ON \
+#  -DKokkos_ENABLE_SERIAL=ON \
+#  -DKokkos_ENABLE_OPENMP=OFF \
+#  -DKokkos_ENABLE_DEBUG=OFF \
+#  -DCMAKE_INSTALL_PREFIX=build-kokkos/install
+# cmake --build build-kokkos -j8 --target install
+
+# Omega_h
+rm omega_h -rf
+rm build-omega_h -rf
+git clone git@github.com:SCOREC/omega_h.git
+cd omega_h && git checkout $branch && cd -
+cmake -S omega_h -B build-omega_h \
+  -DCMAKE_INSTALL_PREFIX=build-omega_h/install \
+  -DCMAKE_BUILD_TYPE=RelWithDebInfo \
+  -DBUILD_SHARED_LIBS=OFF \
+  -DOmega_h_USE_MPI=ON \
+  -DOmega_h_USE_OpenMP=OFF \
+  -DCMAKE_CXX_COMPILER=CC \
+  -DOMEGA_H_USE_HIP=ON \
+  -DOmega_h_USE_CUDA_AWARE_MPI=ON \
+  -DOmega_h_USE_Kokkos=ON \
+  -DKokkos_PREFIX=$PWD/build-kokkos/install \
+  -DBUILD_TESTING=ON
+cmake --build build-omega_h -j24 --target install

.github/workflows/frontier/run.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+
+name=omega_h
+
+cd /lustre/orion/phy122/scratch/castia5/globus-compute/$name-test
+
+module load rocm
+module load craype-accel-amd-gfx90a
+module load cray-mpich
+export CRAYPE_LINK_TYPE=dynamic
+export MPICH_GPU_SUPPORT_ENABLED=1
+
+cd build-$name
+salloc --account=PHY122 --time=00:20:00 -q debug --nodes=1 --ntasks=1 --cpus-per-task=1 --gpus-per-task=1 --gpus=1 ctest
+cat $PWD/Testing/Temporary/LastTest.log

.github/workflows/globus-test.yml

@@ -0,0 +1,20 @@
+name: Globus-Systems
+on:
+  push:
+  schedule:
+    # Monday 6:35 UTC or 02:35 EDT
+    - cron: '35 6 * * 1'
+
+jobs:
+
+  perlmutter-test:
+    uses: SCOREC/github-actions/.github/workflows/globus-test.yml@main
+    secrets: inherit
+    with:
+      machine: "perlmutter"
+
+  frontier-test:
+    uses: SCOREC/github-actions/.github/workflows/globus-test.yml@main
+    secrets: inherit
+    with:
+      machine: "frontier" 

.github/workflows/perlmutter/install.sh

@@ -0,0 +1,57 @@
+# !/bin/bash -e
+
+branch=$1
+
+cd $SCRATCH/globus-compute/omega_h-test
+
+export root=$PWD
+module load PrgEnv-gnu
+module load cudatoolkit
+module load cmake
+
+export kk=$root/build-kokkos/install   # This is where kokkos will be (or is) installed
+export oh=$root/build-omega_h/install  # This is where omega_h will be (or is) installed
+export CMAKE_PREFIX_PATH=$kk:$kk/lib64/cmake:$oh:$CMAKE_PREFIX_PATH
+export MPICH_CXX=$root/kokkos/bin/nvcc_wrapper
+
+export SLURM_CPU_BIND="cores"
+
+
+# #kokkos
+# rm ${kk%%install} -rf
+# rm kokkos -rf
+# git clone -b 4.2.00 https://github.com/kokkos/kokkos.git
+# cmake -S kokkos -B ${kk%%install} \
+#   -DCMAKE_INSTALL_PREFIX=$kk \
+#   -DCMAKE_BUILD_TYPE="Release" \
+#   -DCMAKE_CXX_COMPILER=$root/kokkos/bin/nvcc_wrapper \
+#   -DKokkos_ARCH_AMPERE80=ON \
+#   -DKokkos_ENABLE_SERIAL=ON \
+#   -DKokkos_ENABLE_OPENMP=off \
+#   -DKokkos_ENABLE_CUDA=on \
+#   -DKokkos_ENABLE_CUDA_LAMBDA=on \
+#   -DKokkos_ENABLE_DEBUG=off
+# cmake --build ${kk%%install} -j 24 --target install
+
+#omegah
+rm ${oh%%install} -rf
+rm omega_h -rf
+git clone https://github.com/SCOREC/omega_h.git
+cd omega_h && git checkout $branch && cd -
+cmake -S omega_h -B ${oh%%install} \
+  -DCMAKE_INSTALL_PREFIX=$oh \
+  -DCMAKE_BUILD_TYPE=Release \
+  -DBUILD_SHARED_LIBS=off \
+  -DOmega_h_USE_Kokkos=ON \
+  -DOmega_h_USE_CUDA=on \
+  -DOmega_h_CUDA_ARCH=80 \
+  -DOmega_h_USE_MPI=on  \
+  -DMPIEXEC_EXECUTABLE=srun \
+  -DBUILD_TESTING=on  \
+  -DCMAKE_C_COMPILER=cc \
+  -DCMAKE_CXX_COMPILER=CC \
+  -DKokkos_PREFIX=$kk/lib64/cmake
+cmake --build ${oh%%install} -j 24 --target install
+
+
+# -DOmega_h_USE_CUDA_AWARE_MPI=ON \

.github/workflows/perlmutter/run.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+
+name=omega_h
+
+cd $SCRATCH/globus-compute/$name-test
+
+export root=$PWD
+module load PrgEnv-gnu
+module load cudatoolkit
+module load cmake
+export MPICH_CXX=$root/kokkos/bin/nvcc_wrapper
+export SLURM_CPU_BIND="cores"
+
+cd build-$name
+salloc --time 00:20:00 --constrain=gpu --qos=interactive --nodes=1 --ntasks-per-node=40 --cpus-per-task=1 --gpus=1 --account=m4564 ctest
+cat $PWD/Testing/Temporary/LastTest.log