From e46d4f53972153c0e2437294513a628d42bfc1d4 Mon Sep 17 00:00:00 2001 From: Angel Castillo Date: Fri, 8 Nov 2024 21:20:56 -0500 Subject: [PATCH] Globus Refactor (#121) * test globus * remove push * increment version * fix test * remove push --- .github/workflows/frontier/install.sh | 47 ++++++++++++++++++ .github/workflows/frontier/run.sh | 15 ++++++ .github/workflows/globus-test.yml | 19 ++++++++ .github/workflows/perlmutter/install.sh | 54 +++++++++++++++++++++ .github/workflows/perlmutter/run.sh | 16 +++++++ .github/workflows/systems.yml | 63 ------------------------- .github/workflows/test_on_system.py | 38 --------------- CMakeLists.txt | 2 +- 8 files changed, 152 insertions(+), 102 deletions(-) create mode 100644 .github/workflows/frontier/install.sh create mode 100644 .github/workflows/frontier/run.sh create mode 100644 .github/workflows/globus-test.yml create mode 100644 .github/workflows/perlmutter/install.sh create mode 100644 .github/workflows/perlmutter/run.sh delete mode 100644 .github/workflows/systems.yml delete mode 100644 .github/workflows/test_on_system.py diff --git a/.github/workflows/frontier/install.sh b/.github/workflows/frontier/install.sh new file mode 100644 index 000000000..98db28f08 --- /dev/null +++ b/.github/workflows/frontier/install.sh @@ -0,0 +1,47 @@ +# #!/bin/bash + +branch=$1 + +cd /lustre/orion/phy122/scratch/castia5/globus-compute/omega_h-test + +module load rocm +module load craype-accel-amd-gfx90a +module load cray-mpich +export CRAYPE_LINK_TYPE=dynamic +export MPICH_GPU_SUPPORT_ENABLED=1 + +# # Kokkos +# rm kokkos -rf +# rm build-kokkos -rf +# git clone -b 4.1.00 git@github.com:Kokkos/kokkos.git +# cmake -S kokkos -B build-kokkos \ +# -DCMAKE_BUILD_TYPE=RelWithDebInfo \ +# -DCMAKE_CXX_COMPILER=CC \ +# -DCMAKE_CXX_EXTENSIONS=OFF \ +# -DKokkos_ENABLE_TESTS=OFF \ +# -DKokkos_ENABLE_EXAMPLES=OFF \ +# -DKokkos_ENABLE_HIP=ON \ +# -DKokkos_ARCH_VEGA90A=ON \ +# -DKokkos_ENABLE_SERIAL=ON \ +# -DKokkos_ENABLE_OPENMP=OFF \ +# -DKokkos_ENABLE_DEBUG=OFF \ +# -DCMAKE_INSTALL_PREFIX=build-kokkos/install +# cmake --build build-kokkos -j8 --target install + +# Omega_h +rm omega_h -rf +rm build-omega_h -rf +git clone git@github.com:SCOREC/omega_h.git +cd omega_h && git checkout $branch && cd - +cmake -S omega_h -B build-omega_h \ + -DCMAKE_INSTALL_PREFIX=build-omega_h/install \ + -DCMAKE_BUILD_TYPE=RelWithDebInfo \ + -DBUILD_SHARED_LIBS=OFF \ + -DOmega_h_USE_MPI=ON \ + -DOmega_h_USE_OpenMP=OFF \ + -DCMAKE_CXX_COMPILER=CC \ + -DOMEGA_H_USE_HIP=ON \ + -DOmega_h_USE_Kokkos=ON \ + -DKokkos_PREFIX=$PWD/build-kokkos/install \ + -DBUILD_TESTING=ON +cmake --build build-omega_h -j24 --target install diff --git a/.github/workflows/frontier/run.sh b/.github/workflows/frontier/run.sh new file mode 100644 index 000000000..d5b5a9cf7 --- /dev/null +++ b/.github/workflows/frontier/run.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +name=omega_h + +cd /lustre/orion/phy122/scratch/castia5/globus-compute/$name-test + +module load rocm +module load craype-accel-amd-gfx90a +module load cray-mpich +export CRAYPE_LINK_TYPE=dynamic +export MPICH_GPU_SUPPORT_ENABLED=1 + +cd build-$name +salloc --account=PHY122 --time=00:20:00 -q debug --nodes=1 --ntasks=1 --cpus-per-task=1 --gpus-per-task=1 --gpus=1 ctest +cat $PWD/Testing/Temporary/LastTest.log \ No newline at end of file diff --git a/.github/workflows/globus-test.yml b/.github/workflows/globus-test.yml new file mode 100644 index 000000000..479e8c274 --- /dev/null +++ b/.github/workflows/globus-test.yml @@ -0,0 +1,19 @@ +name: Globus-Systems +on: + schedule: + # Monday 6:35 UTC or 02:35 EDT + - cron: '35 6 * * 1' + +jobs: + + perlmutter-test: + uses: SCOREC/github-actions/.github/workflows/globus-test.yml@main + secrets: inherit + with: + machine: "perlmutter" + + frontier-test: + uses: SCOREC/github-actions/.github/workflows/globus-test.yml@main + secrets: inherit + with: + machine: "frontier" \ No newline at end of file diff --git a/.github/workflows/perlmutter/install.sh b/.github/workflows/perlmutter/install.sh new file mode 100644 index 000000000..e86657e35 --- /dev/null +++ b/.github/workflows/perlmutter/install.sh @@ -0,0 +1,54 @@ +# !/bin/bash -e + +branch=$1 + +cd $SCRATCH/globus-compute/omega_h-test + +export root=$PWD +module load PrgEnv-gnu +module load cudatoolkit +module load cmake + +export kk=$root/build-kokkos/install # This is where kokkos will be (or is) installed +export oh=$root/build-omega_h/install # This is where omega_h will be (or is) installed +export CMAKE_PREFIX_PATH=$kk:$kk/lib64/cmake:$oh:$CMAKE_PREFIX_PATH +export MPICH_CXX=$root/kokkos/bin/nvcc_wrapper + +export SLURM_CPU_BIND="cores" + + +# #kokkos +# rm ${kk%%install} -rf +# rm kokkos -rf +# git clone -b 4.2.00 https://github.com/kokkos/kokkos.git +# cmake -S kokkos -B ${kk%%install} \ +# -DCMAKE_INSTALL_PREFIX=$kk \ +# -DCMAKE_BUILD_TYPE="Release" \ +# -DCMAKE_CXX_COMPILER=$root/kokkos/bin/nvcc_wrapper \ +# -DKokkos_ARCH_AMPERE80=ON \ +# -DKokkos_ENABLE_SERIAL=ON \ +# -DKokkos_ENABLE_OPENMP=off \ +# -DKokkos_ENABLE_CUDA=on \ +# -DKokkos_ENABLE_CUDA_LAMBDA=on \ +# -DKokkos_ENABLE_DEBUG=off +# cmake --build ${kk%%install} -j 24 --target install + +#omegah +rm ${oh%%install} -rf +rm omega_h -rf +git clone https://github.com/SCOREC/omega_h.git +cd omega_h && git checkout $branch && cd - +cmake -S omega_h -B ${oh%%install} \ + -DCMAKE_INSTALL_PREFIX=$oh \ + -DCMAKE_BUILD_TYPE=Release \ + -DBUILD_SHARED_LIBS=off \ + -DOmega_h_USE_Kokkos=ON \ + -DOmega_h_USE_CUDA=on \ + -DOmega_h_CUDA_ARCH=80 \ + -DOmega_h_USE_MPI=on \ + -DMPIEXEC_EXECUTABLE=srun \ + -DBUILD_TESTING=on \ + -DCMAKE_C_COMPILER=cc \ + -DCMAKE_CXX_COMPILER=CC \ + -DKokkos_PREFIX=$kk/lib64/cmake +cmake --build ${oh%%install} -j 24 --target install \ No newline at end of file diff --git a/.github/workflows/perlmutter/run.sh b/.github/workflows/perlmutter/run.sh new file mode 100644 index 000000000..ca4c25710 --- /dev/null +++ b/.github/workflows/perlmutter/run.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +name=omega_h + +cd $SCRATCH/globus-compute/$name-test + +export root=$PWD +module load PrgEnv-gnu +module load cudatoolkit +module load cmake +export MPICH_CXX=$root/kokkos/bin/nvcc_wrapper +export SLURM_CPU_BIND="cores" + +cd build-$name +salloc --time 00:20:00 --constrain=gpu --qos=interactive --nodes=1 --ntasks-per-node=40 --cpus-per-task=1 --gpus=1 --account=m4564 ctest +cat $PWD/Testing/Temporary/LastTest.log \ No newline at end of file diff --git a/.github/workflows/systems.yml b/.github/workflows/systems.yml deleted file mode 100644 index b48a13cc5..000000000 --- a/.github/workflows/systems.yml +++ /dev/null @@ -1,63 +0,0 @@ -name: Systems -on: - schedule: - # Monday 5:05 UTC or 01:05 EDT - - cron: '05 5 * * 1' - -concurrency: - group: systems-omega_h - cancel-in-progress: true - -jobs: - test: - runs-on: ubuntu-latest - timeout-minutes: 30 - strategy: - matrix: - machine: ["Perlmutter", "Frontier"] - - steps: - - - name: checkout omega_h - uses: actions/checkout@v4 - with: - path: omega_h - - - name: setup python - uses: actions/setup-python@v5 - with: - python-version: '3.10' - - - name: install packing - run: sudo apt install python3-packaging - - - name: install globus - run: | - python -m ensurepip --upgrade --user - python -m pip install globus-compute-endpoint - - - name: use globus - working-directory: omega_h/.github/workflows - env: - GLOBUS_ID: ${{ secrets.GLOBUS_COMPUTE_ID }} - GLOBUS_SECRET: ${{ secrets.GLOBUS_COMPUTE_SECRET }} - run: | - export GLOBUS_COMPUTE_CLIENT_ID="$GLOBUS_ID" - export GLOBUS_COMPUTE_CLIENT_SECRET="$GLOBUS_SECRET" - if [ ${{matrix.machine}} == Perlmutter ]; then TARGET_ENDPOINT=0dd4499a-8d76-4977-bae9-841e4bb2f616; fi - if [ ${{matrix.machine}} == Frontier ]; then TARGET_ENDPOINT=d625c6cf-de7a-4228-ac44-56247a642fe0; fi - python test_on_system.py ${{ github.event.repository.name }} ${{ github.sha }} $TARGET_ENDPOINT - - - name: print build log - working-directory: omega_h/.github/workflows - run: cat omega_h-result/Build.log - - - name: print test log - working-directory: omega_h/.github/workflows - run: cat omega_h-result/Test.log - - - name: check failed test - working-directory: omega_h/.github/workflows - run: if grep "Failed" omega_h-result/Test.log; then return 1; fi - - \ No newline at end of file diff --git a/.github/workflows/test_on_system.py b/.github/workflows/test_on_system.py deleted file mode 100644 index 1c1f6d539..000000000 --- a/.github/workflows/test_on_system.py +++ /dev/null @@ -1,38 +0,0 @@ -# How to use -# 1. Login to https://app.globus.org/settings/developers and copy a project app id and secret -# 2. Use the id and secret to create and endpoint https://funcx.readthedocs.io/en/latest/sdk.html#client-credentials-with-clients -# $ export FUNCX_SDK_CLIENT_ID="b0500dab-ebd4-430f-b962-0c85bd43bdbb" -# $ export FUNCX_SDK_CLIENT_SECRET="ABCDEFGHIJKLMNOP0123456789=" -# 3. Set up an endpoint on the computer that will run the tests, using these instructions: https://funcx.readthedocs.io/en/latest/endpoints.html -# 4. Create install-test.sh and run-test.sh on target computer - -from globus_compute_sdk import Executor -import sys -import os - -name = sys.argv[1] -branch = sys.argv[2] -endpoint = sys.argv[3] - -def run_on_endpoint(name, branch): - import subprocess - - install = subprocess.run(["./install-test.sh "+name+" "+branch], shell=True, encoding="utf_8", stdout=subprocess.PIPE, stderr=subprocess.STDOUT) - if install.returncode == 1: - return (install, None) - - result = subprocess.run(["./run-test.sh "+name], shell=True, encoding="utf_8", stdout=subprocess.PIPE, stderr=subprocess.STDOUT) - return (install, result) - -gce = Executor(endpoint_id = endpoint) -future = gce.submit(run_on_endpoint, name, branch) -result = future.result() - -os.popen("mkdir -p "+name+"-result").read() -with open(name+"-result/Build.log", "w") as text_file: - text_file.write("%s" % result[0].stdout) - text_file.close() -if result[0].returncode == 0: - with open(name+"-result/Test.log", "w") as text_file: - text_file.write("%s" % result[1].stdout) - text_file.close() diff --git a/CMakeLists.txt b/CMakeLists.txt index 24a847784..cbd15975e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,6 +1,6 @@ cmake_minimum_required(VERSION 3.18.0...${CMAKE_VERSION}) -project(Omega_h VERSION 10.8.6 LANGUAGES CXX) +project(Omega_h VERSION 10.8.7 LANGUAGES CXX) set(Omega_h_USE_STK_DEFAULT OFF) option(Omega_h_USE_STK "Whether to build the STK interface" ${Omega_h_USE_STK_DEFAULT})