lanl · adamdempsey90 · Nov 22, 2024 · Nov 8, 2024 · Nov 19, 2024 · Nov 20, 2024
diff --git a/CMakePresets.json b/CMakePresets.json
@@ -32,8 +32,8 @@
         "CMAKE_BUILD_TYPE": "Debug",
         "Kokkos_ENABLE_DEBUG_BOUNDS_CHECK": "ON",
         "CMAKE_CXX_COMPILER": "$env{ARTEMIS_HOME}/external/parthenon/external/Kokkos/bin/nvcc_wrapper",
-        "ARTEMIS_ENABLE_CUDA": "On",
-        "Kokkos_ARCH_VOLTA70": "On"
+        "ARTEMIS_ENABLE_CUDA": "ON",
+        "Kokkos_ARCH_VOLTA70": "ON"
       }
     },
     {
@@ -42,8 +42,8 @@
         "CMAKE_MAKE_PROGRAM": "$env{MAKE_PROGRAM}",
         "CMAKE_BUILD_TYPE": "RelWithDebInfo",
         "CMAKE_CXX_COMPILER": "$env{ARTEMIS_HOME}/external/parthenon/external/Kokkos/bin/nvcc_wrapper",
-        "ARTEMIS_ENABLE_CUDA": "On",
-        "Kokkos_ARCH_VOLTA70": "On"
+        "ARTEMIS_ENABLE_CUDA": "ON",
+        "Kokkos_ARCH_VOLTA70": "ON"
       }
     },
     {
@@ -97,10 +97,10 @@
         "CMAKE_BUILD_TYPE": "Debug",
         "Kokkos_ENABLE_DEBUG_BOUNDS_CHECK": "ON",
         "CMAKE_CXX_COMPILER": "$env{ARTEMIS_HOME}/external/parthenon/external/Kokkos/bin/nvcc_wrapper",
-        "ARTEMIS_ENABLE_CUDA": "On",
+        "ARTEMIS_ENABLE_CUDA": "ON",
         "HDF5_INCLUDE_DIR": "$env{HDF5_ROOT}/include",
-        "PARTHENON_DISABLE": "HDF5_COMPRESSION=ON",
-        "Kokkos_ARCH_HOPPER90": "On",
+        "PARTHENON_DISABLE_HDF5_COMPRESSION": "ON",
+        "Kokkos_ARCH_HOPPER90": "ON",
         "NUM_GPU_DEVICES_PER_NODE": "4"
       }
     },
@@ -110,10 +110,10 @@
         "CMAKE_MAKE_PROGRAM": "$env{MAKE_PROGRAM}",
         "CMAKE_BUILD_TYPE": "RelWithDebInfo",
         "CMAKE_CXX_COMPILER": "$env{ARTEMIS_HOME}/external/parthenon/external/Kokkos/bin/nvcc_wrapper",
-        "ARTEMIS_ENABLE_CUDA": "On",
+        "ARTEMIS_ENABLE_CUDA": "ON",
         "HDF5_INCLUDE_DIR": "$env{HDF5_ROOT}/include",
-        "PARTHENON_DISABLE": "HDF5_COMPRESSION=ON",
-        "Kokkos_ARCH_HOPPER90": "On",
+        "PARTHENON_DISABLE_HDF5_COMPRESSION": "ON",
+        "Kokkos_ARCH_HOPPER90": "ON",
         "NUM_GPU_DEVICES_PER_NODE": "4"
       }
     }

diff --git a/env/bash b/env/bash
@@ -27,10 +27,12 @@ if [[ $HOSTNAME == ch-fe*  ||  $SLURM_CLUSTER_NAME == "chicoma" ]]; then
     else
         PARTITION="chicoma-cpu"
     fi
-elif [[ "$HOSTNAME" =~ ^ve-rfe[1-3]$ || ( $SLURM_CLUSTER_NAME == "venado" && $SLURM_JOB_PARTITION == "gpu" ) ]]; then
-    PARTITION="venado-gh"
-elif [[ "$HOSTNAME" =~ ^ve-rfe[4-7]$ || ( $SLURM_CLUSTER_NAME == "venado" && $SLURM_JOB_PARTITION == "cpu" ) ]]; then
-    PARTITION="venado-gg"
+elif [[ "$HOSTNAME" =~ ^ve-rfe[1-7]$ || "$HOSTNAME" =~ ^ve-fe[1-7]$ ]]; then
+    if [[ $SLURM_GPUS_ON_NODE > 0 || ("$HOSTNAME" =~ ^ve-rfe[1-3]$ || "$HOSTNAME" =~ ^ve-fe[1-3]$) ]]; then
+        PARTITION="venado-gh"
+    elif [[ $SLURM_GPUS_ON_NODE == 0 || ("$HOSTNAME" =~ ^ve-rfe[4-7]$ || "$HOSTNAME" =~ ^ve-fe[4-7]$) ]]; then
+        PARTITION="venado-gg"
+    fi
 else # Catch-all for Darwin
     if [ -z "$SLURM_JOB_PARTITION" ]; then
         if [[ $HOSTNAME == darwin-fe* ]]; then
@@ -119,36 +121,43 @@ elif [[ $PARTITION == "darwin-volta-x86" ]]; then
     export ARTEMIS_SUITE=darwin-gpu
     echo "...setup SUCCEEDED"
 elif [[ $PARTITION == "venado-gh" ]]; then
-  module load PrgEnv-gnu
-  module load cray-mpich cray-hdf5-parallel cudatoolkit
-  export MPICH_OFI_NIC_POLICY=GPU   # GPU NUMA ROUND-ROBIN
-  export MPICH_GPU_SUPPORT_ENABLED=1 # Allows GPU Aware MPI
-  export CRAY_ACCEL_TARGET=nvidia90
-  export MPICH_MALLOC_FALLBACK=1
-  export MPICH_SMP_SINGLE_COPY_MODE=NONE
-  export MPICH_MAX_THREAD_SAFETY=multiple
-  export FI_CXI_RX_MATCH_MODE=hybrid
-  export PMI_MMAP_SYNC_WAIT_TIME=600
-  export ARTEMIS_SUITE=venado-gpu
-  module list
-  echo "...setup SUCCEEDED"
+    module load PrgEnv-gnu
+    module load cray-mpich
+    #module load cray-hdf5-parallel # Compiler wrappers fail to find hdf5.h 2024/11/8
+    export HDF5_ROOT=/opt/cray/pe/hdf5-parallel/1.14.3.1/gnu/12.3
+    module load cudatoolkit
+    module load cmake
+    export MPICH_OFI_NIC_POLICY=GPU   # GPU NUMA ROUND-ROBIN
+    export MPICH_GPU_SUPPORT_ENABLED=1 # Allows GPU Aware MPI
+    export CRAY_ACCEL_TARGET=nvidia90
+    export MPICH_MALLOC_FALLBACK=1
+    export MPICH_SMP_SINGLE_COPY_MODE=NONE
+    export MPICH_MAX_THREAD_SAFETY=multiple
+    export FI_CXI_RX_MATCH_MODE=hybrid
+    export PMI_MMAP_SYNC_WAIT_TIME=600
+    export NVCC_WRAPPER_DEFAULT_COMPILER=g++
+    export ARTEMIS_SUITE=venado-gpu
+    module list
+    echo "...setup SUCCEEDED"
 elif [[ $PARTITION == "venado-gg" ]]; then
-  module load PrgEnv-gnu
-  module load cray-mpich cray-hdf5-parallel
-  export MPICH_MALLOC_FALLBACK=1
-  export MPICH_SMP_SINGLE_COPY_MODE=NONE
-  export MPICH_MAX_THREAD_SAFETY=multiple
-  export FI_CXI_RX_MATCH_MODE=hybrid
-  export PMI_MMAP_SYNC_WAIT_TIME=600
-  export ARTEMIS_SUITE=venado-cpu
-  module list
-  echo "...setup SUCCEEDED"
+    module load PrgEnv-gnu
+    module load cray-mpich cray-hdf5-parallel
+    export MPICH_MALLOC_FALLBACK=1
+    export MPICH_SMP_SINGLE_COPY_MODE=NONE
+    export MPICH_MAX_THREAD_SAFETY=multiple
+    export FI_CXI_RX_MATCH_MODE=hybrid
+    export PMI_MMAP_SYNC_WAIT_TIME=600
+    export ARTEMIS_SUITE=venado-cpu
+    module list
+    echo "...setup SUCCEEDED"
 fi
 
-echo ""
-echo "To configure and build the code, you can use the function (-h flag for options)"
-echo "  build_artemis"
-echo ""
+if [[ $PARTITION != "unknown" ]]; then
+    echo ""
+    echo "To configure and build the code, you can use the function (-h flag for options)"
+    echo "  build_artemis"
+    echo ""
+fi
 
 function build_artemis {
 

diff --git a/external/parthenon b/external/parthenon
diff --git a/src/geometry/spherical.hpp b/src/geometry/spherical.hpp
@@ -530,12 +530,6 @@ class Coords<Coordinates::spherical1D>
     // volume average (r,theta)
     const Real rv = x1v();
 
-    // face averaged r on the X2 face
-    const Real rf =
-        2.0 / 3.0 *
-        (bnds.x1[0] * bnds.x1[0] + bnds.x1[0] * bnds.x1[1] + bnds.x1[1] * bnds.x1[1]) /
-        (bnds.x1[0] + bnds.x1[1]);
-
     const Real r2cyl = SQR(rv);
 
     std::array<Real, 2> bx1{r2cyl - SQR(bnds.x1[0]), SQR(bnds.x1[1]) - r2cyl};

diff --git a/src/gravity/point_mass.cpp b/src/gravity/point_mass.cpp
@@ -35,7 +35,7 @@ TaskStatus PointMassGravity(MeshData<Real> *md, const Real time, const Real dt)
   const bool do_dust = artemis_pkg->template Param<bool>("do_dust");
 
   auto &gravity_pkg = pm->packages.Get("gravity");
-  const Real gm = gravity_pkg->template Param<Real>("gm");
+  const Real gm_ = gravity_pkg->template Param<Real>("gm");
   const Real sink_rate = dt * (gravity_pkg->template Param<Real>("sink_rate"));
   const Real sink_rad = gravity_pkg->template Param<Real>("sink");
 
@@ -68,6 +68,9 @@ TaskStatus PointMassGravity(MeshData<Real> *md, const Real time, const Real dt)
         const auto &dx = coords.GetCellCenter();
         const auto &hx = coords.GetScaleFactors();
 
+        // Capture outside constexpr if
+        const Real &gm = gm_;
+
         Real gx1 = 0.0, gx2 = 0.0, gx3 = 0.0;
         Real dr = Null<Real>();
         [[maybe_unused]] Real pos[3] = {pos_[0], pos_[1], pos_[2]};
+18 −6		.github/workflows/check-compilers.yml
+3 −1		.github/workflows/ci-extended.yml
+5 −3		.github/workflows/ci-short.yml
+8 −5		.github/workflows/docs.yml
+17 −11		CHANGELOG.md
+2 −2		CMakeLists.txt
+1 −1		README.md
+3 −0		cmake/machinecfg/GitHubActions.cmake
+40 −1		doc/sphinx/src/development.rst
+3 −1		doc/sphinx/src/outputs.rst
+33 −0		example/advection/advection_package.cpp
+42 −20		example/fine_advection/advection_driver.cpp
+157 −116		example/fine_advection/advection_package.cpp
+92 −54		example/fine_advection/advection_package.hpp
+105 −70		example/fine_advection/parthenon_app_inputs.cpp
+7 −0		example/fine_advection/parthinput.advection
+56 −36		example/fine_advection/stokes.hpp
+16 −14		example/particles/particles.cpp
+1 −1		external/Kokkos
+6 −8		scripts/docker/Dockerfile.hip-rocm
+0 −3		src/bvals/bvals.hpp
+2 −2		src/bvals/comms/bnd_info.cpp
+4 −3		src/bvals/comms/bnd_info.hpp
+3 −2		src/bvals/comms/bvals_utils.hpp
+3 −2		src/interface/mesh_data.hpp
+5 −4		src/interface/sparse_pack_base.cpp
+3 −2		src/interface/sparse_pack_base.hpp
+14 −9		src/interface/swarm.cpp
+4 −1		src/interface/swarm.hpp
+98 −97		src/interface/swarm_comms.cpp
+4 −3		src/interface/swarm_device_context.hpp
+8 −7		src/interface/swarm_pack_base.hpp
+19 −17		src/interface/variable_pack.hpp
+52 −15		src/kokkos_abstraction.hpp
+2 −2		src/mesh/meshblock_pack.hpp
+11 −3		src/outputs/history.cpp
+4 −4		src/outputs/output_utils.hpp
+2 −2		src/outputs/restart_hdf5.cpp
+4 −0		src/parthenon_array_generic.hpp
+1 −2		src/utils/hash.hpp
+15 −2		src/utils/sort.hpp
+12 −0		tst/regression/test_suites/output_hdf5/output_hdf5.py
+3 −1		tst/regression/test_suites/output_hdf5/parthinput.advection
+5 −3		tst/unit/test_pararrays.cpp