Fix more view of views

parthenon-hpc-lab · Nov 7, 2024 · 120ceef · 120ceef
1 parent e632f6b
commit 120ceef
Show file tree

Hide file tree

Showing 10 changed files with 24 additions and 17 deletions.
diff --git a/README.md b/README.md
@@ -30,7 +30,7 @@ Parthenon -- a performance portable block-structured adaptive mesh refinement fr
 
 * CMake 3.16 or greater
 * C++17 compatible compiler
-* Kokkos 4.1.1 or greater
+* Kokkos 4.4.1 or greater
 
 ## Optional (enabling features)
 

diff --git a/src/bvals/bvals.hpp b/src/bvals/bvals.hpp
@@ -101,9 +101,6 @@ class BoundarySwarm : public BoundaryCommunication {
   explicit BoundarySwarm(std::weak_ptr<MeshBlock> pmb, const std::string &label);
   ~BoundarySwarm() = default;
 
-  std::vector<ParArrayND<int>> vars_int;
-  std::vector<ParArrayND<Real>> vars_real;
-
   // (usuallly the std::size_t unsigned integer type)
   std::vector<BoundaryCommunication *>::size_type bswarm_index;
 

diff --git a/src/bvals/comms/bnd_info.cpp b/src/bvals/comms/bnd_info.cpp
@@ -40,7 +40,7 @@
 namespace parthenon {
 
 void ProResCache_t::Initialize(int n_regions, StateDescriptor *pkg) {
-  prores_info = ParArray1D<ProResInfo>("prores_info", n_regions);
+  prores_info = ProResInfoArr_t(ViewOfViewAlloc("prores_info"), n_regions);
   prores_info_h = Kokkos::create_mirror_view(
       Kokkos::view_alloc(Kokkos::SequentialHostInit), prores_info);
   int nref_funcs = pkg->NumRefinementFuncs();

diff --git a/src/bvals/comms/bnd_info.hpp b/src/bvals/comms/bnd_info.hpp
@@ -26,6 +26,7 @@
 #include "bvals/neighbor_block.hpp"
 #include "coordinates/coordinates.hpp"
 #include "interface/variable_state.hpp"
+#include "kokkos_abstraction.hpp"
 #include "mesh/domain.hpp"
 #include "mesh/forest/logical_coordinate_transformation.hpp"
 #include "utils/communication_buffer.hpp"
@@ -130,8 +131,8 @@ int GetBufferSize(MeshBlock *pmb, const NeighborBlock &nb,
 using BndInfoArr_t = Kokkos::View<BndInfo *, LayoutWrapper, DevMemSpace>;
 using BndInfoArrHost_t = typename BndInfoArr_t::HostMirror;
 
-using ProResInfoArr_t = ParArray1D<ProResInfo>;
-using ProResInfoArrHost_t = typename ParArray1D<ProResInfo>::HostMirror;
+using ProResInfoArr_t = Kokkos::View<ProResInfo *, LayoutWrapper, DevMemSpace>;
+using ProResInfoArrHost_t = typename ProResInfoArr_t::HostMirror;
 class StateDescriptor;
 struct ProResCache_t {
   ProResInfoArr_t prores_info{};

diff --git a/src/bvals/comms/bvals_utils.hpp b/src/bvals/comms/bvals_utils.hpp
@@ -28,6 +28,7 @@
 #include "bvals/comms/bnd_info.hpp"
 #include "bvals/comms/bvals_in_one.hpp"
 #include "interface/variable.hpp"
+#include "kokkos_abstraction.hpp"
 #include "mesh/domain.hpp"
 #include "mesh/mesh.hpp"
 #include "mesh/meshblock.hpp"
@@ -215,7 +216,7 @@ inline void RebuildBufferCache(std::shared_ptr<MeshData<Real>> md, int nbound,
   using namespace loops;
   using namespace loops::shorthands;
   BvarsSubCache_t &cache = md->GetBvarsCache().GetSubCache(BOUND_TYPE, SENDER);
-  cache.bnd_info = BndInfoArr_t("bnd_info", nbound);
+  cache.bnd_info = BndInfoArr_t(ViewOfViewAlloc("bnd_info"), nbound);
   cache.bnd_info_h = Kokkos::create_mirror_view(
       Kokkos::view_alloc(Kokkos::SequentialHostInit), cache.bnd_info);
 

diff --git a/src/interface/mesh_data.hpp b/src/interface/mesh_data.hpp
@@ -26,6 +26,7 @@
 #include "interface/sparse_pack_base.hpp"
 #include "interface/swarm_pack_base.hpp"
 #include "interface/variable_pack.hpp"
+#include "kokkos_abstraction.hpp"
 #include "mesh/domain.hpp"
 #include "mesh/meshblock.hpp"
 #include "mesh/meshblock_pack.hpp"
@@ -149,7 +150,8 @@ const MeshBlockPack<P> &PackOnMesh(M &map, BlockDataList_t<Real> &block_data_,
   }
 
   if (make_new_pack) {
-    ParArray1D<P> packs("MeshData::PackVariables::packs", nblocks);
+    Kokkos::View<P *, LayoutWrapper, DevMemSpace> packs(
+        ViewOfViewAlloc("MeshData::PackVariables::packs"), nblocks);
     auto packs_host =
         Kokkos::create_mirror_view(Kokkos::view_alloc(Kokkos::SequentialHostInit), packs);
 

diff --git a/src/interface/sparse_pack_base.cpp b/src/interface/sparse_pack_base.cpp
@@ -30,6 +30,7 @@
 #include "interface/sparse_pack_base.hpp"
 #include "interface/state_descriptor.hpp"
 #include "interface/variable.hpp"
+#include "kokkos_abstraction.hpp"
 #include "utils/utils.hpp"
 namespace parthenon {
 namespace impl {
@@ -151,7 +152,7 @@ SparsePackBase SparsePackBase::Build(T *pmd, const PackDescriptor &desc,
   } else if (contains_face_or_edge) {
     leading_dim += 2;
   }
-  pack.pack_ = pack_t("data_ptr", leading_dim, pack.nblocks_, max_size);
+  pack.pack_ = pack_t(ViewOfViewAlloc("data_ptr"), leading_dim, pack.nblocks_, max_size);
   pack.pack_h_ = Kokkos::create_mirror_view(
       Kokkos::view_alloc(Kokkos::SequentialHostInit), pack.pack_);
 
@@ -168,7 +169,7 @@ SparsePackBase SparsePackBase::Build(T *pmd, const PackDescriptor &desc,
   pack.block_props_ = block_props_t("block_props", nblocks, 27 + 1);
   pack.block_props_h_ = Kokkos::create_mirror_view(pack.block_props_);
 
-  pack.coords_ = coords_t("coords", desc.flat ? max_size : nblocks);
+  pack.coords_ = coords_t(ViewOfViewAlloc("coords"), desc.flat ? max_size : nblocks);
   auto coords_h = Kokkos::create_mirror_view(
       Kokkos::view_alloc(Kokkos::SequentialHostInit), pack.coords_);
 

diff --git a/src/interface/sparse_pack_base.hpp b/src/interface/sparse_pack_base.hpp
@@ -30,6 +30,7 @@
 #include "interface/state_descriptor.hpp"
 #include "interface/variable.hpp"
 #include "interface/variable_state.hpp"
+#include "kokkos_abstraction.hpp"
 #include "utils/utils.hpp"
 
 namespace parthenon {
@@ -55,7 +56,8 @@ class SparsePackBase {
 
   using alloc_t = std::vector<int>;
   using include_t = std::vector<bool>;
-  using pack_t = ParArray3D<ParArray3D<Real, VariableState>>;
+  using pack_t =
+      Kokkos::View<ParArray3D<Real, VariableState> ***, LayoutWrapper, DevMemSpace>;
   using pack_h_t = typename pack_t::HostMirror;
   using bounds_t = ParArray3D<int>;
   using bounds_h_t = typename bounds_t::HostMirror;

diff --git a/src/interface/swarm_pack_base.hpp b/src/interface/swarm_pack_base.hpp
@@ -28,6 +28,7 @@
 #include "interface/state_descriptor.hpp"
 #include "interface/swarm_device_context.hpp"
 #include "interface/variable.hpp"
+#include "kokkos_abstraction.hpp"
 #include "utils/utils.hpp"
 
 namespace parthenon {
@@ -43,10 +44,10 @@ class SwarmPackBase {
   SwarmPackBase() = default;
   virtual ~SwarmPackBase() = default;
 
-  using pack_t = ParArray3D<ParArray1D<TYPE>>;
+  using pack_t = Kokkos::View<ParArray1D<TYPE> ***, LayoutWrapper, DevMemSpace>;
   using bounds_t = ParArray3D<int>;
-  using contexts_t = ParArray1D<SwarmDeviceContext>;
-  using contexts_h_t = typename ParArray1D<SwarmDeviceContext>::HostMirror;
+  using contexts_t = Kokkos::View<SwarmDeviceContext *, LayoutWrapper, DevMemSpace>;
+  using contexts_h_t = typename contexts_t::HostMirror;
   using max_active_indices_t = ParArray1D<int>;
   using desc_t = impl::SwarmPackDescriptor<TYPE>;
   using idx_map_t = std::unordered_map<std::string, std::size_t>;
@@ -108,7 +109,7 @@ class SwarmPackBase {
 
     // Allocate the views
     int leading_dim = 1;
-    pack.pack_ = pack_t("data_ptr", leading_dim, nblocks, max_size);
+    pack.pack_ = pack_t(ViewOfViewAlloc("data_ptr"), leading_dim, nblocks, max_size);
     auto pack_h = Kokkos::create_mirror_view(
         Kokkos::view_alloc(Kokkos::SequentialHostInit), pack.pack_);
 
@@ -154,7 +155,7 @@ class SwarmPackBase {
     Kokkos::deep_copy(pack.pack_, pack_h);
     Kokkos::deep_copy(pack.bounds_, bounds_h);
 
-    pack.contexts_ = contexts_t("contexts", nblocks);
+    pack.contexts_ = contexts_t(ViewOfViewAlloc("contexts"), nblocks);
     pack.contexts_h_ = Kokkos::create_mirror_view(
         Kokkos::view_alloc(Kokkos::SequentialHostInit), pack.contexts_);
     pack.max_active_indices_ = max_active_indices_t("max_active_indices", nblocks);

diff --git a/tst/unit/test_pararrays.cpp b/tst/unit/test_pararrays.cpp
@@ -528,6 +528,8 @@ TEST_CASE("Check registry pressure", "[ParArrayND][performance]") {
 
   // view of views. See:
   // https://github.com/kokkos/kokkos/wiki/View#6232-whats-the-problem-with-a-view-of-views
+  // TODO(PG) depending on the results of the view of view discussion, we should add
+  // destructor or ViewOfViewAlloc with SequentialHostInit
   using view_3d_t =
       Kokkos::View<Real ***, parthenon::LayoutWrapper, parthenon::DevMemSpace>;
   using arrays_t = Kokkos::View<parthenon::ParArray6D<Real> *, UVMSpace>;