diff --git a/README.md b/README.md index f049c44ecd7d..b874e5172889 100644 --- a/README.md +++ b/README.md @@ -30,7 +30,7 @@ Parthenon -- a performance portable block-structured adaptive mesh refinement fr * CMake 3.16 or greater * C++17 compatible compiler -* Kokkos 4.1.1 or greater +* Kokkos 4.4.1 or greater ## Optional (enabling features) diff --git a/src/bvals/bvals.hpp b/src/bvals/bvals.hpp index bc6ed0ebed56..aaa4da22c0f0 100644 --- a/src/bvals/bvals.hpp +++ b/src/bvals/bvals.hpp @@ -101,9 +101,6 @@ class BoundarySwarm : public BoundaryCommunication { explicit BoundarySwarm(std::weak_ptr pmb, const std::string &label); ~BoundarySwarm() = default; - std::vector> vars_int; - std::vector> vars_real; - // (usuallly the std::size_t unsigned integer type) std::vector::size_type bswarm_index; diff --git a/src/bvals/comms/bnd_info.cpp b/src/bvals/comms/bnd_info.cpp index 54a6ae2b50fd..198489c1c1f0 100644 --- a/src/bvals/comms/bnd_info.cpp +++ b/src/bvals/comms/bnd_info.cpp @@ -40,7 +40,7 @@ namespace parthenon { void ProResCache_t::Initialize(int n_regions, StateDescriptor *pkg) { - prores_info = ParArray1D("prores_info", n_regions); + prores_info = ProResInfoArr_t(ViewOfViewAlloc("prores_info"), n_regions); prores_info_h = Kokkos::create_mirror_view( Kokkos::view_alloc(Kokkos::SequentialHostInit), prores_info); int nref_funcs = pkg->NumRefinementFuncs(); diff --git a/src/bvals/comms/bnd_info.hpp b/src/bvals/comms/bnd_info.hpp index 8800f6fd867f..8c6c85d59f93 100644 --- a/src/bvals/comms/bnd_info.hpp +++ b/src/bvals/comms/bnd_info.hpp @@ -26,6 +26,7 @@ #include "bvals/neighbor_block.hpp" #include "coordinates/coordinates.hpp" #include "interface/variable_state.hpp" +#include "kokkos_abstraction.hpp" #include "mesh/domain.hpp" #include "mesh/forest/logical_coordinate_transformation.hpp" #include "utils/communication_buffer.hpp" @@ -130,8 +131,8 @@ int GetBufferSize(MeshBlock *pmb, const NeighborBlock &nb, using BndInfoArr_t = Kokkos::View; using BndInfoArrHost_t = typename BndInfoArr_t::HostMirror; -using ProResInfoArr_t = ParArray1D; -using ProResInfoArrHost_t = typename ParArray1D::HostMirror; +using ProResInfoArr_t = Kokkos::View; +using ProResInfoArrHost_t = typename ProResInfoArr_t::HostMirror; class StateDescriptor; struct ProResCache_t { ProResInfoArr_t prores_info{}; diff --git a/src/bvals/comms/bvals_utils.hpp b/src/bvals/comms/bvals_utils.hpp index 8a47c716aa22..87b55b51019d 100644 --- a/src/bvals/comms/bvals_utils.hpp +++ b/src/bvals/comms/bvals_utils.hpp @@ -28,6 +28,7 @@ #include "bvals/comms/bnd_info.hpp" #include "bvals/comms/bvals_in_one.hpp" #include "interface/variable.hpp" +#include "kokkos_abstraction.hpp" #include "mesh/domain.hpp" #include "mesh/mesh.hpp" #include "mesh/meshblock.hpp" @@ -215,7 +216,7 @@ inline void RebuildBufferCache(std::shared_ptr> md, int nbound, using namespace loops; using namespace loops::shorthands; BvarsSubCache_t &cache = md->GetBvarsCache().GetSubCache(BOUND_TYPE, SENDER); - cache.bnd_info = BndInfoArr_t("bnd_info", nbound); + cache.bnd_info = BndInfoArr_t(ViewOfViewAlloc("bnd_info"), nbound); cache.bnd_info_h = Kokkos::create_mirror_view( Kokkos::view_alloc(Kokkos::SequentialHostInit), cache.bnd_info); diff --git a/src/interface/mesh_data.hpp b/src/interface/mesh_data.hpp index 7d7d1cabcbe5..9a86f0e52014 100644 --- a/src/interface/mesh_data.hpp +++ b/src/interface/mesh_data.hpp @@ -26,6 +26,7 @@ #include "interface/sparse_pack_base.hpp" #include "interface/swarm_pack_base.hpp" #include "interface/variable_pack.hpp" +#include "kokkos_abstraction.hpp" #include "mesh/domain.hpp" #include "mesh/meshblock.hpp" #include "mesh/meshblock_pack.hpp" @@ -149,7 +150,8 @@ const MeshBlockPack

&PackOnMesh(M &map, BlockDataList_t &block_data_, } if (make_new_pack) { - ParArray1D

packs("MeshData::PackVariables::packs", nblocks); + Kokkos::View

packs( + ViewOfViewAlloc("MeshData::PackVariables::packs"), nblocks); auto packs_host = Kokkos::create_mirror_view(Kokkos::view_alloc(Kokkos::SequentialHostInit), packs); diff --git a/src/interface/sparse_pack_base.cpp b/src/interface/sparse_pack_base.cpp index 4ea5a558c3f5..751e88c200c6 100644 --- a/src/interface/sparse_pack_base.cpp +++ b/src/interface/sparse_pack_base.cpp @@ -30,6 +30,7 @@ #include "interface/sparse_pack_base.hpp" #include "interface/state_descriptor.hpp" #include "interface/variable.hpp" +#include "kokkos_abstraction.hpp" #include "utils/utils.hpp" namespace parthenon { namespace impl { @@ -151,7 +152,7 @@ SparsePackBase SparsePackBase::Build(T *pmd, const PackDescriptor &desc, } else if (contains_face_or_edge) { leading_dim += 2; } - pack.pack_ = pack_t("data_ptr", leading_dim, pack.nblocks_, max_size); + pack.pack_ = pack_t(ViewOfViewAlloc("data_ptr"), leading_dim, pack.nblocks_, max_size); pack.pack_h_ = Kokkos::create_mirror_view( Kokkos::view_alloc(Kokkos::SequentialHostInit), pack.pack_); @@ -168,7 +169,7 @@ SparsePackBase SparsePackBase::Build(T *pmd, const PackDescriptor &desc, pack.block_props_ = block_props_t("block_props", nblocks, 27 + 1); pack.block_props_h_ = Kokkos::create_mirror_view(pack.block_props_); - pack.coords_ = coords_t("coords", desc.flat ? max_size : nblocks); + pack.coords_ = coords_t(ViewOfViewAlloc("coords"), desc.flat ? max_size : nblocks); auto coords_h = Kokkos::create_mirror_view( Kokkos::view_alloc(Kokkos::SequentialHostInit), pack.coords_); diff --git a/src/interface/sparse_pack_base.hpp b/src/interface/sparse_pack_base.hpp index 0deca487a20a..25331c384b56 100644 --- a/src/interface/sparse_pack_base.hpp +++ b/src/interface/sparse_pack_base.hpp @@ -30,6 +30,7 @@ #include "interface/state_descriptor.hpp" #include "interface/variable.hpp" #include "interface/variable_state.hpp" +#include "kokkos_abstraction.hpp" #include "utils/utils.hpp" namespace parthenon { @@ -55,7 +56,8 @@ class SparsePackBase { using alloc_t = std::vector; using include_t = std::vector; - using pack_t = ParArray3D>; + using pack_t = + Kokkos::View ***, LayoutWrapper, DevMemSpace>; using pack_h_t = typename pack_t::HostMirror; using bounds_t = ParArray3D; using bounds_h_t = typename bounds_t::HostMirror; diff --git a/src/interface/swarm_pack_base.hpp b/src/interface/swarm_pack_base.hpp index 52a2c3c47fc7..6d0933b8485c 100644 --- a/src/interface/swarm_pack_base.hpp +++ b/src/interface/swarm_pack_base.hpp @@ -28,6 +28,7 @@ #include "interface/state_descriptor.hpp" #include "interface/swarm_device_context.hpp" #include "interface/variable.hpp" +#include "kokkos_abstraction.hpp" #include "utils/utils.hpp" namespace parthenon { @@ -43,10 +44,10 @@ class SwarmPackBase { SwarmPackBase() = default; virtual ~SwarmPackBase() = default; - using pack_t = ParArray3D>; + using pack_t = Kokkos::View ***, LayoutWrapper, DevMemSpace>; using bounds_t = ParArray3D; - using contexts_t = ParArray1D; - using contexts_h_t = typename ParArray1D::HostMirror; + using contexts_t = Kokkos::View; + using contexts_h_t = typename contexts_t::HostMirror; using max_active_indices_t = ParArray1D; using desc_t = impl::SwarmPackDescriptor; using idx_map_t = std::unordered_map; @@ -108,7 +109,7 @@ class SwarmPackBase { // Allocate the views int leading_dim = 1; - pack.pack_ = pack_t("data_ptr", leading_dim, nblocks, max_size); + pack.pack_ = pack_t(ViewOfViewAlloc("data_ptr"), leading_dim, nblocks, max_size); auto pack_h = Kokkos::create_mirror_view( Kokkos::view_alloc(Kokkos::SequentialHostInit), pack.pack_); @@ -154,7 +155,7 @@ class SwarmPackBase { Kokkos::deep_copy(pack.pack_, pack_h); Kokkos::deep_copy(pack.bounds_, bounds_h); - pack.contexts_ = contexts_t("contexts", nblocks); + pack.contexts_ = contexts_t(ViewOfViewAlloc("contexts"), nblocks); pack.contexts_h_ = Kokkos::create_mirror_view( Kokkos::view_alloc(Kokkos::SequentialHostInit), pack.contexts_); pack.max_active_indices_ = max_active_indices_t("max_active_indices", nblocks); diff --git a/tst/unit/test_pararrays.cpp b/tst/unit/test_pararrays.cpp index 9e1816b0e669..865dfbedf5e9 100644 --- a/tst/unit/test_pararrays.cpp +++ b/tst/unit/test_pararrays.cpp @@ -528,6 +528,8 @@ TEST_CASE("Check registry pressure", "[ParArrayND][performance]") { // view of views. See: // https://github.com/kokkos/kokkos/wiki/View#6232-whats-the-problem-with-a-view-of-views + // TODO(PG) depending on the results of the view of view discussion, we should add + // destructor or ViewOfViewAlloc with SequentialHostInit using view_3d_t = Kokkos::View; using arrays_t = Kokkos::View *, UVMSpace>;