Implementation of repartitioning distributed matrix wrapper

CMakeLists.txt

@@ -223,6 +223,7 @@ target_sources(
  src/DevicePersistent/ExecutorHandler/ExecutorHandler.C
  src/MatrixWrapper/HostMatrix.C
  src/MatrixWrapper/SparsityPattern.C
+ src/MatrixWrapper/Distributed.C
  src/Repartitioner.C
  # src/Solver/CG/GKOCG.C src/Solver/BiCGStab/GKOBiCGStab.C
  # src/Solver/GMRES/GKOGMRES.C

cmake/ginkgo.cmake

@@ -7,6 +7,7 @@ include(FetchContent)
 if(NOT ${OGL_USE_EXTERNAL_GINKGO})
  FetchContent_Declare(
  Ginkgo
+ SYSTEM
  QUITE
  GIT_SHALLOW ON
  GIT_REPOSITORY "https://github.com/ginkgo-project/ginkgo.git"

include/OGL/CommunicationPattern.H

@@ -157,6 +157,12 @@ struct CommunicationPattern {
  */
  gko::array<label> total_rank_send_idx() const;
 
+ /* @brief concatenate all separate send idxs arrays into one contiguous
+ * array
+ */
+ gko::array<label> compute_recv_gather_idxs(
+ const ExecutorHandler &exec_handler) const;
+
 
  /** Computes the send and recv pattern for ginkgos distributed matrix SpMV
  */

include/OGL/DevicePersistent/DeviceIdGuard.H

@@ -26,7 +26,8 @@ struct DeviceIdGuardInitFunctor {
  std::shared_ptr<gko::scoped_device_id_guard> init() const
  {
  return std::make_shared<gko::scoped_device_id_guard>(
- std::move(executor_->get_scoped_device_id_guard()));
+ executor_->get_scoped_device_id_guard());
+ // std::move(executor_->get_scoped_device_id_guard()));
  }
 };
 
@@ -41,6 +42,4 @@ public:
  {}
 };
 
-
 } // namespace Foam
- // namespace Foam

include/OGL/DevicePersistent/ExecutorHandler.H

@@ -176,22 +176,21 @@ public:
  MPI_COMM_WORLD, gko_force_host_buffer_);
  }
 
- std::shared_ptr<gko::experimental::mpi::communicator>
+ std::shared_ptr<const gko::experimental::mpi::communicator>
  get_gko_mpi_device_comm() const
  {
  return this->device_comm_;
  }
 
- std::shared_ptr<gko::experimental::mpi::communicator> get_communicator()
-   const
+ std::shared_ptr<const gko::experimental::mpi::communicator>
+ get_communicator() const
  {
  return this->device_comm_;
  }
 
- label get_rank() const { return get_communicator().get()->rank(); };
+ label get_rank() const { return get_communicator()->rank(); };
 };
 
 using PersistentExecutor = ExecutorHandler;
 
 } // namespace Foam
-// namespace Foam

include/OGL/DevicePersistent/Partition.H

@@ -55,14 +55,18 @@ struct PartitionInitFunctor {
 
  std::shared_ptr<part_type> init() const
  {
- auto exec = exec_.get_ref_exec();
- auto comm = exec_.get_gko_mpi_host_comm();
+ FatalErrorInFunction << "Not implemented" << abort(FatalError);
+ // auto exec = exec_.get_ref_exec();
+ // auto comm = exec_.get_gko_mpi_host_comm();
 
- word msg{"initialising partition of size " +
- std::to_string(local_size_)};
- LOG_1(verbose_, msg)
+ // word msg{"initialising partition of size " +
+ //  std::to_string(local_size_)};
+ // LOG_1(verbose_, msg)
 
- if (ranks_per_gpu_ == 1) return {};
+ // if (ranks_per_gpu_ == 1) return {};
+
+
+ return {};
  }
 };
 
@@ -84,8 +88,6 @@ class PersistentPartition
 
  mutable label global_elements_;
 
- const ExecutorHandler &exec_;
-
 public:
  using part_type = gko::experimental::distributed::Partition<label, label>;
  /* PersistentPartition constructor using existing memory
@@ -107,8 +109,7 @@ public:
  communication_pattern),
  false, verbose),
  ranks_per_gpu_(ranks_per_gpu),
- local_elements_(elements),
- exec_(exec)
+ local_elements_(elements)
  {
  auto comm = exec.get_gko_mpi_host_comm();
  label local_elements = local_elements_;

include/OGL/MatrixWrapper/Combination.H

@@ -8,6 +8,7 @@
 
 #include "fvCFD.H"
 
+
 /* @brief The CombinationMatrix class is a wrapper around Ginkgos combination
  * linear operator. It provides an interface for creating linear combinations of
  * a given inner matrix type and a read implementation
@@ -18,7 +19,8 @@ class CombinationMatrix
  public gko::EnableCreateMethod<CombinationMatrix<InnerMatrixType>>,
  public gko::ReadableFromMatrixData<scalar, label>,
  public gko::ConvertibleTo<gko::matrix::Coo<scalar, label>>,
- public gko::ConvertibleTo<gko::matrix::Csr<scalar, label>> {
+ public gko::ConvertibleTo<gko::matrix::Csr<scalar, label>>,
+ public gko::WritableToMatrixData<scalar, label> {
  friend class gko::EnableCreateMethod<CombinationMatrix<InnerMatrixType>>;
  friend class gko::EnablePolymorphicObject<
  CombinationMatrix<InnerMatrixType>, gko::LinOp>;
@@ -66,6 +68,25 @@ public:
  return *this;
  }
 
+
+ void write(gko::matrix_data<scalar> &res) const override
+ {
+ auto exec = this->get_executor();
+ auto operators = this->get_operators();
+
+ std::vector<gko::matrix_data_entry<scalar, label>> data;
+ for (auto op : operators) {
+ auto tmp_mat_data = gko::matrix_data<scalar>();
+ gko::as<InnerMatrixType>(op)->write(tmp_mat_data);
+ data.insert(data.end(), tmp_mat_data.nonzeros.begin(),
+ tmp_mat_data.nonzeros.end());
+ }
+ auto mat_data = gko::matrix_data<scalar>();
+ res.nonzeros = data;
+ res.size = this->get_size();
+ }
+
+
  /**
  * Returns a list of coefficients of the combination.
  *
@@ -176,7 +197,7 @@ protected:
  * */
  void apply_impl(const gko::LinOp *b, gko::LinOp *x) const override
  {
- if (this->get_size()[0] > 0) {
+ if (this->get_size()[0] > 0 && this->get_size()[1] > 0) {
  this->comb_->apply(b, x);
  }
  }
@@ -187,11 +208,44 @@ protected:
  void apply_impl(const gko::LinOp *alpha, const gko::LinOp *b,
  const gko::LinOp *beta, gko::LinOp *x) const override
  {
- if (this->get_size()[0] > 0) {
+ if (this->get_size()[0] > 0 && this->get_size()[1] > 0) {
  this->comb_->apply(alpha, b, beta, x);
  }
  }
 
 private:
  std::shared_ptr<gko::Combination<scalar>> comb_;
 };
+
+
+namespace detail {
+std::shared_ptr<gko::matrix::Coo<scalar, label>> convert_combination_to_coo(
+ std::shared_ptr<const gko::Executor> exec,
+ std::shared_ptr<const gko::LinOp> in)
+{
+ auto out = gko::share(gko::matrix::Coo<scalar, label>::create(exec));
+ gko::as<CombinationMatrix<gko::matrix::Coo<scalar, label>>>(in)->convert_to(
+ out.get());
+
+ return out;
+}
+
+template <typename InnerMatrixType>
+void export_mtx(const word fieldName,
+ std::shared_ptr<CombinationMatrix<InnerMatrixType>> out,
+ const word local, const objectRegistry &db,
+ const word matrixFormat)
+{
+ std::string folder{db.time().timePath()};
+ std::filesystem::create_directories(folder);
+ std::string fn{folder + "/" + fieldName + "_A_" + local + ".mtx"};
+ std::cout << "exporting " << fn << std::endl;
+ std::ofstream stream{fn};
+ stream << std::setprecision(15);
+
+ auto coo = gko::share(
+ gko::matrix::Coo<scalar, label>::create(out->get_executor()));
+ out->convert_to(coo);
+}
+
+} // namespace detail