Implement a combination matrix type with support for read

.github/workflows/build-foam.yml

@@ -64,17 +64,10 @@ jobs:
 
     - name: Cache build folder
       uses: actions/cache@v4
+      if: ${{!contains(github.event.pull_request.labels.*.name, 'Skip-cache')}}
       with:
-        key: build-${{ matrix.OF.path }}-${{env.GINKGO_CHECKOUT_VERSION}}
-        path: |
-          ${{github.workspace}}/build
-
-    - name: Cache FOAM_USER_LIBBIN
-      uses: actions/cache@v4
-      with:
-        key: FOAM_USER_LIBBIN-${{ matrix.OF.version }}-${{ github.sha }}
-        path: |
-          ${{matrix.OF.foam_user_libbin}}
+         key: build_PR_${{ github.event.pull_request.number }}_${{matrix.OF.path}}
+         path: ${{github.workspace}}/build
 
     - name: Configure CMake
       # Use a bash shell so we can use the same syntax for environment variable

CMakeLists.txt

@@ -139,6 +139,7 @@ ginkgo_find_package(Ginkgo "Ginkgo::ginkgo" FALSE 1.8.0)
 include(cmake/ginkgo.cmake)
 
 set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -Wall -O0 -ggdb")
+
 set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} \
   -Wall -Wpedantic -Wextra -march=native -fopenmp -Wno-undefined-var-template")
 
@@ -164,6 +165,7 @@ elseif("${CMAKE_CXX_COMPILER_ID}" MATCHES "GNU")
   set(CMAKE_CXX_FLAGS_DEBUG
       "${CMAKE_CXX_FLAGS_DEBUG} \
       --coverage\
+      --fopenmp \
       -fprofile-arcs \
       -ggdb3 \
       ")
@@ -202,6 +204,8 @@ add_custom_command(
           ${GINKGO_CHECKOUT_VERSION}
   COMMENT "Create a version file")
 
+add_subdirectory(include)
+
 target_sources(
   OGL
   PRIVATE src/common.C
@@ -220,9 +224,8 @@ target_sources(
           src/MatrixWrapper/HostMatrix.C
           src/Solver/CG/GKOCG.C
           src/Solver/BiCGStab/GKOBiCGStab.C
-          src/Solver/GMRES/GKOGMRES.C)
-
-add_subdirectory(include)
+          src/Solver/GMRES/GKOGMRES.C
+)
 
 enable_sanitizers(
   OGL ${OGL_ENABLE_SANITIZE_ADDRESS} ${OGL_ENABLE_SANITIZE_LEAK}

cmake/ginkgo.cmake

@@ -10,7 +10,8 @@ if(NOT ${OGL_USE_EXTERNAL_GINKGO})
     QUITE
     GIT_SHALLOW ON
     GIT_REPOSITORY "https://github.com/ginkgo-project/ginkgo.git"
-    GIT_TAG ${GINKGO_CHECKOUT_VERSION})
+    GIT_TAG ${GINKGO_CHECKOUT_VERSION}
+    )
 
   FetchContent_MakeAvailable(Ginkgo)
 endif()

include/OGL/MatrixWrapper/Combination.H

@@ -0,0 +1,197 @@
+// SPDX-FileCopyrightText: 2024 OGL authors
+//
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#pragma once
+
+#include <ginkgo/ginkgo.hpp>
+
+#include "fvCFD.H"
+
+/* @brief The CombinationMatrix class is a wrapper around Ginkgos combination
+ * linear operator. It provides an interface for creating linear combinations of
+ * a given inner matrix type and a read implementation
+ * */
+template <typename InnerMatrixType>
+class CombinationMatrix
+    : public gko::EnableLinOp<CombinationMatrix<InnerMatrixType>>,
+      public gko::EnableCreateMethod<CombinationMatrix<InnerMatrixType>>,
+      public gko::ReadableFromMatrixData<scalar, label>,
+      public gko::ConvertibleTo<gko::matrix::Coo<scalar, label>>,
+      public gko::ConvertibleTo<gko::matrix::Csr<scalar, label>> {
+    friend class gko::EnableCreateMethod<CombinationMatrix<InnerMatrixType>>;
+    friend class gko::EnablePolymorphicObject<
+        CombinationMatrix<InnerMatrixType>, gko::LinOp>;
+
+public:
+    using gko::EnableLinOp<CombinationMatrix<InnerMatrixType>>::convert_to;
+    using gko::EnableLinOp<CombinationMatrix<InnerMatrixType>>::move_to;
+
+    using matrix_data = gko::matrix_data<scalar, label>;
+
+    void read(const matrix_data &data) override
+    {
+        FatalErrorInFunction << "Not implemented" << abort(FatalError);
+    }
+
+    std::shared_ptr<gko::Combination<scalar>> get_combination() const
+    {
+        return this->comb_;
+    }
+
+    /**
+     * Copy-assigns a CombinationMatrix matrix. Preserves executor, copies
+     * everything else.
+     */
+    CombinationMatrix &operator=(const CombinationMatrix &other)
+    {
+        if (&other != this) {
+            gko::EnableLinOp<CombinationMatrix>::operator=(other);
+            comb_ = other.comb_;
+        }
+        return *this;
+    }
+
+    /**
+     * Move-assigns a CombinationMatrix matrix. Preserves executor, moves the
+     * data and leaves the moved-from object in an empty state (0x0 LinOp with
+     * unchanged executor and strategy, no nonzeros and valid row pointers).
+     */
+    CombinationMatrix &operator=(CombinationMatrix &&other)
+    {
+        if (&other != this) {
+            gko::EnableLinOp<CombinationMatrix>::operator=(std::move(other));
+            comb_ = std::move(other.comb_);
+        }
+        return *this;
+    }
+
+     /**
+     * Returns a list of coefficients of the combination.
+     *
+     * @return a list of coefficients
+     */
+    const std::vector<std::shared_ptr<const gko::LinOp>> &get_coefficients()
+        const noexcept
+    {
+        return comb_->get_coefficients();
+    }
+
+    /**
+     * Returns a list of operators of the combination.
+     *
+     * @return a list of operators
+     */
+    const std::vector<std::shared_ptr<const gko::LinOp>> &get_operators()
+        const noexcept
+    {
+        return comb_->get_operators();
+    }
+
+    /*
+     * @warning assumes that all entries are unique
+     */
+    void convert_to(gko::matrix::Coo<scalar, label> *result) const override
+    {
+        auto exec = this->get_executor();
+        auto operators = this->get_operators();
+
+        std::vector<gko::matrix_data_entry<scalar, label>> data;
+        for (auto op : operators) {
+            auto tmp_mat_data = gko::matrix_data<scalar>();
+            gko::as<InnerMatrixType>(op)->write(tmp_mat_data);
+            data.insert(data.end(), tmp_mat_data.nonzeros.begin(),
+                        tmp_mat_data.nonzeros.end());
+        }
+        auto mat_data = gko::matrix_data<scalar>();
+        mat_data.nonzeros = data;
+        mat_data.size = this->get_size();
+
+        result->read(mat_data);
+    }
+
+    /*
+     * @warning assumes that all entries are unique
+     */
+    void convert_to(gko::matrix::Csr<scalar, label> *result) const override 
+    {
+        auto exec = this->get_executor();
+        auto operators = this->get_operators();
+
+        std::vector<gko::matrix_data_entry<scalar, label>> data;
+        for (auto op : operators) {
+            auto tmp_mat_data = gko::matrix_data<scalar>();
+            gko::as<InnerMatrixType>(op)->write(tmp_mat_data);
+            data.insert(data.end(), tmp_mat_data.nonzeros.begin(),
+                        tmp_mat_data.nonzeros.end());
+        }
+        auto mat_data = gko::matrix_data<scalar>();
+        mat_data.nonzeros = data;
+        mat_data.size = this->get_size();
+
+        result->read(mat_data);
+    }
+
+    /*
+     * @warning assumes that all entries are unique
+     */
+    void move_to(gko::matrix::Coo<scalar, label> *result) override 
+    {
+        this->convert_to(result);
+        this->comb_ = nullptr;
+        this->set_size(gko::dim<2>(0, 0));
+    }
+
+    /*
+     * @warning assumes that all entries are unique
+     */
+    void move_to(gko::matrix::Csr<scalar, label> *result) override 
+    {
+        this->convert_to(result);
+        this->comb_ = nullptr;
+        this->set_size(gko::dim<2>(0, 0));
+    }
+
+protected:
+    CombinationMatrix(std::shared_ptr<const gko::Executor> exec)
+        : gko::EnableLinOp<CombinationMatrix>(exec),
+          comb_(gko::share(gko::Combination<scalar>::create(exec)))
+    {}
+
+    CombinationMatrix(std::shared_ptr<const gko::Executor> exec,
+                      gko::dim<2> size,
+                      std::vector<std::shared_ptr<const gko::LinOp>> operators)
+        : gko::EnableLinOp<CombinationMatrix>(exec),
+          comb_(gko::share(gko::Combination<scalar>::create(exec, size)))
+    {
+        for (auto &op : operators) {
+            this->comb_->add_operators(
+                gko::initialize<gko::matrix::Dense<scalar>>({1}, exec), op);
+        }
+        this->set_size(size);
+    }
+
+    /* Forwarding call of apply which checks if the matrix contains any
+     * submatrices
+     * */
+    void apply_impl(const gko::LinOp *b, gko::LinOp *x) const override
+    {
+        if (this->get_size()[0] > 0) {
+            this->comb_->apply(b, x);
+        }
+    }
+
+    /* Forwarding call of apply which checks if the matrix contains any
+     * submatrices
+     * */
+    void apply_impl(const gko::LinOp *alpha, const gko::LinOp *b,
+                    const gko::LinOp *beta, gko::LinOp *x) const override
+    {
+        if (this->get_size()[0] > 0) {
+            this->comb_->apply(alpha, b, beta, x);
+        }
+    }
+
+private:
+    std::shared_ptr<gko::Combination<scalar>> comb_;
+};

include/OGL/common.H

@@ -90,8 +90,6 @@ namespace Foam {
 
 #define UNUSED(x) (void)(x)
 
-#define OGL_NOT_IMPLEMENTED GKO_NOT_IMPLEMENTED
-
 #define ASSERT_EQ(_val1, _val2)                                                \
     if (_val1 != _val2) {                                                      \
         throw ::gko::ValueMismatch(__FILE__, __LINE__, __func__, _val1, _val2, \

unitTests/CMakeLists.txt

@@ -15,8 +15,14 @@ find_package(MPI REQUIRED)
 
 macro(ogl_unit_test TEST FILE)
   add_executable(${TEST} ${FILE})
-  target_link_libraries(${TEST} OpenFOAM GTest::gtest_main OGL Ginkgo::ginkgo
-                        ${CMAKE_DL_LIBS})
+  target_link_libraries(
+    ${TEST}
+    OpenFOAM
+    GTest::gtest_main
+    OGL
+    Ginkgo::ginkgo
+    ginkgo_hip
+    ${CMAKE_DL_LIBS})
 
   add_test(
     NAME ${TEST}
@@ -25,4 +31,5 @@ macro(ogl_unit_test TEST FILE)
 endmacro()
 
 ogl_unit_test(CommunicationPattern "CommunicationPattern.C")
-ogl_unit_test(HostMatrix "HostMatrix.C")
+
+add_subdirectory(MatrixWrapper)

unitTests/MatrixWrapper/CMakeLists.txt

@@ -0,0 +1,16 @@
+# SPDX-FileCopyrightText: 2024 OGL authors
+#
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+# Copy the data files to the execution directory
+configure_file(data/A_sparse.mtx data/A_sparse.mtx COPYONLY)
+configure_file(data/b.mtx data/b.mtx COPYONLY)
+configure_file(data/L.mtx data/L.mtx COPYONLY)
+configure_file(data/D.mtx data/D.mtx COPYONLY)
+configure_file(data/U.mtx data/U.mtx COPYONLY)
+
+ogl_unit_test(matrixConversion "HostMatrix.C")
+ogl_unit_test(combination "Combination.C")
+
+# Set working directory for the unit test suite of Combination Matrix
+set_tests_properties(combination PROPERTIES WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR})