added bootstrap from MPI

include/flock/flock-bootstrap-mpi.h

@@ -0,0 +1,115 @@
+/*
+ * (C) 2024 The University of Chicago
+ *
+ * See COPYRIGHT in top-level directory.
+ */
+#ifndef __FLOCK_BOOTSTRAP_MPI_H
+#define __FLOCK_BOOTSTRAP_MPI_H
+
+#include <string.h>
+#include <margo.h>
+#include <flock/flock-group-view.h>
+#include <mpi.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/**
+ * @brief This helper function will populate a flock_group_view_t
+ * by using the provided MPI communicator. This function is collective
+ * across all the members of the communicator. Each member may provide
+ * a different provider Id. The resulting view will associate to each
+ * flock_member_t the same rank as in the MPI communicator.
+ *
+ * @important The provided view will be reset in the process, including
+ * clearing any metadata attached to it.
+ *
+ * @param[in] mid Margo instance ID.
+ * @param[in] provider_id Provider ID for the calling process.
+ * @param[in] comm MPI communicator.
+ * @param[out] view View to populate.
+ *
+ * @return FLOCK_SUCCESS or other error codes.
+ */
+static inline flock_return_t flock_bootstrap_from_mpi(
+        margo_instance_id mid,
+        uint16_t provider_id,
+        MPI_Comm comm,
+        flock_group_view_t* view)
+{
+    char      self_addr_str[256];
+    hg_size_t self_addr_size = 256;
+    hg_addr_t self_addr      = HG_ADDR_NULL;
+    hg_return_t hret         = HG_SUCCESS;
+    flock_return_t ret       = FLOCK_SUCCESS;
+    uint16_t* provider_ids   = NULL;
+    char* addresses_buf      = NULL;
+    int size, rank, mret;
+
+    hret = margo_addr_self(mid, &self_addr);
+    if(hret != HG_SUCCESS) {
+        ret = FLOCK_ERR_FROM_MERCURY;
+        goto finish;
+    }
+    hret = margo_addr_to_string(mid, self_addr_str, &self_addr_size, self_addr);
+    if(hret != HG_SUCCESS) {
+        ret = FLOCK_ERR_FROM_MERCURY;
+        goto finish;
+    }
+
+    mret = MPI_Comm_rank(comm, &rank);
+    if(mret != 0) {
+        ret = FLOCK_ERR_FROM_MPI;
+        goto finish;
+    }
+    mret = MPI_Comm_size(comm, &size);
+    if(mret != 0) {
+        ret = FLOCK_ERR_FROM_MPI;
+        goto finish;
+    }
+
+    provider_ids = (uint16_t*)malloc(size*sizeof(provider_id));
+    if(!provider_ids) {
+        ret = FLOCK_ERR_ALLOCATION;
+        goto finish;
+    }
+    addresses_buf = (char*)malloc(size*256);
+    if(!addresses_buf) {
+        ret = FLOCK_ERR_ALLOCATION;
+        goto finish;
+    }
+
+    mret = MPI_Allgather(&provider_id, 1, MPI_UINT16_T, provider_ids, 1, MPI_UINT16_T, comm);
+    if(mret != 0) {
+        ret = FLOCK_ERR_FROM_MPI;
+        goto finish;
+    }
+
+    mret = MPI_Allgather(self_addr_str, 256, MPI_CHAR, addresses_buf, 256, MPI_CHAR, comm);
+    if(mret != 0) {
+        ret = FLOCK_ERR_FROM_MPI;
+        goto finish;
+    }
+
+    flock_group_view_clear(view);
+
+    for(int i = 0; i < size; ++i) {
+        if(!flock_group_view_add_member(view, (uint64_t)i, provider_ids[i], addresses_buf + 256*i)) {
+            ret = FLOCK_ERR_ALLOCATION;
+            goto finish;
+        }
+    }
+
+finish:
+    free(provider_ids);
+    free(addresses_buf);
+    margo_addr_free(mid, self_addr);
+    return ret;
+}
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif

include/flock/flock-common.h

@@ -35,6 +35,7 @@ typedef enum flock_return_t {
     FLOCK_ERR_INVALID_CONFIG,    /* Invalid configuration */
     FLOCK_ERR_FROM_MERCURY,      /* Mercurt error */
     FLOCK_ERR_FROM_ARGOBOTS,     /* Argobots error */
+    FLOCK_ERR_FROM_MPI,          /* MPI error */
     FLOCK_ERR_OP_UNSUPPORTED,    /* Unsupported operation */
     FLOCK_ERR_OP_FORBIDDEN,      /* Forbidden operation */
     FLOCK_ERR_NO_MEMBER,         /* No member at this rank */

include/flock/flock-group-view.h

@@ -16,6 +16,10 @@
 #include <stdlib.h>
 #include <string.h>
 
+#ifdef __cplusplus
+extern "C" {
+#endif
+
 /**
  * @file flock-group-view.h
  *
@@ -599,5 +603,9 @@ static inline hg_return_t hg_proc_flock_protected_group_view_t(hg_proc_t proc, f
     return hret;
 }
 
+#ifdef __cplusplus
+}
+#endif
+
 #endif /* FLOCK_GROUP_VIEW_H */
 

spack.yaml

@@ -5,6 +5,7 @@ spack:
   - mochi-margo
   - json-c
   - mochi-bedrock
+  - mpi
   concretizer:
     unify: true
     reuse: true

tests/CMakeLists.txt

@@ -7,6 +7,12 @@ foreach (test-source ${test-sources})
         ${CMAKE_CURRENT_SOURCE_DIR}/../include
         ${CMAKE_CURRENT_BINARY_DIR}/../src)
     target_link_libraries (${name}
-        PRIVATE Catch2::Catch2WithMain flock::server flock::client coverage_config)
-    add_test (NAME ${name} COMMAND ./${name})
+        Catch2::Catch2WithMain flock::server flock::client coverage_config)
+    if (name MATCHES "^test-mpi.*$")
+        add_test (NAME ${name} COMMAND "${MPIEXEC_EXECUTABLE}" ${MPIEXEC_NUMPROC_FLAG} 5 ./${name})
+        find_package (MPI REQUIRED)
+        target_link_libraries (${name} MPI::MPI_C)
+    else ()
+        add_test (NAME ${name} COMMAND ./${name})
+    endif ()
 endforeach ()

tests/spack.yaml

@@ -5,6 +5,7 @@ spack:
   - mochi-margo ^mercury~boostsys~checksum ^libfabric fabrics=tcp,rxm
   - json-c
   - mochi-bedrock
+  - mpi
   concretizer:
     unify: true
     reuse: true

tests/test-mpi-bootstrap.cpp

@@ -0,0 +1,76 @@
+/*
+ * (C) 2024 The University of Chicago
+ *
+ * See COPYRIGHT in top-level directory.
+ */
+#include <stdio.h>
+#include <margo.h>
+#include <catch2/catch_test_macros.hpp>
+#include <catch2/catch_all.hpp>
+#include <flock/flock-group-view.h>
+#include <unordered_map>
+#include <flock/flock-bootstrap-mpi.h>
+#include <mpi.h>
+
+struct TestContext {
+
+    margo_instance_id mid = MARGO_INSTANCE_NULL;
+    hg_addr_t         addr = HG_ADDR_NULL;
+
+    TestContext() {
+        mid = margo_init("na+sm", MARGO_SERVER_MODE, 0, 0);
+        margo_addr_self(mid, &addr);
+    }
+
+    ~TestContext() {
+        margo_addr_free(mid, addr);
+        margo_finalize(mid);
+    }
+};
+
+struct Member {
+    uint64_t    rank;
+    std::string address;
+    uint16_t    provider_id;
+};
+
+TEST_CASE("Test bootstrap with MPI", "[mpi-bootstrap]") {
+
+    MPI_Init(NULL, NULL);
+    int size, rank;
+    MPI_Comm_rank(MPI_COMM_WORLD, &rank);
+    MPI_Comm_size(MPI_COMM_WORLD, &size);
+
+    // create test context
+    auto context = std::make_unique<TestContext>();
+
+    SECTION("Create group from MPI") {
+
+        flock_group_view_t view = FLOCK_GROUP_VIEW_INITIALIZER;
+        REQUIRE(view.members.size == 0);
+        REQUIRE(view.members.capacity == 0);
+        REQUIRE(view.members.data == nullptr);
+        REQUIRE(view.metadata.size == 0);
+        REQUIRE(view.metadata.capacity == 0);
+        REQUIRE(view.metadata.data == nullptr);
+        REQUIRE(view.digest == 0);
+
+        flock_return_t ret = flock_bootstrap_from_mpi(context->mid, 42+rank, MPI_COMM_WORLD, &view);
+        REQUIRE(ret == FLOCK_SUCCESS);
+
+        REQUIRE(view.members.size == (unsigned)size);
+        REQUIRE(view.members.data != nullptr);
+
+        auto me = flock_group_view_find_member(&view, rank);
+        REQUIRE(me->provider_id == 42 + rank);
+        REQUIRE(me->rank == (unsigned)rank);
+
+        char self_addr[256];
+        hg_size_t self_addr_size = 256;
+        margo_addr_to_string(context->mid, self_addr, &self_addr_size, context->addr);
+
+        REQUIRE(strcmp(me->address, self_addr) == 0);
+    }
+
+    MPI_Finalize();
+}