NA Test: add NA_OFI_TESTING_USE_CXI option for cxi auth key testing

This requires the presence of libcxi on the system

Use cxil_alloc_svc()/cxil_destroy_svc() to allocate VNIs

Testing/common/CMake/FindCXI.cmake

@@ -0,0 +1,25 @@
+# - Try to find CXI
+# Once done this will define
+#  CXI_FOUND - System has CXI
+#  CXI_INCLUDE_DIRS - The CXI include directories
+#  CXI_LIBRARIES - The libraries needed to use CXI
+
+find_package(PkgConfig)
+pkg_check_modules(PC_CXI libcxi)
+
+find_path(CXI_INCLUDE_DIR libcxi/libcxi.h
+  HINTS ${PC_CXI_INCLUDEDIR} ${PC_CXI_INCLUDE_DIRS})
+
+find_library(CXI_LIBRARY NAMES cxi
+  HINTS ${PC_CXI_LIBDIR} ${PC_CXI_LIBRARY_DIRS})
+
+set(CXI_INCLUDE_DIRS ${CXI_INCLUDE_DIR})
+set(CXI_LIBRARIES ${CXI_LIBRARY})
+
+include(FindPackageHandleStandardArgs)
+# handle the QUIETLY and REQUIRED arguments and set CXI_FOUND to TRUE
+# if all listed variables are TRUE
+find_package_handle_standard_args(CXI DEFAULT_MSG
+                                  CXI_INCLUDE_DIR CXI_LIBRARY)
+
+mark_as_advanced(CXI_INCLUDE_DIR CXI_LIBRARY)

Testing/common/CMakeLists.txt

@@ -47,6 +47,26 @@ mark_as_advanced(MERCURY_TESTING_ENABLE_PARALLEL)
 # Detect <sys/prctl.h>
 check_include_files("sys/prctl.h" HG_TEST_HAS_SYSPRCTL_H)
 
+# CXI
+if(NA_USE_OFI)
+  option(NA_OFI_TESTING_USE_CXI
+    "Use CXI library for service configuration." OFF)
+  mark_as_advanced(NA_OFI_TESTING_USE_CXI)
+  if(NA_OFI_TESTING_USE_CXI)
+    find_package(CXI REQUIRED)
+    message(STATUS "CXI include directory: ${CXI_INCLUDE_DIRS}")
+    set(HG_TEST_HAS_CXI 1)
+    set(NA_TEST_COMMON_EXT_INCLUDE_DEPENDENCIES
+      ${NA_TEST_COMMON_EXT_INCLUDE_DEPENDENCIES}
+      ${CXI_INCLUDE_DIRS}
+    )
+    set(NA_TEST_COMMON_EXT_LIB_DEPENDENCIES
+      ${NA_TEST_COMMON_EXT_LIB_DEPENDENCIES}
+      ${CXI_LIBRARIES}
+    )
+  endif()
+endif()
+
 #------------------------------------------------------------------------------
 # Set up test options
 #------------------------------------------------------------------------------
@@ -120,7 +140,7 @@ endif()
 # Mercury test common
 add_library(mercury_test_common STATIC ${MERCURY_TEST_COMMON_SRCS}
   ${MERCURY_TEST_COMMON_PRIVATE_HEADERS})
-target_include_directories(na_test_common
+target_include_directories(mercury_test_common
   PUBLIC  "$<BUILD_INTERFACE:${MERCURY_TEST_COMMON_BUILD_INCLUDE_DEPENDENCIES}>"
 )
 target_include_directories(mercury_test_common

Testing/common/mercury_test_config.h.in

@@ -52,4 +52,7 @@
 /* Define if has <sys/prctl.h> */
 #cmakedefine HG_TEST_HAS_SYSPRCTL_H
 
+/* Define if has <libcxi/libcxi.h> */
+#cmakedefine HG_TEST_HAS_CXI
+
 #endif /* MERCURY_TEST_CONFIG_H */

Testing/common/na_test.c

@@ -56,6 +56,18 @@ na_test_parse_size(const char *str);
 static enum na_traffic_class
 na_test_tclass(const char *str);
 
+#ifdef HG_TEST_HAS_CXI
+static na_return_t
+na_test_alloc_svc(struct na_test_cxi_info *cxi_info, const char *init_str);
+
+static na_return_t
+na_test_print_svc(
+    const struct na_test_cxi_info *cxi_info, char *buf, size_t buf_size);
+
+static na_return_t
+na_test_free_svc(struct na_test_cxi_info *cxi_info);
+#endif
+
 static char *
 na_test_gen_config(struct na_test_info *na_test_info, unsigned int i);
 
@@ -276,6 +288,103 @@ na_test_tclass(const char *str)
     }
 }
 
+/*---------------------------------------------------------------------------*/
+#ifdef HG_TEST_HAS_CXI
+static na_return_t
+na_test_alloc_svc(struct na_test_cxi_info *cxi_info, const char *init_str)
+{
+    struct cxi_svc_fail_info svc_fail_info = {};
+    uint16_t vni_min = 0, vni_max = 0;
+    na_return_t ret;
+    unsigned int i;
+    int rc;
+
+    rc = sscanf(init_str, "0:%" SCNu16 ":%" SCNu16, &vni_min, &vni_max);
+    NA_TEST_CHECK_ERROR(rc != 1 && rc != 2, error, ret, NA_PROTONOSUPPORT,
+        "Invalid CXI auth key range string (%s), format is "
+        "\"0:vni_min<:vni_max>\"",
+        init_str);
+
+    cxi_info->dev = NULL;
+
+    rc = cxil_open_device(0, &cxi_info->dev);
+    NA_TEST_CHECK_ERROR(rc != 0, error, ret, NA_PROTOCOL_ERROR,
+        "cxil_open_device() failed (%d)", rc);
+
+    memset(&cxi_info->svc_desc, 0, sizeof(cxi_info->svc_desc));
+
+    cxi_info->svc_desc.restricted_vnis = 1;
+    cxi_info->svc_desc.enable = 1;
+    cxi_info->svc_desc.num_vld_vnis =
+        (vni_max > vni_min) ? vni_max - vni_min + 1 : 1;
+
+    for (i = 0; i < cxi_info->svc_desc.num_vld_vnis; i++)
+        cxi_info->svc_desc.vnis[i] = vni_min + i;
+
+    rc = cxil_alloc_svc(cxi_info->dev, &cxi_info->svc_desc, &svc_fail_info);
+    NA_TEST_CHECK_ERROR(rc <= 0, error, ret, NA_PROTOCOL_ERROR,
+        "cxil_alloc_svc() failed (%d)", rc)
+
+    cxi_info->svc_desc->svc_id = rc;
+
+    return NA_SUCCESS;
+
+error:
+    (void) na_test_free_svc(cxi_info);
+
+    return ret;
+}
+
+/*---------------------------------------------------------------------------*/
+static na_return_t
+na_test_print_svc(
+    const struct na_test_cxi_info *cxi_info, char *buf, size_t buf_size)
+{
+    na_return_t ret;
+    int rc;
+
+    if (cxi_info->svc_desc.num_vld_vnis == 1)
+        rc = snprintf(buf, buf_size, "%" PRIu32 ":%" PRIu16,
+            cxi_info->svc_desc.svc_id, cxi_info->svc_desc.vnis[0]);
+    else
+        rc = snprintf(buf, buf_size, "%" PRIu32 ":%" PRIu16 ":%" PRIu16,
+            cxi_info->svc_desc.svc_id, cxi_info->svc_desc.vnis[0],
+            cxi_info->svc_desc.vnis[cxi_info->svc_desc.num_vld_vnis - 1]);
+    NA_TEST_CHECK_ERROR(rc < 0 || rc > (int) buf_size, error, ret, NA_OVERFLOW,
+        "snprintf() failed or name truncated, rc: %d (expected %zu)", rc,
+        buf_size);
+
+    return NA_SUCCESS;
+
+error:
+    return ret;
+}
+
+/*---------------------------------------------------------------------------*/
+static na_return_t
+na_test_free_svc(struct na_test_cxi_info *cxi_info)
+{
+    na_return_t ret;
+
+    if (cxi_info->svc_desc.svc_id > 0) {
+        int rc = cxil_destroy_svc(cxi_info->dev, cxi_info->svc_desc.svc_id);
+        NA_TEST_CHECK_ERROR(rc != 0, error, ret, NA_PROTOCOL_ERROR,
+            "cxil_destroy_svc() failed (%d)", rc);
+        cxi_info->svc_desc.svc_id = 0;
+    }
+
+    if (cxi_info->dev != NULL) {
+        cxil_close_device(cxi_info->dev);
+        cxi_info->dev = NULL;
+    }
+
+    return NA_SUCCESS;
+
+error:
+    return ret;
+}
+#endif
+
 /*---------------------------------------------------------------------------*/
 static char *
 na_test_gen_config(struct na_test_info *na_test_info, unsigned int i)
@@ -470,6 +579,9 @@ NA_Test_init(int argc, char *argv[], struct na_test_info *na_test_info)
     struct na_init_info na_init_info = NA_INIT_INFO_INITIALIZER;
     na_return_t ret = NA_SUCCESS;
     const char *log_subsys = getenv("HG_LOG_SUBSYS");
+#ifdef HG_TEST_HAS_CXI
+    char auth_key[64];
+#endif
     size_t i;
 
     if (!log_subsys) {
@@ -492,6 +604,17 @@ NA_Test_init(int argc, char *argv[], struct na_test_info *na_test_info)
         na_test_info->use_threads, na_test_info->mpi_static);
     NA_TEST_CHECK_NA_ERROR(error, ret, "na_test_mpi_init() failed");
 
+#ifdef HG_TEST_HAS_CXI
+    if (na_test_info->key != NULL) {
+        ret = na_test_alloc_svc(&na_test_info->cxi_info, na_test_info->key);
+        NA_TEST_CHECK_NA_ERROR(error, ret, "na_test_alloc_svc() failed");
+
+        ret = na_test_print_svc(
+            na_test_info->cxi_info, auth_key, sizeof(auth_key));
+        NA_TEST_CHECK_NA_ERROR(error, ret, "na_test_print_svc() failed");
+    }
+#endif
+
     if (na_test_info->max_classes == 0)
         na_test_info->max_classes = 1;
 
@@ -504,7 +627,12 @@ NA_Test_init(int argc, char *argv[], struct na_test_info *na_test_info)
         if (na_test_info->mpi_info.rank == 0)
             printf("# Initializing NA in busy wait mode\n");
     }
+#ifdef HG_TEST_HAS_CXI
+    if (na_test_info->key != NULL)
+        na_init_info.auth_key = auth_key;
+#else
     na_init_info.auth_key = na_test_info->key;
+#endif
     if (na_test_info->max_contexts != 0)
         na_init_info.max_contexts = na_test_info->max_contexts;
     na_init_info.max_unexpected_size = (size_t) na_test_info->max_msg_size;
@@ -680,6 +808,9 @@ NA_Test_finalize(struct na_test_info *na_test_info)
         na_test_info->domain = NULL;
     }
     if (na_test_info->key != NULL) {
+#ifdef HG_TEST_HAS_CXI
+        (void) na_test_free_svc(&na_test_info->cxi_info);
+#endif
         free(na_test_info->key);
         na_test_info->key = NULL;
     }

Testing/common/na_test.h

@@ -10,10 +10,21 @@
 
 #include "na_test_mpi.h"
 
+#ifdef HG_TEST_HAS_CXI
+#    include <libcxi/libcxi.h>
+#endif
+
 /*************************************/
 /* Public Type and Struct Definition */
 /*************************************/
 
+#ifdef HG_TEST_HAS_CXI
+struct na_test_cxi_info {
+    struct cxil_dev *dev;
+    struct cxi_svc_desc svc_desc;
+};
+#endif
+
 struct na_test_info {
     na_class_t *na_class;    /* Default NA class */
     na_class_t **na_classes; /* Array of NA classes */
@@ -41,6 +52,9 @@ struct na_test_info {
     size_t buf_count;        /* Buffer count */
     bool verbose;            /* Verbose mode */
     struct na_test_mpi_info mpi_info;
+#ifdef HG_TEST_HAS_CXI
+    struct na_test_cxi_info cxi_info;
+#endif
     bool extern_init;    /* Extern init */
     bool use_threads;    /* Use threads */
     bool force_register; /* Force registration each iteration */