Allowed Caliper to control Kokkos fencing behavior

src/services/kokkos/CMakeLists.txt

@@ -1,3 +1,4 @@
+include_directories(include)
 set(CALIPER_KOKKOS_SOURCES
  KokkosProfilingSymbols.cpp
  KokkosLookup.cpp

src/services/kokkos/KokkosProfilingSymbols.cpp

@@ -3,23 +3,52 @@
 #include "caliper/cali.h"
 #include "types.hpp"
 #include <cstdint>
+#include <impl/Kokkos_Profiling_Interface.hpp>
 cali::kokkos::callbacks kokkosp_callbacks;
 using cali::kokkos::SpaceHandle;
 namespace kokkos {
- cali::ConfigManager mgr;
+cali::ConfigManager mgr;
+bool can_control_fences = false;
+bool config_needs_fences = true;
+Kokkos::Tools::Experimental::ToolProgrammingInterface kokkos_interface;
+template <typename Container, typename FenceIf>
+void fence_if_needed(const Container &in, uint32_t dev_id,
+ const FenceIf &predicate) {
+ if (can_control_fences && !in.empty() && predicate()) {
+ kokkos_interface.fence(dev_id);
+ }
 }
-extern "C" void kokkosp_print_help(char* progName){
+} // namespace kokkos
+extern "C" void kokkosp_print_help(char *progName) {
  std::cerr << "Caliper: available configs: \n";
- for(auto conf: kokkos::mgr.available_config_specs() ) {
- std::cerr << kokkos::mgr.get_documentation_for_spec(conf.c_str()) << std::endl;
+ for (auto conf : kokkos::mgr.available_config_specs()) {
+ std::cerr << kokkos::mgr.get_documentation_for_spec(conf.c_str())
+ << std::endl;
+ }
+}
+extern "C" void kokkosp_request_tool_settings(
+ int num_actions, Kokkos::Tools::Experimental::ToolSettings *settings) {
+ if ((num_actions > 0) && (settings != nullptr)) {
+ settings->requires_global_fencing = false;
  }
 }
+
+extern "C" void kokkosp_provide_tool_programming_interface(
+ int num_actions,
+ Kokkos::Tools::Experimental::ToolProgrammingInterface interface) {
+ kokkos::can_control_fences = true;
+ kokkos::kokkos_interface = interface;
+}
 extern "C" void kokkosp_parse_args(int argc, char *argv_raw[]) {
  if (argc > 2) {
  std::cerr << "Error: the Kokkos Caliper connector takes only one argument"
  << std::endl;
  }
  if (argc == 2) {
+ if (std::string(argv_raw[1]).find("cuda-activity-report") !=
+ std::string::npos) {
+ kokkos::config_needs_fences = false;
+ }
  kokkos::mgr.add(argv_raw[1]);
  if (kokkos::mgr.error()) {
  std::cerr << "Kokkos Caliper connector error: " << kokkos::mgr.error_msg()
@@ -46,40 +75,62 @@ extern "C" void kokkosp_finalize_library() {
 extern "C" void kokkosp_begin_parallel_for(const char *name,
  const uint32_t devID,
  uint64_t *kID) {
+ kokkos::fence_if_needed(kokkosp_callbacks.kokkosp_begin_parallel_for_callback,
+ devID, [=]() { return kokkos::config_needs_fences; });
  kokkosp_callbacks.kokkosp_begin_parallel_for_callback(name, devID, kID);
+ *kID = devID;
 }
 extern "C" void kokkosp_begin_parallel_reduce(const char *name,
  const uint32_t devID,
  uint64_t *kID) {
+ kokkos::fence_if_needed(
+ kokkosp_callbacks.kokkosp_begin_parallel_reduce_callback, devID,
+ [=]() { return kokkos::config_needs_fences; });
  kokkosp_callbacks.kokkosp_begin_parallel_reduce_callback(name, devID, kID);
+ *kID = devID;
 }
 extern "C" void kokkosp_begin_parallel_scan(const char *name,
  const uint32_t devID,
  uint64_t *kID) {
+ kokkos::fence_if_needed(
+ kokkosp_callbacks.kokkosp_begin_parallel_scan_callback, devID,
+ [=]() { return kokkos::config_needs_fences; });
  kokkosp_callbacks.kokkosp_begin_parallel_scan_callback(name, devID, kID);
+ *kID = devID;
 }
 extern "C" void kokkosp_begin_fence(const char *name, const uint32_t devID,
  uint64_t *kID) {
  kokkosp_callbacks.kokkosp_begin_fence_callback(name, devID, kID);
 }
 
 extern "C" void kokkosp_end_parallel_for(const uint64_t kID) {
+ kokkos::fence_if_needed(kokkosp_callbacks.kokkosp_end_parallel_for_callback,
+ kID, [=]() { return kokkos::config_needs_fences; });
  kokkosp_callbacks.kokkosp_end_parallel_for_callback(kID);
 }
 extern "C" void kokkosp_end_parallel_reduce(const uint64_t kID) {
+ kokkos::fence_if_needed(
+ kokkosp_callbacks.kokkosp_end_parallel_reduce_callback, kID,
+ [=]() { return kokkos::config_needs_fences; });
  kokkosp_callbacks.kokkosp_end_parallel_reduce_callback(kID);
 }
 extern "C" void kokkosp_end_parallel_scan(const uint64_t kID) {
+ kokkos::fence_if_needed(kokkosp_callbacks.kokkosp_end_parallel_scan_callback,
+ kID, [=]() { return kokkos::config_needs_fences; });
  kokkosp_callbacks.kokkosp_end_parallel_scan_callback(kID);
 }
 extern "C" void kokkosp_end_fence(const uint64_t kID) {
  kokkosp_callbacks.kokkosp_end_fence_callback(kID);
 }
 
 extern "C" void kokkosp_push_profile_region(char *regionName) {
+ kokkos::fence_if_needed(kokkosp_callbacks.kokkosp_push_region_callback, 0,
+ [=]() { return true; });
  kokkosp_callbacks.kokkosp_push_region_callback(regionName);
 }
 extern "C" void kokkosp_pop_profile_region() {
+ kokkos::fence_if_needed(kokkosp_callbacks.kokkosp_push_region_callback, 0,
+ [=]() { return true; });
  kokkosp_callbacks.kokkosp_pop_region_callback();
 }
 extern "C" void kokkosp_allocate_data(const SpaceHandle space,

src/services/kokkos/KokkosProfilingSymbols.hpp

@@ -1,9 +1,9 @@
 #ifndef CALIPER_SERVICES_KOKKOS_PROFILING_SYMBOLS_HPP
 #define CALIPER_SERVICES_KOKKOS_PROFILING_SYMBOLS_HPP
 
-
+//comment
 #include "types.hpp"
-
+#include <impl/Kokkos_Profiling_Interface.hpp>
 
 #endif