diff --git a/CMakeLists.txt b/CMakeLists.txt index 128f901b3..990f1e6ec 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -42,6 +42,7 @@ option(WITH_GOTCHA "Enable GOTCHA wrapping" TRUE) option(WITH_SOS "Enable SOSFlow data management" FALSE) option(WITH_TAU "Enable TAU service (TAU Performance System)" FALSE) option(WITH_VTUNE "Enable Intel(R) VTune(tm) annotation bindings" FALSE) +option(WITH_GEOPM "Enable GEOPM service" TRUE) option(USE_EXTERNAL_GOTCHA "Use pre-installed gotcha instead of building our own" FALSE) @@ -283,6 +284,22 @@ if (WITH_SAMPLER) endif() endif() +# Find GEOPM +if (WITH_GEOPM) + include(FindGeopm) + if(GEOPM_FOUND) + message(STATUS "Found geopm.h in " ${GEOPM_INCLUDE_DIR}) + message(STATUS "Found libgeopm.so in " ${GEOPM_LIBRARY}) + set(CALIPER_HAVE_GEOPM TRUE) + set(CALIPER_GEOPM_CMAKE_MSG "Yes, using ${GEOPM_LIBRARY}") + list(APPEND CALIPER_EXTERNAL_LIBS ${GEOPM_LIBRARY}) + else() + message(WARNING "GEOPM support was requested but libgeopm.so was not found!\n" + "Set -DGEOPM_INSTALL=" + "and re-run cmake") + endif() +endif() + # PGI 17.x has issues with some constexpr constructors if(CMAKE_CXX_COMPILER_ID MATCHES PGI) set(CALIPER_REDUCED_CONSTEXPR_USAGE TRUE) diff --git a/caliper-config.h.in b/caliper-config.h.in index 2231eeb83..53f8a7b32 100644 --- a/caliper-config.h.in +++ b/caliper-config.h.in @@ -18,6 +18,7 @@ #cmakedefine CALIPER_HAVE_CUPTI #cmakedefine CALIPER_HAVE_LIBDW #cmakedefine CALIPER_HAVE_VTUNE +#cmakedefine CALIPER_HAVE_GEOPM #cmakedefine CALIPER_REDUCED_CONSTEXPR_USAGE diff --git a/cmake/FindGeopm.cmake b/cmake/FindGeopm.cmake new file mode 100644 index 000000000..738aa68a4 --- /dev/null +++ b/cmake/FindGeopm.cmake @@ -0,0 +1,28 @@ +# +# - Find GEOPM +# +# GEOPM_INCLUDE_DIR - Path to geopm.h +# GEOPM_LIBRARY - Name of GEOPM library file +# GEOPM_FOUND - True if GEOPM was found + +#set(GEOPM_INSTALL "" CACHE PATH "GEOPM install directory") + +#if(GEOPM_INSTALL) + # install dir specified, only search them + find_path(GEOPM_INCLUDE_DIR "geopm.h" + NAMES geopm.h + PATHS ${GEOPM_INSTALL} ${GEOPM_INSTALL}/include + NO_DEFAULT_PATH + ) + + find_library(GEOPM_LIBRARY + NAMES geopm + PATHS ${GEOPM_INSTALL} ${GEOPM_INSTALL}/lib + NO_DEFAULT_PATH + ) +#endif() + +include(FindPackageHandleStandardArgs) +find_package_handle_standard_args(GEOPM DEFAULT_MSG GEOPM_LIBRARY GEOPM_INCLUDE_DIR) + +mark_as_advanced(GEOPM_LIBRARY GEOPM_INCLUDE_DIR) diff --git a/src/services/CMakeLists.txt b/src/services/CMakeLists.txt index 053bddb42..1ed3e5415 100644 --- a/src/services/CMakeLists.txt +++ b/src/services/CMakeLists.txt @@ -88,6 +88,10 @@ if (CALIPER_HAVE_VTUNE) add_subdirectory(vtune) endif() +if (CALIPER_HAVE_GEOPM) + add_subdirectory(geopm) +endif() + add_service_sources(Services.cpp) configure_file( diff --git a/src/services/geopm/CMakeLists.txt b/src/services/geopm/CMakeLists.txt new file mode 100644 index 000000000..fb19e1c1d --- /dev/null +++ b/src/services/geopm/CMakeLists.txt @@ -0,0 +1,14 @@ +include_directories(SYSTEM ${GEOPM_INCLUDE_DIR}) + +set(CALIPER_GEOPM_SOURCES + Geopm.cpp + ) + +list(APPEND CALIPER_EXTERNAL_LIBS " -DDEBUGLEVEL=2") +set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -DDEBUGLEVEL=2") +set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DDEBUGLEVEL=2") + +add_library(caliper-libgeopm OBJECT ${CALIPER_GEOPM_SOURCES}) + +add_service_objlib("caliper-libgeopm") +add_caliper_service("geopm") diff --git a/src/services/geopm/Geopm.cpp b/src/services/geopm/Geopm.cpp new file mode 100644 index 000000000..00f1dea67 --- /dev/null +++ b/src/services/geopm/Geopm.cpp @@ -0,0 +1,208 @@ +// Copyright (c) 2015, Lawrence Livermore National Security, LLC. +// Produced at the Lawrence Livermore National Laboratory. +// +// This file is part of Caliper. +// Written by Aniruddha Marathe, marathe1@llnl.gov. +// All rights reserved. +// +// For details, see https://github.com/scalability-llnl/Caliper. +// Please also see the LICENSE file for our additional BSD notice. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// * Redistributions of source code must retain the above copyright notice, this list of +// conditions and the disclaimer below. +// * Redistributions in binary form must reproduce the above copyright notice, this list of +// conditions and the disclaimer (as noted below) in the documentation and/or other materials +// provided with the distribution. +// * Neither the name of the LLNS/LLNL nor the names of its contributors may be used to endorse +// or promote products derived from this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS +// OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// LAWRENCE LIVERMORE NATIONAL SECURITY, LLC, THE U.S. DEPARTMENT OF ENERGY OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, +// WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN +// ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +///@file Geopm.cpp +///@brief GEOPM service to relay updates to phase and loop annotation attributes to GEOPM runtime + +#include "caliper/CaliperService.h" +#include "caliper/Caliper.h" +#include "caliper/SnapshotRecord.h" + +#include "caliper/common/Log.h" +#include "caliper/common/RuntimeConfig.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#define GEOPM_NULL_VAL -1 + +using namespace cali; +using namespace std; + + +int rank; +int size; + +namespace +{ + +std::map geopm_phase_map; +std::map geopm_loop_list; + + +void geopm_init_region(Caliper* c, const Attribute& attr, const Variant& value) { + uint64_t sumatoms_rid; + int err; + Log(1).stream() << "GEOPM service initialized" << endl; +} + +void geopm_set_iteration(Caliper* c, Channel* chn, const Attribute& attr, const Variant& val) { + + std::string sAttrName(attr.name()); + /* Check if the attribute is an iteration */ + if(sAttrName.find(".loopcount") != std::string::npos) { + /* Safe-guarding condition for never-before-seen loop region */ + std::string sLoopName = sAttrName.substr(0, sAttrName.find(".")); + geopm_loop_list.insert(std::pair(sLoopName, val.to_uint())); + } +} + +void geopm_begin_region(Caliper* c, Channel* chn, const Attribute& attr, const Variant& regionname) { + int err; + std::string sRegionName(regionname.to_string()); + std::string sAttrName(attr.name()); + /* Check if the attribute is an iteration, if so, find its name and update + * GEOPM progress + */ + + /* check if the attribute type is annotation, loop or function, and create a region ID. */ + if(attr.name() == "annotation") { + if(geopm_phase_map.end() == geopm_phase_map.find(sRegionName)) { + uint64_t phase_rid = GEOPM_NULL_VAL; + geopm_prof_region(sRegionName.c_str(), GEOPM_REGION_HINT_COMPUTE, &phase_rid); + geopm_phase_map.insert(std::pair(sRegionName, phase_rid)); + } + } + + if(attr.name() == "annotation") { + /* Check if the attribute is already present in the phase map */ + geopm_prof_enter(geopm_phase_map[sRegionName]); + } else if(attr.name() == "loop") { + /* Start of a loop in the application, initialize loop count + * to update fractional progress later + */ + if(regionname.to_string() == "mainloop") { + /* Nothing to do at the start of the mainloop */ + } else if(omp_get_max_threads() > 1) { + geopm_loop_list.insert(std::pair(sRegionName, -1)); + std::hash hasher; + auto hashed = hasher(sRegionName); + } + + } else if(attr.name() == "statement") { + /* Do not handle this case */ + } else if(attr.name() == "function") { + /* Do not handle this case */ + } +} + +void geopm_end_region(Caliper* c, Channel* chn, const Attribute& attr, const Variant& regionname) { + int err; + std::string sRegionName(regionname.to_string()); + std::string sAttrName(attr.name()); + /* Check if the attribute is an iteration */ + if(sAttrName.find("iteration#") != std::string::npos) { + std::string sAttrList(sAttrName); + std::string token(""); + int pos = 0; + while((token != "iteration") && (pos = sAttrList.find("#")) != std::string::npos) { + token = sAttrList.substr(0, pos); + sAttrList.erase(0, pos + 1); + } + + /* If this is the main loop, mark end of timestep */ + std::string sLoopName = sAttrList.substr(0, sAttrList.find("#")); + if(omp_get_max_threads() > 1) { + if(regionname.to_uint() == 1) { + int num_thread = omp_get_max_threads(); + int thread_idx = omp_get_thread_num(); + + /* Caliper scope is not thread-level. Caliper must expose + * thread ID in order for geopm_tprof_* markup to function + * as expected. + */ + geopm_tprof_init_loop(num_thread, thread_idx, geopm_loop_list[sLoopName], 1); + } + + /* If this is an OpenMP loop, mark thread-level progress */ + geopm_tprof_post(); + } else { + /* This is neither the main loop nor an OpenMP loop, + * therefore, mark process-level progress */ + geopm_prof_progress(geopm_phase_map[sRegionName], + regionname.to_double()/(double) geopm_loop_list[sRegionName]); + } + } else { + /* This event marks the end of region */ + if(attr.name() == "annotation" + ) { + /* Check if the attribute is already present in the phase map */ + if(geopm_phase_map.end() == geopm_phase_map.find(sRegionName)) { + /* Missing phase begin, throw error */ + uint64_t phase_rid = GEOPM_NULL_VAL; + geopm_phase_map[sRegionName] = phase_rid; + Log(1).stream() << "GEOPM service: missing phase found. Please add the missing 'begin' mark-up for " << sRegionName << endl; + } + } + /* Check if the attribute is of type 'annotation' */ + if(attr.name() == "annotation") { + if(geopm_phase_map[sRegionName] != GEOPM_NULL_VAL) { + geopm_prof_exit(geopm_phase_map[sRegionName]); + } + } else if(attr.name() == "loop") { + if(regionname.to_string() == "mainloop") { + geopm_prof_epoch(); + } else if(omp_get_max_threads() > 1) { + geopm_loop_list.erase(sRegionName); + } else { + geopm_loop_list.erase(sRegionName); + } + } else if(attr.name() == "statement") { + /* Do not handle this case */ + } else if(attr.name() == "function") { + /* Do not handle this case */ + } + } +} + +/// Initialization handler +void geopm_service_register(Caliper* c, Channel* chn) +{ + chn->events().pre_begin_evt.connect(::geopm_begin_region); + chn->events().pre_end_evt.connect(::geopm_end_region); + chn->events().pre_set_evt.connect(::geopm_set_iteration); + Log(1).stream() << chn->name() << ": Registered GEOPM service" << std::endl; +} + +} // namespace + + +namespace cali +{ + CaliperService geopm_service = { "geopm", &::geopm_service_register }; +} // namespace cali diff --git a/src/services/geopm/README b/src/services/geopm/README new file mode 100644 index 000000000..a193faa9a --- /dev/null +++ b/src/services/geopm/README @@ -0,0 +1,16 @@ + +To build the GEOPM service, use the following cmake command instead +of the command described in the main README of Caliper: + +cmake -DCMAKE_INSTALL_PREFIX= \ + -DCMAKE_C_COMPILER=`which mpicc` \ + -DCMAKE_CXX_COMPILER=`which mpicxx` \ + -DGEOPM_INSTALL= \ + .. + +For example: +cmake -DCMAKE_INSTALL_PREFIX=/home/user/caliper/install \ + -DCMAKE_C_COMPILER=`which mpicc` \ + -DCMAKE_CXX_COMPILER=`which mpicxx` \ + -DGEOPM_INSTALL=/home/user/geopm/install + .. diff --git a/src/services/geopm/doc/GEOPM-mapping.pdf b/src/services/geopm/doc/GEOPM-mapping.pdf new file mode 100644 index 000000000..fb9d02d53 Binary files /dev/null and b/src/services/geopm/doc/GEOPM-mapping.pdf differ