merge-upstream (apache#76)

merge with official mesos upstream Co-authored-by: Jason Zhou @gmail.com Co-authored-by: Benjamin Mahler @apache.org Co-authored-by: Jason Zhou @twitter.com Reviewed-on: https://gitea.aventer.internal:3000/AVENTER/clusterd/pulls/76 Co-authored-by: Andreas Peters <ap@aventer.biz> Co-committed-by: Andreas Peters <ap@aventer.biz>
m3scluster · Nov 27, 2024 · af3e286 · af3e286
1 parent 1866a29
commit af3e286
Show file tree

Hide file tree

Showing 92 changed files with 9,164 additions and 841 deletions.
diff --git a/3rdparty/CMakeLists.txt b/3rdparty/CMakeLists.txt
@@ -50,6 +50,9 @@ set(LIBEVENT_URL        ${FETCH_URL}/downloads/libevent-${LIBEVENT_VERSION}-stab
 
 set(CMAKE_CXX_STANDARD 14)
 set(CMAKE_CXX_STANDARD_REQUIRED ON)
+# NOTE: libevent doesn't come rebundled, so this URL is always the same. But,
+# it's only downloaded if `ENABLE_LIBEVENT` is set.
+set(LIBEVENT_URL ${LIBEVENT_RELEASES}/release-${LIBEVENT_VERSION}/libevent-${LIBEVENT_VERSION}.tar.gz)
 
 if (WIN32)
   # NOTE: These dependencies are only rebundled on Windows because they
@@ -269,7 +272,7 @@ if (WIN32)
     INTERFACE_COMPILE_DEFINITIONS LIBSASL_EXPORTS=1)
 
   if (CMAKE_GENERATOR MATCHES "Visual Studio")
-    set_target_properties(
+set_target_properties(
       sasl2 PROPERTIES
       IMPORTED_LOCATION_DEBUG ${SASL2_ROOT}-build/Debug/libsasl2${LIBRARY_SUFFIX}
       IMPORTED_LOCATION_RELEASE ${SASL2_ROOT}-build/Release/libsasl2${LIBRARY_SUFFIX})
@@ -702,6 +705,22 @@ if (ENABLE_LIBEVENT)
     add_library(libevent ${LIBRARY_LINKAGE} IMPORTED)
     add_dependencies(libevent ${LIBEVENT_TARGET})
 
+    if (ENABLE_SSL)
+      add_library(libevent_openssl ${LIBRARY_LINKAGE} IMPORTED)
+      set_target_properties(
+        libevent_openssl PROPERTIES
+        INTERFACE_INCLUDE_DIRECTORIES "${LIBEVENT_ROOT}/include;${LIBEVENT_ROOT}-build/include"
+      )
+    endif ()
+
+    if (BUILD_SHARED_LIBS)
+      add_library(libevent_pthreads ${LIBRARY_LINKAGE} IMPORTED)
+      set_target_properties(
+        libevent_pthreads PROPERTIES
+        INTERFACE_INCLUDE_DIRECTORIES "${LIBEVENT_ROOT}/include;${LIBEVENT_ROOT}-build/include"
+      )
+    endif ()
+
     set_target_properties(
       libevent PROPERTIES
       INTERFACE_INCLUDE_DIRECTORIES "${LIBEVENT_ROOT}/include;${LIBEVENT_ROOT}-build/include")
@@ -723,16 +742,32 @@ if (ENABLE_LIBEVENT)
         IMPORTED_LOCATION ${LIBEVENT_ROOT}-build/lib/libevent${LIBRARY_SUFFIX})
     endif ()
 
+    if (ENABLE_SSL)
+      set_target_properties(
+        libevent_openssl PROPERTIES
+        IMPORTED_LOCATION ${LIBEVENT_ROOT}-build/lib/libevent_openssl${LIBRARY_SUFFIX})
+    endif ()
+
+    if (BUILD_SHARED_LIBS)
+      set_target_properties(
+        libevent_pthreads PROPERTIES
+        IMPORTED_LOCATION ${LIBEVENT_ROOT}-build/lib/libevent_pthreads${LIBRARY_SUFFIX})
+      set(EVENT__LIBRARY_TYPE SHARED)
+    else ()
+      set(EVENT__LIBRARY_TYPE STATIC)
+    endif ()
+
     set(LIBEVENT_CMAKE_FORWARD_ARGS
       ${CMAKE_C_FORWARD_ARGS}
       ${CMAKE_SSL_FORWARD_ARGS}
       # NOTE: Libevent does not respect the BUILD_SHARED_LIBS global flag.
-      -DEVENT__BUILD_SHARED_LIBRARIES=${BUILD_SHARED_LIBS}
+      -DEVENT__LIBRARY_TYPE=${EVENT__LIBRARY_TYPE}
       -DEVENT__DISABLE_OPENSSL=$<NOT:$<BOOL:${ENABLE_SSL}>>
       -DEVENT__DISABLE_BENCHMARK=ON
       -DEVENT__DISABLE_REGRESS=ON
       -DEVENT__DISABLE_SAMPLES=ON
-      -DEVENT__DISABLE_TESTS=ON)
+      -DEVENT__DISABLE_TESTS=ON
+      -DEVENT__DISABLE_THREAD_SUPPORT=OFF)
 
     if (CMAKE_C_COMPILER_ID MATCHES GNU OR CMAKE_C_COMPILER_ID MATCHES Clang)
       list(APPEND LIBEVENT_CMAKE_FORWARD_ARGS -DCMAKE_C_FLAGS=-fPIC)
@@ -754,6 +789,20 @@ if (ENABLE_LIBEVENT)
       FILES $<TARGET_FILE:libevent>
       DESTINATION ${3RDPARTY_LIBS_INSTALL_DIR})
 
+    if (ENABLE_SSL)
+      install(
+        FILES $<TARGET_FILE:libevent_openssl>
+        DESTINATION ${3RDPARTY_LIBS_INSTALL_DIR}
+      )
+    endif ()
+
+    if (BUILD_SHARED_LIBS)
+      install(
+        FILES $<TARGET_FILE:libevent_pthreads>
+        DESTINATION ${3RDPARTY_LIBS_INSTALL_DIR}
+      )
+    endif ()
+
   else ()
     find_package(LIBEVENT REQUIRED)
     add_library(libevent INTERFACE)

diff --git a/3rdparty/libprocess/include/process/io.hpp b/3rdparty/libprocess/include/process/io.hpp
@@ -17,6 +17,7 @@
 #include <string>
 
 #include <process/future.hpp>
+#include <process/queue.hpp>
 
 #include <stout/nothing.hpp>
 #ifdef __WINDOWS__
@@ -166,6 +167,125 @@ Future<Nothing> redirect(
     size_t chunk = 4096,
     const std::vector<lambda::function<void(const std::string&)>>& hooks = {});
 
+
+// Forward declarations.
+class Watcher;
+namespace testing {
+Future<Nothing> watcher_read_loop(Watcher w);
+} // namespace testing {
+
+
+// This provides a high level interface for cross-platform filesystem watch
+// notifications. Currently, only Linux is supported via inotify, but macOS
+// BSD, and Windows implementations can be added.
+//
+// On Linux, inotify provides a vast set of features and comes with a vast
+// amount of subtleties to deal with and providing a cross-platform filesystem
+// watcher while exposing all these subtleties is quite challenging. Therefore,
+// our initial implementation only provides basic functionality in order to
+// simplify the life of the user, and to make cross platform implementation
+// viable.
+//
+// TODO(bmahler): Add support for directories.
+class Watcher
+{
+public:
+  struct Event
+  {
+    // Path to the file for the event. In the case of a Failure event type,
+    // this will be a failure message instead.
+    std::string path;
+
+    // TODO(bmahler): Add more events (e.g. access events, close events,
+    // attribute changes).
+    enum {
+      // The read loop encountered a unrecoverable failure, the watcher is
+      // no longer running and the caller must create a new watcher if desired!
+      Failure,
+
+      // File was modified, note that more writes may follow.
+      Write,
+
+      // The path was removed; any watches on it will be removed.
+      // Some "remove" operations may trigger a Rename if the file is
+      // actually moved (for example "remove to trash" is often a rename).
+      Remove,
+
+      // The path was renamed to something else; any watches on it will be
+      // removed.
+      Rename,
+    } type;
+  };
+
+  // Adds the file for event monitoring.
+  //
+  // Returns an error if:
+  //   * we don't have read access to the provided path
+  //   * the path has already been watched (and not implicitly or
+  //     explicitly removed)
+  //   * the path doesn't exist
+  //   * the path is a directory (not currently supported)
+  //
+  // In order for the caller to not miss any updates to the file, you
+  // *must* read the file yourself after calling add(). Otherwise, if
+  // you were to read the file first, updates between reading the file
+  // and add() the file will be missed!
+  Try<Nothing> add(const std::string& path);
+
+  // Removes the file for event monitoring, removing an already removed
+  // file is a no-op and also returns Nothing.
+  Try<Nothing> remove(const std::string& path);
+
+  Queue<Event> events();
+
+private:
+  friend Try<Watcher> create_watcher();
+  friend Future<Nothing> testing::watcher_read_loop(Watcher w);
+
+  Watcher(int inotify_fd);
+
+  // Start the inotify read loop.
+  void run();
+
+  struct Data
+  {
+    Data() = default;
+
+    ~Data();
+
+    // Rather than use a process to serialize access to the queue's
+    // internal data we use a 'std::atomic_flag' which will spin lock.
+    std::atomic_flag lock = ATOMIC_FLAG_INIT;
+
+    // We need a bidirectional mapping between watch descriptors and
+    // the path the watch descriptor maps to.
+    hashmap<int, std::string> wd_to_path;
+    hashmap<std::string, int> path_to_wd;
+
+    process::Future<Nothing> read_loop;
+
+    // Queue is already thread safe and doesn't require locking.
+    Queue<Watcher::Event> events;
+  };
+
+  const int inotify_fd;
+  std::shared_ptr<Data> data;
+};
+
+
+// Creates a watcher that can be used to monitor for fs changes.
+Try<Watcher> create_watcher();
+
+
+namespace testing {
+
+// Exposed to test read loop discard.
+inline Future<Nothing> watcher_read_loop(Watcher w)
+{
+  return w.data->read_loop;
+}
+
+} // namespace testing {
 } // namespace io {
 } // namespace process {
 

diff --git a/3rdparty/libprocess/src/CMakeLists.txt b/3rdparty/libprocess/src/CMakeLists.txt
@@ -114,10 +114,21 @@ target_link_libraries(
   $<$<BOOL:${ENABLE_SSL}>:OpenSSL::SSL>
   $<$<BOOL:${ENABLE_SSL}>:OpenSSL::Crypto>)
 
+  # We need openssl and pthreads if we have ENABLE_LIBEVENT
+  # Otherwise libprocess will not link properly and we get undefined reference
+if (ENABLE_LIBEVENT)
+  set(LIBEVENT_DEPENDENCIES libevent libevent_openssl libevent_pthreads)
+else ()
+  if (NOT PLATFORM_ID STREQUAL "Windows")
+    set(LIBEVENT_DEPENDENCIES libev)
+  endif ()
+endif ()
+
+# Link libraries
 target_link_libraries(
   process PRIVATE
   concurrentqueue
-  $<IF:$<BOOL:${ENABLE_LIBEVENT}>,libevent,$<$<NOT:$<PLATFORM_ID:Windows>>:libev>>)
+  ${LIBEVENT_DEPENDENCIES})
 
 target_compile_definitions(
   process PRIVATE