feat(libebpfdiscoveryskel): Add sending BPF logging events to userspace

ebpfdiscoverysrv/src/main.cpp

@@ -1,7 +1,9 @@
 // SPDX-License-Identifier: GPL-2.0
+#include "ebpfdiscovery/Config.h"
 #include "ebpfdiscovery/Discovery.h"
 #include "ebpfdiscovery/DiscoveryBpf.h"
 #include "ebpfdiscovery/DiscoveryBpfLoader.h"
+#include "ebpfdiscovery/DiscoveryBpfLogHandler.h"
 #include "ebpfdiscoveryproto/Translator.h"
 #include "logging/Logger.h"
 
@@ -159,7 +161,7 @@ int main(int argc, char** argv) {
 		return EXIT_FAILURE;
 	}
 
-	LOG_DEBUG("Starting the program.");
+	LOG_INFO("Starting the program. (PID: {})", getpid());
 
 	{
 		LOG_TRACE("Setting up unix signals handling.");
@@ -179,13 +181,21 @@ int main(int argc, char** argv) {
 		return EXIT_FAILURE;
 	}
 
-	ebpfdiscovery::Discovery instance(loader.get());
+	ebpfdiscovery::DiscoveryConfig discoveryConfig;
+	ebpfdiscovery::Discovery instance(loader.get(), discoveryConfig);
+	ebpfdiscovery::DiscoveryBpfLogHandler bpfLogHandler(loader.get(), discoveryConfig);
 	try {
 		instance.start();
 	} catch (const std::runtime_error& e) {
 		LOG_CRITICAL("Couldn't start Discovery: {}", e.what());
 	}
 
+	try {
+		bpfLogHandler.start();
+	} catch (const std::runtime_error& e) {
+		LOG_CRITICAL("Couldn't start Discovery BPF log event handler: {}", e.what());
+	}
+
 	if (!isLaunchTest) {
 		std::thread servicesProvider(servicesProvidingLoop, std::ref(instance), std::chrono::seconds(vm["interval"].as<int>()));
 		std::thread unixSignalThread(runUnixSignalHandlerLoop);
@@ -194,22 +204,29 @@ int main(int argc, char** argv) {
 			programStatusCV.wait(programStatusLock, []() { return programStatus != ProgramStatus::Running; });
 		}
 
-		LOG_TRACE("Waiting for unix signal thread to exit.");
+		LOG_DEBUG("Waiting for unix signal thread to exit.");
 		if (unixSignalThread.joinable()) {
 			unixSignalThread.join();
 		}
-		LOG_TRACE("Waiting for services providing thread to exit.");
+
+		LOG_DEBUG("Waiting for services providing thread to exit.");
 		if (servicesProvider.joinable()) {
 			servicesProvider.join();
 		}
 	}
 
-	LOG_DEBUG("Exiting the program.");
+	LOG_INFO("Exiting the program.");
 	instance.stop();
 
-	LOG_TRACE("Waiting for threads to exit.");
+	LOG_DEBUG("Waiting for Discovery to exit.");
 	instance.wait();
 
-	LOG_TRACE("Finished running the program successfully.");
+	LOG_DEBUG("Stopping the Discovery BPF log event handler.");
+	bpfLogHandler.stop();
+
+	LOG_DEBUG("Waiting for Discovery BPF log event handler to exit.");
+	bpfLogHandler.wait();
+
+	LOG_DEBUG("Finished running the program successfully.");
 	return EXIT_SUCCESS;
 }

libebpfdiscovery/CMakeLists.txt

@@ -1,10 +1,10 @@
 list(
         APPEND
         SOURCES
-        src/Config.cpp
         src/Discovery.cpp
         src/DiscoveryBpf.cpp
         src/DiscoveryBpfLoader.cpp
+        src/DiscoveryBpfLogHandler.cpp
         src/IpAddressChecker.cpp
         src/NetlinkCalls.cpp
         src/Session.cpp

libebpfdiscovery/headers/ebpfdiscovery/Config.h

@@ -7,9 +7,9 @@
 namespace ebpfdiscovery {
 
 struct DiscoveryConfig {
-	std::chrono::milliseconds eventQueuePollInterval;
-
-	DiscoveryConfig() noexcept;
+	std::chrono::milliseconds eventQueuePollInterval{std::chrono::milliseconds(250)};
+	std::chrono::milliseconds logPerfPollInterval{std::chrono::milliseconds(250)};
+	size_t logPerfBufferPages{1024};
 };
 
 } // namespace ebpfdiscovery

libebpfdiscovery/headers/ebpfdiscovery/Discovery.h

@@ -28,7 +28,7 @@ class Discovery {
 	Discovery(DiscoveryBpf discoveryBpf, const DiscoveryConfig config);
 	Discovery(const Discovery&) = delete;
 	Discovery& operator=(const Discovery&) = delete;
-	Discovery(Discovery&&) = default;
+	Discovery(Discovery&&) = delete;
 	Discovery& operator=(Discovery&&) = default;
 	~Discovery() = default;
 

libebpfdiscovery/headers/ebpfdiscovery/DiscoveryBpfLogHandler.h

@@ -0,0 +1,41 @@
+// SPDX-License-Identifier: Apache-2.0
+#pragma once
+
+#include "Config.h"
+#include "DiscoveryBpf.h"
+#include "ebpfdiscoveryshared/Types.h"
+
+#include <bpf/libbpf.h>
+
+#include <atomic>
+#include <thread>
+
+namespace ebpfdiscovery {
+
+class DiscoveryBpfLogHandler {
+public:
+	DiscoveryBpfLogHandler(DiscoveryBpf discoveryBpf, const DiscoveryConfig config);
+	DiscoveryBpfLogHandler(const DiscoveryBpfLogHandler&) = delete;
+	DiscoveryBpfLogHandler(DiscoveryBpfLogHandler&&) = delete;
+	DiscoveryBpfLogHandler& operator=(const DiscoveryBpfLogHandler&) = default;
+	DiscoveryBpfLogHandler& operator=(DiscoveryBpfLogHandler&&) = delete;
+	~DiscoveryBpfLogHandler() = default;
+
+	void start();
+	void stop();
+	void wait();
+
+	DiscoveryConfig config;
+	DiscoveryBpf discoveryBpf;
+
+private:
+	void run();
+
+	std::atomic<bool> running{false};
+	std::atomic<bool> stopReceived{false};
+	std::thread workerThread;
+
+	struct perf_buffer* logPb;
+};
+
+} // namespace ebpfdiscovery

libebpfdiscovery/src/DiscoveryBpfLogHandler.cpp

@@ -0,0 +1,122 @@
+// SPDX-License-Identifier: Apache-2.0
+#include "ebpfdiscovery/DiscoveryBpfLogHandler.h"
+
+#include <bpf/libbpf.h>
+
+#include "ebpfdiscoveryshared/Types.h"
+#include "logging/Logger.h"
+
+namespace ebpfdiscovery {
+
+logging::LogLevel severityToLogLevel(const DiscoveryLogLevel severity) {
+	switch (severity) {
+	case DISCOVERY_LOG_LEVEL_TRACE:
+		return logging::LogLevel::Trace;
+	case DISCOVERY_LOG_LEVEL_DEBUG:
+		return logging::LogLevel::Debug;
+	case DISCOVERY_LOG_LEVEL_INFO:
+		return logging::LogLevel::Info;
+	case DISCOVERY_LOG_LEVEL_WARN:
+		return logging::LogLevel::Warn;
+	case DISCOVERY_LOG_LEVEL_ERROR:
+		return logging::LogLevel::Err;
+	case DISCOVERY_LOG_LEVEL_CRITICAL:
+		return logging::LogLevel::Critical;
+	case DISCOVERY_LOG_LEVEL_OFF:
+		return logging::LogLevel::Off;
+	}
+	return logging::LogLevel::Off;
+}
+
+static void handleLogEvent(const DiscoveryLogEvent* event) {
+	const logging::LogLevel level{severityToLogLevel(event->severity)};
+	std::string message(DISCOVERY_LOG_MAX_MESSAGE_LENGTH, '\0');
+	const int resultLength{snprintf(
+			message.data(),
+			message.size() + 1,
+			event->format,
+			event->args[0],
+			event->args[1],
+			event->args[2],
+			event->args[3],
+			event->args[4],
+			event->args[5],
+			event->args[6],
+			event->args[7])};
+	if (resultLength < 0) {
+		LOG_DEBUG("Failed to format BPF log message.");
+		return;
+	}
+	logging::Logger::getInstance().log(level, "[BPF] [{}] [{}] [{}] {}", event->timestamp, event->pidTgid >> 32, event->cpuId, message);
+};
+
+static void handlePerfLogEvent(void* ctx, int cpu, void* data, __u32 dataSize) {
+	if (dataSize < sizeof(DiscoveryLogEvent)) {
+		LOG_TRACE(
+				"Received event on BPF log per buffer with unexpected data size. (received: {}, expected at least: {})",
+				dataSize,
+				sizeof(DiscoveryLogEvent));
+		return;
+	}
+
+	handleLogEvent(static_cast<DiscoveryLogEvent*>(data));
+}
+
+static void handlePerfLogLostEvents(void* ctx, int cpu, __u64 lostEventsCount) {
+	LOG_DEBUG("{} BPF logging events have been lost.", lostEventsCount);
+}
+
+DiscoveryBpfLogHandler::DiscoveryBpfLogHandler(DiscoveryBpf discoveryBpf, const DiscoveryConfig config)
+		: config(config), discoveryBpf(discoveryBpf) {
+}
+
+void DiscoveryBpfLogHandler::start() {
+	if (running) {
+		return;
+	}
+	running = true;
+
+	logPb = perf_buffer__new(
+			bpf_map__fd(discoveryBpf.skel->maps.logEventsPerfMap),
+			config.logPerfBufferPages,
+			handlePerfLogEvent,
+			handlePerfLogLostEvents,
+			nullptr,
+			nullptr);
+	if (logPb == nullptr) {
+		throw std::runtime_error("Could not open perf buffer: " + std::to_string(-errno));
+	}
+
+	workerThread = std::thread([&]() { run(); });
+}
+
+void DiscoveryBpfLogHandler::run() {
+	if (logPb == nullptr) {
+		return;
+	}
+
+	LOG_TRACE("Discovery BPF log event handler loop is starting.");
+	int err{0};
+	while (!stopReceived) {
+		err = perf_buffer__poll(logPb, config.logPerfPollInterval.count());
+		if (err < 0 && errno != -EINTR) {
+			LOG_ERROR("Error polling BPF perf buffer for logging: {}", std::strerror(-err));
+			break;
+		}
+	}
+
+	LOG_TRACE("Discovery BPF log event handler loop has finished running.");
+	perf_buffer__free(logPb);
+}
+
+void DiscoveryBpfLogHandler::stop() {
+	stopReceived = true;
+}
+
+void DiscoveryBpfLogHandler::wait() {
+	if (workerThread.joinable()) {
+		workerThread.join();
+	}
+}
+
+} // namespace ebpfdiscovery

libebpfdiscoveryshared/headers/ebpfdiscoveryshared/Constants.h

@@ -7,3 +7,7 @@
 
 #define DISCOVERY_MAX_HTTP_REQUEST_LENGTH DISCOVERY_BUFFER_MAX_DATA_SIZE
 #define DISCOVERY_MIN_HTTP_REQUEST_LENGTH 16
+
+#define DISCOVERY_LOG_MAX_FORMAT_LENGTH 128
+#define DISCOVERY_LOG_MAX_MESSAGE_LENGTH DISCOVERY_LOG_MAX_FORMAT_LENGTH + 32
+#define DISCOVERY_LOG_MAX_ARGS_COUNT 8

libebpfdiscoveryshared/headers/ebpfdiscoveryshared/Types.h

@@ -199,3 +199,32 @@ struct DiscoveryAllSessionState {
 struct DiscoveryGlobalState {
 	bool isCollectingDisabled;
 };
+
+/*
+ * eBPF logs and program config
+ */
+
+enum DiscoveryLogLevel {
+	DISCOVERY_LOG_LEVEL_TRACE,
+	DISCOVERY_LOG_LEVEL_DEBUG,
+	DISCOVERY_LOG_LEVEL_INFO,
+	DISCOVERY_LOG_LEVEL_WARN,
+	DISCOVERY_LOG_LEVEL_ERROR,
+	DISCOVERY_LOG_LEVEL_CRITICAL,
+	DISCOVERY_LOG_LEVEL_OFF,
+};
+
+struct DiscoveryConfig {
+	enum DiscoveryLogLevel logLevel;
+};
+
+typedef __u64 DiscoveryLogEventArg;
+
+struct DiscoveryLogEvent {
+	__u64 timestamp;
+	__u64 cpuId;
+	__u64 pidTgid;
+	enum DiscoveryLogLevel severity;
+	char format[DISCOVERY_LOG_MAX_FORMAT_LENGTH];
+	DiscoveryLogEventArg args[DISCOVERY_LOG_MAX_ARGS_COUNT];
+};

libebpfdiscoveryskel/src/Config.h

@@ -1,6 +1,10 @@
 // SPDX-License-Identifier: GPL-2.0
 #pragma once
 
+#include "vmlinux.h"
+
+#include <bpf/bpf_helpers.h>
+
 struct {
 	__uint(type, BPF_MAP_TYPE_ARRAY);
 	__type(key, __u32);
@@ -9,6 +13,6 @@ struct {
 } discoveryConfigMap SEC(".maps");
 
 __attribute__((always_inline)) inline static struct DiscoveryConfig* getDiscoveryConfig() {
-	__u32 zero = 0;
+	static __u32 zero = 0;
 	return (struct DiscoveryConfig*)bpf_map_lookup_elem(&discoveryConfigMap, &zero);
 }

libebpfdiscoveryskel/src/DataFunctions.h

@@ -28,3 +28,13 @@ __attribute__((always_inline)) inline static bool dataProbeIsBeginningOfHttpRequ
 	return len >= DISCOVERY_MIN_HTTP_REQUEST_LENGTH &&
 		   (dataProbeEqualToString(ptr, "GET /", 5) == 5 || dataProbeEqualToString(ptr, "POST /", 6) == 6);
 }
+
+__attribute__((always_inline)) inline static int dataCopyString(const char* src, char* dest, size_t len) {
+	for (size_t i = 0; i < len; ++i) {
+		if (src[i] == '\0') {
+			return i + 1;
+		}
+		dest[i] = src[i];
+	}
+	return len;
+}

libebpfdiscoveryskel/src/DebugPrint.h

@@ -0,0 +1,21 @@
+// SPDX-License-Identifier: GPL-2.0
+#pragma once
+
+#include "vmlinux.h"
+
+#include <bpf/bpf_core_read.h>
+#include <bpf/bpf_helpers.h>
+#include <bpf/bpf_tracing.h>
+
+struct trace_event_raw_bpf_trace_printk___log {};
+
+#define DEBUG_PRINTLN(fmt, ...)                                                    \
+	({                                                                             \
+		static char newFmt[] = "[ebpf-discovery] " fmt "\0";                       \
+		if (bpf_core_type_exists(struct trace_event_raw_bpf_trace_printk___log)) { \
+			bpf_trace_printk(newFmt, sizeof(newFmt) - 1, ##__VA_ARGS__);           \
+		} else {                                                                   \
+			newFmt[sizeof(newFmt) - 2] = '\n';                                     \
+			bpf_trace_printk(newFmt, sizeof(newFmt), ##__VA_ARGS__);               \
+		}                                                                          \
+	})