linux-gen: fdserver: run fd server with a separate image

Refactor existing FD server by moving the server functionality to a
separate process image and launch it with `exec()` after the current
forking step has been done. This way typical issues related to forking a
multithreaded program are avoided.

Fixes OpenDataPlane#563.

Signed-off-by: Tuomas Taipale <tuomas.taipale@nokia.com>

platform/linux-generic/.gitignore

@@ -1,2 +1,3 @@
 libodp-linux.pc
+odp_fdserver_runner
 odp_libconfig_config.h

platform/linux-generic/Makefile.am

@@ -492,6 +492,12 @@ if ODP_PKTIO_PCAP
 __LIB__libodp_linux_la_LIBADD += $(PCAP_LIBS)
 endif
 
+pkglibexec_PROGRAMS = odp_fdserver_runner
+
+odp_fdserver_runner_SOURCES = odp_fdserver_runner.c
+
+AM_CFLAGS += -DFDSERVER_PATH='"@libexecdir@/$(PACKAGE)/odp_fdserver_runner"'
+
 CHECK_GLOBALS_REGEX = " (odp_|_odp_|_deprecated_odp_|miniz_|mz_|tdefl_|tinfl_|mp_hdlr_init_odp_pool_ops)"
 
 TESTS_ENVIRONMENT = \

platform/linux-generic/include/odp_fdserver_internal.h

@@ -1,5 +1,6 @@
 /* SPDX-License-Identifier: BSD-3-Clause
  * Copyright (c) 2016-2018 Linaro Limited
+ * Copyright (c) 2024 Nokia
  */
 
 #ifndef _FD_SERVER_INTERNAL_H
@@ -9,6 +10,29 @@
 extern "C" {
 #endif
 
+#include <odp_debug_internal.h>
+#include <odp_config_internal.h>
+
+#include <stdint.h>
+#include <string.h>
+#include <sys/socket.h>
+
+/* Debug level for the FD server */
+#define FD_DBG 3
+/* define the tables of file descriptors handled by this server: */
+#define FDSERVER_MAX_ENTRIES (CONFIG_SHM_BLOCKS + CONFIG_INTERNAL_SHM_BLOCKS)
+/* possible commands: */
+#define FD_REGISTER_REQ 1 /* client -> server */
+#define FD_REGISTER_ACK 2 /* server -> client */
+#define FD_REGISTER_NACK 3 /* server -> client */
+#define FD_LOOKUP_REQ 4 /* client -> server */
+#define FD_LOOKUP_ACK 5 /* server -> client */
+#define FD_LOOKUP_NACK 6 /* server -> client */
+#define FD_DEREGISTER_REQ 7 /* client -> server */
+#define FD_DEREGISTER_ACK 8 /* server -> client */
+#define FD_DEREGISTER_NACK 9 /* server -> client */
+#define FD_SERVERSTOP_REQ 10 /* client -> server (stops) */
+
 /*
  * the following enum defines the different contexts by which the
  * FD server may be used: In the FD server, the keys used to store/retrieve
@@ -25,10 +49,149 @@ typedef enum fd_server_context {
  FD_SRV_CTX_END, /* upper enum limit */
 } fd_server_context_e;
 
+typedef struct fdentry_s {
+ fd_server_context_e context;
+ uint64_t key;
+ int fd;
+} fdentry_t;
+
+/*
+ * define the message struct used for communication between client and server
+ * (this single message is used in both direction)
+ * The file descriptors are sent out of band as ancillary data for conversion.
+ */
+typedef struct fd_server_msg {
+ int command;
+ fd_server_context_e context;
+ uint64_t key;
+} fdserver_msg_t;
+
 int _odp_fdserver_register_fd(fd_server_context_e context, uint64_t key,
  int fd);
 int _odp_fdserver_deregister_fd(fd_server_context_e context, uint64_t key);
 int _odp_fdserver_lookup_fd(fd_server_context_e context, uint64_t key);
+/*
+ * Client and server function:
+ * Send a fdserver_msg, possibly including a file descriptor, on the socket
+ * This function is used both by:
+ * -the client (sending a FD_REGISTER_REQ with a file descriptor to be shared,
+ * or FD_LOOKUP_REQ/FD_DEREGISTER_REQ without a file descriptor)
+ * -the server (sending FD_REGISTER_ACK/NACK, FD_LOOKUP_NACK,
+ * FD_DEREGISTER_ACK/NACK... without a fd or a
+ * FD_LOOKUP_ACK with a fd)
+ * This function make use of the ancillary data (control data) to pass and
+ * convert file descriptors over UNIX sockets
+ * Return -1 on error, 0 on success.
+ */
+static inline int _odp_send_fdserver_msg(int sock, int command, fd_server_context_e context,
+ uint64_t key, int fd_to_send)
+{
+ struct msghdr socket_message;
+ struct iovec io_vector[1]; /* one msg frgmt only */
+ struct cmsghdr *control_message = NULL;
+ int *fd_location;
+ fdserver_msg_t msg;
+ int res;
+
+ char ancillary_data[CMSG_SPACE(sizeof(int))];
+
+ /* prepare the register request body (single fragment): */
+ msg.command = command;
+ msg.context = context;
+ msg.key = key;
+ io_vector[0].iov_base = &msg;
+ io_vector[0].iov_len = sizeof(fdserver_msg_t);
+
+ /* initialize socket message */
+ memset(&socket_message, 0, sizeof(struct msghdr));
+ socket_message.msg_iov = io_vector;
+ socket_message.msg_iovlen = 1;
+
+ if (fd_to_send >= 0) {
+ /* provide space for the ancillary data */
+ memset(ancillary_data, 0, CMSG_SPACE(sizeof(int)));
+ socket_message.msg_control = ancillary_data;
+ socket_message.msg_controllen = CMSG_SPACE(sizeof(int));
+
+ /* initialize a single ancillary data element for fd passing */
+ control_message = CMSG_FIRSTHDR(&socket_message);
+ control_message->cmsg_level = SOL_SOCKET;
+ control_message->cmsg_type = SCM_RIGHTS;
+ control_message->cmsg_len = CMSG_LEN(sizeof(int));
+ fd_location = (int *)(void *)CMSG_DATA(control_message);
+ *fd_location = fd_to_send;
+ }
+ res = sendmsg(sock, &socket_message, 0);
+ if (res < 0)
+ return -1;
+
+ return 0;
+}
+
+/*
+ * Client and server function
+ * Receive a fdserver_msg, possibly including a file descriptor, on the
+ * given socket.
+ * This function is used both by:
+ * -the server (receiving a FD_REGISTER_REQ with a file descriptor to be shared,
+ * or FD_LOOKUP_REQ, FD_DEREGISTER_REQ without a file descriptor)
+ * -the client (receiving FD_REGISTER_ACK...without a fd or a FD_LOOKUP_ACK with
+ * a fd)
+ * This function make use of the ancillary data (control data) to pass and
+ * convert file descriptors over UNIX sockets.
+ * Return -1 on error, 0 on success.
+ */
+static inline int _odp_recv_fdserver_msg(int sock, int *command, fd_server_context_e *context,
+ uint64_t *key, int *recvd_fd)
+{
+ struct msghdr socket_message;
+ struct iovec io_vector[1]; /* one msg frgmt only */
+ struct cmsghdr *control_message = NULL;
+ int *fd_location;
+ fdserver_msg_t msg;
+ char ancillary_data[CMSG_SPACE(sizeof(int))];
+
+ memset(&socket_message, 0, sizeof(struct msghdr));
+ memset(ancillary_data, 0, CMSG_SPACE(sizeof(int)));
+
+ /* setup a place to fill in message contents */
+ io_vector[0].iov_base = &msg;
+ io_vector[0].iov_len = sizeof(fdserver_msg_t);
+ socket_message.msg_iov = io_vector;
+ socket_message.msg_iovlen = 1;
+
+ /* provide space for the ancillary data */
+ socket_message.msg_control = ancillary_data;
+ socket_message.msg_controllen = CMSG_SPACE(sizeof(int));
+
+ /* receive the message */
+ if (recvmsg(sock, &socket_message, MSG_CMSG_CLOEXEC) < 0)
+ return -1;
+
+ *command = msg.command;
+ *context = msg.context;
+ *key = msg.key;
+
+ /* grab the converted file descriptor (if any) */
+ *recvd_fd = -1;
+
+ if ((socket_message.msg_flags & MSG_CTRUNC) == MSG_CTRUNC)
+ return 0;
+
+ /* iterate ancillary elements to find the file descriptor: */
+ for (control_message = CMSG_FIRSTHDR(&socket_message);
+ control_message != NULL;
+ control_message = CMSG_NXTHDR(&socket_message, control_message)) {
+ if ((control_message->cmsg_level == SOL_SOCKET) &&
+ (control_message->cmsg_type == SCM_RIGHTS)) {
+ fd_location = (int *)(void *)CMSG_DATA(control_message);
+ *recvd_fd = *fd_location;
+ break;
+ }
+ }
+
+ return 0;
+}
 
 #ifdef __cplusplus
 }