linux/efa_nv_peermem: Add efa_nv_peermem kernel module

Add a new kernel module that links to Nvidia P2P API functions and
supplies EFA NV peermem API through EXPORT_SYMBOL_GPL.
This will intentionally work only with Nvidia opensource driver that is
licensed as GPL but at this point in time doesn't export p2p symbols
using EXPORT_SYMBOL_GPL.

Reviewed-by: Daniel Kranzdorf <dkkranzd@amazon.com>
Reviewed-by: Yehuda Yitschak <yehuday@amazon.com>
Signed-off-by: Michael Margolin <mrgolin@amazon.com>

kernel/linux/efa_nv_peermem/CMakeLists.txt

@@ -0,0 +1,36 @@
+# SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause
+# Copyright 2023 Amazon.com, Inc. or its affiliates. All rights reserved.
+
+cmake_minimum_required(VERSION 2.8.11)
+project(efa_nv_peermem C)
+
+set(KERNEL_VER "" CACHE STRING "Kernel version to build for")
+if(NOT KERNEL_VER)
+  execute_process(COMMAND uname -r OUTPUT_VARIABLE uname_r
+                  OUTPUT_STRIP_TRAILING_WHITESPACE)
+  set(KERNEL_DIR "/lib/modules/${uname_r}/build")
+else()
+  set(KERNEL_DIR "/lib/modules/${KERNEL_VER}/build")
+endif()
+
+unset(KERNEL_MAKEFILE CACHE)
+find_file(KERNEL_MAKEFILE Makefile PATHS ${KERNEL_DIR} NO_DEFAULT_PATH)
+if(NOT KERNEL_MAKEFILE)
+  message(FATAL_ERROR "No kernel Makefile")
+endif()
+message("-- Kernel directory - ${KERNEL_DIR}")
+
+set(NVIDIA_DIR "" CACHE PATH "Path to NVIDIA directory")
+if(NVIDIA_DIR)
+  unset(NVIDIA_MODULE_SYMVERS CACHE)
+  find_file(NVIDIA_MODULE_SYMVERS Module.symvers PATHS ${NVIDIA_DIR} REQUIRED NO_DEFAULT_PATH)
+  if(NOT NVIDIA_MODULE_SYMVERS)
+    message(FATAL_ERROR "No NVIDIA Module.symvers")
+  else()
+    message("-- NVIDIA directory - ${NVIDIA_DIR}")
+  endif()
+endif()
+
+set(GCOV_PROFILE OFF CACHE BOOL "Enable GCOV profiling")
+
+add_subdirectory(src)

kernel/linux/efa_nv_peermem/README

@@ -0,0 +1,52 @@
+Linux kernel module for EFA support of Nvidia peermem
+=====================================================
+
+Overview
+========
+EFA NV Peermem kernel module is an extension module for the Elastic Fabric
+Adapter (EFA) that is serving as a link between Nvidia opensource drivers
+and the EFA driver for the needs of Cuda memory GPU direct support.
+
+For more information regarding GPUDirect RDMA, visit:
+https://docs.nvidia.com/cuda/gpudirect-rdma/index.html
+
+EFA NV Peermem supports Nvidia opensource drivers only and it is unable to
+load when such drivers are not avilable in the system.
+
+Driver compilation
+==================
+For list of supported kernels and distributions, please refer to the release
+notes documentation in the same directory.
+
+sudo yum update
+sudo yum install gcc
+sudo yum install kernel-devel-$(uname -r)
+
+Compilation:
+Run:
+mkdir build
+cd build
+cmake ..
+make
+
+efa_nv_peermem.ko is created inside the src/ folder.
+
+To build EFA NV Peermem RPMs run `make` in the rpm/ folder. Your environment
+will need to be setup to build RPMs. The EFA NV Peermem RPM will install the
+efa_nv_peermem kernel driver source, and setup DKMS in order to build the driver
+when the kernel is updated.
+
+Driver installation
+===================
+Loading driver
+--------------
+insmod efa_nv_peermem.ko
+
+EFA NV Peermem doesn't need to be automatically started upon the OS boot,
+and it will be requested by EFA driver whenever it is needed for P2P
+memory registration operations to work.
+
+EFA NV Peermem Source Code Directory Structure (under src/)
+===========================================================
+efa_nv_peermem_main.c - Main Linux kernel driver.
+nv-p2p.h              - NV P2P API

kernel/linux/efa_nv_peermem/RELEASENOTES.md

@@ -0,0 +1,8 @@
+# EFA NV Peermem Linux Kernel Driver Release Notes
+
+## Supported Kernel Versions and Distributions
+https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/efa.html#efa-amis
+
+## r1.1.0 release notes
+
+Initial commit

kernel/linux/efa_nv_peermem/build.sh

@@ -0,0 +1,18 @@
+#!/bin/bash -e
+
+SRCDIR=`dirname $0`
+BUILDDIR="$SRCDIR/build"
+
+mkdir -p "$BUILDDIR"
+
+if hash cmake3 2>/dev/null; then
+    # CentOS users are encouraged to install cmake3 from EPEL
+    CMAKE=cmake3
+else
+    CMAKE=cmake
+fi
+
+cd "$BUILDDIR"
+
+$CMAKE ${EXTRA_CMAKE_FLAGS:-} ..
+make

kernel/linux/efa_nv_peermem/conf/configure-dkms.sh

@@ -0,0 +1,27 @@
+#!/bin/bash
+# Auxiliary script for driver packaging
+
+kernelver=$1
+dkms_source_tree=$2
+extra_flags=
+
+nvidia_path=$(find $dkms_source_tree -maxdepth 1 -type d -name '*nvidia-*' | sort | tail -n 1)
+if [ $nvidia_path ]; then
+	echo "== Building NVIDIA Module.symvers"
+	pushd $nvidia_path
+	make KERNEL_UNAME="$kernelver" clean || failed_nvidia=true
+	make KERNEL_UNAME="$kernelver" -j$(nproc) || failed_nvidia=true
+	popd
+
+	if [ $failed_nvidia ]; then
+		echo "== Failed building NVIDIA Module.symvers, proceeding without"
+	else
+		echo "== Using NVIDIA driver path $nvidia_path"
+		extra_flags="-DNVIDIA_DIR=$nvidia_path"
+	fi
+fi
+
+mkdir -p build
+pushd build
+cmake -DKERNEL_VER=$kernelver $extra_flags ..
+popd

kernel/linux/efa_nv_peermem/conf/dkms.conf

@@ -0,0 +1,13 @@
+PACKAGE_NAME="efa-nv-peermem"
+PACKAGE_VERSION="1.1.0"
+CLEAN="cd build; make modules_clean; make clean"
+PRE_BUILD="./configure-dkms.sh $kernelver $source_tree"
+# Quoted 'make' to suppress DKMS append of KERNELRELEASE
+MAKE="cd build; 'make'"
+BUILT_MODULE_NAME[0]="efa_nv_peermem"
+BUILT_MODULE_LOCATION="build/src/"
+DEST_MODULE_LOCATION="/extra"
+DEST_MODULE_NAME[0]="efa_nv_peermem"
+AUTOINSTALL="yes"
+NO_WEAK_MODULES="yes"
+BUILD_DEPENDS="nvidia"

kernel/linux/efa_nv_peermem/conf/efa_nv_peermem.conf

@@ -0,0 +1,2 @@
+# /etc/modules-load.d/efa_nv_peermem.conf
+nvidia

kernel/linux/efa_nv_peermem/debian/changelog

@@ -0,0 +1,4 @@
+efa-nv-peermem (1.1.0-1.amzn1) unstable; urgency=medium
+  * Initial release of efa_nv_peermem kernel driver Debian package
+
+ -- Michael Margolin <mrgolin@amazon.com>  Wed, 25 Oct 2023 16:42:57 +0000

kernel/linux/efa_nv_peermem/debian/compat

@@ -0,0 +1 @@
+10

kernel/linux/efa_nv_peermem/debian/control

@@ -0,0 +1,12 @@
+Source: efa-nv-peermem
+Maintainer: ec2-efa-maintainers <ec2-efa-maintainers@amazon.com>
+Build-Depends: debhelper
+Standards-Version: 4.3.0
+Homepage: https://github.com/amzn/amzn-drivers
+
+Package: efa-nv-peermem
+Architecture: any
+Depends: ${shlibs:Depends}, ${misc:Depends}, dkms, cmake
+Description: Linux kernel driver for EFA Nvidia peermem
+ Elastic Fabric Adapter (EFA) peermem provides an interface for nv-p2p
+ capabilities for the EFA driver use.

kernel/linux/efa_nv_peermem/debian/copyright

@@ -0,0 +1,10 @@
+Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/
+Upstream-Name: efa-nv-peermem
+Upstream-Contact: ec2-efa-maintainers <ec2-efa-maintainers@amazon.com>
+Source: https://github.com/amzn/amzn-drivers
+
+Files: *
+Copyright:
+ Copyright 2023 Amazon.com, Inc. or its affiliates. All rights reserved.
+License:
+ SPDX-License-Identifier: ((GPL-2.0 WITH Linux-syscall-note) OR BSD-2-Clause)

kernel/linux/efa_nv_peermem/debian/efa-nv-peermem.postinst

@@ -0,0 +1,18 @@
+#!/bin/sh
+
+set -e
+
+NAME=efa-nv-peermem
+DRIVER_VERSION=1.1.0
+INSTALL_PATH=/usr/src/${NAME}-${DRIVER_VERSION}
+
+cd $INSTALL_PATH
+dkms add -m ${NAME} -v ${DRIVER_VERSION}
+for kernel in $(/bin/ls /lib/modules); do
+	if [ -e /lib/modules/$kernel/build/include ]; then
+		dkms build -m ${NAME} -v ${DRIVER_VERSION} -k $kernel
+		dkms install --force -m ${NAME} -v ${DRIVER_VERSION} -k $kernel
+	fi
+done
+
+#DEBHELPER#

kernel/linux/efa_nv_peermem/debian/efa-nv-peermem.prerm

@@ -0,0 +1,10 @@
+#!/bin/sh
+
+set -e
+
+NAME=efa-nv-peermem
+DRIVER_VERSION=1.1.0
+
+dkms remove -m ${NAME} -v ${DRIVER_VERSION} --all
+
+#DEBHELPER#

kernel/linux/efa_nv_peermem/debian/rules

@@ -0,0 +1,42 @@
+#!/usr/bin/make -f
+
+include /usr/share/dpkg/architecture.mk
+include /usr/share/dpkg/buildflags.mk
+include /usr/share/dpkg/pkg-info.mk
+
+# See debhelper(7) (uncomment to enable)
+# output every command that modifies files on the build system.
+export DH_VERBOSE = 1
+
+NAME	= efa-nv-peermem
+VERSION = 1.1.0
+DESTDIR:=`pwd`/debian/${NAME}
+INSTALL_PATH = /usr/src/${NAME}-${VERSION}
+
+
+%:
+	dh $@
+
+override_dh_auto_clean:
+	dh_clean
+
+override_dh_auto_build:
+	echo "Skip build"
+
+override_dh_auto_configure:
+	echo "Skip configure"
+
+override_dh_auto_install:
+	mkdir -p ${DESTDIR}/${INSTALL_PATH}
+	mkdir -p ${DESTDIR}/${INSTALL_PATH}/config
+	mkdir -p ${DESTDIR}/${INSTALL_PATH}/src
+	install -D -m 644 conf/efa_nv_peermem.conf		${DESTDIR}/etc/modules-load.d/efa_nv_peermem.conf
+	install -m 644 conf/dkms.conf				${DESTDIR}/${INSTALL_PATH}
+	install -m 744 conf/configure-dkms.sh			${DESTDIR}/${INSTALL_PATH}
+	install -m 644 src/efa_nv_peermem_main.c		${DESTDIR}/${INSTALL_PATH}/src
+	install -m 644 src/nv-p2p.h				${DESTDIR}/${INSTALL_PATH}/src
+	install -m 644 src/CMakeLists.txt			${DESTDIR}/${INSTALL_PATH}/src
+	install -m 644 src/Kbuild.in				${DESTDIR}/${INSTALL_PATH}/src
+	install -m 644 README					${DESTDIR}/${INSTALL_PATH}
+	install -m 644 RELEASENOTES.md				${DESTDIR}/${INSTALL_PATH}
+	install -m 644 CMakeLists.txt				${DESTDIR}/${INSTALL_PATH}

kernel/linux/efa_nv_peermem/debian/source/format

@@ -0,0 +1 @@
+3.0 (quilt)

kernel/linux/efa_nv_peermem/rpm/Makefile

@@ -0,0 +1,40 @@
+# Makefile for creating rpm of the Amazon EFA NV Peermem driver
+
+NAME = efa-nv-peermem
+SPEC = $(NAME).spec
+VERSION	= 1.1.0
+
+TOPDIR  := $(shell git rev-parse --show-toplevel)
+TAG	?= HEAD
+TARBALL	= $(NAME)-$(VERSION).tar
+
+RPMDEFS = --define '_topdir  %(pwd)' \
+          --define '_ntopdir %(pwd)' \
+          --define '_builddir  %{_ntopdir}/build' \
+          --define '_buildrootdir %{_builddir}' \
+          --define '_sourcedir %{_ntopdir}' \
+          --define '_specdir   %{_ntopdir}' \
+          --define '_rpmdir    %{_ntopdir}' \
+          --define '_srcrpmdir %{_ntopdir}' \
+	  --define "driver_version $(VERSION)"
+
+all : rpm
+
+tarball : $(TARBALL)
+$(TARBALL) : always
+	(cd $(TOPDIR) && \
+	git archive --format=tar --prefix=$(NAME)-$(VERSION)/ $(TAG) \
+	kernel/linux/efa_nv_peermem/) > $@
+
+srpm : $(TARBALL) Makefile
+	rpmbuild -bs $(RPMDEFS) $(SPEC)
+
+rpm : $(TARBALL) Makefile
+	rpmbuild -ba $(RPMDEFS) $(SPEC)
+
+clean :
+	rm -f $(TARBALL) *.src.rpm
+
+always:
+
+.PHONY : srpm clean always

kernel/linux/efa_nv_peermem/rpm/efa-nv-peermem.spec

@@ -0,0 +1,74 @@
+# Copyright 2023 Amazon.com, Inc. or its affiliates. All rights reserved
+
+%define name			efa-nv-peermem
+%define driver_name		efa_nv_peermem
+%define debug_package		%{nil}
+
+Name:		%{name}
+Version:	%{driver_version}
+Release:	1%{?dist}
+Summary:	%{driver_name} kernel module
+
+Group:		System/Kernel
+License:	Dual BSD/GPL
+URL:		https://github.com/amzn/amzn-drivers/
+Source0:	%{name}-%{version}.tar
+
+Requires:	dkms %kernel_module_package_buildreqs cmake
+%if %{defined kernel_module_package_buildreqs}
+Requires: %kernel_module_package_buildreqs
+%endif
+# RHEL 8.4 has a broken dependency between cmake and libarchive which
+# causes libarchive to not be updated properly in the update case. Express the
+# dependency so that our install does not break.
+%if 0%{?rhel} >= 8
+Requires: libarchive >= 3.3.3
+%endif
+
+%define install_path /usr/src/%{name}-%{version}
+
+%description
+%{driver_name} kernel module source and DKMS scripts to build the kernel module.
+
+%prep
+%setup -n %{name}-%{version} -q
+
+%post
+cd %{install_path}
+dkms add -m %{name} -v %{driver_version}
+for kernel in $(/bin/ls /lib/modules); do
+	if [ -e /lib/modules/$kernel/build/include ]; then
+		dkms build -m %{name} -v %{driver_version} -k $kernel
+		dkms install --force -m %{name} -v %{driver_version} -k $kernel
+	fi
+done
+
+%preun
+dkms remove -m %{name} -v %{driver_version} --all
+
+%build
+
+%install
+cd kernel/linux/efa_nv_peermem
+mkdir -p %{buildroot}%{install_path}
+mkdir -p %{buildroot}%{install_path}/config
+mkdir -p %{buildroot}%{install_path}/src
+install -D -m 644 conf/efa_nv_peermem.conf		%{buildroot}/etc/modules-load.d/efa_nv_peermem.conf
+install -m 644 conf/dkms.conf				%{buildroot}%{install_path}
+install -m 744 conf/configure-dkms.sh			%{buildroot}%{install_path}
+install -m 644 CMakeLists.txt				%{buildroot}%{install_path}
+install -m 644 README					%{buildroot}%{install_path}
+install -m 644 RELEASENOTES.md				%{buildroot}%{install_path}
+cd src
+install -m 644 efa_nv_peermem_main.c			%{buildroot}%{install_path}/src
+install -m 644 nv-p2p.h					%{buildroot}%{install_path}/src
+install -m 644 CMakeLists.txt				%{buildroot}%{install_path}/src
+install -m 644 Kbuild.in				%{buildroot}%{install_path}/src
+
+%files
+%{install_path}
+/etc/modules-load.d/efa_nv_peermem.conf
+
+%changelog
+* Wed Oct 25 2023 Michael Margolin <mrgolin@amazon.com> - 1.1.0
+- initial build for RHEL