Build py-ticking wheel using docker on top of a manylinux base image. (…

…deephaven#5267) * Build py-ticking wheel using docker on top of a manylinux base image. * Make the python version configurable. * Move test to ubi8.8. * Add a task for building wheels for multiple python versions. * Use auditwheel to perfect the tags. * Followup to review comments. * More tweaks to quoting. * Added target to test all wheels. * Add py.typed. * Used docker/registry for the new images involved. * Follow image name convention. * Add the registry to imageName for manylinux. * Remove --force from pip install. * Add python 3.12 classifier to setup.py. * Added py.typed emtpy file. * Added registry to ubi8 image name. * Ensure py.typed is added in the image.
lbooker42 · Mar 25, 2024 · 6c8b9ce · 6c8b9ce
1 parent db4c2fa
commit 6c8b9ce
Show file tree

Hide file tree

Showing 12 changed files with 226 additions and 63 deletions.
diff --git a/cpp-client/README.md b/cpp-client/README.md
@@ -47,10 +47,12 @@ on them anymore so we do notguarantee they are current for those platforms.
 
    Edit your local copy of the script if necessary to reflect your selection
    of build tools and build target;
-   defaults point to Ubuntu system's g++, cmake, and a Debug build target for cmake.
-   Comments in the script will help you in identifying customization points.
-   Note however that defaults are tested by any deviation from defaults may require
-   manual modification of other files later, when building the C++ client proper.
+   defaults point to Ubuntu system's g++, cmake, and a `RelWithDebInfo` build target
+   for cmake.
+   Comments in the script will help you identifying customization points.
+   Note however that defaults are tested, variations are not;
+   any deviation from defaults may require manual modification of other files later,
+   when building the C++ client proper.
 
    Example:
    ```

diff --git a/cpp-client/build.gradle b/cpp-client/build.gradle
@@ -22,6 +22,7 @@ spotless {
 // image we will generate here as a base.
 // See https://github.com/deephaven/deephaven-base-images/tree/main/cpp-clients-multi
 evaluationDependsOn Docker.registryProject('cpp-clients-multi-base')
+evaluationDependsOn Docker.registryProject('manylinux2014_x86_64')
 
 configurations {
     cpp {}
@@ -57,6 +58,7 @@ deephavenDocker {
 }
 
 def prefix = '/opt/deephaven'
+def build_type = 'RelWithDebInfo'
 
 def buildCppClientImage = Docker.registerDockerTask(project, 'cppClient') {
     // Only tested on x86-64, and we only build dependencies for x86-64
@@ -91,23 +93,23 @@ def buildCppClientImage = Docker.registerDockerTask(project, 'cppClient') {
         copyFile('deephaven/tests/', "${prefix}/src/deephaven/tests/")
         copyFile('cpp-tests-to-junit.sh', "${prefix}/bin/dhcpp")
         copyFile('build-dependencies.sh', "/tmp")
-        runCommand("PREFIX=${prefix}; " +
+        runCommand("PREFIX='${prefix}'; BUILD_TYPE='${build_type}'; " +
                    '''set -eux; \\
-                      cmp /tmp/build-dependencies.sh ${PREFIX}/build-dependencies.sh; \\
+                      cmp /tmp/build-dependencies.sh "${PREFIX}/build-dependencies.sh"; \\
                       rm -f /tmp/build-dependencies.sh; \\
-                      rm -fr ${PREFIX}/src/deephaven/build; \\
-                      mkdir -p ${PREFIX}/src/deephaven/build; \\
-                      cd ${PREFIX}/src/deephaven/build; \\
-                      . ${PREFIX}/env.sh; \\
+                      rm -fr "${PREFIX}/src/deephaven/build"; \\
+                      mkdir -p "${PREFIX}/src/deephaven/build"; \\
+                      cd "${PREFIX}/src/deephaven/build"; \\
+                      . "${PREFIX}/env.sh"; \\
                       cmake \\
-                        -DCMAKE_INSTALL_PREFIX=${PREFIX} \\
-                        -DCMAKE_BUILD_TYPE=Release \\
+                        -DCMAKE_INSTALL_PREFIX="${PREFIX}" \\
+                        -DCMAKE_BUILD_TYPE="${BUILD_TYPE}" \\
                         -DBUILD_SHARED_LIBS=ON \\
                         .. ; \\
-                      VERBOSE=1 make -j${NCPUS} install 2>&1 | gzip > ${PREFIX}/log/make-install.log.gz; \\
-                      cp -f ${PREFIX}/src/deephaven/build/tests/tests ${PREFIX}/bin/dhcpp/tests; \\
+                      VERBOSE=1 make "-j${NCPUS}" install 2>&1 | gzip > "${PREFIX}/log/make-install.log.gz"; \\
+                      cp -f "${PREFIX}/src/deephaven/build/tests/tests" "${PREFIX}/bin/dhcpp/tests"; \\
                       cd ..; \\
-                      rm -fr ${PREFIX}/src/deephaven/build
+                      rm -fr "${PREFIX}/src/deephaven/build"
                    ''')
         // Note environment variables defined here are inherited by other images
         // using this image as a base ("from").
@@ -139,5 +141,54 @@ def testCppClient = Docker.registerDockerTask(project, 'testCppClient') {
     entrypoint = ["${prefix}/bin/dhcpp/cpp-tests-to-junit.sh", '/out/cpp-test.xml', '/out/cpp-test.log']
 }
 
+def buildCppClientPyImage = Docker.registerDockerTask(project, 'cppClientPy') {
+    // Uses a manylinux base image to support cython wheel building for multiple linux distros.
+    platform = 'linux/amd64'
+    copyIn {
+        from(layout.projectDirectory) {
+            include 'build-dependencies.sh'
+            include 'deephaven/CMakeLists.txt'
+            include 'deephaven/dhcore/**'
+        }
+    }
+
+    dockerfile {
+        from('deephaven/manylinux2014_x86_64:local-build')
+        runCommand("""mkdir -p \\
+                        /out \\
+                        ${prefix} \\
+                        ${prefix}/bin/dhcpp \\
+                        ${prefix}/log
+                   """)
+        copyFile('deephaven/CMakeLists.txt',  "${prefix}/src/deephaven/")
+        copyFile('deephaven/dhcore/',  "${prefix}/src/deephaven/dhcore/")
+        copyFile('build-dependencies.sh', "${prefix}")
+        runCommand("PREFIX='${prefix}'; BUILD_TYPE='${build_type}';" +
+                   '''set -eux; \\
+                      cd "${PREFIX}"; \\
+                      PFX="${PREFIX}" BUILD_TYPE="${BUILD_TYPE}" ./build-dependencies.sh \\
+                        --static-pic immer env 2>&1 | gzip >build-dependencies.log.gz; \\
+                      mkdir -p "${PREFIX}/src/deephaven/build"; \\
+                      cd "${PREFIX}/src/deephaven/build"; \\
+                      . "${PREFIX}/env.sh"; \\
+                      cmake \\
+                        -DDHCORE_ONLY=ON \
+                        -DCMAKE_INSTALL_PREFIX="${PREFIX}" \
+                        -DCMAKE_BUILD_TYPE="${BUILD_TYPE}" \
+                        -DBUILD_SHARED_LIBS=ON \\
+                        .. ; \\
+                      VERBOSE=1 make -j"${NCPUS}" install 2>&1 | gzip >"${PREFIX}/log/make-install.log.gz"; \\
+                      cp dhcore/libdhcore_static.a "${PREFIX}/lib"; \\
+                      cd ..; \\
+                      rm -fr "${PREFIX}/src/deephaven/build"
+                   ''')
+        // Note environment variables defined here are inherited by other images
+        // using this image as a base ("from").
+        environmentVariable 'DH_PREFIX', prefix
+        environmentVariable 'LD_LIBRARY_PATH', "${prefix}/lib"
+    }
+    parentContainers = [ Docker.registryTask(project, 'manylinux2014_x86_64') ]
+}
+
 deephavenDocker.shouldLogIfTaskFails testCppClient
 tasks.check.dependsOn(testCppClient)
diff --git a/cpp-client/deephaven/CMakeLists.txt b/cpp-client/deephaven/CMakeLists.txt
@@ -23,28 +23,34 @@ if(${SANITIZE_ADDRESS} STREQUAL "ON")
   link_libraries("-fsanitize=address")
 endif()
 
-add_subdirectory(dhclient)
-add_subdirectory(dhcore)
-add_subdirectory(tests)
-add_subdirectory(examples)
+option(DHCORE_ONLY "Only build dhcore, skip rest" OFF)
 
-install(DIRECTORY dhclient/include/public/
-    DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}
-    )
+add_subdirectory(dhcore)
+if(NOT DHCORE_ONLY)
+    add_subdirectory(dhclient)
+    add_subdirectory(tests)
+    add_subdirectory(examples)
+endif()
 
 install(DIRECTORY dhcore/include/public/
     DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}
     )
 
-install(TARGETS dhclient dhcore_static dhcore
-    EXPORT deephavenConfig
-    ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}
-    LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
-    RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}
-    INCLUDES DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}
-    )
-
-install(EXPORT deephavenConfig
-    DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/deephaven
-    NAMESPACE deephaven::
-    )
+if(NOT DHCORE_ONLY)
+    install(DIRECTORY dhclient/include/public/
+        DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}
+        )
+
+    install(TARGETS dhclient dhcore_static dhcore
+        EXPORT deephavenConfig
+        ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}
+        LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
+        RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}
+        INCLUDES DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}
+        )
+
+    install(EXPORT deephavenConfig
+        DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/deephaven
+        NAMESPACE deephaven::
+        )
+endif()
diff --git a/docker/registry/fedora/build.gradle b/docker/registry/fedora/build.gradle
@@ -0,0 +1,3 @@
+plugins {
+    id 'io.deephaven.project.register'
+}
diff --git a/docker/registry/fedora/gradle.properties b/docker/registry/fedora/gradle.properties
@@ -0,0 +1,3 @@
+io.deephaven.project.ProjectType=DOCKER_REGISTRY
+deephaven.registry.imageName=fedora:39
+deephaven.registry.imageId=fedora@sha256:61864fd19bbd64d620f338eb11dae9e8759bf7fa97302ac6c43865c48dccd679
diff --git a/docker/registry/manylinux2014_x86_64/build.gradle b/docker/registry/manylinux2014_x86_64/build.gradle
@@ -0,0 +1,3 @@
+plugins {
+    id 'io.deephaven.project.register'
+}
diff --git a/docker/registry/manylinux2014_x86_64/gradle.properties b/docker/registry/manylinux2014_x86_64/gradle.properties
@@ -0,0 +1,4 @@
+io.deephaven.project.ProjectType=DOCKER_REGISTRY
+deephaven.registry.imageName=quay.io/pypa/manylinux2014_x86_64:latest
+deephaven.registry.imageId=quay.io/pypa/manylinux2014_x86_64@sha256:6f0a4ff6decf82b7663bef67fb9a9de5a5047878c670235f3e96041bf0edf206
+deephaven.registry.platform=linux/amd64
diff --git a/docker/registry/ubi-minimal/build.gradle b/docker/registry/ubi-minimal/build.gradle
@@ -0,0 +1,3 @@
+plugins {
+    id 'io.deephaven.project.register'
+}
diff --git a/docker/registry/ubi-minimal/gradle.properties b/docker/registry/ubi-minimal/gradle.properties
@@ -0,0 +1,3 @@
+io.deephaven.project.ProjectType=DOCKER_REGISTRY
+deephaven.registry.imageName=registry.access.redhat.com/ubi8/ubi-minimal:8.8
+deephaven.registry.imageId=registry.access.redhat.com/ubi8/ubi-minimal@sha256:b93deceb59a58588d5b16429fc47f98920f84740a1f2ed6454e33275f0701b59