From 6c8b9ce3cf1a50c014d99036f78d28d32517f12a Mon Sep 17 00:00:00 2001 From: Cristian Ferretti <37232625+jcferretti@users.noreply.github.com> Date: Mon, 25 Mar 2024 16:01:04 -0400 Subject: [PATCH] Build py-ticking wheel using docker on top of a manylinux base image. (#5267) * Build py-ticking wheel using docker on top of a manylinux base image. * Make the python version configurable. * Move test to ubi8.8. * Add a task for building wheels for multiple python versions. * Use auditwheel to perfect the tags. * Followup to review comments. * More tweaks to quoting. * Added target to test all wheels. * Add py.typed. * Used docker/registry for the new images involved. * Follow image name convention. * Add the registry to imageName for manylinux. * Remove --force from pip install. * Add python 3.12 classifier to setup.py. * Added py.typed emtpy file. * Added registry to ubi8 image name. * Ensure py.typed is added in the image. --- cpp-client/README.md | 10 +- cpp-client/build.gradle | 73 +++++++-- cpp-client/deephaven/CMakeLists.txt | 44 +++--- docker/registry/fedora/build.gradle | 3 + docker/registry/fedora/gradle.properties | 3 + .../manylinux2014_x86_64/build.gradle | 3 + .../manylinux2014_x86_64/gradle.properties | 4 + docker/registry/ubi-minimal/build.gradle | 3 + docker/registry/ubi-minimal/gradle.properties | 3 + py/client-ticking/build.gradle | 138 ++++++++++++++---- py/client-ticking/setup.py | 5 +- .../src/pydeephaven_ticking/py.typed | 0 12 files changed, 226 insertions(+), 63 deletions(-) create mode 100644 docker/registry/fedora/build.gradle create mode 100644 docker/registry/fedora/gradle.properties create mode 100644 docker/registry/manylinux2014_x86_64/build.gradle create mode 100644 docker/registry/manylinux2014_x86_64/gradle.properties create mode 100644 docker/registry/ubi-minimal/build.gradle create mode 100644 docker/registry/ubi-minimal/gradle.properties create mode 100644 py/client-ticking/src/pydeephaven_ticking/py.typed diff --git a/cpp-client/README.md b/cpp-client/README.md index 4d389110303..bc3f555637e 100644 --- a/cpp-client/README.md +++ b/cpp-client/README.md @@ -47,10 +47,12 @@ on them anymore so we do notguarantee they are current for those platforms. Edit your local copy of the script if necessary to reflect your selection of build tools and build target; - defaults point to Ubuntu system's g++, cmake, and a Debug build target for cmake. - Comments in the script will help you in identifying customization points. - Note however that defaults are tested by any deviation from defaults may require - manual modification of other files later, when building the C++ client proper. + defaults point to Ubuntu system's g++, cmake, and a `RelWithDebInfo` build target + for cmake. + Comments in the script will help you identifying customization points. + Note however that defaults are tested, variations are not; + any deviation from defaults may require manual modification of other files later, + when building the C++ client proper. Example: ``` diff --git a/cpp-client/build.gradle b/cpp-client/build.gradle index 0b686c24faf..f393eaf4664 100644 --- a/cpp-client/build.gradle +++ b/cpp-client/build.gradle @@ -22,6 +22,7 @@ spotless { // image we will generate here as a base. // See https://github.com/deephaven/deephaven-base-images/tree/main/cpp-clients-multi evaluationDependsOn Docker.registryProject('cpp-clients-multi-base') +evaluationDependsOn Docker.registryProject('manylinux2014_x86_64') configurations { cpp {} @@ -57,6 +58,7 @@ deephavenDocker { } def prefix = '/opt/deephaven' +def build_type = 'RelWithDebInfo' def buildCppClientImage = Docker.registerDockerTask(project, 'cppClient') { // Only tested on x86-64, and we only build dependencies for x86-64 @@ -91,23 +93,23 @@ def buildCppClientImage = Docker.registerDockerTask(project, 'cppClient') { copyFile('deephaven/tests/', "${prefix}/src/deephaven/tests/") copyFile('cpp-tests-to-junit.sh', "${prefix}/bin/dhcpp") copyFile('build-dependencies.sh', "/tmp") - runCommand("PREFIX=${prefix}; " + + runCommand("PREFIX='${prefix}'; BUILD_TYPE='${build_type}'; " + '''set -eux; \\ - cmp /tmp/build-dependencies.sh ${PREFIX}/build-dependencies.sh; \\ + cmp /tmp/build-dependencies.sh "${PREFIX}/build-dependencies.sh"; \\ rm -f /tmp/build-dependencies.sh; \\ - rm -fr ${PREFIX}/src/deephaven/build; \\ - mkdir -p ${PREFIX}/src/deephaven/build; \\ - cd ${PREFIX}/src/deephaven/build; \\ - . ${PREFIX}/env.sh; \\ + rm -fr "${PREFIX}/src/deephaven/build"; \\ + mkdir -p "${PREFIX}/src/deephaven/build"; \\ + cd "${PREFIX}/src/deephaven/build"; \\ + . "${PREFIX}/env.sh"; \\ cmake \\ - -DCMAKE_INSTALL_PREFIX=${PREFIX} \\ - -DCMAKE_BUILD_TYPE=Release \\ + -DCMAKE_INSTALL_PREFIX="${PREFIX}" \\ + -DCMAKE_BUILD_TYPE="${BUILD_TYPE}" \\ -DBUILD_SHARED_LIBS=ON \\ .. ; \\ - VERBOSE=1 make -j${NCPUS} install 2>&1 | gzip > ${PREFIX}/log/make-install.log.gz; \\ - cp -f ${PREFIX}/src/deephaven/build/tests/tests ${PREFIX}/bin/dhcpp/tests; \\ + VERBOSE=1 make "-j${NCPUS}" install 2>&1 | gzip > "${PREFIX}/log/make-install.log.gz"; \\ + cp -f "${PREFIX}/src/deephaven/build/tests/tests" "${PREFIX}/bin/dhcpp/tests"; \\ cd ..; \\ - rm -fr ${PREFIX}/src/deephaven/build + rm -fr "${PREFIX}/src/deephaven/build" ''') // Note environment variables defined here are inherited by other images // using this image as a base ("from"). @@ -139,5 +141,54 @@ def testCppClient = Docker.registerDockerTask(project, 'testCppClient') { entrypoint = ["${prefix}/bin/dhcpp/cpp-tests-to-junit.sh", '/out/cpp-test.xml', '/out/cpp-test.log'] } +def buildCppClientPyImage = Docker.registerDockerTask(project, 'cppClientPy') { + // Uses a manylinux base image to support cython wheel building for multiple linux distros. + platform = 'linux/amd64' + copyIn { + from(layout.projectDirectory) { + include 'build-dependencies.sh' + include 'deephaven/CMakeLists.txt' + include 'deephaven/dhcore/**' + } + } + + dockerfile { + from('deephaven/manylinux2014_x86_64:local-build') + runCommand("""mkdir -p \\ + /out \\ + ${prefix} \\ + ${prefix}/bin/dhcpp \\ + ${prefix}/log + """) + copyFile('deephaven/CMakeLists.txt', "${prefix}/src/deephaven/") + copyFile('deephaven/dhcore/', "${prefix}/src/deephaven/dhcore/") + copyFile('build-dependencies.sh', "${prefix}") + runCommand("PREFIX='${prefix}'; BUILD_TYPE='${build_type}';" + + '''set -eux; \\ + cd "${PREFIX}"; \\ + PFX="${PREFIX}" BUILD_TYPE="${BUILD_TYPE}" ./build-dependencies.sh \\ + --static-pic immer env 2>&1 | gzip >build-dependencies.log.gz; \\ + mkdir -p "${PREFIX}/src/deephaven/build"; \\ + cd "${PREFIX}/src/deephaven/build"; \\ + . "${PREFIX}/env.sh"; \\ + cmake \\ + -DDHCORE_ONLY=ON \ + -DCMAKE_INSTALL_PREFIX="${PREFIX}" \ + -DCMAKE_BUILD_TYPE="${BUILD_TYPE}" \ + -DBUILD_SHARED_LIBS=ON \\ + .. ; \\ + VERBOSE=1 make -j"${NCPUS}" install 2>&1 | gzip >"${PREFIX}/log/make-install.log.gz"; \\ + cp dhcore/libdhcore_static.a "${PREFIX}/lib"; \\ + cd ..; \\ + rm -fr "${PREFIX}/src/deephaven/build" + ''') + // Note environment variables defined here are inherited by other images + // using this image as a base ("from"). + environmentVariable 'DH_PREFIX', prefix + environmentVariable 'LD_LIBRARY_PATH', "${prefix}/lib" + } + parentContainers = [ Docker.registryTask(project, 'manylinux2014_x86_64') ] +} + deephavenDocker.shouldLogIfTaskFails testCppClient tasks.check.dependsOn(testCppClient) diff --git a/cpp-client/deephaven/CMakeLists.txt b/cpp-client/deephaven/CMakeLists.txt index 9f615ecbaa4..df812f11a26 100644 --- a/cpp-client/deephaven/CMakeLists.txt +++ b/cpp-client/deephaven/CMakeLists.txt @@ -23,28 +23,34 @@ if(${SANITIZE_ADDRESS} STREQUAL "ON") link_libraries("-fsanitize=address") endif() -add_subdirectory(dhclient) -add_subdirectory(dhcore) -add_subdirectory(tests) -add_subdirectory(examples) +option(DHCORE_ONLY "Only build dhcore, skip rest" OFF) -install(DIRECTORY dhclient/include/public/ - DESTINATION ${CMAKE_INSTALL_INCLUDEDIR} - ) +add_subdirectory(dhcore) +if(NOT DHCORE_ONLY) + add_subdirectory(dhclient) + add_subdirectory(tests) + add_subdirectory(examples) +endif() install(DIRECTORY dhcore/include/public/ DESTINATION ${CMAKE_INSTALL_INCLUDEDIR} ) -install(TARGETS dhclient dhcore_static dhcore - EXPORT deephavenConfig - ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} - LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} - RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} - INCLUDES DESTINATION ${CMAKE_INSTALL_INCLUDEDIR} - ) - -install(EXPORT deephavenConfig - DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/deephaven - NAMESPACE deephaven:: - ) +if(NOT DHCORE_ONLY) + install(DIRECTORY dhclient/include/public/ + DESTINATION ${CMAKE_INSTALL_INCLUDEDIR} + ) + + install(TARGETS dhclient dhcore_static dhcore + EXPORT deephavenConfig + ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} + LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} + RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} + INCLUDES DESTINATION ${CMAKE_INSTALL_INCLUDEDIR} + ) + + install(EXPORT deephavenConfig + DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/deephaven + NAMESPACE deephaven:: + ) +endif() diff --git a/docker/registry/fedora/build.gradle b/docker/registry/fedora/build.gradle new file mode 100644 index 00000000000..a9bb1568cd2 --- /dev/null +++ b/docker/registry/fedora/build.gradle @@ -0,0 +1,3 @@ +plugins { + id 'io.deephaven.project.register' +} diff --git a/docker/registry/fedora/gradle.properties b/docker/registry/fedora/gradle.properties new file mode 100644 index 00000000000..f6e64071df6 --- /dev/null +++ b/docker/registry/fedora/gradle.properties @@ -0,0 +1,3 @@ +io.deephaven.project.ProjectType=DOCKER_REGISTRY +deephaven.registry.imageName=fedora:39 +deephaven.registry.imageId=fedora@sha256:61864fd19bbd64d620f338eb11dae9e8759bf7fa97302ac6c43865c48dccd679 diff --git a/docker/registry/manylinux2014_x86_64/build.gradle b/docker/registry/manylinux2014_x86_64/build.gradle new file mode 100644 index 00000000000..a9bb1568cd2 --- /dev/null +++ b/docker/registry/manylinux2014_x86_64/build.gradle @@ -0,0 +1,3 @@ +plugins { + id 'io.deephaven.project.register' +} diff --git a/docker/registry/manylinux2014_x86_64/gradle.properties b/docker/registry/manylinux2014_x86_64/gradle.properties new file mode 100644 index 00000000000..2e540be6214 --- /dev/null +++ b/docker/registry/manylinux2014_x86_64/gradle.properties @@ -0,0 +1,4 @@ +io.deephaven.project.ProjectType=DOCKER_REGISTRY +deephaven.registry.imageName=quay.io/pypa/manylinux2014_x86_64:latest +deephaven.registry.imageId=quay.io/pypa/manylinux2014_x86_64@sha256:6f0a4ff6decf82b7663bef67fb9a9de5a5047878c670235f3e96041bf0edf206 +deephaven.registry.platform=linux/amd64 diff --git a/docker/registry/ubi-minimal/build.gradle b/docker/registry/ubi-minimal/build.gradle new file mode 100644 index 00000000000..a9bb1568cd2 --- /dev/null +++ b/docker/registry/ubi-minimal/build.gradle @@ -0,0 +1,3 @@ +plugins { + id 'io.deephaven.project.register' +} diff --git a/docker/registry/ubi-minimal/gradle.properties b/docker/registry/ubi-minimal/gradle.properties new file mode 100644 index 00000000000..120beb2a191 --- /dev/null +++ b/docker/registry/ubi-minimal/gradle.properties @@ -0,0 +1,3 @@ +io.deephaven.project.ProjectType=DOCKER_REGISTRY +deephaven.registry.imageName=registry.access.redhat.com/ubi8/ubi-minimal:8.8 +deephaven.registry.imageId=registry.access.redhat.com/ubi8/ubi-minimal@sha256:b93deceb59a58588d5b16429fc47f98920f84740a1f2ed6454e33275f0701b59 diff --git a/py/client-ticking/build.gradle b/py/client-ticking/build.gradle index 0e815de1101..d56e35bac30 100644 --- a/py/client-ticking/build.gradle +++ b/py/client-ticking/build.gradle @@ -13,6 +13,8 @@ dependencies { } evaluationDependsOn(':cpp-client') +evaluationDependsOn Docker.registryProject('fedora') +evaluationDependsOn Docker.registryProject('ubi-minimal') def prefix = '/opt/deephaven' @@ -26,7 +28,7 @@ deephavenDocker { networkName.set "pydeephaven-network-${randomSuffix}" } -def buildPyClientTicking = Docker.registerDockerTask(project, 'pyClientTicking') { +def buildPyClientTickingManyLinux = { wheelsSet, taskName -> Docker.registerDockerTask(project, taskName) { // Only tested on x86-64, and we only build dependencies for x86-64 platform = 'linux/amd64' @@ -34,7 +36,10 @@ def buildPyClientTicking = Docker.registerDockerTask(project, 'pyClientTicking') from(layout.projectDirectory) { include 'setup.py' include 'README.md' - include 'src/**' + include 'src/**/py.typed' + include 'src/**/*.py' + include 'src/**/*.pyx' + include 'src/**/*.pxd' } from(configurations.pythonWheel) { into 'wheels' @@ -44,59 +49,109 @@ def buildPyClientTicking = Docker.registerDockerTask(project, 'pyClientTicking') into layout.buildDirectory.dir('wheel') } dockerfile { - from('deephaven/cpp-client:local-build') + from('deephaven/cpp-client-py:local-build') runCommand("""mkdir -p \\ /out \\ - ${prefix}/log \\ - ${prefix}/src/py-client-ticking/src \\ - ${prefix}/src/py-client-ticking/in-wheels + '${prefix}/log' \\ + '${prefix}/src/py-client-ticking/src' \\ + '${prefix}/src/py-client-ticking/in-wheels' """) copyFile('setup.py', "${prefix}/src/py-client-ticking") copyFile('README.md', "${prefix}/src/py-client-ticking") copyFile('src/', "${prefix}/src/py-client-ticking/src/") copyFile('wheels/', "${prefix}/src/py-client-ticking/in-wheels") - runCommand("PREFIX=${prefix}; DEEPHAVEN_VERSION=${project.version};" + + runCommand("PREFIX='${prefix}'; WHEELS_SET='${wheelsSet}'; DEEPHAVEN_VERSION='${project.version}';" + '''set -eux ; \ - cd "${PREFIX}/src/py-client-ticking"; \ - . "${PREFIX}/env.sh"; \ - MAKEFLAGS="-j${NCPUS}" \ - CFLAGS="-I${DHCPP}/include" \ - LDFLAGS="-L${DHCPP}/lib" \ - DEEPHAVEN_VERSION="${DEEPHAVEN_VERSION}" \ - python3 setup.py build_ext -i; \ - DEEPHAVEN_VERSION="${DEEPHAVEN_VERSION}" python3 setup.py bdist_wheel; \ - pip3 install in-wheels/*.whl; \ - pip3 install --force --no-deps dist/*.whl; \ - ln dist/*.whl /out; \ - cd /; \ - rm -fr "${PREFIX}/src/py-client-ticking" + cd "${PREFIX}/src/py-client-ticking"; \ + . "${PREFIX}/env.sh"; \ + ORIG_PATH="$PATH"; \ + for spec in ${WHEELS_SET}; do \ + tag=`echo "$spec" | cut -d: -f 2`; \ + rm -f *.cpp *.so; \ + PATH="/opt/python/${tag}/bin:$ORIG_PATH"; \ + pip3 install cython; \ + MAKEFLAGS="-j${NCPUS}" \ + CFLAGS="-I${DHCPP}/include" \ + LDFLAGS="-L${DHCPP}/lib" \ + DEEPHAVEN_VERSION="${DEEPHAVEN_VERSION}" \ + python3 setup.py build_ext -i; \ + DEEPHAVEN_VERSION="${DEEPHAVEN_VERSION}" python3 setup.py bdist_wheel; \ + auditwheel repair dist/pydeephaven_ticking*"${tag}"*.whl; \ + rm -f dist/pydeephaven_ticking*"${tag}"*.whl; \ + mv -f wheelhouse/*.whl dist; \ + pip3 install in-wheels/*.whl; \ + pip3 install --no-deps dist/pydeephaven_ticking*"${tag}"*.whl; \ + done; \ + ln dist/*.whl /out; \ + cd /; \ + rm -fr "${PREFIX}/src/py-client-ticking" ''') environmentVariable 'LD_LIBRARY_PATH', "" // avoid conflict with libarrow.13.0.0.so } - parentContainers = [ project.tasks.getByPath(':cpp-client:cppClient') ] -} + parentContainers = [ project.tasks.getByPath(':cpp-client:cppClientPy') ] +}} + +def checksWheelSet = '3.9:cp39-cp39' +def buildPyClientTicking = buildPyClientTickingManyLinux(checksWheelSet, 'pyClientTicking') -def testPyClientTicking = Docker.registerDockerTask(project, 'testPyClientTicking') { +def testPyClientTickingManyLinux = { wheelsSet, taskName, parentContainer, image -> Docker.registerDockerTask(project, taskName ) { // Only tested on x86-64, and we only build dependencies for x86-64 platform = 'linux/amd64' copyIn { + from(configurations.pythonWheel) { + into 'dep-wheels' + } from(layout.projectDirectory) { include 'tests/**' } + from(layout.buildDirectory.dir('wheel')) { + into 'pyt-wheels' + } } dockerfile { - from('deephaven/py-client-ticking:local-build') - runCommand("PREFIX=${prefix}; " + + from("deephaven/${image}:local-build") + runCommand("WHEELS_SET='${wheelsSet}'; PREFIX='${prefix}'; " + '''set -eux ; \ + DNF=`type microdnf >/dev/null 2>&1 && echo 'microdnf --disableplugin=subscription-manager' || echo 'dnf -q'`; \ + $DNF -y update; \ + $DNF -y install python3; \ + $DNF -y install python3-pip; \ + $DNF -y install python3-virtualenv; \ + for spec in ${WHEELS_SET}; do \ + pyver=`echo "$spec" | cut -d: -f 1`; \ + $DNF -y install "python${pyver}"; \ + "python$pyver" -m venv "/project/$pyver"; \ + source "/project/$pyver/bin/activate"; \ + pip install --upgrade pip; \ + pip install unittest-xml-reporting; \ + deactivate; \ + done; \ + $DNF clean all; \ rm -fr /out; \ mkdir -p \ /out/report \ /project/tests ''') copyFile('tests/', "/project/tests/") + copyFile('dep-wheels/', "/project/dep-wheels") + copyFile('pyt-wheels/', "/project/pyt-wheels") workingDir('/project') + runCommand("WHEELS_SET='${wheelsSet}'; " + + '''set -eux ; \ + for spec in ${WHEELS_SET}; do \ + pyver=`echo "$spec" | cut -d: -f 1`; \ + tag=`echo "$spec" | cut -d: -f 2`; \ + [ -f /project/pyt-wheels/pydeephaven_ticking*"${tag}"*.whl ]; \ + source "/project/$pyver/bin/activate"; \ + pip install unittest-xml-reporting; \ + pip install /project/dep-wheels/*.whl; \ + pip install /project/pyt-wheels/pydeephaven_ticking*"${tag}"*.whl; \ + deactivate; \ + done + ''') + // // Setup for test run. // @@ -106,12 +161,39 @@ def testPyClientTicking = Docker.registerDockerTask(project, 'testPyClientTickin containerDependencies.dependsOn = [deephavenDocker.healthyTask] containerDependencies.finalizedBy = deephavenDocker.endTask network = deephavenDocker.networkName.get() - parentContainers = [ project.tasks.getByName('pyClientTicking') ] - entrypoint = ['python3', '-m', 'xmlrunner', 'discover', 'tests', '-v', '-o', '/out/report'] + parentContainers = [ project.tasks.getByName(parentContainer), + Docker.registryTask(project, "${image}") ] + entrypoint = ['bash', '-c', + "WHEELS_SET='${wheelsSet}'; " + + ''' + for spec in ${WHEELS_SET}; do \ + pyver=`echo "$spec" | cut -d: -f 1`; \ + source "/project/$pyver/bin/activate"; \ + "python$pyver" -m xmlrunner discover tests -v -o "/out/report/$pyver"; \ + deactivate + done + '''] copyOut { into layout.buildDirectory.dir('test-results') } -} +}} + +def testPyClientTicking = testPyClientTickingManyLinux( + checksWheelSet, + 'testPyClientTicking', + 'pyClientTicking', + 'ubi-minimal') + +def wheelsSet = [ '3.8:cp38-cp38', '3.9:cp39-cp39', '3.10:cp310-cp310', '3.11:cp311-cp311', '3.12:cp312-cp312' ] +def pyClientTickingAllWheels = buildPyClientTickingManyLinux(String.join(' ', wheelsSet), 'pyClientTickingAllWheels') + +def testPyClientTickingAllWheels = testPyClientTickingManyLinux( + String.join(' ', wheelsSet), + 'testPyClientTickingAllWheels', + 'pyClientTickingAllWheels', + 'fedora') tasks.getByName('check').dependsOn(testPyClientTicking) +tasks.getByName('testPyClientTickingPrepareDocker').dependsOn(pyClientTicking) +tasks.getByName('testPyClientTickingAllWheelsPrepareDocker').dependsOn(pyClientTickingAllWheels) deephavenDocker.shouldLogIfTaskFails testPyClientTicking diff --git a/py/client-ticking/setup.py b/py/client-ticking/setup.py index 2011447da10..e5d4f402b12 100644 --- a/py/client-ticking/setup.py +++ b/py/client-ticking/setup.py @@ -54,14 +54,17 @@ def _compute_version(): 'Programming Language :: Python :: 3.9', 'Programming Language :: Python :: 3.10', 'Programming Language :: Python :: 3.11', + 'Programming Language :: Python :: 3.12', ], ext_modules = cythonize( [Extension("pydeephaven_ticking._core", sources=["src/pydeephaven_ticking/*.pyx"], + language="c++", extra_compile_args=["-std=c++17"], libraries=["dhcore_static"] )]), python_requires='>=3.8', - install_requires=[f"pydeephaven=={_version}"] + install_requires=[f"pydeephaven=={_version}"], + package_data={'pydeephaven_ticking': ['py.typed']} ) diff --git a/py/client-ticking/src/pydeephaven_ticking/py.typed b/py/client-ticking/src/pydeephaven_ticking/py.typed new file mode 100644 index 00000000000..e69de29bb2d