From 278fc011d77ba8740921bb65f54f23ace2bc1dd1 Mon Sep 17 00:00:00 2001 From: zhaojing Date: Wed, 1 Nov 2023 15:49:08 +0800 Subject: [PATCH] Update python versions for wheel files --- doap/doap_SINGA.rdf | 14 +- java/pom.xml | 6 +- setup.py | 8 +- .../cuda10/Dockerfile.manylinux_2_28_x86_64 | 155 ++++++++++++++++++ ...ckerfile.manylinux_2_28_x86_64_test_wheels | 20 +++ tool/wheel.sh | 14 +- 6 files changed, 197 insertions(+), 20 deletions(-) create mode 100644 tool/docker/devel/centos6/cuda10/Dockerfile.manylinux_2_28_x86_64 create mode 100644 tool/docker/devel/centos6/cuda10/Dockerfile.manylinux_2_28_x86_64_test_wheels mode change 100644 => 100755 tool/wheel.sh diff --git a/doap/doap_SINGA.rdf b/doap/doap_SINGA.rdf index 9203d50091..47a6572e35 100644 --- a/doap/doap_SINGA.rdf +++ b/doap/doap_SINGA.rdf @@ -1,10 +1,3 @@ - - - + + + 2023-09-06 diff --git a/java/pom.xml b/java/pom.xml index 6ff9568537..32d223cfc0 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -90,7 +90,8 @@ doc/_static/*.png doc/_static/*.gif doc/_static/*.ai - doc/_static/images/*.png + doc/_static/images/*.png + CITATION.cff examples/model_selection/TRAILS-Database-Native-Model-Selection/requirement.txt examples/model_selection/TRAILS-Database-Native-Model-Selection/documents/*.png examples/model_selection/TRAILS-Database-Native-Model-Selection/internal/ml/model_selection/init_env @@ -98,7 +99,8 @@ examples/model_selection/TRAILS-Database-Native-Model-Selection/internal/pg_extension/Cargo.toml examples/model_selection/TRAILS-Database-Native-Model-Selection/internal/pg_extension/pg_extension.control examples/singa_easy/examples/data/SampleQuestion.json - examples/model_selection/TRAILS-Database-Native-Model-Selection/internal/ml/model_selection/documents/imgs/*.png + examples/model_selection/TRAILS-Database-Native-Model-Selection/internal/ml/model_selection/documents/imgs/*.png + examples/model_selection/TRAILS-Database-Native-Model-Selection/internal/pg_extension/.cargo/config.toml True diff --git a/setup.py b/setup.py index cfd87d6114..b4c01e419d 100644 --- a/setup.py +++ b/setup.py @@ -377,9 +377,9 @@ def build_extensions(self): 'License :: OSI Approved :: Apache Software License', 'Development Status :: 3 - Alpha', 'Intended Audience :: Developers', - 'Programming Language :: Python :: 3.6', - 'Programming Language :: Python :: 3.7', - 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', + 'Programming Language :: Python :: 3.10', + 'Programming Language :: Python :: 3.11', 'Topic :: Scientific/Engineering :: Artificial Intelligence' ] if sys.platform == 'darwin': @@ -420,7 +420,7 @@ def build_extensions(self): python_requires='>=3', install_requires=[ 'numpy >=1.16,<2.0', #1.16 - 'onnx==1.6', + 'onnx==1.15', 'deprecated', 'pytest', 'unittest-xml-reporting', diff --git a/tool/docker/devel/centos6/cuda10/Dockerfile.manylinux_2_28_x86_64 b/tool/docker/devel/centos6/cuda10/Dockerfile.manylinux_2_28_x86_64 new file mode 100644 index 0000000000..74ad49306b --- /dev/null +++ b/tool/docker/devel/centos6/cuda10/Dockerfile.manylinux_2_28_x86_64 @@ -0,0 +1,155 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# https://quay.io/repository/pypa/manylinux_2_28_x86_64?tab=tags +# https://github.com/pypa/manylinux/tree/main +# FROM quay.io/pypa/manylinux_2_28_x86_64:2023-10-15-e4d9465 +FROM quay.io/pypa/manylinux_2_28_x86_64:2022-05-30-26ca994 + +# install dependencies +RUN yum install -y \ + protobuf-devel \ + openblas-devel \ + # git \ + wget \ + openssh-server \ + pcre-devel \ + cmake \ + rpm-build rpmdevtools \ + && yum clean all \ + && rm -rf /var/cache/yum/* + +# install glog into /usr/local/include/glog /usr/local/lib +# RUN wget https://github.com/google/glog/archive/v0.3.5.tar.gz -P /tmp/\ +# && tar zxf /tmp/v0.3.5.tar.gz -C /tmp/ \ +# && cd /tmp/glog-0.3.5 \ +# && ./configure && make && make install && cd .. && rm -rf glog-0.3.5 + +# install newer glog +RUN wget https://github.com/google/glog/archive/refs/tags/v0.4.0.tar.gz -P /tmp/\ + && tar zxf /tmp/v0.4.0.tar.gz -C /tmp/ \ + && cd /tmp/glog-0.4.0 \ + && ./autogen.sh && ./configure && make && make install && cd .. && rm -rf glog-0.4.0 + +# install dnnl into /usr/local/include /usr/local/lib +RUN wget https://github.com/intel/mkl-dnn/releases/download/v1.1/dnnl_lnx_1.1.0_cpu_gomp.tgz -P /tmp/ \ + && tar zxf /tmp/dnnl_lnx_1.1.0_cpu_gomp.tgz -C /tmp/ \ + && cp -r -H /tmp/dnnl_lnx_1.1.0_cpu_gomp/lib/lib* /usr/local/lib/ \ + && cp -r -H /tmp/dnnl_lnx_1.1.0_cpu_gomp/include/* /usr/local/include/ \ + && rm -rf /tmp/dnnl_lnx_1.1.0_cpu_gomp +# ENV DNNL_ROOT /root/dnnl_lnx_1.1.0_cpu_gomp/ + +# install swig into /usr/local/bin +RUN wget http://prdownloads.sourceforge.net/swig/swig-3.0.12.tar.gz -P /tmp/ \ + && tar zxf /tmp/swig-3.0.12.tar.gz -C /tmp/ \ + && cd /tmp/swig-3.0.12 && ./configure && make && make install && cd .. && rm -rf swig-3.0.12 + +# numpy and python versions should be matched; +# twine works for all python versions +RUN /opt/python/cp39-cp39/bin/pip install numpy twine +RUN /opt/python/cp310-cp310/bin/pip install numpy +RUN /opt/python/cp311-cp311/bin/pip install numpy + +# install cuda and cudnn +# Refer to https://gitlab.com/nvidia/container-images/cuda/-/tree/master/dist for other cuda and cudnn versions +# 10.2-base-centos7 +RUN NVIDIA_GPGKEY_SUM=d0664fbbdb8c32356d45de36c5984617217b2d0bef41b93ccecd326ba3b80c87 && \ + curl -fsSL https://developer.download.nvidia.com/compute/cuda/repos/rhel7/x86_64/D42D0685.pub | sed '/^Version/d' > /etc/pki/rpm-gpg/RPM-GPG-KEY-NVIDIA && \ + echo "$NVIDIA_GPGKEY_SUM /etc/pki/rpm-gpg/RPM-GPG-KEY-NVIDIA" | sha256sum -c --strict - +COPY cuda.repo /etc/yum.repos.d/cuda.repo +ENV CUDA_VERSION 10.2.89 +ENV CUDA_PKG_VERSION 10-2-$CUDA_VERSION-1 +# For libraries in the cuda-compat-* package: https://docs.nvidia.com/cuda/eula/index.html#attachment-a +RUN yum install -y \ + cuda-cudart-$CUDA_PKG_VERSION \ + cuda-compat-10-2 \ + && ln -s cuda-10.2 /usr/local/cuda && \ + rm -rf /var/cache/yum/* + +# nvidia-docker 1.0 +RUN echo "/usr/local/nvidia/lib" >> /etc/ld.so.conf.d/nvidia.conf && \ + echo "/usr/local/nvidia/lib64" >> /etc/ld.so.conf.d/nvidia.conf +ENV PATH /usr/local/nvidia/bin:/usr/local/cuda/bin:${PATH} +ENV LD_LIBRARY_PATH /usr/local/nvidia/lib:/usr/local/nvidia/lib64:$LD_LIBRARY_PATH + +# nvidia-container-runtime +ENV NVIDIA_VISIBLE_DEVICES all +ENV NVIDIA_DRIVER_CAPABILITIES compute,utility +ENV NVIDIA_REQUIRE_CUDA "cuda>=10.2 brand=tesla,driver>=396,driver<397 brand=tesla,driver>=410,driver<411 brand=tesla,driver>=418,driver<419 brand=tesla,driver>=440,driver<441" + +# 10.2-runtime-centos7 +RUN yum install -y \ + cuda-libraries-$CUDA_PKG_VERSION \ + cuda-nvtx-$CUDA_PKG_VERSION \ + libcublas10-10.2.2.89-1 \ + && rm -rf /var/cache/yum/* + +# 10.2-devel-centos7 +RUN yum install -y \ + cuda-nvml-dev-$CUDA_PKG_VERSION \ + cuda-command-line-tools-$CUDA_PKG_VERSION \ + cuda-cudart-dev-$CUDA_PKG_VERSION \ + cuda-libraries-dev-$CUDA_PKG_VERSION \ + cuda-minimal-build-$CUDA_PKG_VERSION \ + && rm -rf /var/cache/yum/* +RUN yum install -y xz && NCCL_DOWNLOAD_SUM=a9ee790c3fc64b0ecbb00db92eddc1525552eda10a8656ff4b7380f66d81bda1 && \ + curl -fsSL https://developer.download.nvidia.com/compute/redist/nccl/v2.7/nccl_2.7.3-1+cuda10.2_x86_64.txz -O && \ + echo "$NCCL_DOWNLOAD_SUM nccl_2.7.3-1+cuda10.2_x86_64.txz" | sha256sum -c - && \ + unxz nccl_2.7.3-1+cuda10.2_x86_64.txz && \ + # tar --no-same-owner --keep-old-files --no-overwrite-dir -xvf nccl_2.7.3-1+cuda10.2_x86_64.tar -C /usr/local/cuda/include/ --strip-components=2 --wildcards '*/include/*' && \ + tar --no-same-owner --keep-old-files -xvf nccl_2.7.3-1+cuda10.2_x86_64.tar -C /usr/local/cuda/include/ --strip-components=2 --wildcards '*/include/*' && \ + # tar --no-same-owner --keep-old-files --no-overwrite-dir -xvf nccl_2.7.3-1+cuda10.2_x86_64.tar -C /usr/local/cuda/lib64/ --strip-components=2 --wildcards '*/lib/libnccl.so' && \ + tar --no-same-owner --keep-old-files -xvf nccl_2.7.3-1+cuda10.2_x86_64.tar -C /usr/local/cuda/lib64/ --strip-components=2 --wildcards '*/lib/libnccl.so' && \ + + rm -f nccl_2.7.3-1+cuda10.2_x86_64.tar && \ + ldconfig +ENV LIBRARY_PATH /usr/local/cuda/lib64/stubs + +# 10.2-cudnn7-devel-centos7 +ENV CUDNN_VERSION 7.6.5.32 +# cuDNN license: https://developer.nvidia.com/cudnn/license_agreement +RUN CUDNN_DOWNLOAD_SUM=600267f2caaed2fd58eb214ba669d8ea35f396a7d19b94822e6b36f9f7088c20 && \ + curl -fsSL http://developer.download.nvidia.com/compute/redist/cudnn/v7.6.5/cudnn-10.2-linux-x64-v7.6.5.32.tgz -O && \ + echo "$CUDNN_DOWNLOAD_SUM cudnn-10.2-linux-x64-v7.6.5.32.tgz" | sha256sum -c - && \ + tar --no-same-owner -xzf cudnn-10.2-linux-x64-v7.6.5.32.tgz -C /usr/local && \ + rm cudnn-10.2-linux-x64-v7.6.5.32.tgz && \ + ldconfig + +# install gcc-8.5.0 for nccl +RUN dnf install gcc +ENV PATH /usr/bin:$PATH +RUN gcc --version + +# install nccl for distributed training +RUN git clone https://github.com/NVIDIA/nccl.git $HOME/nccl \ + && cd $HOME/nccl \ + && git checkout v2.4.8-1 \ + && make BUILDDIR=/usr/local/ -j$(nproc) src.build \ + && rm -rf /usr/local/obj \ + && rm -rf $HOME/nccl + +# install cnmem to /usr/local/include /usr/local/lib +RUN git clone https://github.com/NVIDIA/cnmem.git cnmem \ + && cd cnmem && mkdir build && cd build && cmake .. && make && make install && cd ../.. && rm -rf cnmem + +# install mpich /usr/local/include /usr/local/lib +RUN wget http://www.mpich.org/static/downloads/3.3.2/mpich-3.3.2.tar.gz -P $HOME \ + && cd $HOME \ + && tar xfz mpich-3.3.2.tar.gz \ + && cd mpich-3.3.2 \ + && ./configure --prefix=/usr/local --disable-fortran \ + && make && make install && cd .. && rm -rf mpich-3.3.2 diff --git a/tool/docker/devel/centos6/cuda10/Dockerfile.manylinux_2_28_x86_64_test_wheels b/tool/docker/devel/centos6/cuda10/Dockerfile.manylinux_2_28_x86_64_test_wheels new file mode 100644 index 0000000000..afa72e0377 --- /dev/null +++ b/tool/docker/devel/centos6/cuda10/Dockerfile.manylinux_2_28_x86_64_test_wheels @@ -0,0 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# https://quay.io/repository/pypa/manylinux_2_28_x86_64?tab=tags +# https://github.com/pypa/manylinux/tree/main +FROM quay.io/pypa/manylinux_2_28_x86_64:2022-05-30-26ca994 diff --git a/tool/wheel.sh b/tool/wheel.sh old mode 100644 new mode 100755 index 10e419a80e..d54ab44e9b --- a/tool/wheel.sh +++ b/tool/wheel.sh @@ -24,20 +24,20 @@ rm -rf dist # build cpu only wheel packages rm -rf build -/opt/python/cp36-cp36m/bin/python setup.py bdist_wheel +/opt/python/cp39-cp39/bin/python setup.py bdist_wheel rm -rf build -/opt/python/cp37-cp37m/bin/python setup.py bdist_wheel +/opt/python/cp310-cp310/bin/python setup.py bdist_wheel rm -rf build -/opt/python/cp38-cp38/bin/python setup.py bdist_wheel +/opt/python/cp311-cp311/bin/python setup.py bdist_wheel # build cuda enabled wheel packages export SINGA_CUDA=ON rm -rf build -/opt/python/cp36-cp36m/bin/python setup.py bdist_wheel +/opt/python/cp39-cp39/bin/python setup.py bdist_wheel rm -rf build -/opt/python/cp37-cp37m/bin/python setup.py bdist_wheel +/opt/python/cp310-cp310/bin/python setup.py bdist_wheel rm -rf build -/opt/python/cp38-cp38/bin/python setup.py bdist_wheel +/opt/python/cp311-cp311/bin/python setup.py bdist_wheel # repair the wheel files in dist/*.whl and store the results into wheelhouse/ -/opt/python/cp38-cp38/bin/python setup.py audit \ No newline at end of file +/opt/python/cp311-cp311/bin/python setup.py audit