Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Update python versions for wheel files #1119

Closed
wants to merge 1 commit into from
Closed

Update python versions for wheel files #1119

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 7 additions & 7 deletions doap/doap_SINGA.rdf
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,10 +1,3 @@
<?xml version="1.0"?>
<?xml-stylesheet type="text/xsl"?>
<rdf:RDF xml:lang="en"
xmlns="http://usefulinc.com/ns/doap#"
xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
xmlns:asfext="http://projects.apache.org/ns/asfext#"
xmlns:foaf="http://xmlns.com/foaf/0.1/">
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
Expand All @@ -21,6 +14,13 @@
See the License for the specific language governing permissions and
limitations under the License.
-->
<?xml version="1.0"?>
<?xml-stylesheet type="text/xsl"?>
<rdf:RDF xml:lang="en"
xmlns="http://usefulinc.com/ns/doap#"
xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
xmlns:asfext="http://projects.apache.org/ns/asfext#"
xmlns:foaf="http://xmlns.com/foaf/0.1/">
<Project rdf:about="https://singa.apache.org/">
<created>2023-09-06</created>
<license rdf:resource="https://spdx.org/licenses/Apache-2.0" />
Expand Down
6 changes: 4 additions & 2 deletions java/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -90,15 +90,17 @@
<exclude>doc/_static/*.png</exclude>
<exclude>doc/_static/*.gif</exclude>
<exclude>doc/_static/*.ai</exclude>
<exclude>doc/_static/images/*.png</exclude>
<exclude>doc/_static/images/*.png</exclude>
<exclude>CITATION.cff</exclude>
<exclude>examples/model_selection/TRAILS-Database-Native-Model-Selection/requirement.txt</exclude>
<exclude>examples/model_selection/TRAILS-Database-Native-Model-Selection/documents/*.png</exclude>
<exclude>examples/model_selection/TRAILS-Database-Native-Model-Selection/internal/ml/model_selection/init_env</exclude>
<exclude>examples/model_selection/TRAILS-Database-Native-Model-Selection/internal/ml/model_selection/requirement.txt</exclude>
<exclude>examples/model_selection/TRAILS-Database-Native-Model-Selection/internal/pg_extension/Cargo.toml</exclude>
<exclude>examples/model_selection/TRAILS-Database-Native-Model-Selection/internal/pg_extension/pg_extension.control</exclude>
<exclude>examples/singa_easy/examples/data/SampleQuestion.json</exclude>
<exclude>examples/model_selection/TRAILS-Database-Native-Model-Selection/internal/ml/model_selection/documents/imgs/*.png</exclude>
<exclude>examples/model_selection/TRAILS-Database-Native-Model-Selection/internal/ml/model_selection/documents/imgs/*.png</exclude>
<exclude>examples/model_selection/TRAILS-Database-Native-Model-Selection/internal/pg_extension/.cargo/config.toml</exclude>
</excludes>
<consoleOutput>True</consoleOutput>
</configuration>
Expand Down
8 changes: 4 additions & 4 deletions setup.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -377,9 +377,9 @@ def build_extensions(self):
'License :: OSI Approved :: Apache Software License',
'Development Status :: 3 - Alpha',
'Intended Audience :: Developers',
'Programming Language :: Python :: 3.6',
'Programming Language :: Python :: 3.7',
'Programming Language :: Python :: 3.8',
'Programming Language :: Python :: 3.9',
'Programming Language :: Python :: 3.10',
'Programming Language :: Python :: 3.11',
'Topic :: Scientific/Engineering :: Artificial Intelligence'
]
if sys.platform == 'darwin':
Expand Down Expand Up @@ -420,7 +420,7 @@ def build_extensions(self):
python_requires='>=3',
install_requires=[
'numpy >=1.16,<2.0', #1.16
'onnx==1.6',
'onnx==1.15',
'deprecated',
'pytest',
'unittest-xml-reporting',
Expand Down
155 changes: 155 additions & 0 deletions tool/docker/devel/centos6/cuda10/Dockerfile.manylinux_2_28_x86_64
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,155 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.


# https://quay.io/repository/pypa/manylinux_2_28_x86_64?tab=tags
# https://github.com/pypa/manylinux/tree/main
# FROM quay.io/pypa/manylinux_2_28_x86_64:2023-10-15-e4d9465
FROM quay.io/pypa/manylinux_2_28_x86_64:2022-05-30-26ca994

# install dependencies
RUN yum install -y \
protobuf-devel \
openblas-devel \
# git \
wget \
openssh-server \
pcre-devel \
cmake \
rpm-build rpmdevtools \
&& yum clean all \
&& rm -rf /var/cache/yum/*

# install glog into /usr/local/include/glog /usr/local/lib
# RUN wget https://github.com/google/glog/archive/v0.3.5.tar.gz -P /tmp/\
# && tar zxf /tmp/v0.3.5.tar.gz -C /tmp/ \
# && cd /tmp/glog-0.3.5 \
# && ./configure && make && make install && cd .. && rm -rf glog-0.3.5

# install newer glog
RUN wget https://github.com/google/glog/archive/refs/tags/v0.4.0.tar.gz -P /tmp/\
&& tar zxf /tmp/v0.4.0.tar.gz -C /tmp/ \
&& cd /tmp/glog-0.4.0 \
&& ./autogen.sh && ./configure && make && make install && cd .. && rm -rf glog-0.4.0

# install dnnl into /usr/local/include /usr/local/lib
RUN wget https://github.com/intel/mkl-dnn/releases/download/v1.1/dnnl_lnx_1.1.0_cpu_gomp.tgz -P /tmp/ \
&& tar zxf /tmp/dnnl_lnx_1.1.0_cpu_gomp.tgz -C /tmp/ \
&& cp -r -H /tmp/dnnl_lnx_1.1.0_cpu_gomp/lib/lib* /usr/local/lib/ \
&& cp -r -H /tmp/dnnl_lnx_1.1.0_cpu_gomp/include/* /usr/local/include/ \
&& rm -rf /tmp/dnnl_lnx_1.1.0_cpu_gomp
# ENV DNNL_ROOT /root/dnnl_lnx_1.1.0_cpu_gomp/

# install swig into /usr/local/bin
RUN wget http://prdownloads.sourceforge.net/swig/swig-3.0.12.tar.gz -P /tmp/ \
&& tar zxf /tmp/swig-3.0.12.tar.gz -C /tmp/ \
&& cd /tmp/swig-3.0.12 && ./configure && make && make install && cd .. && rm -rf swig-3.0.12

# numpy and python versions should be matched;
# twine works for all python versions
RUN /opt/python/cp39-cp39/bin/pip install numpy twine
RUN /opt/python/cp310-cp310/bin/pip install numpy
RUN /opt/python/cp311-cp311/bin/pip install numpy

# install cuda and cudnn
# Refer to https://gitlab.com/nvidia/container-images/cuda/-/tree/master/dist for other cuda and cudnn versions
# 10.2-base-centos7
RUN NVIDIA_GPGKEY_SUM=d0664fbbdb8c32356d45de36c5984617217b2d0bef41b93ccecd326ba3b80c87 && \
curl -fsSL https://developer.download.nvidia.com/compute/cuda/repos/rhel7/x86_64/D42D0685.pub | sed '/^Version/d' > /etc/pki/rpm-gpg/RPM-GPG-KEY-NVIDIA && \
echo "$NVIDIA_GPGKEY_SUM /etc/pki/rpm-gpg/RPM-GPG-KEY-NVIDIA" | sha256sum -c --strict -
COPY cuda.repo /etc/yum.repos.d/cuda.repo
ENV CUDA_VERSION 10.2.89
ENV CUDA_PKG_VERSION 10-2-$CUDA_VERSION-1
# For libraries in the cuda-compat-* package: https://docs.nvidia.com/cuda/eula/index.html#attachment-a
RUN yum install -y \
cuda-cudart-$CUDA_PKG_VERSION \
cuda-compat-10-2 \
&& ln -s cuda-10.2 /usr/local/cuda && \
rm -rf /var/cache/yum/*

# nvidia-docker 1.0
RUN echo "/usr/local/nvidia/lib" >> /etc/ld.so.conf.d/nvidia.conf && \
echo "/usr/local/nvidia/lib64" >> /etc/ld.so.conf.d/nvidia.conf
ENV PATH /usr/local/nvidia/bin:/usr/local/cuda/bin:${PATH}
ENV LD_LIBRARY_PATH /usr/local/nvidia/lib:/usr/local/nvidia/lib64:$LD_LIBRARY_PATH

# nvidia-container-runtime
ENV NVIDIA_VISIBLE_DEVICES all
ENV NVIDIA_DRIVER_CAPABILITIES compute,utility
ENV NVIDIA_REQUIRE_CUDA "cuda>=10.2 brand=tesla,driver>=396,driver<397 brand=tesla,driver>=410,driver<411 brand=tesla,driver>=418,driver<419 brand=tesla,driver>=440,driver<441"

# 10.2-runtime-centos7
RUN yum install -y \
cuda-libraries-$CUDA_PKG_VERSION \
cuda-nvtx-$CUDA_PKG_VERSION \
libcublas10-10.2.2.89-1 \
&& rm -rf /var/cache/yum/*

# 10.2-devel-centos7
RUN yum install -y \
cuda-nvml-dev-$CUDA_PKG_VERSION \
cuda-command-line-tools-$CUDA_PKG_VERSION \
cuda-cudart-dev-$CUDA_PKG_VERSION \
cuda-libraries-dev-$CUDA_PKG_VERSION \
cuda-minimal-build-$CUDA_PKG_VERSION \
&& rm -rf /var/cache/yum/*
RUN yum install -y xz && NCCL_DOWNLOAD_SUM=a9ee790c3fc64b0ecbb00db92eddc1525552eda10a8656ff4b7380f66d81bda1 && \
curl -fsSL https://developer.download.nvidia.com/compute/redist/nccl/v2.7/nccl_2.7.3-1+cuda10.2_x86_64.txz -O && \
echo "$NCCL_DOWNLOAD_SUM nccl_2.7.3-1+cuda10.2_x86_64.txz" | sha256sum -c - && \
unxz nccl_2.7.3-1+cuda10.2_x86_64.txz && \
# tar --no-same-owner --keep-old-files --no-overwrite-dir -xvf nccl_2.7.3-1+cuda10.2_x86_64.tar -C /usr/local/cuda/include/ --strip-components=2 --wildcards '*/include/*' && \
tar --no-same-owner --keep-old-files -xvf nccl_2.7.3-1+cuda10.2_x86_64.tar -C /usr/local/cuda/include/ --strip-components=2 --wildcards '*/include/*' && \
# tar --no-same-owner --keep-old-files --no-overwrite-dir -xvf nccl_2.7.3-1+cuda10.2_x86_64.tar -C /usr/local/cuda/lib64/ --strip-components=2 --wildcards '*/lib/libnccl.so' && \
tar --no-same-owner --keep-old-files -xvf nccl_2.7.3-1+cuda10.2_x86_64.tar -C /usr/local/cuda/lib64/ --strip-components=2 --wildcards '*/lib/libnccl.so' && \

rm -f nccl_2.7.3-1+cuda10.2_x86_64.tar && \
ldconfig
ENV LIBRARY_PATH /usr/local/cuda/lib64/stubs

# 10.2-cudnn7-devel-centos7
ENV CUDNN_VERSION 7.6.5.32
# cuDNN license: https://developer.nvidia.com/cudnn/license_agreement
RUN CUDNN_DOWNLOAD_SUM=600267f2caaed2fd58eb214ba669d8ea35f396a7d19b94822e6b36f9f7088c20 && \
curl -fsSL http://developer.download.nvidia.com/compute/redist/cudnn/v7.6.5/cudnn-10.2-linux-x64-v7.6.5.32.tgz -O && \
echo "$CUDNN_DOWNLOAD_SUM cudnn-10.2-linux-x64-v7.6.5.32.tgz" | sha256sum -c - && \
tar --no-same-owner -xzf cudnn-10.2-linux-x64-v7.6.5.32.tgz -C /usr/local && \
rm cudnn-10.2-linux-x64-v7.6.5.32.tgz && \
ldconfig

# install gcc-8.5.0 for nccl
RUN dnf install gcc
ENV PATH /usr/bin:$PATH
RUN gcc --version

# install nccl for distributed training
RUN git clone https://github.com/NVIDIA/nccl.git $HOME/nccl \
&& cd $HOME/nccl \
&& git checkout v2.4.8-1 \
&& make BUILDDIR=/usr/local/ -j$(nproc) src.build \
&& rm -rf /usr/local/obj \
&& rm -rf $HOME/nccl

# install cnmem to /usr/local/include /usr/local/lib
RUN git clone https://github.com/NVIDIA/cnmem.git cnmem \
&& cd cnmem && mkdir build && cd build && cmake .. && make && make install && cd ../.. && rm -rf cnmem

# install mpich /usr/local/include /usr/local/lib
RUN wget http://www.mpich.org/static/downloads/3.3.2/mpich-3.3.2.tar.gz -P $HOME \
&& cd $HOME \
&& tar xfz mpich-3.3.2.tar.gz \
&& cd mpich-3.3.2 \
&& ./configure --prefix=/usr/local --disable-fortran \
&& make && make install && cd .. && rm -rf mpich-3.3.2
20 changes: 20 additions & 0 deletions tool/docker/devel/centos6/cuda10/Dockerfile.manylinux_2_28_x86_64_test_wheels
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
# Licensed to the Apache Software Foundation (ASF) under one
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

remove this file?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

ok, I will remove this one

# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.


# https://quay.io/repository/pypa/manylinux_2_28_x86_64?tab=tags
# https://github.com/pypa/manylinux/tree/main
FROM quay.io/pypa/manylinux_2_28_x86_64:2022-05-30-26ca994
14 changes: 7 additions & 7 deletions tool/wheel.sh
100644 → 100755
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,20 +24,20 @@ rm -rf dist

# build cpu only wheel packages
rm -rf build
/opt/python/cp36-cp36m/bin/python setup.py bdist_wheel
/opt/python/cp39-cp39/bin/python setup.py bdist_wheel
rm -rf build
/opt/python/cp37-cp37m/bin/python setup.py bdist_wheel
/opt/python/cp310-cp310/bin/python setup.py bdist_wheel
rm -rf build
/opt/python/cp38-cp38/bin/python setup.py bdist_wheel
/opt/python/cp311-cp311/bin/python setup.py bdist_wheel

# build cuda enabled wheel packages
export SINGA_CUDA=ON
rm -rf build
/opt/python/cp36-cp36m/bin/python setup.py bdist_wheel
/opt/python/cp39-cp39/bin/python setup.py bdist_wheel
rm -rf build
/opt/python/cp37-cp37m/bin/python setup.py bdist_wheel
/opt/python/cp310-cp310/bin/python setup.py bdist_wheel
rm -rf build
/opt/python/cp38-cp38/bin/python setup.py bdist_wheel
/opt/python/cp311-cp311/bin/python setup.py bdist_wheel

# repair the wheel files in dist/*.whl and store the results into wheelhouse/
/opt/python/cp38-cp38/bin/python setup.py audit
/opt/python/cp311-cp311/bin/python setup.py audit
Loading