From 94548c7be8cba97e1168e2225325cf97b44142d8 Mon Sep 17 00:00:00 2001 From: Jinyu-W <53509467+Jinyu-W@users.noreply.github.com> Date: Tue, 14 Mar 2023 18:36:48 +0800 Subject: [PATCH] Update pymaro build wheel logic (#587) * add libffi-dev, rust toolchain to linux build wheel * try updating build wheel * update python version in build wheel script * ignore install requirements in build wheel workflow * fix python versions in build wheel * Update build wheel action (#586) * avoid install unused package while building wheels * move source package to windows, as it will cause permission issue on linux * fix manylinux python version * remove duplicated definition --------- Co-authored-by: Jinyu Wang Co-authored-by: Chaos Yu --- .github/workflows/build_wheel.yml | 17 ++++++----- scripts/build_maro.bat | 20 ++++++------ scripts/build_maro.sh | 2 +- scripts/build_sdist.bat | 6 ++-- scripts/build_sdist.sh | 2 +- scripts/build_wheel.bat | 28 ++++++++--------- scripts/build_wheel.sh | 2 +- scripts/install_maro.bat | 34 ++++++++++----------- scripts/install_torch.bat | 2 +- scripts/run_tests.bat | 34 ++++++++++----------- scripts/run_tests.sh | 2 +- scripts/show_test_coverage_html.bat | 38 +++++++++++------------ setup.py | 47 +++++++++++++++-------------- 13 files changed, 120 insertions(+), 114 deletions(-) diff --git a/.github/workflows/build_wheel.yml b/.github/workflows/build_wheel.yml index f688b07a9..31cc8cce0 100644 --- a/.github/workflows/build_wheel.yml +++ b/.github/workflows/build_wheel.yml @@ -41,14 +41,22 @@ jobs: run: | python setup.py bdist_wheel + - name: Build source package on Windows + if: runner.os == 'Windows' && matrix.python-version == '3.7' + run: | + python setup.py sdist + - name: Build manylinux wheel if: runner.os == 'Linux' && matrix.python-version == '3.7' uses: RalfG/python-wheels-manylinux-build@v0.3.1-manylinux2010_x86_64 + env: + GITHUB_BUILD_ACTION: True with: - python-versions: 'cp37-cp37m cp38-cp38m cp39-cp39m' + python-versions: 'cp37-cp37m cp38-cp38 cp39-cp39' build-requirements: 'numpy' - pip-wheel-args: '-w ./wheelhouse' # save wheel packages to wheelhouse folder + system-packages: 'libffi-devel' pre-build-command: 'export LD_LIBRARY_PATH=/usr/local/lib/:$LD_LIBRARY_PATH' + pip-wheel-args: '-w ./wheelhouse' # save wheel packages to wheelhouse folder - name: Move valid packages to dist folder for manylinux if: runner.os == 'Linux' && matrix.python-version == '3.7' @@ -56,11 +64,6 @@ jobs: mkdir -p dist cp wheelhouse/pymaro-*-manylinux*.whl dist - - name: Build source package on linux - if: runner.os == 'Linux' && matrix.python-version == '3.7' - run: | - python setup.py sdist - - name: Upload linux wheel to artifact uses: actions/upload-artifact@v2 with: diff --git a/scripts/build_maro.bat b/scripts/build_maro.bat index fa0b2377c..e26d014d5 100644 --- a/scripts/build_maro.bat +++ b/scripts/build_maro.bat @@ -1,10 +1,10 @@ -@ECHO OFF - -rem script to build maro locally on Windows, usually for development - -chdir "%~dp0.." - -rem compile cython files -call scripts\compile_cython.bat - -python setup.py build_ext -i \ No newline at end of file +@ECHO OFF + +rem script to build maro locally on Windows, usually for development + +chdir "%~dp0.." + +rem compile cython files +call scripts\compile_cython.bat + +python setup.py build_ext -i diff --git a/scripts/build_maro.sh b/scripts/build_maro.sh index 63b3c16eb..827a1842c 100644 --- a/scripts/build_maro.sh +++ b/scripts/build_maro.sh @@ -10,4 +10,4 @@ fi # compile cython files first bash ./scripts/compile_cython.sh -python setup.py build_ext -i \ No newline at end of file +python setup.py build_ext -i diff --git a/scripts/build_sdist.bat b/scripts/build_sdist.bat index afd9535cd..808fed4e4 100644 --- a/scripts/build_sdist.bat +++ b/scripts/build_sdist.bat @@ -1,3 +1,3 @@ -chdir "%~dp0.." - -python setup.py sdist \ No newline at end of file +chdir "%~dp0.." + +python setup.py sdist diff --git a/scripts/build_sdist.sh b/scripts/build_sdist.sh index 8b150a43d..53daa1ddd 100644 --- a/scripts/build_sdist.sh +++ b/scripts/build_sdist.sh @@ -10,4 +10,4 @@ fi bash ./scripts/compile_cython.sh -python setup.py sdist \ No newline at end of file +python setup.py sdist diff --git a/scripts/build_wheel.bat b/scripts/build_wheel.bat index ca4d7e2ca..8543da312 100644 --- a/scripts/build_wheel.bat +++ b/scripts/build_wheel.bat @@ -1,14 +1,14 @@ - -rem script to build wheel package on Windows -rem NOTE: Before building the wheels, please make sure you have setup-up the environment. -rem for python 3.6/3.7 we need vs++14 - -chdir "%~dp0.." - -call scripts\compile_cython.bat - -pip install -r maro/requirements.build.txt -pip install wheel -pip install --upgrade setuptools - -python setup.py bdist_wheel \ No newline at end of file + +rem script to build wheel package on Windows +rem NOTE: Before building the wheels, please make sure you have setup-up the environment. +rem for python 3.6/3.7 we need vs++14 + +chdir "%~dp0.." + +call scripts\compile_cython.bat + +pip install -r maro/requirements.build.txt +pip install wheel +pip install --upgrade setuptools + +python setup.py bdist_wheel diff --git a/scripts/build_wheel.sh b/scripts/build_wheel.sh index 6b544235c..690f07339 100644 --- a/scripts/build_wheel.sh +++ b/scripts/build_wheel.sh @@ -8,7 +8,7 @@ set -e -x cd /maro # Compile wheels -for PYVER in 6 7; do +for PYVER in 7 8 9; do PYBIN="/opt/python/cp3${PYVER}-cp3${PYVER}m/bin" "${PYBIN}/pip" install -r maro/requirements.build.txt diff --git a/scripts/install_maro.bat b/scripts/install_maro.bat index da6ad328b..2863d9afd 100644 --- a/scripts/install_maro.bat +++ b/scripts/install_maro.bat @@ -1,17 +1,17 @@ -@ECHO OFF - -rem Script to install MARO in editable mode on Windows, -rem usually for development. - -chdir "%~dp0.." - -rem Install dependencies. -pip install -r .\maro\requirements.build.txt - -rem Compile cython files. -call .\scripts\compile_cython.bat - -call .\scripts\install_torch.bat - -rem Install MARO in editable mode. -pip install -e . \ No newline at end of file +@ECHO OFF + +rem Script to install MARO in editable mode on Windows, +rem usually for development. + +chdir "%~dp0.." + +rem Install dependencies. +pip install -r .\maro\requirements.build.txt + +rem Compile cython files. +call .\scripts\compile_cython.bat + +call .\scripts\install_torch.bat + +rem Install MARO in editable mode. +pip install -e . diff --git a/scripts/install_torch.bat b/scripts/install_torch.bat index c4ab3edcb..a511088c3 100644 --- a/scripts/install_torch.bat +++ b/scripts/install_torch.bat @@ -1 +1 @@ -pip install torch===1.6.0 torchvision===0.7.0 -f https://download.pytorch.org/whl/torch_stable.html \ No newline at end of file +pip install torch===1.6.0 torchvision===0.7.0 -f https://download.pytorch.org/whl/torch_stable.html diff --git a/scripts/run_tests.bat b/scripts/run_tests.bat index 8ef3fbac4..17c41ae52 100644 --- a/scripts/run_tests.bat +++ b/scripts/run_tests.bat @@ -1,17 +1,17 @@ -rem script to run all the test script under tests folder which the file name match test_xxxx.py - -chdir "%~dp0.." - -set "PYTHONPAH=." - -call scripts/build_maro.bat - -rem install requirements - -pip install -r ./tests/requirements.test.txt - -rem show coverage - -coverage run --rcfile=./tests/.coveragerc - -coverage report --rcfile=./tests/.coveragerc \ No newline at end of file +rem script to run all the test script under tests folder which the file name match test_xxxx.py + +chdir "%~dp0.." + +set "PYTHONPAH=." + +call scripts/build_maro.bat + +rem install requirements + +pip install -r ./tests/requirements.test.txt + +rem show coverage + +coverage run --rcfile=./tests/.coveragerc + +coverage report --rcfile=./tests/.coveragerc diff --git a/scripts/run_tests.sh b/scripts/run_tests.sh index 0208f8d23..cc9d6bed1 100644 --- a/scripts/run_tests.sh +++ b/scripts/run_tests.sh @@ -17,4 +17,4 @@ pip install -r ./tests/requirements.test.txt coverage run --rcfile=./tests/.coveragerc -coverage report --rcfile=./tests/.coveragerc \ No newline at end of file +coverage report --rcfile=./tests/.coveragerc diff --git a/scripts/show_test_coverage_html.bat b/scripts/show_test_coverage_html.bat index 2a709fbc6..f3836c3bf 100644 --- a/scripts/show_test_coverage_html.bat +++ b/scripts/show_test_coverage_html.bat @@ -1,19 +1,19 @@ - -chdir "%~dp0.." - -rem remove old htmlcov -@RD /S /Q htmlcov - -rem generate html - -coverage html - -rem host html - -cd htmlcov - -REM python -m http.server 8888 - -start python -m http.server 8888 - -start "" "http://localhost:8888" \ No newline at end of file + +chdir "%~dp0.." + +rem remove old htmlcov +@RD /S /Q htmlcov + +rem generate html + +coverage html + +rem host html + +cd htmlcov + +REM python -m http.server 8888 + +start python -m http.server 8888 + +start "" "http://localhost:8888" diff --git a/setup.py b/setup.py index efc1e7db2..d7832ff5e 100644 --- a/setup.py +++ b/setup.py @@ -25,7 +25,6 @@ # Set environment variable to skip deployment process of MARO os.environ["SKIP_DEPLOYMENT"] = "TRUE" - # root path to backend BASE_SRC_PATH = "./maro/backends" # backend module name @@ -51,7 +50,6 @@ ), ) - include_dirs.append(numpy.get_include()) extensions.append( @@ -84,9 +82,31 @@ ), ) -specific_requires = [] +# It is not necessary to install these packages when using manylinux action, as we only need numpy to build wheels +# NOTE: install following package will cause build error in current(2023-03-14) manylinux image +if "GITHUB_BUILD_ACTION" in os.environ: + install_requires = [] +else: + install_requires = [ + # TODO: use a helper function to collect these + "holidays>=0.10.3", + "numpy>=1.19.5", + "pandas>=0.25.3", + "paramiko>=2.9.2", + "ptvsd>=4.3.2", + "python_dateutil>=2.8.1", + "PyYAML>=5.4.1", + "pyzmq>=19.0.2", + "redis>=3.5.3", + "requests>=2.25.1", + "scipy>=1.7.0", + "tabulate>=0.8.5", + "torch>=1.6.0, <1.14.0", + "tornado>=6.1", + ] + if sys.version.startswith("3.6"): - specific_requires.append("dataclasses>=0.5") + install_requires.append("dataclasses>=0.5") readme = io.open("./maro/README.rst", encoding="utf-8").read() @@ -133,24 +153,7 @@ setup_requires=[ "numpy<1.20.0", ], - install_requires=[ - # TODO: use a helper function to collect these - "holidays>=0.10.3", - "numpy>=1.19.5", - "pandas>=0.25.3", - "paramiko>=2.9.2", - "ptvsd>=4.3.2", - "python_dateutil>=2.8.1", - "PyYAML>=5.4.1", - "pyzmq>=19.0.2", - "redis>=3.5.3", - "requests>=2.25.1", - "scipy>=1.7.0", - "tabulate>=0.8.5", - "torch>=1.6.0, <1.14.0", - "tornado>=6.1", - ] - + specific_requires, + install_requires=install_requires, entry_points={ "console_scripts": [ "maro=maro.cli.maro:main",