From b465ee3dfa1df4819c4be9bb3e62e531873a779b Mon Sep 17 00:00:00 2001 From: Aaron Smith Date: Tue, 23 Aug 2022 06:55:51 -0500 Subject: [PATCH] [MRG] Add M1 mac builds and fix nightly build (#513) * Add cross-compile helper from scikit * Fix lint * Try cibuildwheel * Add CIBW_BUILD env var * Use CIBW_BEFORE_BUILD * Skip 32-bit on windows * Swap '>' to '|' for yaml * Use '>' and '&&' * Add CIBW_BEFORE_TEST * Reinstall deps before testing * Cleanup * Use PYTHON_CROSSENV for mac builds * Skip musl builds * Fix format step? * Fix README check * Customize job names * Remove 'Python' from job names * Move 'make version' to CIBW_BEFORE_ALL * Update nightly to use cibuildwheel * Fix which versions we build * Don't use DEPENDENCIES env var * Skip musllinux builds * Use batch for windows * Try all caps? * Single percent * Try quotes? * Use pip instead of python -m * Back to 2 percents * Just try a bash script * Add slack notifications back * Remove architecture from slack message * Update whats new --- .github/utils/build_and_test_aarch64.sh | 34 ----- .github/workflows/build_and_deploy.yml | 139 +++++++-------------- .github/workflows/nightly_cron.yml | 94 +++++--------- doc/whats_new.rst | 5 + pmdarima/_build_utils/pre_build_helpers.py | 11 +- 5 files changed, 86 insertions(+), 197 deletions(-) delete mode 100644 .github/utils/build_and_test_aarch64.sh diff --git a/.github/utils/build_and_test_aarch64.sh b/.github/utils/build_and_test_aarch64.sh deleted file mode 100644 index 94cd07a95..000000000 --- a/.github/utils/build_and_test_aarch64.sh +++ /dev/null @@ -1,34 +0,0 @@ -#!/bin/bash - -PYTHON=$1 - -# Set up virtual env -$PYTHON -m pip install virtualenv -$PYTHON -m venv .env -source .env/bin/activate - -# Install requirements -pip install --upgrade pip wheel -pip install -r build_tools/build_requirements.txt -pip install -r requirements.txt - -# Make our wheel -make version bdist_wheel - -# Audit our wheel -pip install auditwheel -for whl in dist/*.whl; do - if [[ "$whl" =~ "pmdarima" ]]; then - auditwheel repair "$whl" -w dist/ # repair pmdarima wheel and output to dist/ - fi - rm "$whl" # remove original wheel -done - -pip install --pre --no-index --find-links dist/ pmdarima - -# Can't be in the top-level directory for import or it will fail -cd .github - -# Testing on aarch64 takes too long, so we simply import the package as a spot test -# Don't use $PYTHON here because we are in a virtual env -python -c 'import pmdarima; pmdarima.show_versions()' diff --git a/.github/workflows/build_and_deploy.yml b/.github/workflows/build_and_deploy.yml index 3b050b7e6..efa129610 100644 --- a/.github/workflows/build_and_deploy.yml +++ b/.github/workflows/build_and_deploy.yml @@ -18,16 +18,19 @@ concurrency: jobs: build-and-deploy: - name: Build and Deploy strategy: matrix: - os: [windows-latest, macos-latest] - python-version: ['3.7', '3.8', '3.9', '3.10'] + python3-minor-version: [7, 8, 9, 10] + os: [macos-latest, ubuntu-latest, windows-latest] defaults: run: shell: bash + env: + PYTHON_VERSION: ${{ format('3.{0}', matrix.python3-minor-version) }} + PYTHON_EXECUTABLE: ${{ format('cp3{0}', matrix.python3-minor-version) }} runs-on: ${{ matrix.os }} + name: Build and Deploy (${{ matrix.os }}, 3.${{ matrix.python3-minor-version }}) steps: # This LOOKS like it is checking out 'master', but it is using the 'master' version of the checkout action @@ -35,62 +38,56 @@ jobs: - name: Checkout uses: actions/checkout@master + # Python interpreter used by cibuildwheel, but it uses a different one internally - name: Setting up Python - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python-version }} - - # https://github.com/actions/cache/blob/master/examples.md#multiple-oss-in-a-workflow - - name: Checking for cached pip dependencies (macOS) - if: startsWith(runner.os, 'macOS') - uses: actions/cache@v1 + uses: actions/setup-python@v3 with: - path: ~/Library/Caches/pip - key: ${{ runner.os }}-pip-${{ hashFiles('requirements.txt') }} + python-version: ${{ env.PYTHON_VERSION }} - - name: Checking for cached pip dependencies (Windows) - if: startsWith(runner.os, 'Windows') - uses: actions/cache@v1 + - name: Set up QEMU + if: runner.os == 'Linux' + uses: docker/setup-qemu-action@v1 with: - path: ~\AppData\Local\pip\Cache - key: ${{ runner.os }}-pip-${{ hashFiles('requirements.txt') }} - - - name: Updating pip - run: python -m pip install --upgrade pip - - - name: Installing requirements - run: | - pip install -r build_tools/build_requirements.txt - pip install -r requirements.txt + platforms: all # We build the source archive separately because of this: https://github.com/alkaline-ml/pmdarima/pull/136#discussion_r279781731 # We build it first because of this: https://github.com/alkaline-ml/pmdarima/issues/448 - name: Building source distribution run: make version sdist - - name: Building wheel - run: make version bdist_wheel - - - name: Installing generated wheel - run: pip install --pre --no-index --find-links dist/ pmdarima - - - name: Running unit tests - run: | - if [ "${{ matrix.os }}" == "macos-latest" ]; then - export PMD_MPL_BACKEND=TkAgg - fi - pytest --showlocals --durations=20 --pyargs pmdarima + - name: Install cibuildwheel + run: python -m pip install cibuildwheel==2.9.0 # TODO: Do we want this pinned? - - name: Checking for numpy regression - run: | - pip install --upgrade numpy - if [ "${{ matrix.os }}" == "macos-latest" ]; then - export PMD_MPL_BACKEND=TkAgg - fi - pytest --showlocals --durations=20 --pyargs pmdarima + - name: Building and testing wheels + run: python -m cibuildwheel --output-dir dist + env: + # TODO: Move Linux x86_64 builds to GHA? + CIBW_ARCHS_LINUX: "aarch64" + CIBW_ARCHS_MACOS: "x86_64 arm64" + CIBW_ARCHS_WINDOWS: "AMD64" + CIBW_BEFORE_ALL: make version + CIBW_BEFORE_BUILD: > + pip install -r build_tools/build_requirements.txt && + pip install -r requirements.txt + # Tests are run in a separate virtual env, so we need to re-install deps + CIBW_BEFORE_TEST: > + pip install -r build_tools/build_requirements.txt && + pip install -r requirements.txt + CIBW_BUILD: "${{ env.PYTHON_EXECUTABLE }}-*" + CIBW_ENVIRONMENT_MACOS: PMD_MPL_BACKEND=TkAgg PYTHON_CROSSENV=true + # No support for pypy or musl + CIBW_SKIP: "pp* *-musllinux_*" + # Runs tests and checks for a numpy regression by upgrading numpy and running tests again + CIBW_TEST_COMMAND: > + pytest --showlocals --durations=20 --pyargs pmdarima && + pip install --upgrade numpy && + pytest --showlocals --durations=20 --pyargs pmdarima + # Avoid testing on emulated architectures + CIBW_TEST_SKIP: "*-*linux_{aarch64,ppc64le,s390x} *-macosx_arm64" - name: Checking README compatibility run: | + python -m pip install "twine>=1.13.0" readme_renderer if python -c "from twine.commands.check import check; check(['dist/*'])" | grep "warning"; then echo "README will not render properly on PyPI" exit 1 @@ -115,57 +112,7 @@ jobs: - name: Ensuring sdist can be installed run: pip install dist/$(ls dist | grep tar) - - name: Ensuring VERSION file existis - id: version_check # Need this to refer to output below - run: | - if [ -f "${GITHUB_WORKSPACE}/pmdarima/VERSION" ]; then - echo "VERSION file exists" - echo "::set-output name=version_exists::true" - else - echo "VERSION file does not exist" - echo "::set-output name=version_exists::false" - fi - - - name: Deploying to PyPI - # Only deploy on tags and when VERSION file created - if: startsWith(github.ref, 'refs/tags') && success() && steps.version_check.outputs.version_exists == 'true' - run: | - chmod +x build_tools/github/deploy.sh - ./build_tools/github/deploy.sh - env: - TWINE_USERNAME: ${{ secrets.TWINE_USERNAME }} - TWINE_PASSWORD: ${{ secrets.TWINE_PASSWORD }} - - build-and-deploy-aarch64: - name: 'Build and Deploy (aarch64)' - runs-on: ubuntu-latest - strategy: - matrix: - python-version: [cp37-cp37m, cp38-cp38, cp39-cp39, cp310-cp310] - env: - python: /opt/python/${{ matrix.python-version }}/bin/python - image: quay.io/pypa/manylinux_2_28_aarch64 - defaults: - run: - shell: bash - - steps: - - name: Checkout - uses: actions/checkout@master - - - name: Setup QEMU - uses: docker/setup-qemu-action@v1 - - - name: Build and Test - run: | - chmod +x .github/utils/build_and_test_aarch64.sh - docker run --rm \ - -v ${{ github.workspace }}:/workspace \ - --workdir=/workspace \ - ${{ env.image }} \ - .github/utils/build_and_test_aarch64.sh ${{ env.python }} - - - name: Ensuring VERSION file existis + - name: Ensuring VERSION file exists id: version_check # Need this to refer to output below run: | if [ -f "${GITHUB_WORKSPACE}/pmdarima/VERSION" ]; then diff --git a/.github/workflows/nightly_cron.yml b/.github/workflows/nightly_cron.yml index 22f87d01f..21aa60e6a 100644 --- a/.github/workflows/nightly_cron.yml +++ b/.github/workflows/nightly_cron.yml @@ -24,15 +24,11 @@ jobs: fail-fast: false matrix: os: [windows-latest, macos-latest, ubuntu-latest] - python-version: ['3.9'] - architecture: ['x86', 'x64'] - exclude: - # Don't build 32-bit on Mac or Linux - - os: macos-latest - architecture: x86 - - - os: ubuntu-latest - architecture: x86 + python-version: ['3.10'] + python-executable: ['cp310'] + defaults: + run: + shell: bash runs-on: ${{ matrix.os }} @@ -44,19 +40,13 @@ jobs: uses: actions/checkout@master - name: Setting up Python - uses: actions/setup-python@v2 + uses: actions/setup-python@v3 with: python-version: ${{ matrix.python-version }} - architecture: ${{ matrix.architecture }} - name: Set Run ID id: definition # Needed to retrieve the output of this step later run: echo "::set-output name=run_id::$GITHUB_RUN_ID" - shell: bash - - - name: Updating pip - run: python -m pip install --upgrade pip - shell: bash - name: Collecting naked dependencies id: dependencies # Needed to retrieve the output of this step later @@ -65,60 +55,43 @@ jobs: echo "::set-output name=latest_dependencies::$dependencies" shell: bash - # https://github.com/actions/cache/blob/master/examples.md#using-pip-to-get-cache-location - - name: Get pip cache dir - id: pip-cache-dir-path - run: echo "::set-output name=dir::$(pip cache dir)" + - name: Install cibuildwheel + run: python -m pip install cibuildwheel==2.9.0 # TODO: Do we want this pinned? - - name: Generating cache key file + - name: Generating dependency table id: dependency-table # Needed to set output of job run: | pip install requests tabulate table=$(python .github/utils/get_dependency_releases.py $DEPENDENCIES) - echo $table > dependencies_key.txt # This is used in the next job (if necessary) rather than re-running the above echo "::set-output name=table::$table" - shell: bash env: DEPENDENCIES: ${{ steps.dependencies.outputs.latest_dependencies }} - - name: Check for cache hit - id: pip-cache - uses: actions/cache@v2 - with: - path: ${{ steps.pip-cache-dir-path.outputs.dir }} - key: ${{ runner.os }}-${{ matrix.python-version }}-pip-${{ hashFiles('**/dependencies_key.txt') }} - - # We install one-by-one because some dependencies depend on others - - name: Installing dependencies - if: steps.pip-cache.outputs.cache-hit != 'true' - run: | - for dependency in $DEPENDENCIES; do - python -m pip install $dependency - done - shell: bash + - name: Building and testing wheel + run: python -m cibuildwheel --output-dir dist env: - DEPENDENCIES: ${{ steps.dependencies.outputs.latest_dependencies }} - - - name: Building wheel - if: steps.pip-cache.outputs.cache-hit != 'true' - run: make bdist_wheel - shell: bash - - - name: Installing generated wheel - if: steps.pip-cache.outputs.cache-hit != 'true' - run: pip install --pre --no-index --find-links dist/ pmdarima - shell: bash - - - name: Running unit tests - if: steps.pip-cache.outputs.cache-hit != 'true' - run: | - if [ "${{ matrix.os }}" == "macos-latest" ]; then - export PMD_MPL_BACKEND=TkAgg - fi - pytest --showlocals --durations=20 --pyargs pmdarima - shell: bash + CIBW_ARCHS_LINUX: "x86_64" + CIBW_ARCHS_MACOS: "x86_64" + CIBW_ARCHS_WINDOWS: "AMD64" + CIBW_BEFORE_BUILD: > + for dependency in ${{ steps.dependencies.outputs.latest_dependencies }}; do + pip install $dependency + done + # Windows runs a batch script that I couldn't get to work, so we just force it to run bash + CIBW_BEFORE_BUILD_WINDOWS: bash -c 'for dependency in ${{ steps.dependencies.outputs.latest_dependencies }}; do pip install $dependency; done' + # Tests are run in a separate virtual env, so we need to re-install deps + CIBW_BEFORE_TEST: > + for dependency in ${{ steps.dependencies.outputs.latest_dependencies }}; do + pip install $dependency + done + CIBW_BEFORE_TEST_WINDOWS: bash -c 'for dependency in ${{ steps.dependencies.outputs.latest_dependencies }}; do pip install $dependency; done' + CIBW_BUILD: "${{ matrix.python-executable }}-*" + CIBW_ENVIRONMENT_MACOS: PMD_MPL_BACKEND=TkAGG + # No support for pypy or musl + CIBW_SKIP: "pp* *-musllinux_*" + CIBW_TEST_COMMAND: pytest --showlocals --durations=20 --pyargs pmdarima # https://github.com/marketplace/actions/action-slack#custom-notification - name: Posting to Slack @@ -141,11 +114,6 @@ jobs: value: '${{ matrix.os }}', short: false }, - { - title: 'Architecture', - value: '${{ matrix.architecture }}', - short: false - }, { title: 'Python Version', value: '${{ matrix.python-version }}', diff --git a/doc/whats_new.rst b/doc/whats_new.rst index e9f078369..7f0c07dc7 100644 --- a/doc/whats_new.rst +++ b/doc/whats_new.rst @@ -7,6 +7,11 @@ What's new in pmdarima As new releases of pmdarima are pushed out, the following list (introduced in v0.8.1) will document the latest features. +`v2.0.1 `_ +------------------------------------------------- + +* Add support for macOS with M1 chip + `v2.0.0 `_ ------------------------------------------------- diff --git a/pmdarima/_build_utils/pre_build_helpers.py b/pmdarima/_build_utils/pre_build_helpers.py index e5b9c446e..73bc7859e 100644 --- a/pmdarima/_build_utils/pre_build_helpers.py +++ b/pmdarima/_build_utils/pre_build_helpers.py @@ -50,10 +50,13 @@ def compile_test_program(code, extra_preargs=[], extra_postargs=[]): extra_preargs=extra_preargs, extra_postargs=extra_postargs) - # Run test program - # will raise a CalledProcessError if return code was non-zero - output = subprocess.check_output('./test_program') - output = output.decode(sys.stdout.encoding or 'utf-8').splitlines() + if "PYTHON_CROSSENV" not in os.environ: + # Run test program if not cross compiling + # will raise a CalledProcessError if return code was non-zero + output = subprocess.check_output('./test_program') + output = output.decode(sys.stdout.encoding or 'utf-8').splitlines() # noqa + else: + output = [] except Exception: raise finally: