Skip to content

Build Python Package #498

Build Python Package

Build Python Package #498

name: Build Python Package
on:
workflow_dispatch:
inputs:
incoming_ref:
description: >
The ref from Cantera/cantera to be built. Can be a tag, commit hash,
or branch name.
required: true
default: "main"
upload:
description: Attempt to upload to PyPI
required: true
default: "false"
concurrency:
group: ${{ github.ref }}-${{ github.event.inputs.incoming_ref}}
cancel-in-progress: true
env:
ACTION_URL: "https://github.com/Cantera/pypi-packages/actions/runs/${{ github.run_id }}"
jobs:
dump:
name: Dump the input parameters for the workflow
runs-on: ubuntu-latest
steps:
- name: Dump Event Payload
run: jq . "$GITHUB_EVENT_PATH"
- name: Echo the input variables
run: |
echo "${{ github.event.inputs.incoming_ref }}"
echo "${{ github.event.inputs.upload }}"
post-pending-status:
name: Post a pending workflow status to Cantera/cantera
runs-on: ubuntu-latest
env:
GITHUB_TOKEN: ${{ secrets.CANTERA_REPO_STATUS }}
outputs:
incoming-sha: ${{ steps.get-incoming-sha.outputs.incoming-sha }}
tag-ref: ${{ steps.munge-incoming-ref.outputs.tag-ref }}
steps:
- name: Munge the incoming ref
id: munge-incoming-ref
run: |
import os
import re
from pathlib import Path
INCOMING_REF = "${{ github.event.inputs.incoming_ref }}"
INCOMING_SHA = ""
if INCOMING_REF.startswith("refs/"):
INCOMING_REF = INCOMING_REF.replace("refs/", "")
elif re.match(r"^v\d\.\d\.\d.*$", INCOMING_REF) is not None:
INCOMING_REF = f"tags/{INCOMING_REF}"
elif re.match(r"^[a-f0-9]{6,40}", INCOMING_REF) is not None:
INCOMING_SHA = INCOMING_REF
else:
INCOMING_REF = f"heads/{INCOMING_REF}"
TAG_REF = "false"
if INCOMING_REF.startswith("tags"):
TAG_REF = "true"
Path(os.environ["GITHUB_ENV"]).write_text(
f"INCOMING_REF={INCOMING_REF}\n"
f"TAG_REF={TAG_REF}\n"
f"INCOMING_SHA={INCOMING_SHA}"
)
Path(os.environ["GITHUB_OUTPUT"]).write_text(
f"tag-ref={TAG_REF}"
)
shell: python
- name: Get the SHA associated with the incoming ref
id: get-incoming-sha
run: |
if [[ "${INCOMING_SHA}" == "" ]]; then
INCOMING_SHA=$(gh api repos/cantera/cantera/git/matching-refs/${INCOMING_REF} \
-H "Accept: application/vnd.github.v3+json" --jq ".[0].object.sha")
echo "INCOMING_SHA=${INCOMING_SHA}" >> $GITHUB_ENV
fi
# This needs to be in this step to be output to other jobs.
echo "incoming-sha=${INCOMING_SHA}" >> $GITHUB_OUTPUT
- name: Post the status to the upstream commit
id: set-the-status
if: env.TAG_REF == 'false'
run: |
gh api repos/cantera/cantera/statuses/${INCOMING_SHA} \
-H "Accept: application/vnd.github.v3+json" \
--field state='pending' \
--field target_url=$ACTION_URL \
--field context='PyPI Package Build' \
--field description="Pending build" \
--silent
sdist:
name: Build the sdist
runs-on: ubuntu-24.04
needs:
- "post-pending-status"
outputs:
job-status: ${{ job.status }}
steps:
- uses: actions/checkout@v4
name: Checkout the repository
with:
repository: "Cantera/cantera"
submodules: recursive
ref: ${{ github.event.inputs.incoming_ref }}
- name: Set Up Python 3.12
uses: actions/setup-python@v5
with:
python-version: "3.12"
cache: 'pip'
cache-dependency-path: interfaces/python_sdist/pyproject.toml.in
- name: Install dependencies
run: python3 -m pip install scons build
- name: Build the sdist
run: |
python3 `which scons` sdist
- name: Archive the built sdist
uses: actions/upload-artifact@v4
with:
path: ./build/python_sdist/dist/*.tar.gz
name: cibw-sdist
if-no-files-found: error
# Copied from https://github.com/hynek/build-and-inspect-python-package/
- name: Show SDist contents hierarchically, including metadata.
shell: bash
run: |
mkdir -p /tmp/out/sdist
cp build/python_sdist/dist/*.tar.gz /tmp/
cd /tmp
tar xf *.tar.gz -C out/sdist
echo -e '\n<details><summary>SDist contents</summary>\n' >> $GITHUB_STEP_SUMMARY
(cd /tmp/out/sdist && tree -Da --timefmt="%Y-%m-%dT%H:%M:%SZ" * | sed 's/^/ /' | tee -a $GITHUB_STEP_SUMMARY)
echo -e '\n</details>\n' >> $GITHUB_STEP_SUMMARY
echo ----- Metadata Follows -----
echo -e '\n<details><summary>Metadata</summary>\n' >> $GITHUB_STEP_SUMMARY
cat out/sdist/*/PKG-INFO | sed 's/^/ /' | tee -a $GITHUB_STEP_SUMMARY
echo -e '\n</details>\n' >> $GITHUB_STEP_SUMMARY
echo ----- End of Metadata -----
build-wheels:
name: Build ${{ matrix.os }} ${{ matrix.arch }} for py${{ matrix.python || '-all' }}
runs-on: ${{ matrix.os }}
needs: ["sdist", "post-pending-status"]
outputs:
job-status: ${{ job.status }}
strategy:
matrix:
# Wheel builds are fast except for aarch64, so split that into multiple jobs,
# one for each Python version
os: [ubuntu-latest]
arch: [aarch64]
python: ["3.8", "3.9", "3.10", "3.11", "3.12", "3.13"]
include:
- os: ubuntu-latest
arch: x86_64
- os: windows-2022
arch: AMD64
boost-arch: x86
boost-toolset: msvc
boost-platform-version: 2022
boost-version: "1.86.0"
- os: macos-14
arch: arm64
boost-arch: aarch64
boost-toolset: clang
# Since we only use the headers, we can use the platform version for this
# macos version
boost-platform-version: "14"
boost-version: "1.86.0"
- os: macos-13
arch: x86_64
boost-arch: x86
boost-toolset: clang
# Since we only use the headers, we can use the platform version for this
# macos version
boost-platform-version: "13"
boost-version: "1.86.0"
fail-fast: false
steps:
- name: Checkout the repository
uses: actions/checkout@v4
- name: Download pre-built sdist
uses: actions/download-artifact@v4
with:
name: cibw-sdist
- name: Extract the sdist tarball
run: tar -xvf *.tar.gz --strip-components=1
shell: bash
- name: Download test files
id: download-test-files
uses: ./.github/actions/download-cantera-test
with:
incoming-sha: ${{ needs.post-pending-status.outputs.incoming-sha }}
# Linux steps
- name: Set up QEMU
uses: docker/setup-qemu-action@v3
with:
platforms: all
if: matrix.arch != 'x86_64' && runner.os == 'Linux'
- name: Set up CIBW environment
run: |
PYTHON="${{ matrix.python }}"
if [[ $PYTHON == "" ]]; then PYTHON="*"; fi
CIBW_BUILD="cp${PYTHON//./}-*_${{ matrix.arch }}"
echo "CIBW_BUILD=${CIBW_BUILD}" | tee -a $GITHUB_ENV
if: runner.os == 'Linux'
- name: Install boost
# Our custom manylinux images already have Boost installed
if: runner.os != 'Linux'
uses: MarkusJx/install-boost@v2.4.5
id: install-boost
with:
# REQUIRED: Specify the required boost version
# A list of supported versions can be found here:
# https://github.com/MarkusJx/prebuilt-boost/blob/main/versions-manifest.json
boost_version: ${{ matrix.boost-version }}
# OPTIONAL: Specify a custon install location
boost_install_dir: ${{ runner.temp }}
toolset: ${{ matrix.boost-toolset }}
platform_version: ${{ matrix.boost-platform-version }}
arch: ${{ matrix.boost-arch }}
- name: Cache built libraries
id: cache-built-libraries
# Our custom manylinux images already have hdf5 installed
if: runner.os != 'Linux'
uses: actions/cache@v4
with:
key: ${{ matrix.os }}-${{ matrix.arch }}-0
path: ${{ runner.temp }}/cache
# Windows-only steps
- name: Set Up Nuget
uses: nuget/setup-nuget@v2
if: runner.os == 'Windows'
- uses: actions/setup-python@v5
with:
python-version: 3.12
if: runner.os == 'Windows'
- run: bash ./cibw_before_all_windows.sh "${{ runner.temp }}"
if: runner.os == 'Windows'
- name: Set up CIBW environment
# On Windows, Boost_ROOT needs to have \ replaced by / because that's what
# cibuildwheel says. CANTERA_TEST_DIR doesn't need the replacement because
# it will be substituted in a cmd or pwsh session.
run: |
$BOOST_ROOT = "${{ steps.install-boost.outputs.BOOST_ROOT }}" -replace "\\", "/"
echo "Boost_ROOT=$BOOST_ROOT" | Out-File -FilePath $Env:GITHUB_ENV -Encoding utf8 -Append
echo "CANTERA_TEST_DIR=${{ steps.download-test-files.outputs.test-root }}" | Out-File -FilePath $Env:GITHUB_ENV -Encoding utf8 -Append
echo "CIBW_BUILD=cp*-*" | Out-File -FilePath $Env:GITHUB_ENV -Encoding utf8 -Append
shell: pwsh
if: runner.os == 'Windows'
# macOS-only steps
- name: Build required libraries
run: bash ./cibw_before_all_macos.sh "${{ runner.temp }}"
if: runner.os == 'macOS'
# Force installation to resolve Python 3.8 bug (https://github.com/pypa/cibuildwheel/pull/1871#issuecomment-2161613619)
- name: Hack for 3.8 bug
uses: actions/setup-python@v5
with:
python-version: 3.8
if: runner.os == 'macOS' && matrix.arch == 'arm64'
- name: Set up CIBW environment
run: |
echo "Boost_ROOT=${{ steps.install-boost.outputs.BOOST_ROOT }}" >> $GITHUB_ENV
echo "CANTERA_TEST_DIR=${{ steps.download-test-files.outputs.test-root }}" >> $GITHUB_ENV
echo "CIBW_BUILD=cp*-*" >> $GITHUB_ENV
if: runner.os == 'macOS'
- name: Build wheels
uses: pypa/cibuildwheel@v2.21.3
env:
CIBW_ENVIRONMENT_LINUX: CT_SKIP_SLOW=1 CANTERA_TEST_DIR=/host/${{ steps.download-test-files.outputs.test-root }}
CIBW_ENVIRONMENT_WINDOWS: CT_SKIP_SLOW=1 CMAKE_BUILD_PARALLEL_LEVEL=4
CIBW_ENVIRONMNET_MACOS: CT_SKIP_SLOW=1
CIBW_BUILD: ${{ env.CIBW_BUILD }}
CIBW_ARCHS: ${{ matrix.arch }}
- name: Archive the built wheels
uses: actions/upload-artifact@v4
with:
path: ./wheelhouse/*.whl
name: cibw-wheels-${{ runner.os }}-${{ strategy.job-index }}
publish-files-to-pypi:
name: Publish distribution files to PyPI
runs-on: ubuntu-22.04
outputs:
job-status: ${{ job.status }}
needs:
- "sdist"
- "build-wheels"
if: github.event.inputs.upload == 'true'
permissions:
id-token: write
environment: pypi
steps:
- name: Download pre-built wheels
uses: actions/download-artifact@v4
with:
path: dist
pattern: cibw-*
merge-multiple: true
- name: pypi-publish
uses: pypa/gh-action-pypi-publish@release/v1
send_status_to_cantera:
name: Send jobs status to Cantera/cantera
runs-on: ubuntu-22.04
needs:
- "post-pending-status"
- "sdist"
- "build-wheels"
- "publish-files-to-pypi"
if: always()
steps:
- name: Collect statuses
run: |
from collections import Counter
import os
statuses = {
"sdist": "${{needs.sdist.outputs.job-status}}",
"wheels": "${{needs.build-wheels.outputs.job-status}}",
"publish": "${{needs.publish-files-to-pypi.outputs.job-status}}",
}
# This is a deliberate comparison to the empty string.
if statuses["publish"] == "" and "${{ github.event.inputs.upload }}" == "false":
publish = statuses.pop("publish")
else:
publish = ""
if all(v == "success" for v in statuses.values()):
overall_status = "success"
elif any(v in ("cancelled", "") for v in statuses.values()):
overall_status = "error"
elif any(v == "failure" for v in statuses.values()):
overall_status = "failure"
status_counts = Counter(statuses.values())
status_counts.update([publish])
description = []
if overall_status in ("error", "failure"):
if status_counts.get("success") is not None:
description.append(f"{status_counts['success']} succeeded")
if status_counts.get("cancelled") is not None:
description.append(f"{status_counts['cancelled']} cancelled")
if status_counts.get("failure") is not None:
description.append(f"{status_counts['failure']} failed")
if status_counts.get("") is not None:
description.append(f"{status_counts['']} skipped")
description = ", ".join(description)
else:
description = "Successfully built Python wheels!"
with open(os.environ["GITHUB_ENV"], "a") as gh_env:
gh_env.write(f"OVERALL_STATUS={overall_status}\nDESCRIPTION={description}")
shell: python
- name: Post the status to the upstream commit
if: needs.post-pending-status.outputs.tag-ref == 'false'
run: |
INCOMING_SHA=${{ needs.post-pending-status.outputs.incoming-sha }}
gh api repos/cantera/cantera/statuses/${INCOMING_SHA} \
-H "Accept: application/vnd.github.v3+json" \
--field state="${OVERALL_STATUS}" \
--field target_url=$ACTION_URL \
--field context='PyPI Package Build' \
--field description="${DESCRIPTION}" \
--silent
env:
GITHUB_TOKEN: ${{ secrets.CANTERA_REPO_STATUS }}