Skip to content

Implement new sdist feature to download rust toolchain #4226

Implement new sdist feature to download rust toolchain

Implement new sdist feature to download rust toolchain #4226

Workflow file for this run

name: Test
on:
pull_request:
paths-ignore:
- "guide/**"
- "sysconfig/**"
- "**.md"
- ".cirrus.yml"
- ".github/ISSUE_TEMPLATE/**"
- ".github/dependabot.yml"
merge_group:
workflow_dispatch:
concurrency:
group: ${{ github.workflow }}-${{ github.ref_name }}-${{ github.event.pull_request.number }}
cancel-in-progress: true
jobs:
generate-matrix:
name: Generate Matrix
runs-on: ubuntu-latest
outputs:
platform: ${{ steps.generate-matrix.outputs.platform }}
fail-fast: ${{ steps.generate-matrix.outputs.fail-fast }}
steps:
- uses: actions/setup-node@v4
with:
node-version: 16
- run: npm install js-yaml
- name: Generate matrix
id: generate-matrix
uses: actions/github-script@v7
with:
script: |
const yaml = require('js-yaml')
const OS = yaml.load(process.env.OS_MATRIX)
const PYTHON_VERSIONS = yaml.load(process.env.PYTHON_VERSION)
let platforms = []
OS.forEach(os => {
PYTHON_VERSIONS.forEach(pythonVersion => {
if (os == "macos-14") {
if (pythonVersion.startsWith("pypy")) {
// Skip PyPy on macOS M1 runner because they are built for x86_64
return
} else if (parseInt(pythonVersion.slice(2)) < 11) {
// macOS M1 runner only have Python 3.11+
return
}
}
platforms.push({
"os": os,
"python-version": pythonVersion
})
})
})
if (context.eventName == 'workflow_dispatch' || context.eventName == 'merge_group') {
// Run all of them on workflow dispatch or merge group
core.setOutput('fail-fast', 'false')
} else if (context.eventName == 'pull_request') {
const { data: { labels: labels } } = await github.rest.pulls.get({
owner: context.repo.owner,
repo: context.repo.repo,
pull_number: context.payload.pull_request.number
})
const labelNames = labels.map(label => label.name)
if (labelNames.includes('CI-no-fail-fast')) {
core.setOutput('fail-fast', 'false')
}
// Only run latest CPython and PyPy tests on pull requests
const firstPyPy = PYTHON_VERSIONS.findIndex(version => version.startsWith('pypy'))
const pythonVersions = [PYTHON_VERSIONS[firstPyPy - 1], PYTHON_VERSIONS[PYTHON_VERSIONS.length - 1]]
// Only use macOS M1 runner on pull requests
platforms = platforms.filter(
platform => platform.os != 'macos-13' && pythonVersions.indexOf(platform["python-version"]) != -1
)
}
core.setOutput('platform', platforms)
env:
OS_MATRIX: |
- ubuntu-latest
- macos-13
- macos-14
- windows-latest
PYTHON_VERSION: |
- '3.8'
- '3.9'
- '3.10'
- '3.11'
- '3.12'
- 'pypy3.9'
- 'pypy3.10'
test:
name: Test
needs: [generate-matrix]
strategy:
fail-fast: ${{ needs.generate-matrix.outputs.fail-fast != 'false' }}
matrix:
platform: ${{ fromJson(needs.generate-matrix.outputs.platform) }}
runs-on: ${{ matrix.platform.os }}
env:
RUST_BACKTRACE: "1"
SCCACHE_GHA_ENABLED: "true"
RUSTC_WRAPPER: "sccache"
steps:
- name: Enable long paths
if: startsWith(matrix.platform.os, 'windows')
run: |
reg add HKEY_LOCAL_MACHINE\SYSTEM\CurrentControlSet\Control\FileSystem /v LongPathsEnabled /t REG_DWORD /d 1 /f
- name: Cleanup Disk
if: ${{ !startsWith(matrix.platform.os, 'windows') }}
run: |
sudo rm -rf /usr/share/dotnet
sudo rm -rf /opt/ghc
- name: Sccache Setup
uses: mozilla-actions/sccache-action@v0.0.6
with:
version: "v0.7.6"
- uses: actions/checkout@v4
- uses: conda-incubator/setup-miniconda@v3
with:
auto-activate-base: "false"
activate-environment: ""
miniconda-version: "latest"
- uses: actions/setup-python@v5
with:
python-version: ${{ matrix.platform.python-version }}
architecture: "x64"
- name: Set PYTHON_VERSION env var
shell: bash
run: |
set -ex
# remove -dev suffix
python_version=$(echo ${{ matrix.platform.python-version }} | sed -e s/-dev//)
echo "PYTHON_VERSION=$python_version" >> "${GITHUB_ENV}"
- name: Install cffi
if: ${{ !contains(matrix.platform.python-version, 'pypy') }}
run: pip install cffi
- name: Install python packages
run: pip install virtualenv ziglang~=0.10.0 twine uniffi-bindgen==0.28.0
- uses: dtolnay/rust-toolchain@stable
id: rustup
with:
targets: wasm32-wasi # Additional target
- name: Install cargo-nextest
uses: taiki-e/install-action@nextest
- name: Install additional Rust target
if: startsWith(matrix.platform.os, 'macos')
run: rustup target add aarch64-apple-darwin x86_64-apple-darwin
- name: Setup Xcode env
if: startsWith(matrix.platform.os, 'macos')
shell: bash
run: |
set -ex
sudo xcode-select -s /Applications/Xcode.app
bindir="$(xcode-select --print-path)/Toolchains/XcodeDefault.xctoolchain/usr/bin"
echo "CC=${bindir}/clang" >> "${GITHUB_ENV}"
echo "CXX=${bindir}/clang++" >> "${GITHUB_ENV}"
echo "SDKROOT=$(xcrun --sdk macosx --show-sdk-path)" >> "${GITHUB_ENV}"
# Caching
# Install gnu-tar because BSD tar is buggy
# https://github.com/actions/cache/issues/403
- name: Add GNU tar to PATH (macOS)
if: startsWith(matrix.platform.os, 'macos')
run: |
echo "/usr/local/opt/gnu-tar/libexec/gnubin" >> $GITHUB_PATH
- name: Set MATURIN_TEST_PYTHON for PyPy
shell: bash
if: contains(matrix.platform.python-version, 'pypy')
run: echo "MATURIN_TEST_PYTHON=pypy3" >> $GITHUB_ENV
# To save disk space
- name: Disable debuginfo on Windows
if: startsWith(matrix.platform.os, 'windows')
run: echo "RUSTFLAGS="-C debuginfo=0"" >> $GITHUB_ENV
- name: cargo test
run: cargo nextest run --features password-storage
# TODO: https://github.com/PyO3/maturin/issues/2263
#- name: test cross compiling with zig
# if: ${{ !contains(matrix.platform.python-version, '-dev') }}
# shell: bash
# run: |
# set -ex
# rustup target add aarch64-unknown-linux-gnu
# rustup target add aarch64-unknown-linux-musl
# rustup target add aarch64-apple-darwin
#
# # abi3
# cargo run -- build -i $PYTHON_VERSION -m test-crates/pyo3-pure/Cargo.toml --target aarch64-unknown-linux-gnu --zig
# cargo run -- build -i $PYTHON_VERSION -m test-crates/pyo3-pure/Cargo.toml --target aarch64-unknown-linux-musl --zig
# if [[ "$PYTHON_VERSION" != "pypy"* ]]; then
# cargo run -- build -i $PYTHON_VERSION -m test-crates/pyo3-pure/Cargo.toml --target aarch64-apple-darwin --zig
# fi
#
# if [[ "$PYTHON_VERSION" == "3.1"* ]]; then
# # Check abi3 wheels with abi3audit on CPython only
# pip install abi3audit
# abi3audit test-crates/pyo3-pure/target/wheels/*.whl
# fi
# # Check wheels with twine
# twine check --strict test-crates/pyo3-pure/target/wheels/*.whl
#
# # non-abi3
# cargo run -- build -i $PYTHON_VERSION -m test-crates/pyo3-mixed/Cargo.toml --target aarch64-unknown-linux-gnu --zig
# if [[ "$PYTHON_VERSION" != "pypy"* ]]; then
# cargo run -- build -i $PYTHON_VERSION -m test-crates/pyo3-mixed/Cargo.toml --target aarch64-apple-darwin --zig
# fi
# # Check wheels with twine
# twine check --strict test-crates/pyo3-mixed/target/wheels/*.whl
- name: test compiling with PYO3_CONFIG_FILE
shell: bash
run: |
set -ex
rustup target add x86_64-unknown-linux-gnu
export PYO3_CONFIG_FILE=$(pwd)/test-crates/pyo3-mixed/pyo3-config.txt
cargo run -- build -m test-crates/pyo3-mixed/Cargo.toml --target x86_64-unknown-linux-gnu --zig
- name: test maturin new
shell: bash
run: |
set -ex
cargo run new -b pyo3 test-crates/pyo3-new
cargo run build -m test-crates/pyo3-new/Cargo.toml --target-dir test-crates/targets/
cargo run new --mixed -b pyo3 test-crates/pyo3-new-mixed
cargo run build -m test-crates/pyo3-new-mixed/Cargo.toml --target-dir test-crates/targets/
test-windows-cross:
name: Test windows cross
needs: [generate-matrix]
strategy:
fail-fast: ${{ needs.generate-matrix.outputs.fail-fast != 'false' }}
runs-on: ubuntu-latest
env:
RUST_BACKTRACE: "1"
SCCACHE_GHA_ENABLED: "true"
RUSTC_WRAPPER: "sccache"
steps:
- uses: actions/checkout@v4
- name: Sccache Setup
uses: mozilla-actions/sccache-action@v0.0.6
with:
version: "v0.7.6"
- uses: actions/setup-python@v5
with:
python-version: 3.12
architecture: "x64"
- name: Set PYTHON_VERSION env var
shell: bash
run: |
set -ex
# remove -dev suffix
python_version=$(echo "3.12" | sed -e s/-dev//)
echo "PYTHON_VERSION=$python_version" >> "${GITHUB_ENV}"
- uses: dtolnay/rust-toolchain@stable
id: rustup
- name: test cross compiling windows wheel
run: |
set -ex
sudo apt-get install -y mingw-w64
rustup component add llvm-tools-preview
rustup target add x86_64-pc-windows-gnu
rustup target add x86_64-pc-windows-msvc
# abi3
cargo run -- build -m test-crates/pyo3-pure/Cargo.toml --target x86_64-pc-windows-gnu
cargo run -- build -m test-crates/pyo3-pure/Cargo.toml --target x86_64-pc-windows-msvc
# no-abi3
cargo run -- build -i "python${PYTHON_VERSION}" -m test-crates/pyo3-mixed/Cargo.toml --target x86_64-pc-windows-msvc
test-emscripten:
name: Test Emscripten
if: github.event_name != 'pull_request'
runs-on: ubuntu-latest
env:
PYODIDE_VERSION: "0.23.4"
PYTHON_VERSION: "3.11.2"
NODE_VERSION: 18
steps:
- uses: actions/checkout@v4
- uses: actions/setup-python@v5
id: setup-python
with:
python-version: ${{ env.PYTHON_VERSION }}
- uses: actions/setup-node@v4
with:
node-version: ${{ env.NODE_VERSION }}
- uses: dtolnay/rust-toolchain@nightly
with:
components: rust-src
targets: wasm32-unknown-emscripten
- uses: Swatinem/rust-cache@v2
with:
shared-key: maturin-build
- run: pip install nox
- name: Setup Pyodide
run: nox -s setup-pyodide
- uses: mymindstorm/setup-emsdk@v14
with:
version: ${{ env.EMSCRIPTEN_VERSION }}
actions-cache-folder: emsdk-cache
- uses: actions/setup-node@v4
with:
node-version: ${{ env.NODE_VERSION }}
- name: Run tests
run: nox -s test-emscripten
test-alpine:
name: Test Alpine Linux
if: github.event_name != 'pull_request'
runs-on: ubuntu-latest
env:
RUST_BACKTRACE: "1"
CARGO_INCREMENTAL: "0"
CARGO_TERM_COLOR: always
container: alpine:latest
steps:
- name: Install build requirements
run: |
set -ex
apk add cargo python3-dev libffi-dev py3-pip curl \
bash tar zstd git patchelf py3-cffi py3-virtualenv
- uses: actions/checkout@v4
- name: Fix git permissions
run: git config --global --add safe.directory $GITHUB_WORKSPACE
- name: Cache cargo build
uses: Swatinem/rust-cache@v2
with:
shared-key: maturin-build
- name: Install cargo-nextest
uses: taiki-e/install-action@nextest
- name: cargo test
run: |
# unset GITHUB_ACTIONS env var to disable zig related tests
env -u GITHUB_ACTIONS cargo nextest run --features password-storage
test-auditwheel:
name: Test Auditwheel
if: github.event_name != 'pull_request'
runs-on: ubuntu-latest
strategy:
fail-fast: ${{ !contains(github.event.pull_request.labels.*.name, 'CI-no-fail-fast') }}
matrix:
manylinux: ["manylinux_2_28"]
container: quay.io/pypa/${{ matrix.manylinux }}_x86_64
steps:
- uses: actions/checkout@v4
- uses: dtolnay/rust-toolchain@stable
id: rustup
# Caching
- name: Cache cargo build
uses: Swatinem/rust-cache@v2
with:
shared-key: maturin-build
- name: Compliant Build
run: tests/manylinux_compliant.sh ${{ matrix.manylinux }}
- name: Incompliant Build
if: matrix.manylinux == 'manylinux_2_28'
run: tests/manylinux_incompliant.sh ${{ matrix.manylinux }}
test-docker:
name: Test Docker
if: github.event_name != 'pull_request'
runs-on: ubuntu-latest
env:
CARGO_INCREMENTAL: "0"
CARGO_TERM_COLOR: always
steps:
- uses: actions/checkout@v4
- uses: actions/setup-python@v5
with:
python-version: "3.11"
- name: Login to GitHub Container Registry
uses: docker/login-action@v3
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- uses: docker/setup-buildx-action@v3
- name: Build
uses: docker/build-push-action@v6
with:
context: .
push: false
tags: maturin
load: true
cache-from: type=registry,ref=ghcr.io/pyo3/maturin:buildcache
cache-to: type=registry,ref=ghcr.io/pyo3/maturin:buildcache,mode=max
- name: Test the Docker container
run: ./test-dockerfile.sh
test-cross-compile:
name: Test Cross Compile
if: github.event_name != 'pull_request'
runs-on: ubuntu-latest
container: ${{ matrix.platform.container }}
strategy:
fail-fast: ${{ !contains(github.event.pull_request.labels.*.name, 'CI-no-fail-fast') }}
matrix:
platform:
# CPython
- target: aarch64-unknown-linux-gnu
abi: cp38-cp38
python: python3.8
container: ghcr.io/rust-cross/manylinux2014-cross:aarch64
- target: armv7-unknown-linux-gnueabihf
abi: cp39-cp39
python: python3.9
container: ghcr.io/rust-cross/manylinux2014-cross:armv7
- target: s390x-unknown-linux-gnu
abi: cp310-cp310
python: python3.10
container: ghcr.io/rust-cross/manylinux2014-cross:s390x
# PyPy
- target: aarch64-unknown-linux-gnu
abi: pp39-pypy39_pp73
python: pypy3.9
container: ghcr.io/rust-cross/manylinux2014-cross:aarch64
- target: aarch64-unknown-linux-gnu
abi: pp310-pypy310_pp73
python: pypy3.10
container: ghcr.io/rust-cross/manylinux2014-cross:aarch64
steps:
- uses: actions/checkout@v4
- uses: dtolnay/rust-toolchain@stable
with:
targets: ${{ matrix.platform.target }}
- name: Build wheels
run: |
set -ex
# Use bundled sysconfig
cargo run --target x86_64-unknown-linux-gnu -- build -i ${{ matrix.platform.python }} --release --out dist --target ${{ matrix.platform.target }} -m test-crates/pyo3-mixed/Cargo.toml
# Use PYO3_CROSS_LIB_DIR
export PYO3_CROSS_LIB_DIR=/opt/python/${{ matrix.platform.abi }}
cargo run --target x86_64-unknown-linux-gnu -- build -i python3.9 --release --out dist --target ${{ matrix.platform.target }} -m test-crates/pyo3-mixed/Cargo.toml
unset PYO3_CROSS_LIB_DIR
# Test abi3
cargo run --target x86_64-unknown-linux-gnu -- build -i ${{ matrix.platform.python }} --release --out dist --target ${{ matrix.platform.target }} -m test-crates/pyo3-pure/Cargo.toml
# --find-interpreter
cargo run --target x86_64-unknown-linux-gnu -- build --find-interpreter --release --out dist --target ${{ matrix.platform.target }} -m test-crates/pyo3-mixed/Cargo.toml
test-bootstrap:
name: Test Bootstrap
runs-on: ${{ matrix.os }}
env:
SCCACHE_GHA_ENABLED: "true"
RUSTC_WRAPPER: "sccache"
strategy:
fail-fast: ${{ !contains(github.event.pull_request.labels.*.name, 'CI-no-fail-fast') }}
matrix:
os: [ubuntu-latest, macos-14, windows-latest]
steps:
- uses: actions/checkout@v4
- uses: dorny/paths-filter@v3
id: changes
with:
filters: |
changed:
- 'maturin/**'
- 'pyproject.toml'
- 'setup.py'
- 'MANIFEST.in'
- 'Cargo.toml'
- '.github/workflows/test.yml'
- uses: dtolnay/rust-toolchain@stable
if: ${{ steps.changes.outputs.changed == 'true' || contains(github.event.pull_request.labels.*.name, 'release') }}
# Caching
- name: Sccache Setup
if: ${{ steps.changes.outputs.changed == 'true' || contains(github.event.pull_request.labels.*.name, 'release') }}
uses: mozilla-actions/sccache-action@v0.0.6
with:
version: "v0.7.6"
- uses: actions/setup-python@v5
if: ${{ steps.changes.outputs.changed == 'true' || contains(github.event.pull_request.labels.*.name, 'release') }}
with:
python-version: "3.10"
- name: Build and install
shell: bash
run: |
set -ex
cargo run sdist -o dist
pip install -v dist/maturin-*.tar.gz
if: ${{ steps.changes.outputs.changed == 'true' || contains(github.event.pull_request.labels.*.name, 'release') }}
- run: maturin --version
if: ${{ steps.changes.outputs.changed == 'true' || contains(github.event.pull_request.labels.*.name, 'release') }}
- run: python3 -m maturin --version
if: ${{ steps.changes.outputs.changed == 'true' || contains(github.event.pull_request.labels.*.name, 'release') }}
- name: Upload wheel artifacts
if: ${{ steps.changes.outputs.changed == 'true' || contains(github.event.pull_request.labels.*.name, 'release') }}
uses: actions/upload-artifact@v4
with:
name: wheels-${{ runner.os }}
path: dist
test-msrv:
name: Test MSRV
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: dtolnay/rust-toolchain@1.74.0
# Caching
- name: Cache cargo build
uses: Swatinem/rust-cache@v2
- run: cargo build --no-default-features
- name: cargo build
run: cargo build --all
test-downstream:
name: Test downstream - ${{ matrix.downstream.repository }}
if: ${{ contains(github.event.pull_request.labels.*.name, 'sdist') && github.event_name == 'pull_request' }}
uses: ./.github/workflows/downstream.yml
with:
repository: ${{ matrix.downstream.repository }}
manifest-dir: ${{ matrix.downstream.manifest-dir }}
strategy:
matrix:
downstream:
- repository: "pola-rs/polars"
manifest-dir: "py-polars"
- repository: "astral-sh/ruff"
manifest-dir: "crates/ruff_cli"
- repository: "crate-ci/typos"
manifest-dir: "crates/typos-cli"
- repository: "ast-grep/ast-grep"
manifest-dir: "crates/pyo3"
- repository: "oxigraph/oxigraph"
manifest-dir: "python"
build-maturin-wheel:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- name: Setup Python
uses: actions/setup-python@v5
with:
python-version: "3.10"
- uses: dtolnay/rust-toolchain@stable
id: rustup
- name: Install python build tools
run: pip install setuptools wheel>=0.36.2 setuptools-rust>=1.4.0 tomli>=1.1.0
- name: Build maturin wheel
run: |
export MATURIN_SETUP_ARGS="--features rustls"
python setup.py bdist_wheel
- name: install maturin wheel
run: pip install --force-reinstall ./dist/*.whl
- name: test
run: maturin sdist --manifest-path ./test-crates/hello-world/Cargo.toml -o ./target/hello-world/sdist
- name: Upload sdist
id: upload-sdist
uses: actions/upload-artifact@v3
with:
name: hello-world-sdist
path: ./target/hello-world/sdist/*.tar.gz
- name: Upload wheels
id: upload-wheel
uses: actions/upload-artifact@v3
with:
name: maturin-wheel
path: ./dist/*.whl
test-pip-install-without-toolchain:
runs-on: ubuntu-latest
needs: [build-maturin-wheel]
steps:
- uses: actions/checkout@v4
- name: Setup Python
uses: actions/setup-python@v5
with:
python-version: "3.10"
- name: Download sdist from previous job
uses: actions/download-artifact@v3
with:
name: hello-world-sdist
path: ./target/hello-world/sdist
- name: Download wheel from previous job
uses: actions/download-artifact@v3
with:
name: maturin-wheel
path: ./dist/*.whl
- name: Install maturin whl
run: |
WHEEL_FILE=$(ls ./dist/*.whl)
pip install --force-reinstall "$WHEEL_FILE"
- name: pip install
run: |
export MATURIN_PEP517_ARGS="--verbose"
export RUST_LOG="debug"
pip install --no-build-isolation --verbose ./target/hello-world/sdist/hello_world-0.1.0.tar.gz
check:
name: Check ${{ matrix.platform.target }}
if: github.event_name != 'pull_request'
strategy:
fail-fast: ${{ !contains(github.event.pull_request.labels.*.name, 'CI-no-fail-fast') }}
matrix:
platform:
- target: armv5te-unknown-linux-gnueabi
apt-packages: gcc-arm-linux-gnueabi
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- name: Install packages
run: |
sudo apt-get update
sudo apt-get install -y ${{ matrix.platform.apt-packages }}
- uses: dtolnay/rust-toolchain@stable
with:
targets: ${{ matrix.platform.target }}
- name: Cache cargo build
uses: Swatinem/rust-cache@v2
- name: cargo check
run: cargo check --target ${{ matrix.platform.target }}
conclusion:
needs:
- test
- test-emscripten
- test-alpine
- test-auditwheel
- test-docker
- test-cross-compile
- test-bootstrap
- test-msrv
- test-downstream
- check
if: always()
runs-on: ubuntu-latest
steps:
- name: Result
run: |
jq -C <<< "${needs}"
# Check if all needs were successful or skipped.
"$(jq -r 'all(.result as $result | (["success", "skipped"] | contains([$result])))' <<< "${needs}")"
env:
needs: ${{ toJson(needs) }}