Use uv in Makefile and CI
#5399
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: ci | |
| on: | |
| push: | |
| branches: | |
| - main | |
| tags: | |
| - '**' | |
| pull_request: {} | |
| env: | |
| COLUMNS: 150 | |
| jobs: | |
| coverage: | |
| runs-on: ubuntu-20.04 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: install rust nightly | |
| uses: dtolnay/rust-toolchain@nightly | |
| - id: cache-rust | |
| name: cache rust | |
| uses: Swatinem/rust-cache@v2 | |
| - run: cargo install rustfilt coverage-prepare | |
| if: steps.cache-rust.outputs.cache-hit != 'true' | |
| - run: rustup component add llvm-tools-preview | |
| - name: install uv | |
| uses: astral-sh/setup-uv@v3 | |
| with: | |
| enable-cache: true | |
| - name: install deps | |
| run: uv sync --python 3.13 --group testing | |
| - run: rustc --version --verbose | |
| - run: uv pip install -e . | |
| env: | |
| RUST_BACKTRACE: 1 | |
| RUSTFLAGS: '-C instrument-coverage' | |
| - run: uv pip freeze | |
| - run: uv run coverage run -m pytest | |
| - run: ls -lha | |
| - run: uv run coverage xml | |
| - run: coverage-prepare lcov python/pydantic_core/*.so | |
| - uses: codecov/codecov-action@v4 | |
| # See https://github.com/PyO3/pyo3/discussions/2781 | |
| # tests intermittently segfault with pypy and cpython 3.7 when using `coverage run ...`, hence separate job | |
| test-python: | |
| name: test ${{ matrix.python-version }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| python-version: | |
| - '3.8' | |
| - '3.9' | |
| - '3.10' | |
| - '3.11' | |
| - '3.12' | |
| - '3.13' | |
| - 'pypy3.9' | |
| - 'pypy3.10' | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: install rust stable | |
| uses: dtolnay/rust-toolchain@stable | |
| - name: cache rust | |
| uses: Swatinem/rust-cache@v2 | |
| with: | |
| key: test-v3 | |
| - name: install uv | |
| uses: astral-sh/setup-uv@v3 | |
| with: | |
| enable-cache: true | |
| - name: install deps | |
| run: uv sync --python ${{ matrix.python-version }} --group testing | |
| - run: uv pip install -e . | |
| env: | |
| RUST_BACKTRACE: 1 | |
| - run: uv pip freeze | |
| - run: uv run pytest | |
| env: | |
| HYPOTHESIS_PROFILE: slow | |
| test-os: | |
| name: test on ${{ matrix.os }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| os: [ubuntu, macos, windows] | |
| runs-on: ${{ matrix.os }}-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: install rust stable | |
| uses: dtolnay/rust-toolchain@stable | |
| - name: cache rust | |
| uses: Swatinem/rust-cache@v2 | |
| - name: install uv | |
| uses: astral-sh/setup-uv@v3 | |
| with: | |
| enable-cache: true | |
| - name: install deps | |
| run: uv sync --python 3.13 --group testing | |
| - run: uv pip install -e . | |
| env: | |
| RUST_BACKTRACE: 1 | |
| - run: uv pip freeze | |
| - run: uv run pytest | |
| - run: cargo test | |
| test-msrv: | |
| name: test MSRV | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: install uv | |
| uses: astral-sh/setup-uv@v3 | |
| with: | |
| enable-cache: true | |
| - name: resolve MSRV | |
| id: resolve-msrv | |
| run: echo MSRV=`python -c 'import tomllib; print(tomllib.load(open("Cargo.toml", "rb"))["package"]["rust-version"])'` >> $GITHUB_OUTPUT | |
| - name: install rust MSRV | |
| uses: dtolnay/rust-toolchain@master | |
| with: | |
| toolchain: ${{ steps.resolve-msrv.outputs.MSRV }} | |
| - name: cache rust | |
| uses: Swatinem/rust-cache@v2 | |
| - name: install deps | |
| run: uv sync --python 3.13 --group testing | |
| - run: uv pip install -e . | |
| env: | |
| RUST_BACKTRACE: 1 | |
| - run: uv pip freeze | |
| - run: uv run pytest | |
| - run: cargo test | |
| # test with a debug build as it picks up errors which optimised release builds do not | |
| test-debug: | |
| name: test-debug ${{ matrix.python-version }} | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| python-version: | |
| - '3.13' | |
| - 'pypy3.10' | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: install uv | |
| uses: astral-sh/setup-uv@v3 | |
| with: | |
| enable-cache: true | |
| - name: install rust stable | |
| uses: dtolnay/rust-toolchain@stable | |
| - name: cache rust | |
| uses: Swatinem/rust-cache@v2 | |
| - name: install deps | |
| run: uv sync --python ${{ matrix.python-version }} --group testing | |
| - run: make build-dev | |
| - run: uv pip freeze | |
| - run: uv run pytest | |
| test-pydantic-integration: | |
| runs-on: ubuntu-latest | |
| continue-on-error: true | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| repository: pydantic/pydantic | |
| path: pydantic | |
| - uses: actions/checkout@v4 | |
| with: | |
| path: pydantic-core | |
| - name: install rust stable | |
| uses: dtolnay/rust-toolchain@stable | |
| - name: cache rust | |
| uses: Swatinem/rust-cache@v2 | |
| - name: install uv | |
| uses: astral-sh/setup-uv@v3 | |
| with: | |
| enable-cache: true | |
| - name: set up venv | |
| run: | | |
| uv sync --python 3.13 --project pydantic --extra timezone | |
| uv pip install maturin | |
| uv run bash -c 'cd ../pydantic-core && make build-dev' | |
| env: | |
| UV_PROJECT_ENVIRONMENT: .venv | |
| - run: uv --version && uv pip list | |
| # Run pytest with lax xfail because we often add tests to pydantic | |
| # which xfail on a pending release of pydantic-core | |
| - run: uv run pytest --override-ini=xfail_strict=False | |
| working-directory: pydantic | |
| env: | |
| PYDANTIC_PRIVATE_ALLOW_UNHANDLED_SCHEMA_TYPES: 1 | |
| UV_PROJECT_ENVIRONMENT: ../.venv | |
| lint: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: install rust stable | |
| uses: dtolnay/rust-toolchain@stable | |
| with: | |
| components: rustfmt, clippy | |
| - name: cache rust | |
| uses: Swatinem/rust-cache@v2 | |
| # used to lint js code | |
| - uses: actions/setup-node@v4 | |
| with: | |
| node-version: '18' | |
| - name: install uv | |
| uses: astral-sh/setup-uv@v3 | |
| with: | |
| enable-cache: true | |
| - name: install deps | |
| run: | | |
| uv sync --python 3.13 --group linting | |
| make build-dev | |
| uv pip freeze | |
| - run: make lint | |
| - run: make pyright | |
| - run: npm install | |
| - run: npm run lint | |
| bench: | |
| name: rust benchmarks | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: install rust nightly | |
| uses: dtolnay/rust-toolchain@nightly | |
| - name: cache rust | |
| uses: Swatinem/rust-cache@v2 | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.13' | |
| - run: pip install typing_extensions | |
| - run: cargo bench | |
| build-wasm-emscripten: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: install rust nightly | |
| uses: dtolnay/rust-toolchain@nightly | |
| with: | |
| components: rust-src | |
| targets: wasm32-unknown-emscripten | |
| - name: cache rust | |
| uses: Swatinem/rust-cache@v2 | |
| - uses: mymindstorm/setup-emsdk@v14 | |
| with: | |
| # NOTE!: as per https://github.com/pydantic/pydantic-core/pull/149 this version needs to match the version | |
| # in node_modules/pyodide/repodata.json, to get the version, run: | |
| # `cat node_modules/pyodide/repodata.json | python -m json.tool | rg platform` | |
| version: '3.1.58' | |
| actions-cache-folder: emsdk-cache | |
| - name: install uv | |
| uses: astral-sh/setup-uv@v3 | |
| with: | |
| enable-cache: true | |
| - name: install deps | |
| run: uv sync --python 3.12 --group wasm | |
| - name: build wheels | |
| run: make build-wasm | |
| - uses: actions/setup-node@v4 | |
| with: | |
| node-version: '18' | |
| - run: npm install | |
| - run: npm run test | |
| - run: | | |
| ls -lh dist/ | |
| ls -l dist/ | |
| - uses: actions/upload-artifact@v4 | |
| with: | |
| name: wasm_wheels | |
| path: dist | |
| # https://github.com/marketplace/actions/alls-green#why used for branch protection checks | |
| check: | |
| if: always() | |
| needs: [coverage, test-python, test-os, test-debug, lint, bench, build-wasm-emscripten] | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Decide whether the needed jobs succeeded or failed | |
| uses: re-actors/alls-green@release/v1 | |
| with: | |
| jobs: ${{ toJSON(needs) }} | |
| allowed-failures: coverage | |
| build-sdist: | |
| name: build sdist | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.13' | |
| - uses: PyO3/maturin-action@v1 | |
| with: | |
| command: sdist | |
| args: --out dist | |
| rust-toolchain: stable | |
| - uses: actions/upload-artifact@v4 | |
| with: | |
| name: pypi_files_sdist | |
| path: dist | |
| build: | |
| name: build on ${{ matrix.os }} (${{ matrix.target }} - ${{ matrix.interpreter || 'all' }}${{ matrix.os == 'linux' && format(' - {0}', matrix.manylinux == 'auto' && 'manylinux' || matrix.manylinux) || '' }}) | |
| # only run on push to main and on release | |
| if: startsWith(github.ref, 'refs/tags/') || github.ref == 'refs/heads/main' || contains(github.event.pull_request.labels.*.name, 'Full Build') | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| os: [linux, macos, windows] | |
| target: [x86_64, aarch64] | |
| manylinux: [auto] | |
| include: | |
| # manylinux for various platforms, plus x86_64 pypy | |
| - os: linux | |
| manylinux: auto | |
| target: i686 | |
| - os: linux | |
| manylinux: auto | |
| target: aarch64 | |
| - os: linux | |
| manylinux: auto | |
| target: armv7 | |
| interpreter: 3.8 3.9 3.10 3.11 3.12 3.13 | |
| - os: linux | |
| manylinux: auto | |
| target: ppc64le | |
| interpreter: 3.8 3.9 3.10 3.11 3.12 3.13 | |
| - os: linux | |
| manylinux: auto | |
| target: s390x | |
| interpreter: 3.8 3.9 3.10 3.11 3.12 3.13 | |
| - os: linux | |
| manylinux: auto | |
| target: x86_64 | |
| interpreter: pypy3.9 pypy3.10 | |
| # musllinux | |
| - os: linux | |
| manylinux: musllinux_1_1 | |
| target: x86_64 | |
| - os: linux | |
| manylinux: musllinux_1_1 | |
| target: aarch64 | |
| - os: linux | |
| manylinux: musllinux_1_1 | |
| target: armv7 | |
| # macos; | |
| # all versions x86_64 | |
| # arm pypy and older pythons which can't be run on the arm hardware for PGO | |
| - os: macos | |
| target: x86_64 | |
| - os: macos | |
| target: aarch64 | |
| interpreter: 3.8 3.9 pypy3.9 pypy3.10 | |
| # windows; | |
| # x86_64 pypy builds are not PGO optimized | |
| # i686 not supported by pypy | |
| # aarch64 only 3.11 and up, also not PGO optimized | |
| - os: windows | |
| target: x86_64 | |
| interpreter: pypy3.9 pypy3.10 | |
| - os: windows | |
| target: i686 | |
| python-architecture: x86 | |
| interpreter: 3.8 3.9 3.10 3.11 3.12 3.13 | |
| - os: windows | |
| target: aarch64 | |
| interpreter: 3.11 3.12 3.13 | |
| exclude: | |
| # See above; disabled for now. | |
| - os: windows | |
| target: aarch64 | |
| runs-on: ${{ (matrix.os == 'linux' && 'ubuntu') || matrix.os }}-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: set up python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.13' | |
| architecture: ${{ matrix.python-architecture || 'x64' }} | |
| - run: pip install -U twine 'ruff==0.5.0' typing_extensions | |
| # generate self-schema now, so we don't have to do so inside docker in maturin build | |
| - run: python generate_self_schema.py | |
| - name: build wheels | |
| uses: PyO3/maturin-action@v1 | |
| with: | |
| target: ${{ matrix.target }} | |
| manylinux: ${{ matrix.manylinux }} | |
| args: --release --out dist --interpreter ${{ matrix.interpreter || '3.8 3.9 3.10 3.11 3.12 3.13 pypy3.9 pypy3.10' }} | |
| # Limit windows builds to 1.77 to keep Windows 7 support. | |
| # FIXME: Unpin when Python 3.8 support is dropped. (3.9 requires Windows 10) | |
| rust-toolchain: ${{ (matrix.os == 'windows' && '1.77') || 'stable' }} | |
| docker-options: -e CI | |
| - run: ${{ (matrix.os == 'windows' && 'dir') || 'ls -lh' }} dist/ | |
| - run: twine check --strict dist/* | |
| - uses: actions/upload-artifact@v4 | |
| with: | |
| name: pypi_files_${{ matrix.os }}_${{ matrix.target }}_${{ matrix.interpreter || 'all' }}_${{ matrix.manylinux }} | |
| path: dist | |
| build-pgo: | |
| name: build pgo-optimized on ${{ matrix.os }} / ${{ matrix.interpreter }} | |
| # only run on push to main and on release | |
| if: startsWith(github.ref, 'refs/tags/') || github.ref == 'refs/heads/main' || contains(github.event.pull_request.labels.*.name, 'Full Build') | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| os: [linux, windows, macos] | |
| interpreter: ['3.8', '3.9', '3.10', '3.11', '3.12', '3.13'] | |
| include: | |
| # standard runners with override for macos arm | |
| - os: linux | |
| runs-on: ubuntu-latest | |
| - os: windows | |
| ls: dir | |
| runs-on: windows-latest | |
| - os: macos | |
| runs-on: macos-latest-xlarge | |
| exclude: | |
| # macos arm only supported from 3.10 and up | |
| - os: macos | |
| interpreter: '3.8' | |
| - os: macos | |
| interpreter: '3.9' | |
| runs-on: ${{ matrix.runs-on }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: set up python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ matrix.interpreter }} | |
| allow-prereleases: true | |
| - name: install rust stable | |
| id: rust-toolchain | |
| uses: dtolnay/rust-toolchain@master | |
| with: | |
| components: llvm-tools | |
| # Limit windows builds to 1.77 to keep Windows 7 support. | |
| # FIXME: Unpin when Python 3.8 support is dropped. (3.9 requires Windows 10) | |
| toolchain: ${{ (matrix.os == 'windows' && '1.77') || 'stable' }} | |
| - run: pip install -U 'ruff==0.5.0' typing_extensions | |
| # generate self-schema now, so we don't have to do so inside docker in maturin build | |
| - run: python generate_self_schema.py | |
| - name: build initial wheel | |
| uses: PyO3/maturin-action@v1 | |
| with: | |
| manylinux: auto | |
| args: > | |
| --release | |
| --out pgo-wheel | |
| --interpreter ${{ matrix.interpreter }} | |
| rust-toolchain: ${{ steps.rust-toolchain.outputs.name }} | |
| docker-options: -e CI | |
| env: | |
| RUSTFLAGS: '-Cprofile-generate=${{ github.workspace }}/profdata' | |
| - name: detect rust host | |
| run: echo RUST_HOST=$(rustc -Vv | grep host | cut -d ' ' -f 2) >> "$GITHUB_ENV" | |
| shell: bash | |
| - name: generate pgo data | |
| run: | | |
| pip install -U pip | |
| pip install -r tests/requirements.txt | |
| pip install pydantic-core --no-index --no-deps --find-links pgo-wheel --force-reinstall | |
| pytest tests/benchmarks | |
| rustup run ${{ steps.rust-toolchain.outputs.name }} bash -c 'echo LLVM_PROFDATA=$RUSTUP_HOME/toolchains/$RUSTUP_TOOLCHAIN/lib/rustlib/${{ env.RUST_HOST }}/bin/llvm-profdata >> "$GITHUB_ENV"' | |
| - name: merge pgo data | |
| run: ${{ env.LLVM_PROFDATA }} merge -o ${{ github.workspace }}/merged.profdata ${{ github.workspace }}/profdata | |
| - name: build pgo-optimized wheel | |
| uses: PyO3/maturin-action@v1 | |
| with: | |
| manylinux: auto | |
| args: > | |
| --release | |
| --out dist | |
| --interpreter ${{ matrix.interpreter }} | |
| rust-toolchain: ${{steps.rust-toolchain.outputs.name}} | |
| docker-options: -e CI | |
| env: | |
| RUSTFLAGS: '-Cprofile-use=${{ github.workspace }}/merged.profdata' | |
| - run: ${{ matrix.ls || 'ls -lh' }} dist/ | |
| - uses: actions/upload-artifact@v4 | |
| with: | |
| name: pypi_files_${{ matrix.os }}_${{ matrix.interpreter }} | |
| path: dist | |
| inspect-pypi-assets: | |
| needs: [build, build-sdist, build-pgo] | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: get dist artifacts | |
| uses: actions/download-artifact@v4 | |
| with: | |
| pattern: pypi_files_* | |
| merge-multiple: true | |
| path: dist | |
| - name: list dist files | |
| run: | | |
| ls -lh dist/ | |
| ls -l dist/ | |
| echo "`ls dist | wc -l` files" | |
| - name: extract and list sdist file | |
| run: | | |
| mkdir sdist-files | |
| tar -xvf dist/*.tar.gz -C sdist-files | |
| tree -a sdist-files | |
| - name: extract and list wheel file | |
| run: | | |
| ls dist/*cp310-manylinux*x86_64.whl | head -n 1 | |
| python -m zipfile --list `ls dist/*cp310-manylinux*x86_64.whl | head -n 1` | |
| test-builds-arch: | |
| name: test build on ${{ matrix.target }}-${{ matrix.distro }} | |
| needs: [build] | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| target: [aarch64, armv7, s390x, ppc64le] | |
| distro: ['ubuntu22.04'] | |
| include: | |
| - target: aarch64 | |
| distro: alpine_latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: get dist artifacts | |
| uses: actions/download-artifact@v4 | |
| with: | |
| pattern: pypi_files_linux_* | |
| merge-multiple: true | |
| path: dist | |
| - uses: uraimo/run-on-arch-action@v2.8.1 | |
| name: install & test | |
| with: | |
| arch: ${{ matrix.target }} | |
| distro: ${{ matrix.distro }} | |
| githubToken: ${{ github.token }} | |
| install: | | |
| set -x | |
| if command -v apt-get &> /dev/null; then | |
| echo "installing python & pip with apt-get..." | |
| apt-get update | |
| apt-get install -y --no-install-recommends python3 python3-pip python3-venv git | |
| else | |
| echo "installing python & pip with apk..." | |
| apk update | |
| apk add python3 py3-pip git | |
| fi | |
| run: | | |
| set -x | |
| python3 -m venv venv | |
| source venv/bin/activate | |
| python3 -m pip install -r tests/requirements.txt | |
| python3 -m pip install pydantic-core --no-index --no-deps --find-links dist --force-reinstall | |
| python3 -m pytest --ignore=tests/test_docstrings.py | |
| python3 -c 'import pydantic_core._pydantic_core; print(pydantic_core._pydantic_core.__version__)' | |
| test-builds-os: | |
| name: test build on ${{ matrix.os }} | |
| needs: [build, build-pgo] | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| os: [ubuntu, macos, windows] | |
| runs-on: ${{ matrix.os }}-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: set up python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.13' | |
| - name: get dist artifacts | |
| uses: actions/download-artifact@v4 | |
| with: | |
| pattern: pypi_files_* | |
| merge-multiple: true | |
| path: dist | |
| - run: pip install -r tests/requirements.txt | |
| - run: pip install pydantic-core --no-index --no-deps --find-links dist --force-reinstall | |
| - run: pytest --ignore=tests/test_docstrings.py | |
| release: | |
| needs: [test-builds-arch, test-builds-os, build-sdist, check] | |
| if: success() && startsWith(github.ref, 'refs/tags/') | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: set up python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.13' | |
| - run: pip install -U twine | |
| - name: check package version | |
| run: python .github/check_version.py | |
| - name: get dist artifacts | |
| uses: actions/download-artifact@v4 | |
| with: | |
| pattern: pypi_files_* | |
| merge-multiple: true | |
| path: dist | |
| - run: twine check --strict dist/* | |
| - name: upload to pypi | |
| run: twine upload dist/* | |
| env: | |
| TWINE_USERNAME: __token__ | |
| TWINE_PASSWORD: ${{ secrets.pypi_token }} | |
| - name: get wasm dist artifacts | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: wasm_wheels | |
| path: wasm | |
| - name: upload to github release | |
| uses: softprops/action-gh-release@v2 | |
| with: | |
| files: | | |
| wasm/*.whl | |
| prerelease: ${{ contains(github.ref, 'alpha') || contains(github.ref, 'beta') }} |