From b95a374c13be2c7fd8dc6badafd0c53ba5049a40 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Thu, 20 Jun 2024 08:12:05 -0500 Subject: [PATCH] PYTHON-4388 Add SSDLC workflows (#1691) Signed-off-by: mongodb-dbx-release-bot[bot] <167856002+mongodb-dbx-release-bot[bot]@users.noreply.github.com> Co-authored-by: mongodb-dbx-release-bot[bot] <167856002+mongodb-dbx-release-bot[bot]@users.noreply.github.com> (cherry picked from commit 25cbc7e2a5f6d97fd37ce34b2a06e4071181fc4e) --- .github/workflows/codeql.yml | 7 + .github/workflows/dist.yml | 140 +++++++++++++++++++ .github/workflows/release-python.yml | 198 +++++++++------------------ pyproject.toml | 1 + 4 files changed, 212 insertions(+), 134 deletions(-) create mode 100644 .github/workflows/dist.yml diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 0d2551d76b..abdd98b722 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -5,6 +5,11 @@ on: branches: [ "master", "v*"] tags: ['*'] pull_request: + workflow_call: + inputs: + ref: + required: true + type: string schedule: - cron: '17 10 * * 2' @@ -35,6 +40,8 @@ jobs: steps: - name: Checkout repository uses: actions/checkout@v4 + with: + ref: ${{ inputs.ref }} - uses: actions/setup-python@v3 # Initializes the CodeQL tools for scanning. diff --git a/.github/workflows/dist.yml b/.github/workflows/dist.yml new file mode 100644 index 0000000000..8ac1d00a6b --- /dev/null +++ b/.github/workflows/dist.yml @@ -0,0 +1,140 @@ +name: Python Dist + +on: + push: + tags: + - "[0-9]+.[0-9]+.[0-9]+" + - "[0-9]+.[0-9]+.[0-9]+.post[0-9]+" + - "[0-9]+.[0-9]+.[0-9]+[a-b][0-9]+" + - "[0-9]+.[0-9]+.[0-9]+rc[0-9]+" + workflow_dispatch: + pull_request: + workflow_call: + +concurrency: + group: dist-${{ github.ref }} + cancel-in-progress: true + +defaults: + run: + shell: bash -eux {0} + +jobs: + build_wheels: + name: Build wheels for ${{ matrix.buildplat[1] }} + runs-on: ${{ matrix.buildplat[0] }} + strategy: + # Ensure that a wheel builder finishes even if another fails + fail-fast: false + matrix: + # Github Actions doesn't support pairing matrix values together, let's improvise + # https://github.com/github/feedback/discussions/7835#discussioncomment-1769026 + buildplat: + - [ubuntu-20.04, "manylinux_x86_64", "cp3*-manylinux_x86_64"] + - [ubuntu-20.04, "manylinux_aarch64", "cp3*-manylinux_aarch64"] + - [ubuntu-20.04, "manylinux_ppc64le", "cp3*-manylinux_ppc64le"] + - [ubuntu-20.04, "manylinux_s390x", "cp3*-manylinux_s390x"] + - [ubuntu-20.04, "manylinux_i686", "cp3*-manylinux_i686"] + - [windows-2019, "win_amd6", "cp3*-win_amd64"] + - [windows-2019, "win32", "cp3*-win32"] + - [macos-14, "macos", "cp*-macosx_*"] + + steps: + - name: Checkout pymongo + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - uses: actions/setup-python@v5 + with: + cache: 'pip' + python-version: 3.8 + cache-dependency-path: 'pyproject.toml' + allow-prereleases: true + + - name: Set up QEMU + if: runner.os == 'Linux' + uses: docker/setup-qemu-action@v3 + with: + platforms: all + + - name: Install cibuildwheel + # Note: the default manylinux is manylinux2014 + run: | + python -m pip install -U pip + python -m pip install "cibuildwheel>=2.17,<3" + + - name: Build wheels + env: + CIBW_BUILD: ${{ matrix.buildplat[2] }} + run: python -m cibuildwheel --output-dir wheelhouse + + - name: Build manylinux1 wheels + if: ${{ matrix.buildplat[1] == 'manylinux_x86_64' || matrix.buildplat[1] == 'manylinux_i686' }} + env: + CIBW_MANYLINUX_X86_64_IMAGE: manylinux1 + CIBW_MANYLINUX_I686_IMAGE: manylinux1 + CIBW_BUILD: "cp38-${{ matrix.buildplat[1] }} cp39-${{ matrix.buildplat[1] }}" + run: python -m cibuildwheel --output-dir wheelhouse + + - name: Assert all versions in wheelhouse + if: ${{ ! startsWith(matrix.buildplat[1], 'macos') }} + run: | + ls wheelhouse/*cp38*.whl + ls wheelhouse/*cp39*.whl + ls wheelhouse/*cp310*.whl + ls wheelhouse/*cp311*.whl + ls wheelhouse/*cp312*.whl + + - uses: actions/upload-artifact@v4 + with: + name: wheel-${{ matrix.buildplat[1] }} + path: ./wheelhouse/*.whl + if-no-files-found: error + + make_sdist: + name: Make SDist + runs-on: macos-13 + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - uses: actions/setup-python@v5 + with: + # Build sdist on lowest supported Python + python-version: '3.8' + + - name: Build SDist + run: | + set -ex + python -m pip install -U pip build + python -m build --sdist . + + - name: Test SDist + run: | + python -m pip install dist/*.gz + cd .. + python -c "from pymongo import has_c; assert has_c()" + + - uses: actions/upload-artifact@v4 + with: + name: "sdist" + path: ./dist/*.tar.gz + + collect_dist: + runs-on: ubuntu-latest + needs: [build_wheels, make_sdist] + name: Download Wheels + steps: + - name: Download all workflow run artifacts + uses: actions/download-artifact@v4 + - name: Flatten directory + working-directory: . + run: | + find . -mindepth 2 -type f -exec mv {} . \; + find . -type d -empty -delete + - uses: actions/upload-artifact@v4 + with: + name: all-dist-${{ github.run_id }} + path: "./*" diff --git a/.github/workflows/release-python.yml b/.github/workflows/release-python.yml index c3ee0d4eb1..8ce4eaa84f 100644 --- a/.github/workflows/release-python.yml +++ b/.github/workflows/release-python.yml @@ -1,156 +1,86 @@ -name: Python Wheels +name: Release on: - push: - tags: - - "[0-9]+.[0-9]+.[0-9]+" - - "[0-9]+.[0-9]+.[0-9]+.post[0-9]+" - - "[0-9]+.[0-9]+.[0-9]+[a-b][0-9]+" - - "[0-9]+.[0-9]+.[0-9]+rc[0-9]+" workflow_dispatch: - pull_request: - -concurrency: - group: wheels-${{ github.ref }} - cancel-in-progress: true + inputs: + version: + description: "The new version to set" + required: true + following_version: + description: "The post (dev) version to set" + required: true + dry_run: + description: "Dry Run?" + default: false + type: boolean + +env: + # Changes per repo + PRODUCT_NAME: PyMongo + # Changes per branch + SILK_ASSET_GROUP: mongodb-python-driver defaults: run: shell: bash -eux {0} jobs: - build_wheels: - name: Build wheels for ${{ matrix.buildplat[1] }} - runs-on: ${{ matrix.buildplat[0] }} - strategy: - # Ensure that a wheel builder finishes even if another fails - fail-fast: false - matrix: - # Github Actions doesn't support pairing matrix values together, let's improvise - # https://github.com/github/feedback/discussions/7835#discussioncomment-1769026 - buildplat: - - [ubuntu-20.04, "manylinux_x86_64", "cp3*-manylinux_x86_64"] - - [ubuntu-20.04, "manylinux_aarch64", "cp3*-manylinux_aarch64"] - - [ubuntu-20.04, "manylinux_ppc64le", "cp3*-manylinux_ppc64le"] - - [ubuntu-20.04, "manylinux_s390x", "cp3*-manylinux_s390x"] - - [ubuntu-20.04, "manylinux_i686", "cp3*-manylinux_i686"] - - [windows-2019, "win_amd6", "cp3*-win_amd64"] - - [windows-2019, "win32", "cp3*-win32"] - - [macos-14, "macos", "cp*-macosx_*"] - + pre-publish: + environment: release + runs-on: ubuntu-latest + permissions: + id-token: write + contents: write steps: - - name: Checkout pymongo - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - uses: actions/setup-python@v5 - with: - cache: 'pip' - python-version: 3.8 - cache-dependency-path: 'pyproject.toml' - allow-prereleases: true - - - name: Set up QEMU - if: runner.os == 'Linux' - uses: docker/setup-qemu-action@v3 - with: - platforms: all - - - name: Install cibuildwheel - # Note: the default manylinux is manylinux2014 - run: | - python -m pip install -U pip - python -m pip install "cibuildwheel>=2.17,<3" - - - name: Build wheels - env: - CIBW_BUILD: ${{ matrix.buildplat[2] }} - run: python -m cibuildwheel --output-dir wheelhouse - - - name: Build manylinux1 wheels - if: ${{ matrix.buildplat[1] == 'manylinux_x86_64' || matrix.buildplat[1] == 'manylinux_i686' }} - env: - CIBW_MANYLINUX_X86_64_IMAGE: manylinux1 - CIBW_MANYLINUX_I686_IMAGE: manylinux1 - CIBW_BUILD: "cp38-${{ matrix.buildplat[1] }} cp39-${{ matrix.buildplat[1] }}" - run: python -m cibuildwheel --output-dir wheelhouse - - - name: Assert all versions in wheelhouse - if: ${{ ! startsWith(matrix.buildplat[1], 'macos') }} - run: | - ls wheelhouse/*cp38*.whl - ls wheelhouse/*cp39*.whl - ls wheelhouse/*cp310*.whl - ls wheelhouse/*cp311*.whl - ls wheelhouse/*cp312*.whl - - - uses: actions/upload-artifact@v4 + - uses: mongodb-labs/drivers-github-tools/secure-checkout@v2 with: - name: wheel-${{ matrix.buildplat[1] }} - path: ./wheelhouse/*.whl - if-no-files-found: error - - make_sdist: - name: Make SDist - runs-on: macos-13 - steps: - - uses: actions/checkout@v4 + app_id: ${{ vars.APP_ID }} + private_key: ${{ secrets.APP_PRIVATE_KEY }} + - uses: mongodb-labs/drivers-github-tools/setup@v2 with: - fetch-depth: 0 - - - uses: actions/setup-python@v5 + aws_role_arn: ${{ secrets.AWS_ROLE_ARN }} + aws_region_name: ${{ vars.AWS_REGION_NAME }} + aws_secret_id: ${{ secrets.AWS_SECRET_ID }} + artifactory_username: ${{ vars.ARTIFACTORY_USERNAME }} + - uses: mongodb-labs/drivers-github-tools/python/pre-publish@v2 with: - # Build sdist on lowest supported Python - python-version: '3.8' - - - name: Build SDist - run: | - set -ex - python -m pip install -U pip build - python -m build --sdist . - - - name: Test SDist - run: | - python -m pip install dist/*.gz - cd .. - python -c "from pymongo import has_c; assert has_c()" + version: ${{ inputs.version }} + dry_run: ${{ inputs.dry_run }} - - uses: actions/upload-artifact@v4 - with: - name: "sdist" - path: ./dist/*.tar.gz + build-dist: + needs: [pre-publish] + uses: ./.github/workflows/dist.yml - collect_dist: - runs-on: ubuntu-latest - needs: [build_wheels, make_sdist] - name: Download Wheels - steps: - - name: Download all workflow run artifacts - uses: actions/download-artifact@v4 - - name: Flatten directory - working-directory: . - run: | - find . -mindepth 2 -type f -exec mv {} . \; - find . -type d -empty -delete - - uses: actions/upload-artifact@v4 - with: - name: all-dist-${{ github.run_id }} - path: "./*" + static-scan: + needs: [pre-publish] + uses: ./.github/workflows/codeql.yml + with: + ref: ${{ inputs.version }} publish: - # https://packaging.python.org/en/latest/guides/publishing-package-distribution-releases-using-github-actions-ci-cd-workflows/#publishing-the-distribution-to-pypi - needs: [collect_dist] - if: startsWith(github.ref, 'refs/tags/') + needs: [build-dist, static-scan] runs-on: ubuntu-latest environment: release permissions: id-token: write + contents: write + security-events: write steps: - - name: Download all the dists - uses: actions/download-artifact@v4 - with: - name: all-dist-${{ github.run_id }} - path: dist/ - - name: Publish distribution 📦 to PyPI - uses: pypa/gh-action-pypi-publish@release/v1 + - uses: mongodb-labs/drivers-github-tools/secure-checkout@v2 + with: + app_id: ${{ vars.APP_ID }} + private_key: ${{ secrets.APP_PRIVATE_KEY }} + - uses: mongodb-labs/drivers-github-tools/setup@v2 + with: + aws_role_arn: ${{ secrets.AWS_ROLE_ARN }} + aws_region_name: ${{ vars.AWS_REGION_NAME }} + aws_secret_id: ${{ secrets.AWS_SECRET_ID }} + artifactory_username: ${{ vars.ARTIFACTORY_USERNAME }} + - uses: mongodb-labs/drivers-github-tools/python/publish@v2 + with: + version: ${{ inputs.version }} + following_version: ${{ inputs.following_version }} + product_name: ${{ env.PRODUCT_NAME }} + silk_asset_group: ${{ env.SILK_ASSET_GROUP }} + token: ${{ github.token }} + dry_run: ${{ inputs.dry_run }} diff --git a/pyproject.toml b/pyproject.toml index f49ee881bd..d208f6a439 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -50,6 +50,7 @@ Tracker = "https://jira.mongodb.org/projects/PYTHON/issues" [tool.hatch.version] path = "pymongo/_version.py" +validate-bump = false [tool.hatch.build.targets.wheel] packages = ["bson","gridfs", "pymongo"]