Skip to content
CI/CD

CI/CD #1478

Sign in to view logs

CI/CD

CI/CD #1478

Workflow file for this run

.github/workflows/ci-cd.yml at 83aa96e
name: CI/CD
on:
push:
branches-ignore:
- dependabot/**
pull_request:
workflow_dispatch:
inputs:
release-version:
# github.event_name == 'workflow_dispatch'
# && github.event.inputs.release-version
description: >-
Target PEP440-compliant version to release.
Please, don't prepend `v`.
required: true
release-commitish:
# github.event_name == 'workflow_dispatch'
# && github.event.inputs.release-commitish
default: ''
description: >-
The commit to be released to PyPI and tagged
in Git as `release-version`. Normally, you
should keep this empty.
required: false
YOLO:
default: false
description: >-
Flag whether test results should block the
release (true/false). Only use this under
extraordinary circumstances to ignore the
test failures and cut the release regardless.
required: false
schedule:
- cron: 1 0 * * * # Run daily at 0:01 UTC
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
cancel-in-progress: true
jobs:
pre-setup:
name: ⚙️ Pre-set global build settings
runs-on: ubuntu-latest
defaults:
run:
shell: python
outputs:
dist-version: >-
${{
steps.request-check.outputs.release-requested == 'true'
&& github.event.inputs.release-version
|| steps.scm-version.outputs.dist-version
}}
is-untagged-devel: >-
${{ steps.untagged-check.outputs.is-untagged-devel || false }}
release-requested: >-
${{
steps.request-check.outputs.release-requested || false
}}
cache-key-files: >-
${{ steps.calc-cache-key-files.outputs.files-hash-key }}
git-tag: ${{ steps.git-tag.outputs.tag }}
sdist-artifact-name: ${{ steps.artifact-name.outputs.sdist }}
wheel-artifact-name: ${{ steps.artifact-name.outputs.wheel }}
steps:
- name: Switch to using Python 3.10 by default
uses: actions/setup-python@v4
with:
python-version: >-
3.10
- name: >-
Mark the build as untagged '${{
github.event.repository.default_branch
}}' branch build
id: untagged-check
if: >-
github.event_name == 'push' &&
github.ref == format(
'refs/heads/{0}', github.event.repository.default_branch
)
run: >-
echo "is-untagged-devel=true" >> "$GITHUB_OUTPUT"
shell: bash
- name: Mark the build as "release request"
id: request-check
if: github.event_name == 'workflow_dispatch'
run: >-
echo "release-requested=true" >> "$GITHUB_OUTPUT"
shell: bash
- name: Check out src from Git
if: >-
steps.request-check.outputs.release-requested != 'true'
uses: actions/checkout@v3
with:
fetch-depth: 0
ref: ${{ github.event.inputs.release-commitish }}
- name: >-
Calculate Python interpreter version hash value
for use in the cache key
if: >-
steps.request-check.outputs.release-requested != 'true'
id: calc-cache-key-py
run: |
from hashlib import sha512
from os import environ
from sys import version
hash = sha512(version.encode()).hexdigest()
with open(environ['GITHUB_OUTPUT'], mode='a') as github_output:
print(f'py-hash-key={hash}', file=github_output)
- name: >-
Calculate dependency files' combined hash value
for use in the cache key
if: >-
steps.request-check.outputs.release-requested != 'true'
id: calc-cache-key-files
run: >-
echo "files-hash-key=${{
hashFiles(
'requirements-dev.txt',
'setup.cfg',
'pyproject.toml'
)
}}" >> "$GITHUB_OUTPUT"
shell: bash
- name: Get pip cache dir
id: pip-cache-dir
if: >-
steps.request-check.outputs.release-requested != 'true'
run: >-
echo "dir=$(python -m pip cache dir)" >> "$GITHUB_OUTPUT"
shell: bash
- name: Set up pip cache
if: >-
steps.request-check.outputs.release-requested != 'true'
uses: actions/cache@v3.3.1
with:
path: ${{ steps.pip-cache-dir.outputs.dir }}
key: >-
${{ runner.os }}-pip-${{
steps.calc-cache-key-py.outputs.py-hash-key }}-${{
steps.calc-cache-key-files.outputs.files-hash-key }}
restore-keys: |
${{ runner.os }}-pip-${{
steps.calc-cache-key-py.outputs.py-hash-key
}}-
${{ runner.os }}-pip-
${{ runner.os }}-
- name: Drop Git tags from HEAD for non-release requests
if: >-
steps.request-check.outputs.release-requested != 'true'
run: >-
git tag --points-at HEAD
|
xargs git tag --delete
shell: bash
- name: Set up versioning prerequisites
if: >-
steps.request-check.outputs.release-requested != 'true'
run: >-
python -m
pip install
--user
--upgrade
setuptools-scm
shell: bash
- name: Set the current dist version from Git
if: steps.request-check.outputs.release-requested != 'true'
id: scm-version
run: |
from os import environ
import setuptools_scm
ver = setuptools_scm.get_version(
${{
steps.untagged-check.outputs.is-untagged-devel == 'true'
&& 'local_scheme="no-local-version"' || ''
}}
)
with open(environ['GITHUB_OUTPUT'], mode='a') as github_output:
print('dist-version={ver}'.format(ver=ver), file=github_output)
- name: Set the target Git tag
id: git-tag
run: >-
echo "tag=v${{
steps.request-check.outputs.release-requested == 'true'
&& github.event.inputs.release-version
|| steps.scm-version.outputs.dist-version
}}" >> "$GITHUB_OUTPUT"
shell: bash
- name: Set the expected dist artifact names
id: artifact-name
run: |
echo "sdist=aiomysql-${{
steps.request-check.outputs.release-requested == 'true'
&& github.event.inputs.release-version
|| steps.scm-version.outputs.dist-version
}}.tar.gz" >> "$GITHUB_OUTPUT"
echo "wheel=aiomysql-${{
steps.request-check.outputs.release-requested == 'true'
&& github.event.inputs.release-version
|| steps.scm-version.outputs.dist-version
}}-py3-none-any.whl" >> "$GITHUB_OUTPUT"
shell: bash
build:
name: >-
👷 dists ${{ needs.pre-setup.outputs.git-tag }}
[mode: ${{
fromJSON(needs.pre-setup.outputs.is-untagged-devel)
&& 'nightly' || ''
}}${{
fromJSON(needs.pre-setup.outputs.release-requested)
&& 'release' || ''
}}${{
(
!fromJSON(needs.pre-setup.outputs.is-untagged-devel)
&& !fromJSON(needs.pre-setup.outputs.release-requested)
) && 'test' || ''
}}]
needs:
- pre-setup # transitive, for accessing settings
runs-on: ubuntu-latest
env:
PY_COLORS: 1
steps:
- name: Switch to using Python v3.10
uses: actions/setup-python@v4
with:
python-version: >-
3.10
- name: >-
Calculate Python interpreter version hash value
for use in the cache key
id: calc-cache-key-py
run: |
from hashlib import sha512
from os import environ
from sys import version
hash = sha512(version.encode()).hexdigest()
with open(environ['GITHUB_OUTPUT'], mode='a') as github_output:
print(f'py-hash-key={hash}', file=github_output)
shell: python
- name: Get pip cache dir
id: pip-cache-dir
run: >-
echo "dir=$(python -m pip cache dir)" >> "$GITHUB_OUTPUT"
- name: Set up pip cache
uses: actions/cache@v3.3.1
with:
path: ${{ steps.pip-cache-dir.outputs.dir }}
key: >-
${{ runner.os }}-pip-${{
steps.calc-cache-key-py.outputs.py-hash-key }}-${{
needs.pre-setup.outputs.cache-key-files }}
restore-keys: |
${{ runner.os }}-pip-${{
steps.calc-cache-key-py.outputs.py-hash-key
}}-
${{ runner.os }}-pip-
- name: Install build tools
run: >-
python -m
pip install
--user
--upgrade
build
- name: Grab the source from Git
uses: actions/checkout@v3
with:
fetch-depth: >-
${{
steps.request-check.outputs.release-requested == 'true'
&& 1 || 0
}}
ref: ${{ github.event.inputs.release-commitish }}
- name: Setup git user as [bot]
if: >-
fromJSON(needs.pre-setup.outputs.is-untagged-devel)
|| fromJSON(needs.pre-setup.outputs.release-requested)
uses: fregante/setup-git-user@77c1b5542f14ab6db4b8462d6857e31deb988b09
- name: >-
Tag the release in the local Git repo
as ${{ needs.pre-setup.outputs.git-tag }}
for setuptools-scm to set the desired version
if: >-
fromJSON(needs.pre-setup.outputs.is-untagged-devel)
|| fromJSON(needs.pre-setup.outputs.release-requested)
run: >-
git tag
-m '${{ needs.pre-setup.outputs.git-tag }}'
'${{ needs.pre-setup.outputs.git-tag }}'
--
${{ github.event.inputs.release-commitish }}
- name: Build dists
run: >-
python
-m
build
- name: Verify that the artifacts with expected names got created
run: >-
ls -1
'dist/${{ needs.pre-setup.outputs.sdist-artifact-name }}'
'dist/${{ needs.pre-setup.outputs.wheel-artifact-name }}'
- name: Store the distribution packages
uses: actions/upload-artifact@v3
with:
name: python-package-distributions
# NOTE: Exact expected file names are specified here
# NOTE: as a safety measure — if anything weird ends
# NOTE: up being in this dir or not all dists will be
# NOTE: produced, this will fail the workflow.
path: |
dist/${{ needs.pre-setup.outputs.sdist-artifact-name }}
dist/${{ needs.pre-setup.outputs.wheel-artifact-name }}
retention-days: 30
lint:
name: 🧹 Lint
needs:
- build
- pre-setup # transitive, for accessing settings
runs-on: ubuntu-latest
env:
PY_COLORS: 1
steps:
- name: Switch to using Python 3.10 by default
uses: actions/setup-python@v4
with:
python-version: >-
3.10
- name: >-
Calculate Python interpreter version hash value
for use in the cache key
id: calc-cache-key-py
run: |
from hashlib import sha512
from os import environ
from sys import version
hash = sha512(version.encode()).hexdigest()
with open(environ['GITHUB_OUTPUT'], mode='a') as github_output:
print(f'py-hash-key={hash}', file=github_output)
shell: python
- name: Get pip cache dir
id: pip-cache-dir
run: >-
echo "dir=$(python -m pip cache dir)" >> "$GITHUB_OUTPUT"
- name: Set up pip cache
uses: actions/cache@v3.3.1
with:
path: ${{ steps.pip-cache-dir.outputs.dir }}
key: >-
${{ runner.os }}-pip-${{
steps.calc-cache-key-py.outputs.py-hash-key }}-${{
needs.pre-setup.outputs.cache-key-files }}
restore-keys: |
${{ runner.os }}-pip-${{
steps.calc-cache-key-py.outputs.py-hash-key
}}-
${{ runner.os }}-pip-
- name: Grab the source from Git
uses: actions/checkout@v3
with:
ref: ${{ github.event.inputs.release-commitish }}
- name: Download all the dists
uses: actions/download-artifact@v3
with:
name: python-package-distributions
path: dist/
- name: Install build tools
run: >-
python -m
pip install
--user
--requirement requirements-dev.txt
- name: flake8 Lint
uses: py-actions/flake8@v2.2.1
with:
flake8-version: 5.0.4
path: aiomysql
args: tests examples
- name: Check package description
run: |
python -m twine check --strict dist/*
tests:
name: >-
🧪 🐍${{
matrix.py
}} @ ${{
matrix.os
}} on ${{
join(matrix.db, '-')
}}
needs:
- build
- pre-setup # transitive, for accessing settings
strategy:
# when updating matrix jobs make sure to adjust the expected reports in
# codecov.notify.after_n_builds in .codecov.yml
matrix:
# service containers are only supported on ubuntu currently
os:
- ubuntu-latest
py:
- '3.7'
- '3.8'
- '3.9'
- '3.10'
- '3.11'
db:
- [mysql, '5.7']
- [mysql, '8.0']
- [mariadb, '10.4']
- [mariadb, '10.5']
- [mariadb, '10.6']
- [mariadb, '10.9']
- [mariadb, '10.10']
- [mariadb, '10.11']
fail-fast: false
runs-on: ${{ matrix.os }}
timeout-minutes: 15
continue-on-error: >-
${{
(
(
needs.pre-setup.outputs.release-requested == 'true' &&
!toJSON(github.event.inputs.YOLO)
) ||
contains(matrix.py, '-dev')
) && true || false
}}
env:
MYSQL_ROOT_PASSWORD: rootpw
PY_COLORS: 1
services:
mysql:
image: "${{ join(matrix.db, ':') }}"
ports:
- 3306:3306
volumes:
- "/tmp/run-${{ join(matrix.db, '-') }}/:/socket-mount/"
options: '--name=mysqld'
env:
MYSQL_ROOT_PASSWORD: rootpw
steps:
- name: Setup Python ${{ matrix.py }}
id: python-install
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.py }}
- name: Figure out if the interpreter ABI is stable
id: py-abi
run: |
from os import environ
from sys import version_info
is_stable_abi = version_info.releaselevel == 'final'
with open(environ['GITHUB_OUTPUT'], mode='a') as github_output:
print(
'is-stable-abi={is_stable_abi}'.
format(is_stable_abi=str(is_stable_abi).lower()),
file=github_output,
)
shell: python
- name: >-
Calculate Python interpreter version hash value
for use in the cache key
if: fromJSON(steps.py-abi.outputs.is-stable-abi)
id: calc-cache-key-py
run: |
from os import environ
from hashlib import sha512
from sys import version
hash = sha512(version.encode()).hexdigest()
with open(environ['GITHUB_OUTPUT'], mode='a') as github_output:
print('py-hash-key={hash}'.format(hash=hash), file=github_output)
shell: python
- name: Get pip cache dir
if: fromJSON(steps.py-abi.outputs.is-stable-abi)
id: pip-cache-dir
run: >-
echo "dir=$(python -m pip cache dir)" >> "$GITHUB_OUTPUT"
- name: Set up pip cache
if: fromJSON(steps.py-abi.outputs.is-stable-abi)
uses: actions/cache@v3.3.1
with:
path: ${{ steps.pip-cache-dir.outputs.dir }}
key: >-
${{ runner.os }}-pip-${{
steps.calc-cache-key-py.outputs.py-hash-key }}-${{
needs.pre-setup.outputs.cache-key-files }}
restore-keys: |
${{ runner.os }}-pip-${{
steps.calc-cache-key-py.outputs.py-hash-key
}}-
${{ runner.os }}-pip-
- name: Update pip
run: >-
python -m
pip install
--user
--upgrade
pip
- name: Grab the source from Git
uses: actions/checkout@v3
with:
ref: ${{ github.event.inputs.release-commitish }}
- name: Remove aiomysql source to avoid accidentally using it
run: >-
rm -rf aiomysql
- name: Download all the dists
uses: actions/download-artifact@v3
with:
name: python-package-distributions
path: dist/
- name: Install dependencies
run: >-
python -m
pip install
--user
--requirement requirements-dev.txt
- name: Install previously built wheel
run: >-
python -m
pip install
--user
'dist/${{ needs.pre-setup.outputs.wheel-artifact-name }}'
- name: >-
Log platform.platform()
run: >-
python -m platform
- name: >-
Log platform.version()
run: >-
python -c "import platform;
print(platform.version())"
- name: >-
Log platform.uname()
run: >-
python -c "import platform;
print(platform.uname())"
- name: >-
Log platform.release()
run: >-
python -c "import platform;
print(platform.release())"
- name: Log stdlib OpenSSL version
run: >-
python -c
"import ssl; print('\nOPENSSL_VERSION: '
+ ssl.OPENSSL_VERSION + '\nOPENSSL_VERSION_INFO: '
+ repr(ssl.OPENSSL_VERSION_INFO)
+ '\nOPENSSL_VERSION_NUMBER: '
+ repr(ssl.OPENSSL_VERSION_NUMBER))"
# this ensures our database is ready. typically by the time the preparations have completed its first start logic.
# unfortunately we need this hacky workaround as GitHub Actions service containers can't reference data from our repo.
- name: Prepare mysql
run: |
# ensure server is started up
while :
do
sleep 1
mysql -h127.0.0.1 -uroot "-p$MYSQL_ROOT_PASSWORD" -e 'select version()' && break
done
# inject tls configuration
docker container stop mysqld
docker container cp "${{ github.workspace }}/tests/ssl_resources/ssl" mysqld:/etc/mysql/ssl
docker container cp "${{ github.workspace }}/tests/ssl_resources/tls.cnf" mysqld:/etc/mysql/conf.d/aiomysql-tls.cnf
# use custom socket path
# we need to ensure that the socket path is writable for the user running the DB process in the container
sudo chmod 0777 /tmp/run-${{ join(matrix.db, '-') }}
docker container cp "${{ github.workspace }}/tests/ssl_resources/socket.cnf" mysqld:/etc/mysql/conf.d/aiomysql-socket.cnf
docker container start mysqld
# ensure server is started up
while :
do
sleep 1
mysql -h127.0.0.1 -uroot "-p$MYSQL_ROOT_PASSWORD" -e 'select version()' && break
done
mysql -h127.0.0.1 -uroot "-p$MYSQL_ROOT_PASSWORD" -e "SET GLOBAL local_infile=on"
- name: Run tests
run: |
# timeout ensures a more or less clean stop by sending a KeyboardInterrupt which will still provide useful logs
timeout --preserve-status --signal=INT --verbose 570s \
pytest --capture=no --verbosity 2 --cov-report term --cov-report xml --cov aiomysql --cov tests ./tests --mysql-unix-socket "unix-${{ join(matrix.db, '') }}=/tmp/run-${{ join(matrix.db, '-') }}/mysql.sock" --mysql-address "tcp-${{ join(matrix.db, '') }}=127.0.0.1:3306"
env:
PYTHONUNBUFFERED: 1
timeout-minutes: 10
- name: Upload coverage
if: ${{ github.event_name != 'schedule' }}
uses: codecov/codecov-action@v3.1.4
with:
file: ./coverage.xml
flags: >-
CI-GHA,
OS-${{ runner.os }},
VM-${{ matrix.os }},
Py-${{ steps.python-install.outputs.python-version }},
DB-${{ join(matrix.db, '-') }},
${{ matrix.os }}_${{ matrix.py }}_${{ join(matrix.db, '-') }}
fail_ci_if_error: true
check: # This job does nothing and is only used for the branch protection
if: always()
needs:
- lint
- tests
runs-on: ubuntu-latest
steps:
- name: Decide whether the needed jobs succeeded or failed
uses: re-actors/alls-green@v1.2.2
with:
jobs: ${{ toJSON(needs) }}
publish-pypi:
name: Publish 🐍📦 ${{ needs.pre-setup.outputs.git-tag }} to PyPI
needs:
- check
- pre-setup # transitive, for accessing settings
if: >-
fromJSON(needs.pre-setup.outputs.release-requested)
runs-on: ubuntu-latest
environment:
name: pypi
url: >-
https://pypi.org/project/aiomysql/${{
needs.pre-setup.outputs.dist-version
}}
permissions:
id-token: write # this permission is mandatory for trusted publishing
steps:
- name: Download all the dists
uses: actions/download-artifact@v3
with:
name: python-package-distributions
path: dist/
- name: >-
Publish 🐍📦 ${{ needs.pre-setup.outputs.git-tag }} to PyPI
uses: pypa/gh-action-pypi-publish@v1.8.6
with:
print-hash: true
publish-testpypi:
name: Publish 🐍📦 ${{ needs.pre-setup.outputs.git-tag }} to TestPyPI
needs:
- check
- pre-setup # transitive, for accessing settings
if: >-
fromJSON(needs.pre-setup.outputs.is-untagged-devel)
|| fromJSON(needs.pre-setup.outputs.release-requested)
runs-on: ubuntu-latest
environment:
name: testpypi
url: >-
https://test.pypi.org/project/aiomysql/${{
needs.pre-setup.outputs.dist-version
}}
permissions:
id-token: write # this permission is mandatory for trusted publishing
steps:
- name: Download all the dists
uses: actions/download-artifact@v3
with:
name: python-package-distributions
path: dist/
- name: >-
Publish 🐍📦 ${{ needs.pre-setup.outputs.git-tag }} to TestPyPI
uses: pypa/gh-action-pypi-publish@v1.8.6
with:
repository-url: https://test.pypi.org/legacy/
print-hash: true
post-release-repo-update:
name: >-
Publish post-release Git tag
for ${{ needs.pre-setup.outputs.git-tag }}
needs:
- publish-pypi
- pre-setup # transitive, for accessing settings
runs-on: ubuntu-latest
steps:
- name: Fetch the src snapshot
uses: actions/checkout@v3
with:
fetch-depth: 1
ref: ${{ github.event.inputs.release-commitish }}
- name: Setup git user as [bot]
uses: fregante/setup-git-user@77c1b5542f14ab6db4b8462d6857e31deb988b09
- name: >-
Tag the release in the local Git repo
as v${{ needs.pre-setup.outputs.git-tag }}
run: >-
git tag
-m '${{ needs.pre-setup.outputs.git-tag }}'
'${{ needs.pre-setup.outputs.git-tag }}'
--
${{ github.event.inputs.release-commitish }}
- name: >-
Push ${{ needs.pre-setup.outputs.git-tag }} tag corresponding
to the just published release back to GitHub
run: >-
git push --atomic origin '${{ needs.pre-setup.outputs.git-tag }}'
publish-github-release:
name: >-
Publish a tag and GitHub release for
${{ needs.pre-setup.outputs.git-tag }}
needs:
- post-release-repo-update
- pre-setup # transitive, for accessing settings
runs-on: ubuntu-latest
permissions:
contents: write
discussions: write
steps:
- name: Fetch the src snapshot
uses: actions/checkout@v3
with:
fetch-depth: 1
ref: ${{ github.event.inputs.release-commitish }}
- name: Download all the dists
uses: actions/download-artifact@v3
with:
name: python-package-distributions
path: dist/
- name: >-
Publish a GitHub Release for
${{ needs.pre-setup.outputs.git-tag }}
uses: ncipollo/release-action@a2e71bdd4e7dab70ca26a852f29600c98b33153e
with:
artifacts: |
dist/${{ needs.pre-setup.outputs.sdist-artifact-name }}
dist/${{ needs.pre-setup.outputs.wheel-artifact-name }}
artifactContentType: raw # Because whl and tgz are of different types
# FIXME: Use Towncrier once it is integrated.
bodyFile: CHANGES.txt
discussionCategory: Announcements
name: ${{ needs.pre-setup.outputs.git-tag }}
tag: ${{ needs.pre-setup.outputs.git-tag }}