From 11ae325bfeb7ecb430487859c4ba5970635f4f7f Mon Sep 17 00:00:00 2001 From: yxd92326 Date: Wed, 14 Aug 2024 16:54:51 +0100 Subject: [PATCH] Crude initial attempt to migrate to github-actions --- .azure-pipelines/azure-pipelines.yml | 167 ------------------------- .azure-pipelines/ci.yml | 76 ----------- .azure-pipelines/update-orm.yml | 109 ---------------- .github/workflows/ci.yml | 83 ++++++++++++ .github/workflows/test-and-publish.yml | 146 +++++++++++++++++++++ .github/workflows/update-orm.yml | 110 ++++++++++++++++ 6 files changed, 339 insertions(+), 352 deletions(-) delete mode 100644 .azure-pipelines/azure-pipelines.yml delete mode 100644 .azure-pipelines/ci.yml delete mode 100644 .azure-pipelines/update-orm.yml create mode 100644 .github/workflows/ci.yml create mode 100644 .github/workflows/test-and-publish.yml create mode 100644 .github/workflows/update-orm.yml diff --git a/.azure-pipelines/azure-pipelines.yml b/.azure-pipelines/azure-pipelines.yml deleted file mode 100644 index 1c48c4d..0000000 --- a/.azure-pipelines/azure-pipelines.yml +++ /dev/null @@ -1,167 +0,0 @@ -variables: - DATABASE_SCHEMA: 4.1.0 - -trigger: - branches: - include: - - '*' - tags: - include: - - '*' - -resources: - containers: - - container: mariadb - image: mariadb:10.6 - env: - MYSQL_DATABASE: ispybtest - MYSQL_ROOT_PASSWORD: mysql_root_pwd - ports: - - 3306:3306 - -stages: -- stage: static - displayName: Static Analysis - jobs: - - job: checks - displayName: static code analysis - pool: - vmImage: ubuntu-22.04 - steps: - # Use Python >=3.7 for syntax validation - - task: UsePythonVersion@0 - displayName: Set up python - inputs: - versionSpec: 3.7 - - - bash: | - python .azure-pipelines/syntax-validation.py - displayName: Syntax validation - - - bash: | - pip install flake8 - python .azure-pipelines/flake8-validation.py - displayName: Flake8 validation - -- stage: build - displayName: Build - dependsOn: - jobs: - - job: build - displayName: build package - pool: - vmImage: ubuntu-22.04 - steps: - - task: UsePythonVersion@0 - displayName: Set up python - inputs: - versionSpec: 3.9 - - - bash: | - pip install -U pip - pip install collective.checkdocs wheel - displayName: Install dependencies - - - bash: | - set -ex - python setup.py sdist bdist_wheel - mkdir -p dist/pypi - shopt -s extglob - mv -v dist/!(pypi) dist/pypi - git archive HEAD | gzip > dist/repo-source.tar.gz - ls -laR dist - displayName: Build python package - - - bash: | - wget -t 3 --waitretry=20 https://github.com/DiamondLightSource/ispyb-database/releases/download/v$(DATABASE_SCHEMA)/ispyb-database-$(DATABASE_SCHEMA).tar.gz -O dist/ispyb-database.tar.gz - displayName: Download ISPyB DB schema v$(DATABASE_SCHEMA) for tests - - - task: PublishBuildArtifacts@1 - displayName: Store artifact - inputs: - pathToPublish: dist/ - artifactName: package - - - bash: python setup.py checkdocs - displayName: Check package description - -- stage: tests - displayName: Run unit tests - dependsOn: - - static - - build - jobs: - - job: linux - pool: - vmImage: ubuntu-22.04 - strategy: - matrix: - python37: - PYTHON_VERSION: 3.7 - python38: - PYTHON_VERSION: 3.8 - python39: - PYTHON_VERSION: 3.9 - python310: - PYTHON_VERSION: 3.10 - python311: - PYTHON_VERSION: 3.11 - services: - db: mariadb - steps: - - template: ci.yml - -- stage: update_ORM - displayName: Update ORM - dependsOn: - - build - jobs: - - job: update_ORM - displayName: Update ORM - pool: - vmImage: ubuntu-22.04 - services: - db: mariadb - steps: - - template: update-orm.yml - -- stage: deploy - displayName: Publish release - dependsOn: - - tests - - static - condition: and(succeeded(), startsWith(variables['Build.SourceBranch'], 'refs/tags/')) - jobs: - - job: pypi - displayName: Publish pypi release - pool: - vmImage: ubuntu-22.04 - steps: - - checkout: none - - - task: UsePythonVersion@0 - displayName: Set up python - inputs: - versionSpec: 3.11 - - - task: DownloadBuildArtifacts@0 - displayName: Get pre-built package - inputs: - buildType: 'current' - downloadType: 'single' - artifactName: 'package' - downloadPath: '$(System.ArtifactsDirectory)' - - - script: | - pip install -U pip - pip install twine - displayName: Install twine - - - task: TwineAuthenticate@1 - displayName: Set up credentials - inputs: - pythonUploadServiceConnection: pypi-ispyb - - - bash: | - python -m twine upload -r pypi-ispyb --config-file $(PYPIRC_PATH) $(System.ArtifactsDirectory)/package/pypi/*.tar.gz $(System.ArtifactsDirectory)/package/pypi/*.whl - displayName: Publish package diff --git a/.azure-pipelines/ci.yml b/.azure-pipelines/ci.yml deleted file mode 100644 index 8791c4b..0000000 --- a/.azure-pipelines/ci.yml +++ /dev/null @@ -1,76 +0,0 @@ -steps: -- checkout: none - -- task: UsePythonVersion@0 - inputs: - versionSpec: '$(PYTHON_VERSION)' - displayName: 'Use Python $(PYTHON_VERSION)' - -- task: DownloadBuildArtifacts@0 - displayName: Get pre-built package - inputs: - buildType: 'current' - downloadType: 'single' - artifactName: 'package' - downloadPath: '$(System.ArtifactsDirectory)' - -- task: ExtractFiles@1 - displayName: Checkout sources - inputs: - archiveFilePatterns: "$(System.ArtifactsDirectory)/package/repo-source.tar.gz" - destinationFolder: "$(Pipeline.Workspace)/src" - -- script: | - set -eux - pip install -r "$(Pipeline.Workspace)/src/requirements_dev.txt" - pip install "$(Pipeline.Workspace)/src" - displayName: Install package - -- script: | - set -eu - cat >~/.my.cnf </dev/null; do printf '.'; sleep 0.5; done - printf '\n' - - mysql -e "SET GLOBAL log_bin_trust_function_creators = 1;" - for f in schemas/ispyb/tables.sql \ - schemas/ispyb/lookups.sql \ - schemas/ispyb/data.sql \ - schemas/ispyb/routines.sql \ - grants/ispyb_processing.sql \ - grants/ispyb_import.sql; do - echo Importing ${f}... - mysql < $f - done - mysql -e "CREATE USER ispyb_api@'%' IDENTIFIED BY 'password_1234'; GRANT ispyb_processing to ispyb_api@'%'; GRANT ispyb_import to ispyb_api@'%'; SET DEFAULT ROLE ispyb_processing FOR ispyb_api@'%';" - mysql -e "CREATE USER ispyb_api_future@'%' IDENTIFIED BY 'password_4321'; GRANT SELECT ON ispybtest.* to ispyb_api_future@'%';" - mysql -e "CREATE USER ispyb_api_sqlalchemy@'%' IDENTIFIED BY 'password_5678'; GRANT SELECT ON ispybtest.* to ispyb_api_sqlalchemy@'%'; GRANT INSERT ON ispybtest.* to ispyb_api_sqlalchemy@'%'; GRANT UPDATE ON ispybtest.* to ispyb_api_sqlalchemy@'%';" - rm ~/.my.cnf - displayName: Set up test database - -- script: | - export ISPYB_CREDENTIALS="$(Pipeline.Workspace)/src/conf/config.cfg" - PYTHONDEVMODE=1 pytest -ra --cov=ispyb --cov-report=xml --cov-branch - displayName: Run tests - workingDirectory: $(Pipeline.Workspace)/src - -- bash: bash <(curl -s https://codecov.io/bash) -n "Python $(PYTHON_VERSION) $(Agent.OS)" - displayName: 'Publish coverage stats' - continueOnError: True - workingDirectory: $(Pipeline.Workspace)/src - timeoutInMinutes: 2 diff --git a/.azure-pipelines/update-orm.yml b/.azure-pipelines/update-orm.yml deleted file mode 100644 index c89e32b..0000000 --- a/.azure-pipelines/update-orm.yml +++ /dev/null @@ -1,109 +0,0 @@ -steps: -- task: UsePythonVersion@0 - inputs: - versionSpec: 3.10 - displayName: Use Python 3.10 - -- script: | - set -eu - CURRENT_VERSION=$(grep __schema_version__ _auto_db_schema.py | cut -d'"' -f2) - diff -a <(echo ${CURRENT_VERSION}) <(echo $(DATABASE_SCHEMA)) >/dev/null && { - echo "##[section]ORM is up to date (${CURRENT_VERSION})" - echo "##vso[task.setvariable variable=UpdateRequired;isOutput=true]False" - } || { - echo "##[warning]ORM needs to be updated (${CURRENT_VERSION} -> $(DATABASE_SCHEMA)). You can download an automatically generated version as Azure artifact." - echo "##vso[task.setvariable variable=UpdateRequired;isOutput=true]True" - } - displayName: Check if update is required - workingDirectory: $(Build.SourcesDirectory)/src/ispyb/sqlalchemy - name: ORMCheck - -- script: | - set -eux - pip install -r requirements_orm.txt - pip install pre-commit - displayName: Install sqlacodegen - condition: and(succeeded(), - eq(variables['ORMCheck.UpdateRequired'], 'True')) - -- task: DownloadBuildArtifacts@0 - displayName: Get database schema - inputs: - buildType: 'current' - downloadType: 'single' - artifactName: 'package' - downloadPath: '$(System.ArtifactsDirectory)' - condition: and(succeeded(), - eq(variables['ORMCheck.UpdateRequired'], 'True')) - -- script: | - set -eu - cat >~/.my.cnf </dev/null; do printf '.'; sleep 0.5; done - printf '\n' - - echo "Installing reference database" - ./build.sh - - echo - echo "Installed tables:" - mysql -D ispyb_build -e "SHOW TABLES" - displayName: Set up reference database schema - workingDirectory: $(System.ArtifactsDirectory) - condition: and(succeeded(), - eq(variables['ORMCheck.UpdateRequired'], 'True')) - -- bash: | - set -eux - sqlacodegen mysql+mysqlconnector://root:mysql_root_pwd@127.0.0.1/ispyb_build --noinflect --nojoined --outfile _auto_db_schema.py.in - # This code produces false positives due to non-deterministic ordering. - # Add an identifier to the file, and only update/PR the changes when the - # identifier indicates the file is out of date. - cat <(echo "__schema_version__ = \"$(DATABASE_SCHEMA)\"") _auto_db_schema.py.in > _auto_db_schema.py - rm _auto_db_schema.py.in - pre-commit run --files _auto_db_schema.py || echo pre-commit exited with non-zero return code - displayName: Generate ORM - workingDirectory: $(Build.SourcesDirectory)/src/ispyb/sqlalchemy - condition: and(succeeded(), - eq(variables['ORMCheck.UpdateRequired'], 'True')) - -- script: | - git diff - displayName: Show differences - workingDirectory: $(Build.SourcesDirectory) - condition: and(succeeded(), - eq(variables['ORMCheck.UpdateRequired'], 'True')) - -- task: PublishBuildArtifacts@1 - displayName: Store artifact - inputs: - pathToPublish: $(Build.SourcesDirectory)/src/ispyb/sqlalchemy/_auto_db_schema.py - artifactName: ORM - condition: and(succeeded(), - eq(variables['ORMCheck.UpdateRequired'], 'True')) - -- script: .azure-pipelines/create-orm-update-pull-request - displayName: Create ORM pull request - condition: and(succeeded(), - eq(variables['Build.SourceBranch'], 'refs/heads/main'), - eq(variables['ORMCheck.UpdateRequired'], 'True')) - workingDirectory: $(Build.SourcesDirectory) - env: - GITHUB_TOKEN: $(GITHUB_TOKEN) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..eae7ed6 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,83 @@ +on: + workflow_call: + inputs: + PYTHON_VERSION: + required: true + type: string + +jobs: + test: + name: Build distribution and run tests + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + - name: Use Python ${{ inputs.PYTHON_VERSION }} + uses: actions/setup-python@v5 + with: + python-version: ${{ inputs.PYTHON_VERSION }} + + - name: Get pre-built package + uses: actions/download-artifact@v4 + with: + name: python-package-distributions + path: dist/ + + - name: Checkout sources + inputs: + archiveFilePatterns: "dist/package/repo-source.tar.gz" + destinationFolder: "./src" + + - name: Install package + run: | + set -eux + pip install -r "./src/requirements_dev.txt" + pip install "./src" + + - name: Set up test database + run: | + set -eu + cat >~/.my.cnf </dev/null; do printf '.'; sleep 0.5; done + printf '\n' + + mysql -e "SET GLOBAL log_bin_trust_function_creators = 1;" + for f in schemas/ispyb/tables.sql \ + schemas/ispyb/lookups.sql \ + schemas/ispyb/data.sql \ + schemas/ispyb/routines.sql \ + grants/ispyb_processing.sql \ + grants/ispyb_import.sql; do + echo Importing ${f}... + mysql < $f + done + mysql -e "CREATE USER ispyb_api@'%' IDENTIFIED BY 'password_1234'; GRANT ispyb_processing to ispyb_api@'%'; GRANT ispyb_import to ispyb_api@'%'; SET DEFAULT ROLE ispyb_processing FOR ispyb_api@'%';" + mysql -e "CREATE USER ispyb_api_future@'%' IDENTIFIED BY 'password_4321'; GRANT SELECT ON ispybtest.* to ispyb_api_future@'%';" + mysql -e "CREATE USER ispyb_api_sqlalchemy@'%' IDENTIFIED BY 'password_5678'; GRANT SELECT ON ispybtest.* to ispyb_api_sqlalchemy@'%'; GRANT INSERT ON ispybtest.* to ispyb_api_sqlalchemy@'%'; GRANT UPDATE ON ispybtest.* to ispyb_api_sqlalchemy@'%';" + rm ~/.my.cnf + + - name: Run tests + run: | + export ISPYB_CREDENTIALS="./src/conf/config.cfg" + PYTHONDEVMODE=1 pytest tests -ra --cov=ispyb --cov-report=xml --cov-branch + + - name: Publish coverage stats + run: bash <(curl -s https://codecov.io/bash) -n "Python ${{ inputs.PYTHON_VERSION }}" + continue-on-error: true + working-directory: ./src + timeout-inutes: 2 diff --git a/.github/workflows/test-and-publish.yml b/.github/workflows/test-and-publish.yml new file mode 100644 index 0000000..1d4d88a --- /dev/null +++ b/.github/workflows/test-and-publish.yml @@ -0,0 +1,146 @@ +name: Run tests and publish Python distribution to PyPI + +on: + [push, pull_request] + +env: + DATABASE_SCHEMA: 4.1.0 + +permissions: + contents: read + +jobs: + mariadb: + name: mariadb + runs-on: ubuntu-latest + container: mariadb + services: + image: mariadb:10.6 + env: + MYSQL_DATABASE: ispybtest + MYSQL_ROOT_PASSWORD: mysql_root_pwd + ports: + - 3306:3306 + + build: + name: Build package + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + - name: Set up Python 3.9 + uses: actions/setup-python@v5 + with: + python-version: "3.9" + - name: Install dependencies + run: | + pip install -U pip + pip install collective.checkdocs wheel + - name: Build python package + run: | + set -ex + python setup.py sdist bdist_wheel + mkdir -p dist/pypi + shopt -s extglob + mv -v dist/!(pypi) dist/pypi + git archive HEAD | gzip > dist/repo-source.tar.gz + ls -laR dist + - name: Download ISPyB DB schema v${{ DATABASE_SCHEMA }} for tests + run: wget -t 3 --waitretry=20 https://github.com/DiamondLightSource/ispyb-database/releases/download/v${{ DATABASE_SCHEMA }}/ispyb-database-$${{ DATABASE_SCHEMA }}.tar.gz -O dist/ispyb-database.tar.gz + - name: Store artifact + uses: actions/upload-artifact@v4 + with: + name: python-package-distributions + path: dist/ + - name: Check package description + run: python setup.py checkdocs + + test: + name: Run tests + runs-on: ubuntu-latest + needs: + - build + - mariadb + strategy: + matrix: + python-version: [ "3.7", "3.8", "3.9", "3.10", "3.11" ] + steps: + - uses: ./.github/workflows/ci.yml + with: + PYTHON_VERSION: ${{ matrix.python-version }} + + update_ORM: + name: Update ORM + runs-on: ubuntu-latest + needs: + - build + - mariadb + steps: + - uses: ./.github/workflows/update-orm.yml + with: + DATABASE_SCHEMA: ${{ DATABASE_SCHEMA }} + + publish-to-pypi: + name: >- + Publish Python distribution to PyPI + if: startsWith(github.ref, 'refs/tags/') # only publish to PyPI on tag pushes + needs: + - build + runs-on: ubuntu-latest + environment: + name: pypi + url: https://pypi.org/p/ispyb + permissions: + id-token: write # IMPORTANT: mandatory for trusted publishing + + steps: + - name: Download all the dists + uses: actions/download-artifact@v4 + with: + name: python-package-distributions + path: dist/ + - name: Publish distribution to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 + + github-release: + name: >- + Sign the Python distribution with Sigstore + and upload them to GitHub Release + needs: + - publish-to-pypi + runs-on: ubuntu-latest + + permissions: + contents: write # IMPORTANT: mandatory for making GitHub Releases + id-token: write # IMPORTANT: mandatory for sigstore + + steps: + - name: Download all the dists + uses: actions/download-artifact@v4 + with: + name: python-package-distributions + path: dist/ + - name: Sign the dists with Sigstore + uses: sigstore/gh-action-sigstore-python@v2.1.1 + with: + inputs: >- + ./dist/*.tar.gz + ./dist/*.whl + - name: Create GitHub Release + env: + GITHUB_TOKEN: ${{ github.token }} + run: >- + gh release create + '${{ github.ref_name }}' + --repo '${{ github.repository }}' + --notes "" + - name: Upload artifact signatures to GitHub Release + env: + GITHUB_TOKEN: ${{ github.token }} + # Upload to GitHub Release using the `gh` CLI. + # `dist/` contains the built packages, and the + # sigstore-produced signatures and certificates. + run: >- + gh release upload + '${{ github.ref_name }}' dist/** + --repo '${{ github.repository }}' diff --git a/.github/workflows/update-orm.yml b/.github/workflows/update-orm.yml new file mode 100644 index 0000000..00b2ddc --- /dev/null +++ b/.github/workflows/update-orm.yml @@ -0,0 +1,110 @@ +on: + workflow_call: + inputs: + DATABASE_SCHEMA: + required: true + type: string + +jobs: + orm-update: + name: ORMCheck + runs-on: ubuntu-latest + outputs: + output1: ${{ steps.checkUpdate.needsUpdate }} + + steps: + - task: UsePythonVersion@0 + inputs: + versionSpec: 3.10 + displayName: Use Python 3.10 + + - name: Check if update is required + id: checkUpdate + run: | + set -eu + CURRENT_VERSION=$(grep __schema_version__ _auto_db_schema.py | cut -d'"' -f2) + diff -a <(echo ${CURRENT_VERSION}) <(echo ${{ inputs.DATABASE_SCHEMA }}) >/dev/null && { + echo "##[section]ORM is up to date (${CURRENT_VERSION})" + echo "needsUpdate=false" >> $GITHUB_OUTPUT + } || { + echo "##[warning]ORM needs to be updated (${CURRENT_VERSION} -> ${{ inputs.DATABASE_SCHEMA }})." + echo "needsUpdate=talse" >> $GITHUB_OUTPUT + } + working-directory: ./src/ispyb/sqlalchemy + + - name: Install sqlacodegen + if: ${{ steps.checkUpdate.needsUpdate == 'true' }} + run: | + set -eux + pip install -r requirements_orm.txt + pip install pre-commit + + - name: Get database schema + if: ${{ steps.checkUpdate.needsUpdate == 'true' }} + uses: actions/download-artifact@v4 + with: + name: ispyb-database.tar.gz + path: dist/ + + - name: Set up reference database schema + if: ${{ steps.checkUpdate.needsUpdate == 'true' }} + run: | + set -eu + cat >~/.my.cnf </dev/null; do printf '.'; sleep 0.5; done + printf '\n' + + echo "Installing reference database" + ./build.sh + + echo + echo "Installed tables:" + mysql -D ispyb_build -e "SHOW TABLES" + working-directory: dist + + - name: Generate ORM + if: ${{ steps.checkUpdate.needsUpdate == 'true' }} + run: | + set -eux + sqlacodegen mysql+mysqlconnector://root:mysql_root_pwd@127.0.0.1/ispyb_build --noinflect --nojoined --outfile _auto_db_schema.py.in + # This code produces false positives due to non-deterministic ordering. + # Add an identifier to the file, and only update/PR the changes when the + # identifier indicates the file is out of date. + cat <(echo "__schema_version__ = \"$(DATABASE_SCHEMA)\"") _auto_db_schema.py.in > _auto_db_schema.py + rm _auto_db_schema.py.in + pre-commit run --files _auto_db_schema.py || echo pre-commit exited with non-zero return code + working-directory: ./src/ispyb/sqlalchemy + + - name: Show differences + if: ${{ steps.checkUpdate.needsUpdate == 'true' }} + run: git diff + + - name: Store artifact + if: ${{ steps.checkUpdate.needsUpdate == 'true' }} + uses: actions/upload-artifact@v4 + with: + name: ORM + path: ./src/ispyb/sqlalchemy/_auto_db_schema.py + + - name: Create ORM pull request + if: ${{ steps.checkUpdate.needsUpdate == 'true' && github.ref_name == 'main' }} + run: .azure-pipelines/create-orm-update-pull-request + env: + GITHUB_TOKEN: ${{ github.token }}