From 2350613af6febef8d5e3d3ce9ea3eefcfeda3220 Mon Sep 17 00:00:00 2001 From: Stephen Riggs <122790971+stephen-riggs@users.noreply.github.com> Date: Wed, 21 Aug 2024 16:12:10 +0100 Subject: [PATCH] Change from using azure pipeline to GitHub workflows (#215) * Crude initial attempt to migrate to github-actions * Ensure all stages have a run or use action * Don't try and reuse mariadb container * Explicit env definition * Missed a ref * Missed repo checkout * Actions need subfolders * Try with github.event.inputs * Try going back to ci.yml file * Typo * Misplaced dependency * Return orm update to yml file * Rearrange file locations and install order * Typo * Ugly attempt to work around env issue * Try and fix python errors * File locations * Don't make schema dir * Run in container * Revert previous, and start mysql * Code rearangement * Double bracket * Use bash * Single equals * Install sudo * Restore static analysis * Missed image * Find commands available * Move all files to workflows dir * Try alterative mysql method * Install mysql manually * look for files * systemctl start mysql * Look for mysql service * Look for systemd paths * Install systemctl * Try installing systemctl again * Alternative service start method * Install mysql * Typo * Typo * Remove mysql before installing again * Force y * Hard clean mysql before reinstalling * Deletion failed * Force y * Try starting mariadb not mysql * Still need sudo installed * Remove apt statements, and replace mysql with mariadb commands * Add more env variables, and health check * Specify port * Specify file with configuration * Change host env and login user * Remove hostname again * Revert to 10.6 * Force host and port * Add mysql setup action * Longer wait * Try as localhost * Find socks, and start mariadb * Initialise mariadb? * Create databse * Use cfg file in initialisation * Restore db creation * Don't change datadir, and find sqls * Restore datadir * No new user * skip-test-db flag * Revert back to 127.0.0.1 * Debug console * Ensure mariadb-client installed * Failed on specific version * Set protocol * Revert to MYSQL variables, and re-add setup action * Use debug console to look at env * Force ubuntu-22.04 image as a test * Remove spurious extra container * Cleanup debugging items * Update database schema version to 4.2.1 --- .azure-pipelines/azure-pipelines.yml | 167 ------------------ .azure-pipelines/ci.yml | 76 -------- .azure-pipelines/update-orm.yml | 109 ------------ .github/workflows/ci.yml | 90 ++++++++++ .../scripts}/create-orm-update-pull-request | 0 .../workflows/scripts}/flake8-validation.py | 0 .../workflows/scripts}/syntax-validation.py | 0 .github/workflows/test-and-publish.yml | 150 ++++++++++++++++ .github/workflows/update-orm.yml | 122 +++++++++++++ 9 files changed, 362 insertions(+), 352 deletions(-) delete mode 100644 .azure-pipelines/azure-pipelines.yml delete mode 100644 .azure-pipelines/ci.yml delete mode 100644 .azure-pipelines/update-orm.yml create mode 100644 .github/workflows/ci.yml rename {.azure-pipelines => .github/workflows/scripts}/create-orm-update-pull-request (100%) rename {.azure-pipelines => .github/workflows/scripts}/flake8-validation.py (100%) rename {.azure-pipelines => .github/workflows/scripts}/syntax-validation.py (100%) create mode 100644 .github/workflows/test-and-publish.yml create mode 100644 .github/workflows/update-orm.yml diff --git a/.azure-pipelines/azure-pipelines.yml b/.azure-pipelines/azure-pipelines.yml deleted file mode 100644 index ac56e6ce..00000000 --- a/.azure-pipelines/azure-pipelines.yml +++ /dev/null @@ -1,167 +0,0 @@ -variables: - DATABASE_SCHEMA: 4.2.0 - -trigger: - branches: - include: - - '*' - tags: - include: - - '*' - -resources: - containers: - - container: mariadb - image: mariadb:10.6 - env: - MYSQL_DATABASE: ispybtest - MYSQL_ROOT_PASSWORD: mysql_root_pwd - ports: - - 3306:3306 - -stages: -- stage: static - displayName: Static Analysis - jobs: - - job: checks - displayName: static code analysis - pool: - vmImage: ubuntu-22.04 - steps: - # Use Python >=3.7 for syntax validation - - task: UsePythonVersion@0 - displayName: Set up python - inputs: - versionSpec: 3.7 - - - bash: | - python .azure-pipelines/syntax-validation.py - displayName: Syntax validation - - - bash: | - pip install flake8 - python .azure-pipelines/flake8-validation.py - displayName: Flake8 validation - -- stage: build - displayName: Build - dependsOn: - jobs: - - job: build - displayName: build package - pool: - vmImage: ubuntu-22.04 - steps: - - task: UsePythonVersion@0 - displayName: Set up python - inputs: - versionSpec: 3.9 - - - bash: | - pip install -U pip - pip install collective.checkdocs wheel - displayName: Install dependencies - - - bash: | - set -ex - python setup.py sdist bdist_wheel - mkdir -p dist/pypi - shopt -s extglob - mv -v dist/!(pypi) dist/pypi - git archive HEAD | gzip > dist/repo-source.tar.gz - ls -laR dist - displayName: Build python package - - - bash: | - wget -t 3 --waitretry=20 https://github.com/DiamondLightSource/ispyb-database/releases/download/v$(DATABASE_SCHEMA)/ispyb-database-$(DATABASE_SCHEMA).tar.gz -O dist/ispyb-database.tar.gz - displayName: Download ISPyB DB schema v$(DATABASE_SCHEMA) for tests - - - task: PublishBuildArtifacts@1 - displayName: Store artifact - inputs: - pathToPublish: dist/ - artifactName: package - - - bash: python setup.py checkdocs - displayName: Check package description - -- stage: tests - displayName: Run unit tests - dependsOn: - - static - - build - jobs: - - job: linux - pool: - vmImage: ubuntu-22.04 - strategy: - matrix: - python37: - PYTHON_VERSION: 3.7 - python38: - PYTHON_VERSION: 3.8 - python39: - PYTHON_VERSION: 3.9 - python310: - PYTHON_VERSION: 3.10 - python311: - PYTHON_VERSION: 3.11 - services: - db: mariadb - steps: - - template: ci.yml - -- stage: update_ORM - displayName: Update ORM - dependsOn: - - build - jobs: - - job: update_ORM - displayName: Update ORM - pool: - vmImage: ubuntu-22.04 - services: - db: mariadb - steps: - - template: update-orm.yml - -- stage: deploy - displayName: Publish release - dependsOn: - - tests - - static - condition: and(succeeded(), startsWith(variables['Build.SourceBranch'], 'refs/tags/')) - jobs: - - job: pypi - displayName: Publish pypi release - pool: - vmImage: ubuntu-22.04 - steps: - - checkout: none - - - task: UsePythonVersion@0 - displayName: Set up python - inputs: - versionSpec: 3.11 - - - task: DownloadBuildArtifacts@0 - displayName: Get pre-built package - inputs: - buildType: 'current' - downloadType: 'single' - artifactName: 'package' - downloadPath: '$(System.ArtifactsDirectory)' - - - script: | - pip install -U pip - pip install twine - displayName: Install twine - - - task: TwineAuthenticate@1 - displayName: Set up credentials - inputs: - pythonUploadServiceConnection: pypi-ispyb - - - bash: | - python -m twine upload -r pypi-ispyb --config-file $(PYPIRC_PATH) $(System.ArtifactsDirectory)/package/pypi/*.tar.gz $(System.ArtifactsDirectory)/package/pypi/*.whl - displayName: Publish package diff --git a/.azure-pipelines/ci.yml b/.azure-pipelines/ci.yml deleted file mode 100644 index 8791c4bf..00000000 --- a/.azure-pipelines/ci.yml +++ /dev/null @@ -1,76 +0,0 @@ -steps: -- checkout: none - -- task: UsePythonVersion@0 - inputs: - versionSpec: '$(PYTHON_VERSION)' - displayName: 'Use Python $(PYTHON_VERSION)' - -- task: DownloadBuildArtifacts@0 - displayName: Get pre-built package - inputs: - buildType: 'current' - downloadType: 'single' - artifactName: 'package' - downloadPath: '$(System.ArtifactsDirectory)' - -- task: ExtractFiles@1 - displayName: Checkout sources - inputs: - archiveFilePatterns: "$(System.ArtifactsDirectory)/package/repo-source.tar.gz" - destinationFolder: "$(Pipeline.Workspace)/src" - -- script: | - set -eux - pip install -r "$(Pipeline.Workspace)/src/requirements_dev.txt" - pip install "$(Pipeline.Workspace)/src" - displayName: Install package - -- script: | - set -eu - cat >~/.my.cnf </dev/null; do printf '.'; sleep 0.5; done - printf '\n' - - mysql -e "SET GLOBAL log_bin_trust_function_creators = 1;" - for f in schemas/ispyb/tables.sql \ - schemas/ispyb/lookups.sql \ - schemas/ispyb/data.sql \ - schemas/ispyb/routines.sql \ - grants/ispyb_processing.sql \ - grants/ispyb_import.sql; do - echo Importing ${f}... - mysql < $f - done - mysql -e "CREATE USER ispyb_api@'%' IDENTIFIED BY 'password_1234'; GRANT ispyb_processing to ispyb_api@'%'; GRANT ispyb_import to ispyb_api@'%'; SET DEFAULT ROLE ispyb_processing FOR ispyb_api@'%';" - mysql -e "CREATE USER ispyb_api_future@'%' IDENTIFIED BY 'password_4321'; GRANT SELECT ON ispybtest.* to ispyb_api_future@'%';" - mysql -e "CREATE USER ispyb_api_sqlalchemy@'%' IDENTIFIED BY 'password_5678'; GRANT SELECT ON ispybtest.* to ispyb_api_sqlalchemy@'%'; GRANT INSERT ON ispybtest.* to ispyb_api_sqlalchemy@'%'; GRANT UPDATE ON ispybtest.* to ispyb_api_sqlalchemy@'%';" - rm ~/.my.cnf - displayName: Set up test database - -- script: | - export ISPYB_CREDENTIALS="$(Pipeline.Workspace)/src/conf/config.cfg" - PYTHONDEVMODE=1 pytest -ra --cov=ispyb --cov-report=xml --cov-branch - displayName: Run tests - workingDirectory: $(Pipeline.Workspace)/src - -- bash: bash <(curl -s https://codecov.io/bash) -n "Python $(PYTHON_VERSION) $(Agent.OS)" - displayName: 'Publish coverage stats' - continueOnError: True - workingDirectory: $(Pipeline.Workspace)/src - timeoutInMinutes: 2 diff --git a/.azure-pipelines/update-orm.yml b/.azure-pipelines/update-orm.yml deleted file mode 100644 index c89e32b0..00000000 --- a/.azure-pipelines/update-orm.yml +++ /dev/null @@ -1,109 +0,0 @@ -steps: -- task: UsePythonVersion@0 - inputs: - versionSpec: 3.10 - displayName: Use Python 3.10 - -- script: | - set -eu - CURRENT_VERSION=$(grep __schema_version__ _auto_db_schema.py | cut -d'"' -f2) - diff -a <(echo ${CURRENT_VERSION}) <(echo $(DATABASE_SCHEMA)) >/dev/null && { - echo "##[section]ORM is up to date (${CURRENT_VERSION})" - echo "##vso[task.setvariable variable=UpdateRequired;isOutput=true]False" - } || { - echo "##[warning]ORM needs to be updated (${CURRENT_VERSION} -> $(DATABASE_SCHEMA)). You can download an automatically generated version as Azure artifact." - echo "##vso[task.setvariable variable=UpdateRequired;isOutput=true]True" - } - displayName: Check if update is required - workingDirectory: $(Build.SourcesDirectory)/src/ispyb/sqlalchemy - name: ORMCheck - -- script: | - set -eux - pip install -r requirements_orm.txt - pip install pre-commit - displayName: Install sqlacodegen - condition: and(succeeded(), - eq(variables['ORMCheck.UpdateRequired'], 'True')) - -- task: DownloadBuildArtifacts@0 - displayName: Get database schema - inputs: - buildType: 'current' - downloadType: 'single' - artifactName: 'package' - downloadPath: '$(System.ArtifactsDirectory)' - condition: and(succeeded(), - eq(variables['ORMCheck.UpdateRequired'], 'True')) - -- script: | - set -eu - cat >~/.my.cnf </dev/null; do printf '.'; sleep 0.5; done - printf '\n' - - echo "Installing reference database" - ./build.sh - - echo - echo "Installed tables:" - mysql -D ispyb_build -e "SHOW TABLES" - displayName: Set up reference database schema - workingDirectory: $(System.ArtifactsDirectory) - condition: and(succeeded(), - eq(variables['ORMCheck.UpdateRequired'], 'True')) - -- bash: | - set -eux - sqlacodegen mysql+mysqlconnector://root:mysql_root_pwd@127.0.0.1/ispyb_build --noinflect --nojoined --outfile _auto_db_schema.py.in - # This code produces false positives due to non-deterministic ordering. - # Add an identifier to the file, and only update/PR the changes when the - # identifier indicates the file is out of date. - cat <(echo "__schema_version__ = \"$(DATABASE_SCHEMA)\"") _auto_db_schema.py.in > _auto_db_schema.py - rm _auto_db_schema.py.in - pre-commit run --files _auto_db_schema.py || echo pre-commit exited with non-zero return code - displayName: Generate ORM - workingDirectory: $(Build.SourcesDirectory)/src/ispyb/sqlalchemy - condition: and(succeeded(), - eq(variables['ORMCheck.UpdateRequired'], 'True')) - -- script: | - git diff - displayName: Show differences - workingDirectory: $(Build.SourcesDirectory) - condition: and(succeeded(), - eq(variables['ORMCheck.UpdateRequired'], 'True')) - -- task: PublishBuildArtifacts@1 - displayName: Store artifact - inputs: - pathToPublish: $(Build.SourcesDirectory)/src/ispyb/sqlalchemy/_auto_db_schema.py - artifactName: ORM - condition: and(succeeded(), - eq(variables['ORMCheck.UpdateRequired'], 'True')) - -- script: .azure-pipelines/create-orm-update-pull-request - displayName: Create ORM pull request - condition: and(succeeded(), - eq(variables['Build.SourceBranch'], 'refs/heads/main'), - eq(variables['ORMCheck.UpdateRequired'], 'True')) - workingDirectory: $(Build.SourcesDirectory) - env: - GITHUB_TOKEN: $(GITHUB_TOKEN) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 00000000..95ff9c2b --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,90 @@ +name: Run tests + +on: + workflow_call: + +jobs: + test: + name: Run tests + runs-on: ubuntu-latest + strategy: + matrix: + python-version: [ "3.7", "3.8", "3.9", "3.10", "3.11" ] + services: + mariadb: + image: mariadb:10.6 + env: + MARIADB_DATABASE: ispybtest + MARIADB_ROOT_PASSWORD: mariadb_root_pwd + ports: + - 3306:3306 + options: --health-cmd="healthcheck.sh --connect --innodb_initialized" --health-interval=10s --health-timeout=5s --health-retries=3 + steps: + - uses: actions/checkout@v4 + - name: Use Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + + - name: Install package + run: | + set -eux + python -m pip install -r ./requirements_dev.txt + python -m pip install . + + - name: Get pre-built package + uses: actions/download-artifact@v4 + with: + name: package-distributions + path: dist/ + + - uses: shogo82148/actions-setup-mysql@v1 + with: + distribution: 'mariadb' + mysql-version: '10.6' + auto-start: false + + - name: Set up test database + run: | + set -eu + cat > .my.cnf </dev/null; do printf '.'; sleep 10; done + printf '\n' + + mariadb --defaults-file=.my.cnf -e "SET GLOBAL log_bin_trust_function_creators = 1;" + for f in schemas/ispyb/tables.sql \ + schemas/ispyb/lookups.sql \ + schemas/ispyb/data.sql \ + schemas/ispyb/routines.sql \ + grants/ispyb_processing.sql \ + grants/ispyb_import.sql; do + echo Importing ${f}... + mariadb --defaults-file=.my.cnf < $f + done + mariadb --defaults-file=.my.cnf -e "CREATE USER ispyb_api@'%' IDENTIFIED BY 'password_1234'; GRANT ispyb_processing to ispyb_api@'%'; GRANT ispyb_import to ispyb_api@'%'; SET DEFAULT ROLE ispyb_processing FOR ispyb_api@'%';" + mariadb --defaults-file=.my.cnf -e "CREATE USER ispyb_api_future@'%' IDENTIFIED BY 'password_4321'; GRANT SELECT ON ispybtest.* to ispyb_api_future@'%';" + mariadb --defaults-file=.my.cnf -e "CREATE USER ispyb_api_sqlalchemy@'%' IDENTIFIED BY 'password_5678'; GRANT SELECT ON ispybtest.* to ispyb_api_sqlalchemy@'%'; GRANT INSERT ON ispybtest.* to ispyb_api_sqlalchemy@'%'; GRANT UPDATE ON ispybtest.* to ispyb_api_sqlalchemy@'%';" + rm .my.cnf + + - name: Run tests + run: | + export ISPYB_CREDENTIALS="./conf/config.cfg" + PYTHONDEVMODE=1 pytest tests -ra --cov=ispyb --cov-report=xml --cov-branch + + - name: Publish coverage stats + run: bash <(curl -s https://codecov.io/bash) -n "Python ${{ matrix.python-version }}" + continue-on-error: true + timeout-minutes: 2 diff --git a/.azure-pipelines/create-orm-update-pull-request b/.github/workflows/scripts/create-orm-update-pull-request similarity index 100% rename from .azure-pipelines/create-orm-update-pull-request rename to .github/workflows/scripts/create-orm-update-pull-request diff --git a/.azure-pipelines/flake8-validation.py b/.github/workflows/scripts/flake8-validation.py similarity index 100% rename from .azure-pipelines/flake8-validation.py rename to .github/workflows/scripts/flake8-validation.py diff --git a/.azure-pipelines/syntax-validation.py b/.github/workflows/scripts/syntax-validation.py similarity index 100% rename from .azure-pipelines/syntax-validation.py rename to .github/workflows/scripts/syntax-validation.py diff --git a/.github/workflows/test-and-publish.yml b/.github/workflows/test-and-publish.yml new file mode 100644 index 00000000..8f173e49 --- /dev/null +++ b/.github/workflows/test-and-publish.yml @@ -0,0 +1,150 @@ +name: Run tests and publish Python distribution to PyPI + +on: + [push, pull_request] + +env: + DATABASE_SCHEMA: 4.2.1 + +permissions: + contents: read + +jobs: + static: + name: Static Analysis + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Set up Python 3.9 + uses: actions/setup-python@v5 + with: + python-version: "3.9" + - name: Syntax validation + run: | + python .github/workflows/scripts/syntax-validation.py + - name: Flake8 validation + run: | + pip install flake8 + python .github/workflows/scripts/flake8-validation.py + + build: + name: Build package + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Set up Python 3.9 + uses: actions/setup-python@v5 + with: + python-version: "3.9" + - name: Install dependencies + run: | + pip install -U pip + pip install collective.checkdocs wheel + - name: Build python package + run: | + set -ex + python setup.py sdist bdist_wheel + mkdir -p dist/pypi + shopt -s extglob + mv -v dist/!(pypi) dist/pypi + git archive HEAD | gzip > dist/repo-source.tar.gz + ls -laR dist + - name: Download ISPyB DB schema v${{ env.DATABASE_SCHEMA }} for tests + run: wget -t 3 --waitretry=20 https://github.com/DiamondLightSource/ispyb-database/releases/download/v${{ env.DATABASE_SCHEMA }}/ispyb-database-${{ env.DATABASE_SCHEMA }}.tar.gz -O dist/ispyb-database.tar.gz + - name: Store artifact + uses: actions/upload-artifact@v4 + with: + name: package-distributions + path: dist/ + - name: Check package description + run: python setup.py checkdocs + + tests: + name: Call ci unit tests workflow + uses: ./.github/workflows/ci.yml + needs: + - build + - static + + get-env-vars: + name: Get environment variables + runs-on: ubuntu-latest + outputs: + ENV_DB_SCHEMA: ${{ env.DATABASE_SCHEMA }} + steps: + - run: echo "null" + + update_ORM: + name: Call orm update workflow + needs: + - build + - get-env-vars + uses: ./.github/workflows/update-orm.yml + with: + DATABASE_SCHEMA: ${{ needs.get-env-vars.outputs.ENV_DB_SCHEMA }} + + + publish-to-pypi: + name: >- + Publish Python distribution to PyPI + if: startsWith(github.ref, 'refs/tags/') # only publish to PyPI on tag pushes + needs: + - tests + runs-on: ubuntu-latest + environment: + name: pypi + url: https://pypi.org/p/ispyb + permissions: + id-token: write # IMPORTANT: mandatory for trusted publishing + + steps: + - name: Download all the dists + uses: actions/download-artifact@v4 + with: + name: package-distributions + path: dist/ + - name: Publish distribution to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 + + github-release: + name: >- + Sign the Python distribution with Sigstore + and upload them to GitHub Release + needs: + - publish-to-pypi + runs-on: ubuntu-latest + + permissions: + contents: write # IMPORTANT: mandatory for making GitHub Releases + id-token: write # IMPORTANT: mandatory for sigstore + + steps: + - name: Download all the dists + uses: actions/download-artifact@v4 + with: + name: package-distributions + path: dist/ + - name: Sign the dists with Sigstore + uses: sigstore/gh-action-sigstore-python@v2.1.1 + with: + inputs: >- + ./dist/*.tar.gz + ./dist/*.whl + - name: Create GitHub Release + env: + GITHUB_TOKEN: ${{ github.token }} + run: >- + gh release create + '${{ github.ref_name }}' + --repo '${{ github.repository }}' + --notes "" + - name: Upload artifact signatures to GitHub Release + env: + GITHUB_TOKEN: ${{ github.token }} + # Upload to GitHub Release using the `gh` CLI. + # `dist/` contains the built packages, and the + # sigstore-produced signatures and certificates. + run: >- + gh release upload + '${{ github.ref_name }}' dist/** + --repo '${{ github.repository }}' diff --git a/.github/workflows/update-orm.yml b/.github/workflows/update-orm.yml new file mode 100644 index 00000000..bb52ecdc --- /dev/null +++ b/.github/workflows/update-orm.yml @@ -0,0 +1,122 @@ +name: Check if ORM update is needed + +on: + workflow_call: + inputs: + DATABASE_SCHEMA: + required: true + type: string + +jobs: + orm-update: + name: Update ORM + runs-on: ubuntu-latest + container: mariadb + services: + mariadb: + image: mariadb:10.6 + env: + MYSQL_DATABASE: ispybtest + MYSQL_ROOT_PASSWORD: mysql_root_pwd + ports: + - 3306:3306 + outputs: + output1: ${{ steps.checkUpdate.needsUpdate }} + + steps: + - uses: actions/checkout@v4 + - name: Use Python 3.10 + uses: actions/setup-python@v5 + with: + python-version: "3.10" + + - name: Check if update is required + id: checkUpdate + run: | + set -eu + CURRENT_VERSION=$(grep __schema_version__ _auto_db_schema.py | cut -d'"' -f2) + if [ ${CURRENT_VERSION} = ${{ inputs.DATABASE_SCHEMA }} ]; then + echo "##[section]ORM is up to date (${CURRENT_VERSION})" + echo "needsUpdate=false" >> $GITHUB_OUTPUT + else + echo "##[warning]ORM needs to be updated (${CURRENT_VERSION} -> ${{ inputs.DATABASE_SCHEMA }})." + echo "needsUpdate=true" >> $GITHUB_OUTPUT + fi + working-directory: ./src/ispyb/sqlalchemy + + - name: Install sqlacodegen + if: ${{ steps.checkUpdate.needsUpdate == 'true' }} + run: | + set -eux + pip install -r requirements_orm.txt + pip install pre-commit + + - name: Get database schema + if: ${{ steps.checkUpdate.needsUpdate == 'true' }} + uses: actions/download-artifact@v4 + with: + name: ispyb-database.tar.gz + path: dist/ + + - name: Set up reference database schema + if: ${{ steps.checkUpdate.needsUpdate == 'true' }} + run: | + set -eu + cat >~/.my.cnf </dev/null; do printf '.'; sleep 0.5; done + printf '\n' + + echo "Installing reference database" + ./build.sh + + echo + echo "Installed tables:" + mysql -D ispyb_build -e "SHOW TABLES" + working-directory: dist + + - name: Generate ORM + if: ${{ steps.checkUpdate.needsUpdate == 'true' }} + run: | + set -eux + sqlacodegen mysql+mysqlconnector://root:mysql_root_pwd@127.0.0.1/ispyb_build --noinflect --nojoined --outfile _auto_db_schema.py.in + # This code produces false positives due to non-deterministic ordering. + # Add an identifier to the file, and only update/PR the changes when the + # identifier indicates the file is out of date. + cat <(echo "__schema_version__ = \"${{ inputs.DATABASE_SCHEMA }}\"") _auto_db_schema.py.in > _auto_db_schema.py + rm _auto_db_schema.py.in + pre-commit run --files _auto_db_schema.py || echo pre-commit exited with non-zero return code + working-directory: ./src/ispyb/sqlalchemy + + - name: Show differences + if: ${{ steps.checkUpdate.needsUpdate == 'true' }} + run: git diff + + - name: Store artifact + if: ${{ steps.checkUpdate.needsUpdate == 'true' }} + uses: actions/upload-artifact@v4 + with: + name: ORM + path: ./src/ispyb/sqlalchemy/_auto_db_schema.py + + - name: Create ORM pull request + if: ${{ steps.checkUpdate.needsUpdate == 'true' && github.ref_name == 'main' }} + run: .github/workflows/scripts/create-orm-update-pull-request + env: + GITHUB_TOKEN: ${{ github.token }}