From b67de741e40c363eb36a21d92453d70525c92da5 Mon Sep 17 00:00:00 2001 From: IceS2 Date: Fri, 10 May 2024 11:40:30 +0200 Subject: [PATCH] MINOR: Small refactor to the Make update_all scripts (#16204) * Small refactor to the Make update_all scripts * Extract regex update method * decouple release from workflows * Fix issue with docker-openmetadata-db workflow referencing non existent action --- .../action.yml | 64 ++++++ .github/workflows/docker-openmetadata-db.yml | 42 +--- ...ocker-openmetadata-ingestion-base-slim.yml | 56 +---- .../docker-openmetadata-ingestion-base.yml | 56 +---- .../docker-openmetadata-ingestion.yml | 56 +---- .../docker-openmetadata-postgres.yml | 43 +--- .../workflows/docker-openmetadata-server.yml | 82 ++----- Makefile | 65 ++---- scripts/update_version.py | 213 +++++++++--------- 9 files changed, 261 insertions(+), 416 deletions(-) create mode 100644 .github/actions/prepare-for-docker-build-and-push/action.yml diff --git a/.github/actions/prepare-for-docker-build-and-push/action.yml b/.github/actions/prepare-for-docker-build-and-push/action.yml new file mode 100644 index 000000000000..c219d1dffae6 --- /dev/null +++ b/.github/actions/prepare-for-docker-build-and-push/action.yml @@ -0,0 +1,64 @@ +name: Prepare for Docker Build&Push +description: Set up Docker Build&Push dependencies and generate Docker Tags. + +inputs: + image: + description: image name + required: true + tag: + description: Docker tag to use + required: true + push_latest: + description: true will push the 'latest' tag. + required: true + default: "false" + is_ingestion: + description: true if we are building an Ingestion image, false otherwise + required: true + default: "false" + +outputs: + tags: + description: Generated Docker Tags + value: ${{ steps.generate-tags.outputs.tags }} + +runs: + using: composite + steps: + - name: Set up QEMU + uses: docker/setup-qemu-action@v2 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + + - name: Login to DockerHub + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKERHUB_OPENMETADATA_USERNAME }} + password: ${{ secrets.DOCKERHUB_OPENMETADATA_TOKEN }} + + - name: Install Ubuntu dependencies + if: ${{ inputs.is_ingestion == "true" }} + shell: bash + run: | + sudo apt-get install -y python3-venv + + - name: Install open-metadata dependencies + if: ${{ inputs.is_ingestion == "true" }} + shell: bash + run: | + python3 -m venv env + source env/bin/activate + pip install --upgrade pip + sudo make install_antlr_cli + make install_dev generate + + - name: Process Docker Tags + id: generate-tags + shell: bash + run: | + if ${{ inputs.push_latest == "true" }}; then + echo "tags=${{ inputs.image }}:${{ inputs.tag }},${{ inputs.image }}:latest" >> $GITHUB_OUTPUT + else + echo "tags=${{ inputs.image }}:${{ inputs.tag }}" >> $GITHUB_OUTPUT + fi diff --git a/.github/workflows/docker-openmetadata-db.yml b/.github/workflows/docker-openmetadata-db.yml index a2277cb0bf33..731be366048c 100644 --- a/.github/workflows/docker-openmetadata-db.yml +++ b/.github/workflows/docker-openmetadata-db.yml @@ -19,51 +19,29 @@ on: push_latest_tag_to_release: description: "Do you want to update docker image latest tag as well ?" type: boolean - release: - types: [published] jobs: push_to_docker_hub: runs-on: ubuntu-latest - env: - input: ${{ github.event.inputs.tag }} - + steps: - - name: Check trigger type - if: ${{ env.input == '' }} - run: echo "input=1.4.0-SNAPSHOT" >> $GITHUB_ENV - - name: Check out the Repo uses: actions/checkout@v3 - - name: Set up QEMU - uses: docker/setup-qemu-action@v2 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 - - - name: Login to DockerHub - uses: docker/login-action@v2 + - name: Prepare for Docker Build&Push + id: prepare + uses: ./.github/actions/prepare-for-docker-build-and-push with: - username: ${{ secrets.DOCKERHUB_OPENMETADATA_USERNAME }} - password: ${{ secrets.DOCKERHUB_OPENMETADATA_TOKEN }} + image: openmetadata/db + tag: ${{ inputs.tag }} + push_latest: ${{ inputs.push_latest_tag_to_release }} - - name: Process Docker Tags - id: input_check - run: | - if ${{ github.event_name == 'release' }}; then - echo "tags=openmetadata/db:${{ env.input }},openmetadata/db:latest" >> $GITHUB_OUTPUT - elif ${{ github.event_name == 'workflow_dispatch' && inputs.push_latest_tag_to_release }}; then - echo "tags=openmetadata/db:${{inputs.tag}},openmetadata/db:latest" >> $GITHUB_OUTPUT - else - echo "tags=openmetadata/db:${{ inputs.tag }}" >> $GITHUB_OUTPUT - fi - name: Build and push if event is workflow_dispatch and input is checked uses: docker/build-push-action@v3 with: context: . platforms: linux/amd64,linux/arm64 - push: ${{ github.event_name == 'release' || github.event_name == 'workflow_dispatch' }} - tags: ${{ steps.input_check.outputs.tags }} - file: ./docker/mysql/Dockerfile_mysql \ No newline at end of file + push: true + tags: ${{ steps.prepare.outputs.tags }} + file: ./docker/mysql/Dockerfile_mysql diff --git a/.github/workflows/docker-openmetadata-ingestion-base-slim.yml b/.github/workflows/docker-openmetadata-ingestion-base-slim.yml index 0cf5a8cb3096..bf194aac02f8 100644 --- a/.github/workflows/docker-openmetadata-ingestion-base-slim.yml +++ b/.github/workflows/docker-openmetadata-ingestion-base-slim.yml @@ -19,65 +19,31 @@ on: push_latest_tag_to_release: description: "Do you want to update docker image latest tag as well ?" type: boolean - release: - types: [published] jobs: push_to_docker_hub: runs-on: ubuntu-latest - env: - input: ${{ github.event.inputs.tag }} - + steps: - - name: Check trigger type - if: ${{ env.input == '' }} - run: echo "input=1.4.0-SNAPSHOT" >> $GITHUB_ENV - - name: Check out the Repo uses: actions/checkout@v3 - - name: Set up QEMU - uses: docker/setup-qemu-action@v2 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 - - - name: Login to DockerHub - uses: docker/login-action@v2 + - name: Prepare for Docker Build&Push + id: prepare + uses: ./.github/actions/prepare-for-docker-build-and-push with: - username: ${{ secrets.DOCKERHUB_OPENMETADATA_USERNAME }} - password: ${{ secrets.DOCKERHUB_OPENMETADATA_TOKEN }} - - - name: Install Ubuntu dependencies - run: | - sudo apt-get install -y python3-venv - - - name: Install open-metadata dependencies - run: | - python3 -m venv env - source env/bin/activate - pip install --upgrade pip - sudo make install_antlr_cli - make install_dev generate - - - name: Process Docker Tags - id: input_check - run: | - if ${{ github.event_name == 'release' }}; then - echo "tags=openmetadata/ingestion-base-slim:${{ env.input }},openmetadata/ingestion-base-slim:latest" >> $GITHUB_OUTPUT - elif ${{ github.event_name == 'workflow_dispatch' && inputs.push_latest_tag_to_release }}; then - echo "tags=openmetadata/ingestion-base-slim:${{inputs.tag}},openmetadata/ingestion-base-slim:latest" >> $GITHUB_OUTPUT - else - echo "tags=openmetadata/ingestion-base-slim:${{ inputs.tag }}" >> $GITHUB_OUTPUT - fi + image: openmetadata/ingestion-base-slim + tag: ${{ inputs.tag }} + push_latest: ${{ inputs.push_latest_tag_to_release }} + is_ingestion: true - name: Build and push if event is workflow_dispatch and input is checked uses: docker/build-push-action@v3 with: context: . platforms: linux/amd64,linux/arm64 - push: ${{ github.event_name == 'release' || github.event_name == 'workflow_dispatch' }} - tags: ${{ steps.input_check.outputs.tags }} + push: true + tags: ${{ steps.prepare.outputs.tags }} file: ./ingestion/operators/docker/Dockerfile build-args: | - INGESTION_DEPENDENCY=slim \ No newline at end of file + INGESTION_DEPENDENCY=slim diff --git a/.github/workflows/docker-openmetadata-ingestion-base.yml b/.github/workflows/docker-openmetadata-ingestion-base.yml index cd34badb2482..31b5666a3465 100644 --- a/.github/workflows/docker-openmetadata-ingestion-base.yml +++ b/.github/workflows/docker-openmetadata-ingestion-base.yml @@ -19,63 +19,29 @@ on: push_latest_tag_to_release: description: "Do you want to update docker image latest tag as well ?" type: boolean - release: - types: [published] jobs: push_to_docker_hub: runs-on: ubuntu-latest - env: - input: ${{ github.event.inputs.tag }} - + steps: - - name: Check trigger type - if: ${{ env.input == '' }} - run: echo "input=1.4.0-SNAPSHOT" >> $GITHUB_ENV - - name: Check out the Repo uses: actions/checkout@v3 - - name: Set up QEMU - uses: docker/setup-qemu-action@v2 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 - - - name: Login to DockerHub - uses: docker/login-action@v2 + - name: Prepare for Docker Build&Push + id: prepare + uses: ./.github/actions/prepare-for-docker-build-and-push with: - username: ${{ secrets.DOCKERHUB_OPENMETADATA_USERNAME }} - password: ${{ secrets.DOCKERHUB_OPENMETADATA_TOKEN }} - - - name: Install Ubuntu dependencies - run: | - sudo apt-get install -y python3-venv - - - name: Install open-metadata dependencies - run: | - python3 -m venv env - source env/bin/activate - pip install --upgrade pip - sudo make install_antlr_cli - make install_dev generate - - - name: Process Docker Tags - id: input_check - run: | - if ${{ github.event_name == 'release' }}; then - echo "tags=openmetadata/ingestion-base:${{ env.input }},openmetadata/ingestion-base:latest" >> $GITHUB_OUTPUT - elif ${{ github.event_name == 'workflow_dispatch' && inputs.push_latest_tag_to_release }}; then - echo "tags=openmetadata/ingestion-base:${{inputs.tag}},openmetadata/ingestion-base:latest" >> $GITHUB_OUTPUT - else - echo "tags=openmetadata/ingestion-base:${{ inputs.tag }}" >> $GITHUB_OUTPUT - fi + image: openmetadata/ingestion-base + tag: ${{ inputs.tag }} + push_latest: ${{ inputs.push_latest_tag_to_release }} + is_ingestion: true - name: Build and push if event is workflow_dispatch and input is checked uses: docker/build-push-action@v3 with: context: . platforms: linux/amd64,linux/arm64 - push: ${{ github.event_name == 'release' || github.event_name == 'workflow_dispatch' }} - tags: ${{ steps.input_check.outputs.tags }} - file: ./ingestion/operators/docker/Dockerfile \ No newline at end of file + push: true + tags: ${{ steps.prepare.outputs.tags }} + file: ./ingestion/operators/docker/Dockerfile diff --git a/.github/workflows/docker-openmetadata-ingestion.yml b/.github/workflows/docker-openmetadata-ingestion.yml index d2c97c330d2c..72f1bfad8694 100644 --- a/.github/workflows/docker-openmetadata-ingestion.yml +++ b/.github/workflows/docker-openmetadata-ingestion.yml @@ -19,63 +19,29 @@ on: push_latest_tag_to_release: description: "Do you want to update docker image latest tag as well ?" type: boolean - release: - types: [published] jobs: push_to_docker_hub: runs-on: ubuntu-latest - env: - input: ${{ github.event.inputs.tag }} - + steps: - - name: Check trigger type - if: ${{ env.input == '' }} - run: echo "input=1.4.0-SNAPSHOT" >> $GITHUB_ENV - - name: Check out the Repo uses: actions/checkout@v3 - - name: Set up QEMU - uses: docker/setup-qemu-action@v2 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 - - - name: Login to DockerHub - uses: docker/login-action@v2 + - name: Prepare for Docker Build&Push + id: prepare + uses: ./.github/actions/prepare-for-docker-build-and-push with: - username: ${{ secrets.DOCKERHUB_OPENMETADATA_USERNAME }} - password: ${{ secrets.DOCKERHUB_OPENMETADATA_TOKEN }} - - - name: Install Ubuntu dependencies - run: | - sudo apt-get install -y python3-venv - - - name: Install open-metadata dependencies - run: | - python3 -m venv env - source env/bin/activate - pip install --upgrade pip - sudo make install_antlr_cli - make install_dev generate - - - name: Process Docker Tags - id: input_check - run: | - if ${{ github.event_name == 'release' }}; then - echo "tags=openmetadata/ingestion:${{ env.input }},openmetadata/ingestion:latest" >> $GITHUB_OUTPUT - elif ${{ github.event_name == 'workflow_dispatch' && inputs.push_latest_tag_to_release }}; then - echo "tags=openmetadata/ingestion:${{inputs.tag}},openmetadata/ingestion:latest" >> $GITHUB_OUTPUT - else - echo "tags=openmetadata/ingestion:${{ inputs.tag }}" >> $GITHUB_OUTPUT - fi + image: openmetadata/ingestion + tag: ${{ inputs.tag }} + push_latest: ${{ inputs.push_latest_tag_to_release }} + is_ingestion: true - name: Build and push if event is workflow_dispatch and input is checked uses: docker/build-push-action@v3 with: context: . platforms: linux/amd64,linux/arm64 - push: ${{ github.event_name == 'release' || github.event_name == 'workflow_dispatch' }} - tags: ${{ steps.input_check.outputs.tags }} - file: ./ingestion/Dockerfile \ No newline at end of file + push: true + tags: ${{ steps.prepare.outputs.tags }} + file: ./ingestion/Dockerfile diff --git a/.github/workflows/docker-openmetadata-postgres.yml b/.github/workflows/docker-openmetadata-postgres.yml index 19ccc3f6077d..b4fe7f1fe536 100644 --- a/.github/workflows/docker-openmetadata-postgres.yml +++ b/.github/workflows/docker-openmetadata-postgres.yml @@ -19,51 +19,28 @@ on: push_latest_tag_to_release: description: "Do you want to update docker image latest tag as well ?" type: boolean - release: - types: [published] jobs: push_to_docker_hub: runs-on: ubuntu-latest - env: - input: ${{ github.event.inputs.tag }} - + steps: - - name: Check trigger type - if: ${{ env.input == '' }} - run: echo "input=1.4.0-SNAPSHOT" >> $GITHUB_ENV - - name: Check out the Repo uses: actions/checkout@v3 - - name: Set up QEMU - uses: docker/setup-qemu-action@v2 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 - - - name: Login to DockerHub - uses: docker/login-action@v2 + - name: Prepare for Docker Build&Push + id: prepare + uses: ./.github/actions/prepare-for-docker-build-and-push with: - username: ${{ secrets.DOCKERHUB_OPENMETADATA_USERNAME }} - password: ${{ secrets.DOCKERHUB_OPENMETADATA_TOKEN }} - - - name: Process Docker Tags - id: input_check - run: | - if ${{ github.event_name == 'release' }}; then - echo "tags=openmetadata/postgresql:${{ env.input }},openmetadata/postgresql:latest" >> $GITHUB_OUTPUT - elif ${{ github.event_name == 'workflow_dispatch' && inputs.push_latest_tag_to_release }}; then - echo "tags=openmetadata/postgresql:${{inputs.tag}},openmetadata/postgresql:latest" >> $GITHUB_OUTPUT - else - echo "tags=openmetadata/postgresql:${{ inputs.tag }}" >> $GITHUB_OUTPUT - fi + image: openmetadata/postgresql + tag: ${{ inputs.tag }} + push_latest: ${{ inputs.push_latest_tag_to_release }} - name: Build and push if event is workflow_dispatch and input is checked uses: docker/build-push-action@v3 with: context: . platforms: linux/amd64,linux/arm64 - push: ${{ github.event_name == 'release' || github.event_name == 'workflow_dispatch' }} - tags: ${{ steps.input_check.outputs.tags }} - file: ./docker/postgresql/Dockerfile_postgres \ No newline at end of file + push: true + tags: ${{ steps.prepare.outputs.tags }} + file: ./docker/postgresql/Dockerfile_postgres diff --git a/.github/workflows/docker-openmetadata-server.yml b/.github/workflows/docker-openmetadata-server.yml index 32d36cabfc43..6c4eae9ffd0e 100644 --- a/.github/workflows/docker-openmetadata-server.yml +++ b/.github/workflows/docker-openmetadata-server.yml @@ -19,13 +19,11 @@ on: push_latest_tag_to_release: description: "Do you want to update docker image latest tag as well ?" type: boolean - release: - types: [published] jobs: maven-build: runs-on: ubuntu-latest steps: - + - name: Checkout uses: actions/checkout@v3 @@ -34,7 +32,7 @@ jobs: with: java-version: 17 distribution: 'temurin' - + - name: Install antlr cli run: | sudo make install_antlr_cli @@ -55,38 +53,7 @@ jobs: with: name: openmetadata-binary path: /home/runner/work/OpenMetadata/OpenMetadata/openmetadata-dist/target/*.tar.gz - - release-project-event-release: - if: github.event_name == 'release' - name: Release app with release event - runs-on: ubuntu-latest - needs: maven-build - steps: - - name: Check trigger type - id: check_trigger - run: echo "DOCKER_RELEASE_TAG=1.4.0-SNAPSHOT" >> $GITHUB_OUTPUT - - - name: Download application from Artifiact - uses: actions/download-artifact@v2 - with: - name: openmetadata-binary - - - name: Get release data - id: get_release - uses: bruceadams/get-release@v1.3.2 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - name: Upload app tar to Github Release - uses: actions/upload-release-asset@v1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - upload_url: ${{ steps.get_release.outputs.upload_url }} - asset_path: ./openmetadata-${{ steps.check_trigger.outputs.DOCKER_RELEASE_TAG }}.tar.gz - asset_name: openmetadata-${{ steps.check_trigger.outputs.DOCKER_RELEASE_TAG }}.tar.gz - asset_content_type: application/tar.gz - release-project-event-workflow_dispatch: if: github.event_name == 'workflow_dispatch' name: Release app with workflow_dispatch event @@ -118,52 +85,31 @@ jobs: push_to_docker_hub: runs-on: ubuntu-latest - env: - DOCKER_RELEASE_TAG: ${{ github.event.inputs.DOCKER_RELEASE_TAG }} if: ${{ always() && contains(join(needs.*.result, ','), 'success') }} - needs: [release-project-event-workflow_dispatch, release-project-event-release] + needs: [release-project-event-workflow_dispatch] steps: - - name: Set build arguments - run: | - echo "BUILD_DATE=$(date -u +'%Y-%m-%dT%H:%M:%SZ')" >> $GITHUB_ENV - - name: Check trigger type - id: check_trigger - if: ${{ env.DOCKER_RELEASE_TAG == '' }} - run: echo "DOCKER_RELEASE_TAG=1.4.0-SNAPSHOT" >> $GITHUB_ENV - - name: Check out the Repo uses: actions/checkout@v3 - - name: Set up QEMU - uses: docker/setup-qemu-action@v2 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 + - name: Set build arguments + run: | + echo "BUILD_DATE=$(date -u +'%Y-%m-%dT%H:%M:%SZ')" >> $GITHUB_ENV - - name: Login to DockerHub - uses: docker/login-action@v2 + - name: Prepare for Docker Build&Push + id: prepare + uses: ./.github/actions/prepare-for-docker-build-and-push with: - username: ${{ secrets.DOCKERHUB_OPENMETADATA_USERNAME }} - password: ${{ secrets.DOCKERHUB_OPENMETADATA_TOKEN }} - - - name: Process Docker Tags - id: input_check - run: | - if ${{ github.event_name == 'release' }}; then - echo "tags=openmetadata/server:${{ env.DOCKER_RELEASE_TAG }},openmetadata/server:latest" >> $GITHUB_OUTPUT - elif ${{ github.event_name == 'workflow_dispatch' && inputs.push_latest_tag_to_release }}; then - echo "tags=openmetadata/server:${{ env.DOCKER_RELEASE_TAG }},openmetadata/server:latest" >> $GITHUB_OUTPUT - else - echo "tags=openmetadata/server:${{ env.DOCKER_RELEASE_TAG }}" >> $GITHUB_OUTPUT - fi + image: openmetadata/server + tag: ${{ inputs.DOCKER_RELEASE_TAG }} + push_latest: ${{ inputs.push_latest_tag_to_release }} - name: Build and push if event is workflow_dispatch and input is checked uses: docker/build-push-action@v3 with: context: . platforms: linux/amd64,linux/arm64 - push: ${{ github.event_name == 'release' || github.event_name == 'workflow_dispatch' }} - tags: ${{ steps.input_check.outputs.tags }} + push: true + tags: ${{ steps.prepare.outputs.tags }} file: ./docker/docker-compose-quickstart/Dockerfile build-args: | BUILD_DATE=${{ env.BUILD_DATE }} diff --git a/Makefile b/Makefile index f2754a17e1d2..80bfa58314d3 100644 --- a/Makefile +++ b/Makefile @@ -148,62 +148,38 @@ build-ingestion-base-local: ## Builds the ingestion DEV docker operator with th generate-schema-docs: ## Generates markdown files for documenting the JSON Schemas @echo "Generating Schema docs" # Installing "0.4.0" version for simpler formatting - python3 -m pip install "jsonschema2md==0.4.0" + python3 -m pip install "jsonschema2md==0.4.0" python3 scripts/generate_docs_schemas.py -#Upgrade release automation scripts below +#Upgrade release automation scripts below .PHONY: update_all -update_all: ## To update all the release related files run make update_all RELEASE_VERSION=2.2.2 PY_RELEASE_VERSION=2.2.2.2 +update_all: ## To update all the release related files run make update_all RELEASE_VERSION=2.2.2 @echo "The release version is: $(RELEASE_VERSION)" ; \ - echo "The python metadata release version: $(PY_RELEASE_VERSION)" ; \ $(MAKE) update_maven ; \ - $(MAKE) update_github_action_paths ; \ - $(MAKE) update_python_release_paths ; \ + $(MAKE) update_pyproject_version ; \ $(MAKE) update_dockerfile_version ; \ - $(MAKE) update_ingestion_dockerfile_version ; \ - + $(MAKE) update_dockerfile_ri_version ; \ #remove comment and use the below section when want to use this sub module "update_all" independently to update github actions -#make update_all RELEASE_VERSION=2.2.2 PY_RELEASE_VERSION=2.2.2.2 +#make update_all RELEASE_VERSION=2.2.2 .PHONY: update_maven update_maven: ## To update the common and pom.xml maven version - @echo "Updating Maven projects to version $(MVN_RELEASE_VERSION)..."; \ - mvn versions:set -DnewVersion=$(MVN_RELEASE_VERSION) + @echo "Updating Maven projects to version $(RELEASE_VERSION)..."; \ + mvn versions:set -DnewVersion=$(RELEASE_VERSION) #remove comment and use the below section when want to use this sub module "update_maven" independently to update github actions #make update_maven RELEASE_VERSION=2.2.2 -.PHONY: update_github_action_paths -update_github_action_paths: ## To update the github action ci docker files - @echo "Updating docker github action release version to $(RELEASE_VERSION)... "; \ - file_paths="docker/docker-compose-quickstart/Dockerfile \ - .github/workflows/docker-openmetadata-db.yml \ - .github/workflows/docker-openmetadata-ingestion-base.yml \ - .github/workflows/docker-openmetadata-ingestion-base-slim.yml \ - .github/workflows/docker-openmetadata-ingestion.yml \ - .github/workflows/docker-openmetadata-postgres.yml \ - .github/workflows/docker-openmetadata-server.yml"; \ - for file_path in $$file_paths; do \ - python3 scripts/update_version.py 1 $$file_path -s $(RELEASE_VERSION) ; \ - done; \ - file_paths1="docker/docker-compose-quickstart/Dockerfile"; \ - for file_path in $$file_paths1; do \ - python3 scripts/update_version.py 4 $$file_path -s $(RELEASE_VERSION) ; \ - done - -#remove comment and use the below section when want to use this sub module "update_github_action_paths" independently to update github actions -#make update_github_action_paths RELEASE_VERSION=2.2.2 - -.PHONY: update_python_release_paths -update_python_release_paths: ## To update the setup.py files +.PHONY: update_pyproject_version +update_pyproject_version: ## To update the pyproject.toml files file_paths="ingestion/pyproject.toml \ openmetadata-airflow-apis/pyproject.toml"; \ - echo "Updating Python setup file versions to $(PY_RELEASE_VERSION)... "; \ + echo "Updating pyproject.toml versions to $(RELEASE_VERSION)... "; \ for file_path in $$file_paths; do \ - python3 scripts/update_version.py 2 $$file_path -s $(PY_RELEASE_VERSION) ; \ + python3 scripts/update_version.py update_pyproject_version -f $$file_path -v $(RELEASE_VERSION) ; \ done -# Commented section for independent usage of the module update_python_release_paths independently to update github actions -#make update_python_release_paths PY_RELEASE_VERSION=2.2.2.2 +# Commented section for independent usage of the module update_pyproject_version independently to update github actions +#make update_pyproject_version RELEASE_VERSION=2.2.2 .PHONY: update_dockerfile_version update_dockerfile_version: ## To update the dockerfiles version @@ -213,20 +189,21 @@ update_dockerfile_version: ## To update the dockerfiles version docker/docker-compose-quickstart/docker-compose.yml"; \ echo "Updating docker github action release version to $(RELEASE_VERSION)... "; \ for file_path in $$file_paths; do \ - python3 scripts/update_version.py 3 $$file_path -s $(RELEASE_VERSION) ; \ + python3 scripts/update_version.py update_docker_tag -f $$file_path -t $(RELEASE_VERSION) ; \ done #remove comment and use the below section when want to use this sub module "update_dockerfile_version" independently to update github actions #make update_dockerfile_version RELEASE_VERSION=2.2.2 -.PHONY: update_ingestion_dockerfile_version -update_ingestion_dockerfile_version: ## To update the ingestion dockerfiles version +.PHONY: update_dockerfile_ri_version +update_dockerfile_ri_version: ## To update the dockerfile RI_VERSION argument @file_paths="ingestion/Dockerfile \ ingestion/operators/docker/Dockerfile"; \ echo "Updating ingestion dockerfile release version to $(PY_RELEASE_VERSION)... "; \ for file_path in $$file_paths; do \ - python3 scripts/update_version.py 4 $$file_path -s $(PY_RELEASE_VERSION) ; \ + python3 scripts/update_version.py update_ri_version -f $$file_path -v $(RELEASE_VERSION) --with-python-version ; \ done -#remove comment and use the below section when want to use this sub module "update_ingestion_dockerfile_version" independently to update github actions -#make update_ingestion_dockerfile_version PY_RELEASE_VERSION=2.2.2.2 + python3 scripts/update_version.py update_ri_version -f docker/docker-compose-quickstart/Dockerfile -v $(RELEASE_VERSION) +#remove comment and use the below section when want to use this sub module "update_dockerfile_ri_version" independently to update github actions +#make update_dockerfile_ri_version RELEASE_VERSION=2.2.2 #Upgrade release automation scripts above diff --git a/scripts/update_version.py b/scripts/update_version.py index d7f75097eecc..282551833d12 100644 --- a/scripts/update_version.py +++ b/scripts/update_version.py @@ -1,118 +1,123 @@ -import sys import argparse -import fileinput -import os import re import logging -# Configure the logger logging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s") logger = logging.getLogger() -# Function to update the Github workflow with search pattern as "input=" or "DOCKER_RELEASE_TAG=" -def update_github_action(file_path, release_version): - logger.info(f"Updating Github workflow's Docker version in {file_path} to version {release_version}\n") - try: - with open(file_path, 'r') as file: - content = file.read() - - # Update the input pattern - input_pattern = r'input=\d+(\.\d+)*(\.\d+)?' - input_replacement = f'input={release_version}' - updated_content = re.sub(input_pattern, input_replacement, content) - - # Update the DOCKER_RELEASE_TAG pattern - docker_release_tag_pattern = r'DOCKER_RELEASE_TAG=\d+(\.\d+)*(\.\d+)?' - docker_release_tag_replacement = f'DOCKER_RELEASE_TAG={release_version}' - updated_content = re.sub(docker_release_tag_pattern, docker_release_tag_replacement, updated_content) - - with open(file_path, 'w') as file: - file.write(updated_content) - - logger.info(f"Patterns updated to {release_version} in {file_path}") - except Exception as e: - logger.error(f"An error occurred: {e}") - -# Function to update the Python files in ingestion with search pattern as "version=" -def update_python_files(file_path, release_version): - # Logic for updating Python files - logger.info(f"Updating version numbers in {file_path} to {release_version}\n") - try: - with open(file_path, 'r') as file: - content = file.read() - - pattern = r'version\s*=\s*"([^"]+)"' - updated_content = re.sub(pattern, f'version = "{release_version}"', content) - - with open(file_path, 'w') as file: - file.write(updated_content) - - logger.info(f"Version numbers updated to {release_version} in {file_path}") - except Exception as e: - logger.error(f"An error occurred: {e}") - -# Function to update the image version in Docker compose files with search pattern where image, docker, getcollate, and openmetadata are used. -def update_dockerfile_version(file_path, release_version): - # Logic for updating Docker compose version - try: - with open(file_path, 'r') as file: - content = file.read() - - # Update image versions using regular expression - updated_content = re.sub( - r'(image: docker\.getcollate\.io/openmetadata/.*?):[\d.]+', - rf'\1:{release_version}', - content - ) - - with open(file_path, 'w') as file: - file.write(updated_content) - - logger.info(f"Updated image versions in {file_path}") - except Exception as e: - logger.error(f"An error occurred while updating {file_path}: {e}") - -# Function to update the DOCKERFILE used to create the images, search pattern used as "RI_VERSION" -def update_ingestion_version(file_path, release_version): - logger.info(f"Updating ingestion version in {file_path} to version {release_version}\n") - try: - with open(file_path, 'r') as file: - content = file.read() - - pattern = r'RI_VERSION="[\d\.]+"' - replacement = f'RI_VERSION="{release_version}"' - updated_content = re.sub(pattern, replacement, content) - - with open(file_path, 'w') as file: - file.write(updated_content) - - logger.info(f"RI_VERSION updated to {release_version} in {file_path}") - except Exception as e: - logger.error(f"An error occurred: {e}") + +def get_python_version(version: str) -> str: + "Get Python Package formatted version" + + if "-rc" in version: + version_parts = version.split("-") + return f"{version_parts[0]}.0{version_parts[1]}" + return f"{version}.0" + + +def regex_sub(file_path: str, pattern: str, substitution: str): + with open(file_path, "r") as f: + content = f.read() + + updated_content = re.sub( + pattern, + substitution, + content + ) + + with open(file_path, "w") as f: + f.write(updated_content) + + +def update_dockerfile_arg(arg, file_path, value): + """Updates a Dockerfile ARG.""" + + logger.info(f"Updating ARG {arg} in {file_path} to {value}\n") + + regex_sub( + file_path, + rf"(ARG\s+{arg}=).+", + rf"\1={value}", + + ) + + +def update_docker_tag(args): + """Updates the Docker Tag on docker-compose files.""" + + file_path = args.file_path + tag = args.tag + + logger.info(f"Updating Docker Tag in {file_path} to {tag}\n") + + regex_sub( + file_path, + r'(image: docker\.getcollate\.io/openmetadata/.*?):.+', + rf'\1:{tag}', + ) + + +def update_ri_version(args): + """Updates a Dockerfile RI_VERSION ARG.""" + + version = args.version + with_python_version = args.with_python_version + file_path = args.file_path + + if with_python_version: + version = get_python_version(version) + + update_dockerfile_arg( + arg="RI_VERSION", + file_path=file_path, + value=version + ) + + +def update_pyproject_version(args): + """Updates pyproject version.""" + + file_path = args.file_path + version = args.version + + version = get_python_version(version) + + + logger.info(f"Updating {file_path} version to {version}\n") + + regex_sub( + file_path, + r'version\s*=\s*"[^"]+"', + f'version = "{version}"', + ) + def main(): - parser = argparse.ArgumentParser(description="Update version information in files.") - parser.add_argument("action_type", type=int, choices=range(1, 5), help="Type of action to perform") - parser.add_argument("file_path", type=str, help="Path to the file to update") - parser.add_argument("-s", dest="release_version", required=True, help="Release version to set") + parser = argparse.ArgumentParser(description="Update files for release.") + subparsers = parser.add_subparsers(required=True) + + # Update Docker tag + parser_udt = subparsers.add_parser("update_docker_tag") + parser_udt.add_argument("--file-path", "-f", type=str, help="Docker compose file to update.") + parser_udt.add_argument("--tag", "-t", type=str, help="Tag to update the file to.") + parser_udt.set_defaults(func=update_docker_tag) + + # Update Dockerfile ARG + parser_urv = subparsers.add_parser("update_ri_version") + parser_urv.add_argument("--file-path", "-f", type=str, help="Dockerfile file to update.") + parser_urv.add_argument("--version", "-v", type=str, help="Verision to set for the argument") + parser_urv.add_argument("--with-python-version", action="store_true") + parser_urv.set_defaults(func=update_ri_version) + + # Update pyproject.toml Version + parser_upv = subparsers.add_parser("update_pyproject_version") + parser_upv.add_argument("--file-path", "-f", type=str, help="pyproject.toml file to update.") + parser_upv.add_argument("--version", "-v", type=str, help="Version to update to") + parser_upv.set_defaults(func=update_pyproject_version) args = parser.parse_args() + args.func(args) - action_type = args.action_type - file_path = args.file_path - release_version = args.release_version - - if action_type == 1: - update_github_action(file_path, release_version) - elif action_type == 2: - update_python_files(file_path, release_version) - elif action_type == 3: - update_dockerfile_version(file_path, release_version) - elif action_type == 4: - update_ingestion_version(file_path, release_version) - else: - logger.error("Invalid action type") - sys.exit(1) if __name__ == "__main__": - main() \ No newline at end of file + main()