MINOR: Small refactor to the Make update_all scripts (#16204)

* Small refactor to the Make update_all scripts * Extract regex update method * decouple release from workflows * Fix issue with docker-openmetadata-db workflow referencing non existent action
open-metadata · May 10, 2024 · b67de74 · b67de74
1 parent 7fe0935
commit b67de74
Show file tree

Hide file tree

Showing 9 changed files with 261 additions and 416 deletions.
diff --git a/.github/actions/prepare-for-docker-build-and-push/action.yml b/.github/actions/prepare-for-docker-build-and-push/action.yml
@@ -0,0 +1,64 @@
+name: Prepare for Docker Build&Push
+description: Set up Docker Build&Push dependencies and generate Docker Tags.
+
+inputs:
+ image:
+ description: image name
+ required: true
+ tag:
+ description: Docker tag to use
+ required: true
+ push_latest:
+ description: true will push the 'latest' tag.
+ required: true
+ default: "false"
+ is_ingestion:
+ description: true if we are building an Ingestion image, false otherwise
+ required: true
+ default: "false"
+
+outputs:
+ tags:
+ description: Generated Docker Tags
+ value: ${{ steps.generate-tags.outputs.tags }}
+
+runs:
+ using: composite
+ steps:
+ - name: Set up QEMU
+ uses: docker/setup-qemu-action@v2
+
+ - name: Set up Docker Buildx
+ uses: docker/setup-buildx-action@v2
+
+ - name: Login to DockerHub
+ uses: docker/login-action@v2
+ with:
+ username: ${{ secrets.DOCKERHUB_OPENMETADATA_USERNAME }}
+ password: ${{ secrets.DOCKERHUB_OPENMETADATA_TOKEN }}
+
+ - name: Install Ubuntu dependencies
+ if: ${{ inputs.is_ingestion == "true" }}
+ shell: bash
+ run: |
+ sudo apt-get install -y python3-venv
+
+ - name: Install open-metadata dependencies
+ if: ${{ inputs.is_ingestion == "true" }}
+ shell: bash
+ run: |
+ python3 -m venv env
+ source env/bin/activate
+ pip install --upgrade pip
+ sudo make install_antlr_cli
+ make install_dev generate
+
+ - name: Process Docker Tags
+ id: generate-tags
+ shell: bash
+ run: |
+ if ${{ inputs.push_latest == "true" }}; then
+ echo "tags=${{ inputs.image }}:${{ inputs.tag }},${{ inputs.image }}:latest" >> $GITHUB_OUTPUT
+ else
+ echo "tags=${{ inputs.image }}:${{ inputs.tag }}" >> $GITHUB_OUTPUT
+ fi
diff --git a/.github/workflows/docker-openmetadata-db.yml b/.github/workflows/docker-openmetadata-db.yml
@@ -19,51 +19,29 @@ on:
  push_latest_tag_to_release:
  description: "Do you want to update docker image latest tag as well ?"
  type: boolean
- release:
- types: [published]
 
 jobs:
  push_to_docker_hub:
  runs-on: ubuntu-latest
- env:
- input: ${{ github.event.inputs.tag }}
-
+
  steps:
- - name: Check trigger type
- if: ${{ env.input == '' }}
- run: echo "input=1.4.0-SNAPSHOT" >> $GITHUB_ENV
-
  - name: Check out the Repo
  uses: actions/checkout@v3
 
- - name: Set up QEMU
- uses: docker/setup-qemu-action@v2
-
- - name: Set up Docker Buildx
- uses: docker/setup-buildx-action@v2
-
- - name: Login to DockerHub
- uses: docker/login-action@v2
+ - name: Prepare for Docker Build&Push
+ id: prepare
+ uses: ./.github/actions/prepare-for-docker-build-and-push
  with:
- username: ${{ secrets.DOCKERHUB_OPENMETADATA_USERNAME }}
- password: ${{ secrets.DOCKERHUB_OPENMETADATA_TOKEN }}
+ image: openmetadata/db
+ tag: ${{ inputs.tag }}
+ push_latest: ${{ inputs.push_latest_tag_to_release }}
 
- - name: Process Docker Tags
- id: input_check
- run: |
- if ${{ github.event_name == 'release' }}; then
- echo "tags=openmetadata/db:${{ env.input }},openmetadata/db:latest" >> $GITHUB_OUTPUT
- elif ${{ github.event_name == 'workflow_dispatch' && inputs.push_latest_tag_to_release }}; then
- echo "tags=openmetadata/db:${{inputs.tag}},openmetadata/db:latest" >> $GITHUB_OUTPUT
- else
- echo "tags=openmetadata/db:${{ inputs.tag }}" >> $GITHUB_OUTPUT
- fi
 
  - name: Build and push if event is workflow_dispatch and input is checked
  uses: docker/build-push-action@v3
  with:
  context: .
  platforms: linux/amd64,linux/arm64
- push: ${{ github.event_name == 'release' || github.event_name == 'workflow_dispatch' }}
- tags: ${{ steps.input_check.outputs.tags }}
- file: ./docker/mysql/Dockerfile_mysql
+ push: true
+ tags: ${{ steps.prepare.outputs.tags }}
+ file: ./docker/mysql/Dockerfile_mysql
diff --git a/.github/workflows/docker-openmetadata-ingestion-base-slim.yml b/.github/workflows/docker-openmetadata-ingestion-base-slim.yml
@@ -19,65 +19,31 @@ on:
  push_latest_tag_to_release:
  description: "Do you want to update docker image latest tag as well ?"
  type: boolean
- release:
- types: [published]
 
 jobs:
  push_to_docker_hub:
  runs-on: ubuntu-latest
- env:
- input: ${{ github.event.inputs.tag }}
-
+
  steps:
- - name: Check trigger type
- if: ${{ env.input == '' }}
- run: echo "input=1.4.0-SNAPSHOT" >> $GITHUB_ENV
-
  - name: Check out the Repo
  uses: actions/checkout@v3
 
- - name: Set up QEMU
- uses: docker/setup-qemu-action@v2
-
- - name: Set up Docker Buildx
- uses: docker/setup-buildx-action@v2
-
- - name: Login to DockerHub
- uses: docker/login-action@v2
+ - name: Prepare for Docker Build&Push
+ id: prepare
+ uses: ./.github/actions/prepare-for-docker-build-and-push
  with:
- username: ${{ secrets.DOCKERHUB_OPENMETADATA_USERNAME }}
- password: ${{ secrets.DOCKERHUB_OPENMETADATA_TOKEN }}
-
- - name: Install Ubuntu dependencies
- run: |
- sudo apt-get install -y python3-venv
-
- - name: Install open-metadata dependencies
- run: |
- python3 -m venv env
- source env/bin/activate
- pip install --upgrade pip
- sudo make install_antlr_cli
- make install_dev generate
-
- - name: Process Docker Tags
- id: input_check
- run: |
- if ${{ github.event_name == 'release' }}; then
- echo "tags=openmetadata/ingestion-base-slim:${{ env.input }},openmetadata/ingestion-base-slim:latest" >> $GITHUB_OUTPUT
- elif ${{ github.event_name == 'workflow_dispatch' && inputs.push_latest_tag_to_release }}; then
- echo "tags=openmetadata/ingestion-base-slim:${{inputs.tag}},openmetadata/ingestion-base-slim:latest" >> $GITHUB_OUTPUT
- else
- echo "tags=openmetadata/ingestion-base-slim:${{ inputs.tag }}" >> $GITHUB_OUTPUT
- fi
+ image: openmetadata/ingestion-base-slim
+ tag: ${{ inputs.tag }}
+ push_latest: ${{ inputs.push_latest_tag_to_release }}
+ is_ingestion: true
 
  - name: Build and push if event is workflow_dispatch and input is checked
  uses: docker/build-push-action@v3
  with:
  context: .
  platforms: linux/amd64,linux/arm64
- push: ${{ github.event_name == 'release' || github.event_name == 'workflow_dispatch' }}
- tags: ${{ steps.input_check.outputs.tags }}
+ push: true
+ tags: ${{ steps.prepare.outputs.tags }}
  file: ./ingestion/operators/docker/Dockerfile
  build-args: |
- INGESTION_DEPENDENCY=slim
+ INGESTION_DEPENDENCY=slim
diff --git a/.github/workflows/docker-openmetadata-ingestion-base.yml b/.github/workflows/docker-openmetadata-ingestion-base.yml
@@ -19,63 +19,29 @@ on:
  push_latest_tag_to_release:
  description: "Do you want to update docker image latest tag as well ?"
  type: boolean
- release:
- types: [published]
 
 jobs:
  push_to_docker_hub:
  runs-on: ubuntu-latest
- env:
- input: ${{ github.event.inputs.tag }}
-
+
  steps:
- - name: Check trigger type
- if: ${{ env.input == '' }}
- run: echo "input=1.4.0-SNAPSHOT" >> $GITHUB_ENV
-
  - name: Check out the Repo
  uses: actions/checkout@v3
 
- - name: Set up QEMU
- uses: docker/setup-qemu-action@v2
-
- - name: Set up Docker Buildx
- uses: docker/setup-buildx-action@v2
-
- - name: Login to DockerHub
- uses: docker/login-action@v2
+ - name: Prepare for Docker Build&Push
+ id: prepare
+ uses: ./.github/actions/prepare-for-docker-build-and-push
  with:
- username: ${{ secrets.DOCKERHUB_OPENMETADATA_USERNAME }}
- password: ${{ secrets.DOCKERHUB_OPENMETADATA_TOKEN }}
-
- - name: Install Ubuntu dependencies
- run: |
- sudo apt-get install -y python3-venv
-
- - name: Install open-metadata dependencies
- run: |
- python3 -m venv env
- source env/bin/activate
- pip install --upgrade pip
- sudo make install_antlr_cli
- make install_dev generate
-
- - name: Process Docker Tags
- id: input_check
- run: |
- if ${{ github.event_name == 'release' }}; then
- echo "tags=openmetadata/ingestion-base:${{ env.input }},openmetadata/ingestion-base:latest" >> $GITHUB_OUTPUT
- elif ${{ github.event_name == 'workflow_dispatch' && inputs.push_latest_tag_to_release }}; then
- echo "tags=openmetadata/ingestion-base:${{inputs.tag}},openmetadata/ingestion-base:latest" >> $GITHUB_OUTPUT
- else
- echo "tags=openmetadata/ingestion-base:${{ inputs.tag }}" >> $GITHUB_OUTPUT
- fi
+ image: openmetadata/ingestion-base
+ tag: ${{ inputs.tag }}
+ push_latest: ${{ inputs.push_latest_tag_to_release }}
+ is_ingestion: true
 
  - name: Build and push if event is workflow_dispatch and input is checked
  uses: docker/build-push-action@v3
  with:
  context: .
  platforms: linux/amd64,linux/arm64
- push: ${{ github.event_name == 'release' || github.event_name == 'workflow_dispatch' }}
- tags: ${{ steps.input_check.outputs.tags }}
- file: ./ingestion/operators/docker/Dockerfile
+ push: true
+ tags: ${{ steps.prepare.outputs.tags }}
+ file: ./ingestion/operators/docker/Dockerfile
diff --git a/.github/workflows/docker-openmetadata-ingestion.yml b/.github/workflows/docker-openmetadata-ingestion.yml
@@ -19,63 +19,29 @@ on:
  push_latest_tag_to_release:
  description: "Do you want to update docker image latest tag as well ?"
  type: boolean
- release:
- types: [published]
 
 jobs:
  push_to_docker_hub:
  runs-on: ubuntu-latest
- env:
- input: ${{ github.event.inputs.tag }}
-
+
  steps:
- - name: Check trigger type
- if: ${{ env.input == '' }}
- run: echo "input=1.4.0-SNAPSHOT" >> $GITHUB_ENV
-
  - name: Check out the Repo
  uses: actions/checkout@v3
 
- - name: Set up QEMU
- uses: docker/setup-qemu-action@v2
-
- - name: Set up Docker Buildx
- uses: docker/setup-buildx-action@v2
-
- - name: Login to DockerHub
- uses: docker/login-action@v2
+ - name: Prepare for Docker Build&Push
+ id: prepare
+ uses: ./.github/actions/prepare-for-docker-build-and-push
  with:
- username: ${{ secrets.DOCKERHUB_OPENMETADATA_USERNAME }}
- password: ${{ secrets.DOCKERHUB_OPENMETADATA_TOKEN }}
-
- - name: Install Ubuntu dependencies
- run: |
- sudo apt-get install -y python3-venv
-
- - name: Install open-metadata dependencies
- run: |
- python3 -m venv env
- source env/bin/activate
- pip install --upgrade pip
- sudo make install_antlr_cli
- make install_dev generate
-
- - name: Process Docker Tags
- id: input_check
- run: |
- if ${{ github.event_name == 'release' }}; then
- echo "tags=openmetadata/ingestion:${{ env.input }},openmetadata/ingestion:latest" >> $GITHUB_OUTPUT
- elif ${{ github.event_name == 'workflow_dispatch' && inputs.push_latest_tag_to_release }}; then
- echo "tags=openmetadata/ingestion:${{inputs.tag}},openmetadata/ingestion:latest" >> $GITHUB_OUTPUT
- else
- echo "tags=openmetadata/ingestion:${{ inputs.tag }}" >> $GITHUB_OUTPUT
- fi
+ image: openmetadata/ingestion
+ tag: ${{ inputs.tag }}
+ push_latest: ${{ inputs.push_latest_tag_to_release }}
+ is_ingestion: true
 
  - name: Build and push if event is workflow_dispatch and input is checked
  uses: docker/build-push-action@v3
  with:
  context: .
  platforms: linux/amd64,linux/arm64
- push: ${{ github.event_name == 'release' || github.event_name == 'workflow_dispatch' }}
- tags: ${{ steps.input_check.outputs.tags }}
- file: ./ingestion/Dockerfile
+ push: true
+ tags: ${{ steps.prepare.outputs.tags }}
+ file: ./ingestion/Dockerfile