Merge branch 'main' into service_ingestion_pydantic_fix

.github/actions/setup-openmetadata-test-environment/action.yml

@@ -5,6 +5,14 @@ inputs:
  python-version:
  description: Python Version to install
  required: true
+ args:
+ description: Arguments to pass to run_local_docker.sh
+ required: false
+ default: "-m no-ui"
+ ingestion_dependency:
+ description: Ingestion dependency to pass to run_local_docker.sh
+ required: false
+ default: "mysql,elasticsearch,sample-data"
 
 runs:
  using: composite
@@ -50,9 +58,9 @@ runs:
  - name: Start Server and Ingest Sample Data
  uses: nick-fields/retry@v2.8.3
  env:
- INGESTION_DEPENDENCY: "mysql,elasticsearch,sample-data"
+ INGESTION_DEPENDENCY: ${{ inputs.ingestion_dependency }}
  with:
  timeout_minutes: 60
  max_attempts: 2
  retry_on: error
- command: ./docker/run_local_docker.sh -m no-ui
+ command: ./docker/run_local_docker.sh ${{ inputs.args }}

.github/workflows/playwright-integration-tests-mysql.yml

@@ -85,32 +85,12 @@ jobs:
  key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }}
  restore-keys: |
  ${{ runner.os }}-yarn-
- - name: Set up JDK 17
- if: steps.cache-output.outputs.exit-code == 0
- uses: actions/setup-java@v3
- with:
- java-version: '17'
- distribution: 'temurin'
- - name: Set up Python
- uses: actions/setup-python@v4
+ - name: Setup Openmetadata Test Environment
+ uses: ./.github/actions/setup-openmetadata-test-environment
  with:
  python-version: '3.9'
- - name: Install Ubuntu dependencies
- run: |
- sudo apt-get update && sudo apt-get install -y unixodbc-dev python3-venv librdkafka-dev gcc libsasl2-dev build-essential libssl-dev libffi-dev \
- unixodbc-dev libevent-dev python3-dev libkrb5-dev
- - name: Install Python Dependencies and Generate Data Models
- run: |
- python3 -m venv env
- source env/bin/activate
- pip install --upgrade pip
- sudo make install_antlr_cli
- make install_dev generate install_e2e_tests
- - name: Start Server and Ingest Sample Data
- env:
- INGESTION_DEPENDENCY: "all"
- run: ./docker/run_local_docker.sh -d mysql
- timeout-minutes: 30
+ args: "-d mysql"
+ ingestion_dependency: "all"
  - name: Run Playwright Integration Tests with browser ${{ matrix.browser-type }} 
  env:
  E2E_REDSHIFT_HOST_PORT: ${{ secrets.E2E_REDSHIFT_HOST_PORT }}

.github/workflows/playwright-integration-tests-postgres.yml

@@ -85,32 +85,12 @@ jobs:
  key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }}
  restore-keys: |
  ${{ runner.os }}-yarn-
- - name: Set up JDK 17
- if: steps.cache-output.outputs.exit-code == 0
- uses: actions/setup-java@v3
- with:
- java-version: '17'
- distribution: 'temurin'
- - name: Set up Python
- uses: actions/setup-python@v4
+ - name: Setup Openmetadata Test Environment
+ uses: ./.github/actions/setup-openmetadata-test-environment
  with:
  python-version: '3.9'
- - name: Install Ubuntu dependencies
- run: |
- sudo apt-get update && sudo apt-get install -y unixodbc-dev python3-venv librdkafka-dev gcc libsasl2-dev build-essential libssl-dev libffi-dev \
- unixodbc-dev libevent-dev python3-dev libkrb5-dev
- - name: Install Python Dependencies and Generate Data Models
- run: |
- python3 -m venv env
- source env/bin/activate
- pip install --upgrade pip
- sudo make install_antlr_cli
- make install_dev generate install_e2e_tests
- - name: Start Server and Ingest Sample Data
- env:
- INGESTION_DEPENDENCY: "all"
- run: ./docker/run_local_docker.sh -d postgresql
- timeout-minutes: 30
+ args: "-d postgresql"
+ ingestion_dependency: "all"
  - name: Run Playwright Integration Tests with browser ${{ matrix.browser-type }} 
  env:
  E2E_REDSHIFT_HOST_PORT: ${{ secrets.E2E_REDSHIFT_HOST_PORT }}

.github/workflows/playwright-mysql-e2e.yml

@@ -96,24 +96,12 @@ jobs:
  restore-keys: |
  ${{ runner.os }}-yarn-
 
- - name: Set up JDK 17
- if: steps.cache-output.outputs.exit-code == 0
- uses: actions/setup-java@v3
+ - name: Setup Openmetadata Test Environment
+ uses: ./.github/actions/setup-openmetadata-test-environment
  with:
- java-version: "17"
- distribution: "temurin"
-
- - name: Generating Data Models
- run: |
- pip install --upgrade pip
- sudo make install_antlr_cli
- make install_dev generate
-
- - name: Start Server and Ingest Sample Data
- env:
- INGESTION_DEPENDENCY: "all"
- run: ./docker/run_local_docker.sh -d mysql
- timeout-minutes: 60
+ python-version: '3.9'
+ args: "-d mysql"
+ ingestion_dependency: "all"
 
  - name: Install dependencies
  working-directory: openmetadata-ui/src/main/resources/ui/

.github/workflows/playwright-postgresql-e2e.yml

@@ -96,24 +96,12 @@ jobs:
  restore-keys: |
  ${{ runner.os }}-yarn-
 
- - name: Set up JDK 17
- if: steps.cache-output.outputs.exit-code == 0
- uses: actions/setup-java@v3
+ - name: Setup Openmetadata Test Environment
+ uses: ./.github/actions/setup-openmetadata-test-environment
  with:
- java-version: "17"
- distribution: "temurin"
-
- - name: Generating Data Models
- run: |
- pip install --upgrade pip
- sudo make install_antlr_cli
- make install_dev generate
-
- - name: Start Server and Ingest Sample Data
- env:
- INGESTION_DEPENDENCY: "all"
- run: ./docker/run_local_docker.sh -d postgresql
- timeout-minutes: 60
+ python-version: '3.9'
+ args: "-d postgresql"
+ ingestion_dependency: "all"
 
  - name: Install dependencies
  working-directory: openmetadata-ui/src/main/resources/ui/

.github/workflows/py-cli-e2e-tests.yml

@@ -24,11 +24,22 @@ on:
  required: False
  default: "false"
 
+env:
+ DEBUG: ${{ inputs.debug || 'false' }}
+
 permissions:
  id-token: write
  contents: read
 
 jobs:
+ # Needed since env is not available on the job context: https://github.com/actions/runner/issues/2372
+ check-debug:
+ runs-on: ubuntu-latest
+ outputs:
+ DEBUG: ${{ env.DEBUG }}
+ steps:
+ - run: echo "null"
+
  py-cli-e2e-tests:
  runs-on: ubuntu-latest
  strategy:
@@ -152,21 +163,21 @@ jobs:
  coverage report --rcfile ingestion/pyproject.toml --data-file .coverage.$E2E_TEST || true
 
  - name: Upload coverage artifact for Python tests
- if: matrix.e2e-test == 'python' && steps.python-e2e-test.outcome == 'success' && inputs.debug == 'false'
+ if: matrix.e2e-test == 'python' && steps.python-e2e-test.outcome == 'success' && env.DEBUG == 'false'
  uses: actions/upload-artifact@v3
  with:
  name: coverage-${{ matrix.e2e-test }}
  path: .coverage
 
  - name: Upload coverage artifact for CLI E2E tests
- if: matrix.e2e-test != 'python' && steps.e2e-test.outcome == 'success' && inputs.debug == 'false'
+ if: matrix.e2e-test != 'python' && steps.e2e-test.outcome == 'success' && env.DEBUG == 'false'
  uses: actions/upload-artifact@v3
  with:
  name: coverage-${{ matrix.e2e-test }}
  path: .coverage.${{ matrix.e2e-test }}
 
  - name: Upload tests artifact
- if: steps.e2e-test.outcome == 'success' || steps.python-e2e-test.outcome == 'success' && inputs.debug == 'false'
+ if: steps.e2e-test.outcome == 'success' || steps.python-e2e-test.outcome == 'success' && env.DEBUG == 'false'
  uses: actions/upload-artifact@v3
  with:
  name: tests-${{ matrix.e2e-test }}
@@ -179,7 +190,7 @@ jobs:
  sudo rm -rf ${PWD}/docker-volume
 
  - name: Slack on Failure
- if: steps.e2e-test.outcome != 'success' && steps.python-e2e-test.outcome != 'success' && inputs.debug == 'false'
+ if: steps.e2e-test.outcome != 'success' && steps.python-e2e-test.outcome != 'success' && env.DEBUG == 'false'
  uses: slackapi/slack-github-action@v1.23.0
  with:
  payload: |
@@ -197,8 +208,10 @@ jobs:
 
  sonar-cloud-coverage-upload:
  runs-on: ubuntu-latest
- needs: py-cli-e2e-tests
- if: inputs.debug == 'false'
+ needs:
+ - py-cli-e2e-tests
+ - check-debug
+ if: needs.check-debug.outputs.DEBUG == 'false'
  steps:
 
  - name: Checkout

.github/workflows/py-tests.yml

@@ -78,6 +78,8 @@ jobs:
  uses: ./.github/actions/setup-openmetadata-test-environment
  with:
  python-version: ${{ matrix.py-version}}
+ args: "-m no-ui"
+ ingestion_dependency: "mysql,elasticsearch,sample-data"
 
  - name: Run Python Tests
  if: ${{ matrix.py-version != '3.9' }}

ingestion/src/metadata/ingestion/models/patch_request.py

@@ -12,6 +12,8 @@
 Pydantic definition for storing entities for patching
 """
 import json
+import logging
+import traceback
 from typing import Dict, List, Optional, Tuple
 
 import jsonpatch
@@ -21,6 +23,8 @@
 from metadata.ingestion.ometa.mixins.patch_mixin_utils import PatchOperation
 from metadata.ingestion.ometa.utils import model_str
 
+logger = logging.getLogger("metadata")
+
 
 class PatchRequest(BaseModel):
  """
@@ -326,57 +330,63 @@ def build_patch(
  Returns
  Updated Entity
  """
-
- # remove change descriptions from entities
- if remove_change_description:
- source = _remove_change_description(source)
- destination = _remove_change_description(destination)
-
- if array_entity_fields:
- _sort_array_entity_fields(
- source=source,
- destination=destination,
- array_entity_fields=array_entity_fields,
- )
-
- # Get the difference between source and destination
- if allowed_fields:
- patch = jsonpatch.make_patch(
- json.loads(
- source.model_dump_json(
- exclude_unset=True,
- exclude_none=True,
- include=allowed_fields,
- )
- ),
- json.loads(
- destination.model_dump_json(
- exclude_unset=True,
- exclude_none=True,
- include=allowed_fields,
- )
- ),
- )
- else:
- patch: jsonpatch.JsonPatch = jsonpatch.make_patch(
- json.loads(source.model_dump_json(exclude_unset=True, exclude_none=True)),
- json.loads(
- destination.model_dump_json(exclude_unset=True, exclude_none=True)
- ),
- )
- if not patch:
+ try:
+ # remove change descriptions from entities
+ if remove_change_description:
+ source = _remove_change_description(source)
+ destination = _remove_change_description(destination)
+
+ if array_entity_fields:
+ _sort_array_entity_fields(
+ source=source,
+ destination=destination,
+ array_entity_fields=array_entity_fields,
+ )
+
+ # Get the difference between source and destination
+ if allowed_fields:
+ patch = jsonpatch.make_patch(
+ json.loads(
+ source.model_dump_json(
+ exclude_unset=True,
+ exclude_none=True,
+ include=allowed_fields,
+ )
+ ),
+ json.loads(
+ destination.model_dump_json(
+ exclude_unset=True,
+ exclude_none=True,
+ include=allowed_fields,
+ )
+ ),
+ )
+ else:
+ patch: jsonpatch.JsonPatch = jsonpatch.make_patch(
+ json.loads(
+ source.model_dump_json(exclude_unset=True, exclude_none=True)
+ ),
+ json.loads(
+ destination.model_dump_json(exclude_unset=True, exclude_none=True)
+ ),
+ )
+ if not patch:
+ return None
+
+ # For a user editable fields like descriptions, tags we only want to support "add" operation in patch
+ # we will remove the other operations.
+ if restrict_update_fields:
+ updated_operations = JsonPatchUpdater.from_restrict_update_fields(
+ restrict_update_fields
+ ).update(patch)
+ patch.patch = updated_operations
+
+ return patch
+ except Exception:
+ logger.debug(traceback.format_exc())
+ logger.warning("Couldn't build patch for Entity.")
  return None
 
- # For a user editable fields like descriptions, tags we only want to support "add" operation in patch
- # we will remove the other operations.
- if restrict_update_fields:
- updated_operations = JsonPatchUpdater.from_restrict_update_fields(
- restrict_update_fields
- ).update(patch)
- patch.patch = updated_operations
-
- return patch
-
 
 def _sort_array_entity_fields(
  source: T,