Merge branch 'main' into issue-16620

.github/actions/prepare-for-docker-build-and-push/action.yml

@@ -71,5 +71,5 @@ runs:
  ${{ inputs.image }}
  sep-tags: ','
  tags: |
- type=raw,value=${{ inputs.release_version }},enable=${{ inputs.is_ingestion }}
+ type=raw,value=${{ inputs.release_version }},enable=${{ inputs.is_ingestion == 'true' }}
  type=raw,${{ inputs.tag }}

.github/teams.yml

@@ -18,13 +18,13 @@
  - '@mohityadav766'
  - '@sureshms'
  - '@akash-jain-10'
- - '@dhruvinmaniar123'
- - '@ShilpaVernekar'
  - '@tutte'
  - '@IceS2'
  - '@sushi30'
  - '@snyk-bot'
  - '@dependabot'
+ - '@harshsoni2024'
+ - '@SumanMaharana'
 
 ingestion:
  - '@ayush-shah'
@@ -34,6 +34,8 @@ ingestion:
  - '@pmbrull'
  - '@IceS2'
  - '@sushi30'
+ - '@harshsoni2024'
+ - '@SumanMaharana'
 
 UI:
  - '@aniketkatkar97'
@@ -51,8 +53,4 @@ backend:
 
 devops:
  - '@akash-jain-10'
- - '@dhruvinmaniar123'
  - '@tutte'
-
-documentation:
- - '@ShilpaVernekar'

.github/workflows/docker-openmetadata-ingestion-base-slim.yml

@@ -14,10 +14,10 @@ on:
  workflow_dispatch:
  inputs:
  docker_release_tag:
- description: "Ingestion Base Slim Docker Image Tag"
+ description: "Ingestion Base Slim Docker Image Tag (3 digit docker image tag)"
  required: true
- release_version:
- description: "Provide the Release Version"
+ pypi_release_version:
+ description: "Provide the Release Version (4 digit docker image tag)"
  required: true
  push_latest_tag_to_release:
  description: "Do you want to update docker image latest tag as well ?"
@@ -39,7 +39,7 @@ jobs:
  tag: ${{ inputs.docker_release_tag }}
  push_latest: ${{ inputs.push_latest_tag_to_release }}
  is_ingestion: true
- release_version: ${{ inputs.release_version }}
+ release_version: ${{ inputs.pypi_release_version }}
  dockerhub_username: ${{ secrets.DOCKERHUB_OPENMETADATA_USERNAME }}
  dockerhub_token: ${{ secrets.DOCKERHUB_OPENMETADATA_TOKEN }}
 

.github/workflows/docker-openmetadata-ingestion-base.yml

@@ -14,10 +14,10 @@ on:
  workflow_dispatch:
  inputs:
  docker_release_tag:
- description: "Ingestion Base Docker Image Tag"
+ description: "Ingestion Base Docker Image Tag (3 digit docker image tag)"
  required: true
- release_version:
- description: "Provide the Release Version"
+ pypi_release_version:
+ description: "Provide the Release Version (4 digit docker image tag)"
  required: true
  push_latest_tag_to_release:
  description: "Do you want to update docker image latest tag as well ?"
@@ -39,7 +39,7 @@ jobs:
  tag: ${{ inputs.docker_release_tag }}
  push_latest: ${{ inputs.push_latest_tag_to_release }}
  is_ingestion: true
- release_version: ${{ inputs.release_version }}
+ release_version: ${{ inputs.pypi_release_version }}
  dockerhub_username: ${{ secrets.DOCKERHUB_OPENMETADATA_USERNAME }}
  dockerhub_token: ${{ secrets.DOCKERHUB_OPENMETADATA_TOKEN }}
 

.github/workflows/docker-openmetadata-ingestion.yml

@@ -14,10 +14,10 @@ on:
  workflow_dispatch:
  inputs:
  docker_release_tag:
- description: "Ingestion Docker Image Tag"
+ description: "Ingestion Docker Image Tag (3 digit docker image tag)"
  required: true
- release_version:
- description: "Provide the Release Version"
+ pypi_release_version:
+ description: "Provide the Release Version (4 digit docker image tag)"
  required: true
  push_latest_tag_to_release:
  description: "Mark this as latest tag as well ?"
@@ -39,7 +39,7 @@ jobs:
  tag: ${{ inputs.docker_release_tag }}
  push_latest: ${{ inputs.push_latest_tag_to_release }}
  is_ingestion: true
- release_version: ${{ inputs.release_version }}
+ release_version: ${{ inputs.pypi_release_version }}
  dockerhub_username: ${{ secrets.DOCKERHUB_OPENMETADATA_USERNAME }}
  dockerhub_token: ${{ secrets.DOCKERHUB_OPENMETADATA_TOKEN }}
 

bootstrap/sql/migrations/native/1.5.0/mysql/postDataMigrationSQLScript.sql

@@ -18,12 +18,66 @@ WHERE name IN (
  'columnValueMedianToBeBetween',
  'columnValueMinToBeBetween',
  'columnValueStdDevToBeBetween',
- 'columnValuesLengthsToBeBetween',
+ 'columnValueLengthsToBeBetween',
  'columnValuesSumToBeBetween',
  'columnValuesToBeBetween',
  'tableRowCountToBeBetween'
 );
 
+
+-- KPI Migrations
+UPDATE entity_relationship
+SET toid = (SELECT id
+ FROM di_chart_entity
+ WHERE NAME = 'percentage_of_data_asset_with_owner_kpi'),
+ toentity = 'dataInsightCustomChart'
+WHERE toid = (SELECT id
+ FROM data_insight_chart dic
+ WHERE NAME = 'PercentageOfEntitiesWithOwnerByType')
+ AND fromId IN (SELECT id from kpi_entity WHERE JSON_EXTRACT(json, '$.metricType') = 'PERCENTAGE')
+ AND toentity = 'dataInsightChart'
+ AND fromentity = 'kpi';
+
+
+UPDATE entity_relationship
+SET toid = (SELECT id
+ FROM di_chart_entity
+ WHERE NAME = 'number_of_data_asset_with_owner_kpi'),
+ toentity = 'dataInsightCustomChart'
+WHERE toid = (SELECT id
+ FROM data_insight_chart dic
+ WHERE NAME = 'PercentageOfEntitiesWithOwnerByType')
+ AND fromId IN (SELECT id from kpi_entity WHERE JSON_EXTRACT(json, '$.metricType') = 'NUMBER')
+ AND toentity = 'dataInsightChart'
+ AND fromentity = 'kpi';
+
+
+UPDATE entity_relationship
+SET toid = (SELECT id
+ FROM di_chart_entity
+ WHERE NAME = 'percentage_of_data_asset_with_description_kpi'),
+ toentity = 'dataInsightCustomChart'
+WHERE toid = (SELECT id
+ FROM data_insight_chart dic
+ WHERE NAME = 'PercentageOfEntitiesWithDescriptionByType')
+ AND fromId IN (SELECT id from kpi_entity WHERE JSON_EXTRACT(json, '$.metricType') = 'PERCENTAGE')
+ AND toentity = 'dataInsightChart'
+ AND fromentity = 'kpi';
+
+
+UPDATE entity_relationship
+SET toid = (SELECT id
+ FROM di_chart_entity
+ WHERE NAME = 'number_of_data_asset_with_description_kpi'),
+ toentity = 'dataInsightCustomChart'
+WHERE toid = (SELECT id
+ FROM data_insight_chart dic
+ WHERE NAME = 'PercentageOfEntitiesWithDescriptionByType')
+ AND fromId IN (SELECT id from kpi_entity WHERE JSON_EXTRACT(json, '$.metricType') = 'NUMBER')
+ AND toentity = 'dataInsightChart'
+ AND fromentity = 'kpi';
+-- KPI MIgrations end
+
 -- Update schedule type for applications
 UPDATE installed_apps
 SET json = JSON_MERGE_PATCH(json, '{"scheduleType": "ScheduledOrManual"}')
@@ -32,3 +86,24 @@ WHERE JSON_UNQUOTE(json->'$.scheduleType') = 'Scheduled';
 -- recreate all scheduled apps
 DELETE FROM apps_marketplace
 WHERE JSON_UNQUOTE(json->'$.scheduleType') = 'Scheduled';
+
+ALTER table thread_entity DROP COLUMN entityId;
+
+-- Add entityRef column to thread_entity table
+UPDATE thread_entity
+SET json = JSON_SET(
+ JSON_REMOVE(
+ JSON_REMOVE(json, '$.entityId'),
+ '$.entityType'
+ ),
+ '$.entityRef',
+ JSON_OBJECT(
+ 'id', JSON_UNQUOTE(JSON_EXTRACT(json, '$.entityId')),
+ 'type', JSON_UNQUOTE(JSON_EXTRACT(json, '$.entityType'))
+ )
+)
+WHERE JSON_CONTAINS_PATH(json, 'one', '$.entityId') OR JSON_CONTAINS_PATH(json, 'one', '$.entityType');
+
+-- Add entityId and type column to thread_entity table
+ALTER table thread_entity ADD COLUMN entityId VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.entityRef.id');
+ALTER table thread_entity ADD COLUMN entityType VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.entityRef.type');

bootstrap/sql/migrations/native/1.5.0/mysql/schemaChanges.sql

@@ -1,3 +1,37 @@
+-- Add a new table di_chart_entity
+CREATE TABLE IF NOT EXISTS di_chart_entity (
+ id VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.id') NOT NULL,
+ name VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.name') NOT NULL,
+ fullyQualifiedName VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.fullyQualifiedName') NOT NULL,
+ json JSON NOT NULL,
+ updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.updatedAt') NOT NULL,
+ updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL,
+ fqnHash varchar(768) CHARACTER SET ascii COLLATE ascii_bin DEFAULT NULL,
+ deleted BOOLEAN GENERATED ALWAYS AS (json -> '$.deleted'),
+ UNIQUE(name),
+ INDEX name_index (name)
+);
+
+-- Update the KPI entity to remove the targetDefinition and set the targetValue to the value of the targetDefinition
+UPDATE kpi_entity
+SET json = JSON_REMOVE(
+ JSON_SET(json, 
+ '$.targetValue', 
+ CAST(JSON_UNQUOTE(JSON_EXTRACT(json, '$.targetDefinition[0].value')) AS DECIMAL) * 100 
+ ),
+ '$.targetDefinition'
+ )
+WHERE JSON_UNQUOTE(JSON_EXTRACT(json, '$.metricType')) = 'PERCENTAGE';
+
+UPDATE kpi_entity
+SET json = JSON_REMOVE(
+ JSON_SET(json, 
+ '$.targetValue', 
+ CAST(JSON_UNQUOTE(JSON_EXTRACT(json, '$.targetDefinition[0].value')) AS DECIMAL) 
+ ),
+ '$.targetDefinition'
+ )
+WHERE JSON_UNQUOTE(JSON_EXTRACT(json, '$.metricType')) = 'NUMBER';
 -- Update DeltaLake service due to connection schema changes to enable DeltaLake ingestion from Storage
 UPDATE dbservice_entity dbse
 SET
@@ -148,3 +182,46 @@ JSON_EXTRACT(json, '$.sourceConfig.config.dbtConfigSource.dbtSecurityConfig.gcpC
 JSON_EXTRACT(json, '$.sourceConfig.config.dbtConfigSource.dbtSecurityConfig.gcpConfig.externalType') OR 
 JSON_EXTRACT(json, '$.sourceConfig.config.dbtConfigSource.dbtSecurityConfig.gcpConfig.path')
 ) is NULL AND JSON_EXTRACT(json, '$.sourceConfig.config.dbtConfigSource.dbtSecurityConfig.gcpConfig') is not null;
+
+-- Update Owner Field to Owners
+DELETE from event_subscription_entity where name = 'ActivityFeedAlert';
+
+-- Update thread_entity to move previousOwner and updatedOwner to array
+UPDATE thread_entity
+SET json = JSON_SET(
+ json,
+ '$.feedInfo.entitySpecificInfo.previousOwner',
+ JSON_ARRAY(
+ JSON_EXTRACT(json, '$.feedInfo.entitySpecificInfo.previousOwner')
+ )
+)
+WHERE JSON_CONTAINS_PATH(json, 'one', '$.feedInfo.entitySpecificInfo.previousOwner')
+AND JSON_TYPE(JSON_EXTRACT(json, '$.feedInfo.entitySpecificInfo.previousOwner')) <> 'ARRAY';
+
+UPDATE thread_entity
+SET json = JSON_SET(
+ json,
+ '$.feedInfo.entitySpecificInfo.updatedOwner',
+ JSON_ARRAY(
+ JSON_EXTRACT(json, '$.feedInfo.entitySpecificInfo.updatedOwner')
+ )
+)
+WHERE JSON_CONTAINS_PATH(json, 'one', '$.feedInfo.entitySpecificInfo.updatedOwner')
+AND JSON_TYPE(JSON_EXTRACT(json, '$.feedInfo.entitySpecificInfo.updatedOwner')) <> 'ARRAY';
+
+-- Update entity_extension to move owner to array
+UPDATE entity_extension
+SET json = JSON_SET(
+ json,
+ '$.owner',
+ JSON_ARRAY(
+ JSON_EXTRACT(json, '$.owner')
+ )
+)
+WHERE JSON_CONTAINS_PATH(json, 'one', '$.owner')
+AND JSON_TYPE(JSON_EXTRACT(json, '$.owner')) <> 'ARRAY';
+
+-- set templates to fetch emailTemplates
+UPDATE openmetadata_settings
+SET json = JSON_SET(json, '$.templates', 'openmetadata')
+WHERE configType = 'emailConfiguration';

bootstrap/sql/migrations/native/1.5.0/postgres/postDataMigrationSQLScript.sql

@@ -17,17 +17,90 @@ WHERE name IN (
  'columnValueMedianToBeBetween',
  'columnValueMinToBeBetween',
  'columnValueStdDevToBeBetween',
- 'columnValuesLengthsToBeBetween',
+ 'columnValueLengthsToBeBetween',
  'columnValuesSumToBeBetween',
  'columnValuesToBeBetween',
  'tableRowCountToBeBetween'
 );
 
+
+-- KPI Migrations
+UPDATE entity_relationship
+SET toid = (SELECT id
+ FROM di_chart_entity
+ WHERE NAME = 'percentage_of_data_asset_with_owner_kpi'),
+ toentity = 'dataInsightCustomChart'
+WHERE toid = (SELECT id
+ FROM data_insight_chart dic
+ WHERE NAME = 'PercentageOfEntitiesWithOwnerByType')
+ AND fromId IN (SELECT id from kpi_entity WHERE json ->> 'metricType' = 'PERCENTAGE')
+ AND toentity = 'dataInsightChart'
+ AND fromentity = 'kpi';
+
+
+UPDATE entity_relationship
+SET toid = (SELECT id
+ FROM di_chart_entity
+ WHERE NAME = 'number_of_data_asset_with_owner_kpi'),
+ toentity = 'dataInsightCustomChart'
+WHERE toid = (SELECT id
+ FROM data_insight_chart dic
+ WHERE NAME = 'PercentageOfEntitiesWithOwnerByType')
+ AND fromId IN (SELECT id from kpi_entity WHERE json ->> 'metricType' = 'NUMBER')
+ AND toentity = 'dataInsightChart'
+ AND fromentity = 'kpi';
+
+
+UPDATE entity_relationship
+SET toid = (SELECT id
+ FROM di_chart_entity
+ WHERE NAME = 'percentage_of_data_asset_with_description_kpi'),
+ toentity = 'dataInsightCustomChart'
+WHERE toid = (SELECT id
+ FROM data_insight_chart dic
+ WHERE NAME = 'PercentageOfEntitiesWithDescriptionByType')
+ AND fromId IN (SELECT id from kpi_entity WHERE json ->> 'metricType' = 'PERCENTAGE')
+ AND toentity = 'dataInsightChart'
+ AND fromentity = 'kpi';
+
+
+UPDATE entity_relationship
+SET toid = (SELECT id
+ FROM di_chart_entity
+ WHERE NAME = 'number_of_data_asset_with_description_kpi'),
+ toentity = 'dataInsightCustomChart'
+WHERE toid = (SELECT id
+ FROM data_insight_chart dic
+ WHERE NAME = 'PercentageOfEntitiesWithDescriptionByType')
+ AND fromId IN (SELECT id from kpi_entity WHERE json ->> 'metricType' = 'NUMBER')
+ AND toentity = 'dataInsightChart'
+ AND fromentity = 'kpi';
+-- KPI MIgrations end
+
 -- Update schedule type for applications
 UPDATE installed_apps
 SET json = json || '{"scheduleType": "ScheduledOrManual"}'
 WHERE json->>'scheduleType' = 'Scheduled';
 
 -- recreate all scheduled apps
 DELETE FROM apps_marketplace
-WHERE json->>'scheduleType' = 'Scheduled';
+WHERE json->>'scheduleType' = 'Scheduled';
+
+ALTER TABLE thread_entity DROP COLUMN entityId;
+
+-- Add entityRef column to thread_entity table
+UPDATE thread_entity
+SET json = jsonb_set(
+ json - 'entityId' - 'entityType',
+ '{entityRef}',
+ jsonb_build_object(
+ 'id', json->>'entityId',
+ 'type', json->>'entityType'
+ ),
+ true
+)
+WHERE jsonb_exists(json, 'entityId') OR jsonb_exists(json, 'entityType');
+
+-- Add entityId and type column to thread_entity table
+ALTER TABLE thread_entity ADD COLUMN entityId VARCHAR(36) GENERATED ALWAYS AS (json->'entityRef'->>'id') STORED;
+ALTER TABLE thread_entity ADD COLUMN entityType VARCHAR(36) GENERATED ALWAYS AS (json->'entityRef'->>'type') STORED;