Fixes 16199: Check the totaVotes values for OpenSearch

.github/workflows/docker-openmetadata-db.yml

@@ -31,7 +31,7 @@ jobs:
  steps:
  - name: Check trigger type
  if: ${{ env.input == '' }}
- run: echo "input=1.4.0-SNAPSHOT" >> $GITHUB_ENV
+ run: echo "input=1.3.4" >> $GITHUB_ENV
 
  - name: Check out the Repo
  uses: actions/checkout@v3

.github/workflows/docker-openmetadata-ingestion-base-slim.yml

@@ -31,7 +31,7 @@ jobs:
  steps:
  - name: Check trigger type
  if: ${{ env.input == '' }}
- run: echo "input=1.4.0-SNAPSHOT" >> $GITHUB_ENV
+ run: echo "input=1.3.4" >> $GITHUB_ENV
 
  - name: Check out the Repo
  uses: actions/checkout@v3

.github/workflows/docker-openmetadata-ingestion-base.yml

@@ -31,7 +31,7 @@ jobs:
  steps:
  - name: Check trigger type
  if: ${{ env.input == '' }}
- run: echo "input=1.4.0-SNAPSHOT" >> $GITHUB_ENV
+ run: echo "input=1.3.4" >> $GITHUB_ENV
 
  - name: Check out the Repo
  uses: actions/checkout@v3

.github/workflows/docker-openmetadata-ingestion.yml

@@ -31,7 +31,7 @@ jobs:
  steps:
  - name: Check trigger type
  if: ${{ env.input == '' }}
- run: echo "input=1.4.0-SNAPSHOT" >> $GITHUB_ENV
+ run: echo "input=1.3.4" >> $GITHUB_ENV
 
  - name: Check out the Repo
  uses: actions/checkout@v3

.github/workflows/docker-openmetadata-postgres.yml

@@ -31,7 +31,7 @@ jobs:
  steps:
  - name: Check trigger type
  if: ${{ env.input == '' }}
- run: echo "input=1.4.0-SNAPSHOT" >> $GITHUB_ENV
+ run: echo "input=1.3.4" >> $GITHUB_ENV
 
  - name: Check out the Repo
  uses: actions/checkout@v3

.github/workflows/docker-openmetadata-server.yml

@@ -64,7 +64,7 @@ jobs:
  steps:
  - name: Check trigger type
  id: check_trigger
- run: echo "DOCKER_RELEASE_TAG=1.4.0-SNAPSHOT" >> $GITHUB_OUTPUT
+ run: echo "DOCKER_RELEASE_TAG=1.3.4" >> $GITHUB_OUTPUT
 
  - name: Download application from Artifiact
  uses: actions/download-artifact@v2
@@ -129,7 +129,7 @@ jobs:
  - name: Check trigger type
  id: check_trigger
  if: ${{ env.DOCKER_RELEASE_TAG == '' }} 
- run: echo "DOCKER_RELEASE_TAG=1.4.0-SNAPSHOT" >> $GITHUB_ENV
+ run: echo "DOCKER_RELEASE_TAG=1.3.4" >> $GITHUB_ENV
 
  - name: Check out the Repo
  uses: actions/checkout@v3

bootstrap/sql/migrations/native/1.3.1/mysql/postDataMigrationSQLScript.sql

@@ -0,0 +1,13 @@
+-- Update the relation between testDefinition and testCase to 0 (CONTAINS)
+UPDATE entity_relationship
+SET relation = 0
+WHERE fromEntity = 'testDefinition' AND toEntity = 'testCase' AND relation != 0;
+
+-- Update the test definition provider
+-- If the test definition has OpenMetadata as a test platform, then the provider is system, else it is user
+UPDATE test_definition
+SET json = CASE
+ WHEN JSON_CONTAINS(json, '"OpenMetadata"', '$.testPlatforms') THEN JSON_INSERT(json,'$.provider','system')
+ ELSE JSON_INSERT(json,'$.provider','user')
+ END
+;

bootstrap/sql/migrations/native/1.3.1/postgres/postDataMigrationSQLScript.sql

@@ -0,0 +1,13 @@
+-- Update the relation between testDefinition and testCase to 0 (CONTAINS)
+UPDATE entity_relationship
+SET relation = 0
+WHERE fromEntity = 'testDefinition' AND toEntity = 'testCase' AND relation != 0;
+
+-- Update the test definition provider
+-- If the test definition has OpenMetadata as a test platform, then the provider is system, else it is user
+UPDATE test_definition
+SET json =
+ case
+ when json->'testPlatforms' @> '"OpenMetadata"' then jsonb_set(json,'{provider}','"system"',true)
+ else jsonb_set(json,'{provider}','"user"', true)
+ end;

bootstrap/sql/migrations/native/1.3.2/mysql/schemaChanges.sql

@@ -0,0 +1,3 @@
+ALTER TABLE test_case ADD COLUMN status VARCHAR(56) GENERATED ALWAYS AS (json ->> '$.testCaseResult.testCaseStatus') STORED NULL;
+ALTER TABLE test_case ADD COLUMN entityLink VARCHAR(512) GENERATED ALWAYS AS (json ->> '$.entityLink') STORED NOT NULL;
+

bootstrap/sql/migrations/native/1.3.2/postgres/schemaChanges.sql

@@ -0,0 +1,2 @@
+ALTER TABLE test_case ADD COLUMN status VARCHAR(56) GENERATED ALWAYS AS (json -> 'testCaseResult' ->> 'testCaseStatus') STORED NULL;
+ALTER TABLE test_case ADD COLUMN entityLink VARCHAR(512) GENERATED ALWAYS AS (json ->> 'entityLink') STORED NOT NULL;

bootstrap/sql/migrations/native/1.3.3/mysql/schemaChanges.sql

@@ -0,0 +1,24 @@
+-- Change scheduleType to scheduleTimeline
+
+UPDATE installed_apps
+SET json = JSON_INSERT(
+ JSON_REMOVE(json, '$.appSchedule.scheduleType'),
+ '$.appSchedule.scheduleTimeline',
+ JSON_EXTRACT(json, '$.appSchedule.scheduleType')
+ );
+delete from apps_extension_time_series;
+
+
+-- Change systemApp to system
+UPDATE installed_apps
+SET json = JSON_INSERT(
+ JSON_REMOVE(json, '$.systemApp'),
+ '$.system',
+ JSON_EXTRACT(json, '$.systemApp')
+ );
+UPDATE apps_marketplace
+SET json = JSON_INSERT(
+ JSON_REMOVE(json, '$.systemApp'),
+ '$.system',
+ JSON_EXTRACT(json, '$.systemApp')
+ );

bootstrap/sql/migrations/native/1.3.3/postgres/schemaChanges.sql

@@ -0,0 +1,32 @@
+-- change scheduleType to scheduleTimeline, this was failing earlier in 1.3.2 so updating it here
+UPDATE installed_apps
+SET json = jsonb_set(
+ json::jsonb,
+ '{appSchedule}',
+ jsonb_set(
+ json->'appSchedule',
+ '{scheduleTimeline}',
+ json->'appSchedule'->'scheduleType'
+ ) - 'scheduleType',
+ true
+ )
+WHERE json->'appSchedule'->>'scheduleType' IS NOT NULL;
+
+delete from apps_extension_time_series;
+
+-- Change systemApp to system, this was failing earlier in 1.3.2 so updating it here
+UPDATE installed_apps
+SET json = jsonb_set(
+ json::jsonb,
+ '{system}',
+ json->'systemApp'
+ ) - 'systemApp'
+WHERE jsonb_exists(json::jsonb, 'systemApp') = true;
+
+UPDATE apps_marketplace
+SET json = jsonb_set(
+ json::jsonb,
+ '{system}',
+ json->'systemApp'
+ ) - 'systemApp'
+WHERE jsonb_exists(json::jsonb, 'systemApp') = true;

common/pom.xml

@@ -18,7 +18,7 @@
  <parent>
  <artifactId>platform</artifactId>
  <groupId>org.open-metadata</groupId>
- <version>1.4.0-SNAPSHOT</version>
+ <version>1.3.4</version>
  </parent>
  <modelVersion>4.0.0</modelVersion>
  <properties>

common/src/main/java/org/openmetadata/common/utils/CommonUtil.java

@@ -17,6 +17,7 @@
 
 import java.io.File;
 import java.io.IOException;
+import java.lang.reflect.Method;
 import java.net.URI;
 import java.net.URISyntaxException;
 import java.nio.charset.StandardCharsets;
@@ -207,4 +208,24 @@ public static URI getUri(String uri) {
  }
  return null;
  }
+
+ public static <T> boolean findChildren(List<?> list, String methodName, String fqn) {
+ if (list == null || list.isEmpty()) return false;
+ try {
+ Method getChildren = list.get(0).getClass().getMethod(methodName);
+ Method getFQN = list.get(0).getClass().getMethod("getFullyQualifiedName");
+ return list.stream()
+ .anyMatch(
+ o -> {
+ try {
+ return getFQN.invoke(o).equals(fqn)
+ || findChildren((List<?>) getChildren.invoke(o), methodName, fqn);
+ } catch (Exception e) {
+ return false;
+ }
+ });
+ } catch (Exception e) {
+ return false;
+ }
+ }
 }

conf/openmetadata.yaml

@@ -163,6 +163,7 @@ authorizerConfiguration:
  enableSecureSocketConnection : ${AUTHORIZER_ENABLE_SECURE_SOCKET:-false}
 
 authenticationConfiguration:
+ clientType: ${AUTHENTICATION_CLIENT_TYPE:-public}
  provider: ${AUTHENTICATION_PROVIDER:-basic}
  # This is used by auth provider provide response as either id_token or code
  responseType: ${AUTHENTICATION_RESPONSE_TYPE:-id_token}
@@ -174,6 +175,22 @@ authenticationConfiguration:
  callbackUrl: ${AUTHENTICATION_CALLBACK_URL:-""}
  jwtPrincipalClaims: ${AUTHENTICATION_JWT_PRINCIPAL_CLAIMS:-[email,preferred_username,sub]}
  enableSelfSignup : ${AUTHENTICATION_ENABLE_SELF_SIGNUP:-true}
+ oidcConfiguration:
+ id: ${OIDC_CLIENT_ID:-""}
+ type: ${OIDC_TYPE:-""} # google, azure etc.
+ secret: ${OIDC_CLIENT_SECRET:-""}
+ scope: ${OIDC_SCOPE:-"openid email profile"}
+ discoveryUri: ${OIDC_DISCOVERY_URI:-""}
+ useNonce: ${OIDC_USE_NONCE:-true}
+ preferredJwsAlgorithm: ${OIDC_PREFERRED_JWS:-"RS256"}
+ responseType: ${OIDC_RESPONSE_TYPE:-"code"}
+ disablePkce: ${OIDC_DISABLE_PKCE:-true}
+ callbackUrl: ${OIDC_CALLBACK:-"http://localhost:8585/callback"}
+ serverUrl: ${OIDC_SERVER_URL:-"http://localhost:8585"}
+ clientAuthenticationMethod: ${OIDC_CLIENT_AUTH_METHOD:-"client_secret_post"}
+ tenant: ${OIDC_TENANT:-""}
+ maxClockSkew: ${OIDC_MAX_CLOCK_SKEW:-""}
+ customParams: ${OIDC_CUSTOM_PARAMS:-}
  samlConfiguration:
  debugMode: ${SAML_DEBUG_MODE:-false}
  idp:
@@ -263,7 +280,7 @@ eventMonitoringConfiguration:
  eventMonitor: ${EVENT_MONITOR:-prometheus} # Possible values are "prometheus", "cloudwatch"
  batchSize: ${EVENT_MONITOR_BATCH_SIZE:-10}
  pathPattern: ${EVENT_MONITOR_PATH_PATTERN:-["/api/v1/tables/*", "/api/v1/health-check"]}
- latency: ${EVENT_MONITOR_LATENCY:-[]} # For value p99=0.99, p90=0.90, p50=0.50 etc.
+ latency: ${EVENT_MONITOR_LATENCY:-[0.99, 0.90]} # For value p99=0.99, p90=0.90, p50=0.50 etc.
  # it will use the default auth provider for AWS services if parameters are not set
  # parameters:
  # region: ${OM_MONITOR_REGION:-""}

docker/development/docker-compose-postgres.yml

@@ -98,6 +98,23 @@ services:
  AUTHENTICATION_CALLBACK_URL: ${AUTHENTICATION_CALLBACK_URL:-""}
  AUTHENTICATION_JWT_PRINCIPAL_CLAIMS: ${AUTHENTICATION_JWT_PRINCIPAL_CLAIMS:-[email,preferred_username,sub]}
  AUTHENTICATION_ENABLE_SELF_SIGNUP: ${AUTHENTICATION_ENABLE_SELF_SIGNUP:-true}
+ AUTHENTICATION_CLIENT_TYPE: ${AUTHENTICATION_CLIENT_TYPE:-public}
+ #For OIDC Authentication, when client is confidential
+ OIDC_CLIENT_ID: ${OIDC_CLIENT_ID:-""}
+ OIDC_TYPE: ${OIDC_TYPE:-""} # google, azure etc.
+ OIDC_CLIENT_SECRET: ${OIDC_CLIENT_SECRET:-""}
+ OIDC_SCOPE: ${OIDC_SCOPE:-"openid email profile"}
+ OIDC_DISCOVERY_URI: ${OIDC_DISCOVERY_URI:-""}
+ OIDC_USE_NONCE: ${OIDC_USE_NONCE:-true}
+ OIDC_PREFERRED_JWS: ${OIDC_PREFERRED_JWS:-"RS256"}
+ OIDC_RESPONSE_TYPE: ${OIDC_RESPONSE_TYPE:-"code"}
+ OIDC_DISABLE_PKCE: ${OIDC_DISABLE_PKCE:-true}
+ OIDC_CALLBACK: ${OIDC_CALLBACK:-"http://localhost:8585/callback"}
+ OIDC_SERVER_URL: ${OIDC_SERVER_URL:-"http://localhost:8585"}
+ OIDC_CLIENT_AUTH_METHOD: ${OIDC_CLIENT_AUTH_METHOD:-"client_secret_post"}
+ OIDC_TENANT: ${OIDC_TENANT:-""}
+ OIDC_MAX_CLOCK_SKEW: ${OIDC_MAX_CLOCK_SKEW:-""}
+ OIDC_CUSTOM_PARAMS: ${OIDC_CUSTOM_PARAMS:-{}}
  # For SAML Authentication
  # SAML_DEBUG_MODE: ${SAML_DEBUG_MODE:-false}
  # SAML_IDP_ENTITY_ID: ${SAML_IDP_ENTITY_ID:-""}
@@ -292,6 +309,23 @@ services:
  AUTHENTICATION_CALLBACK_URL: ${AUTHENTICATION_CALLBACK_URL:-""}
  AUTHENTICATION_JWT_PRINCIPAL_CLAIMS: ${AUTHENTICATION_JWT_PRINCIPAL_CLAIMS:-[email,preferred_username,sub]}
  AUTHENTICATION_ENABLE_SELF_SIGNUP : ${AUTHENTICATION_ENABLE_SELF_SIGNUP:-true}
+ AUTHENTICATION_CLIENT_TYPE: ${AUTHENTICATION_CLIENT_TYPE:-public}
+ #For OIDC Authentication, when client is confidential
+ OIDC_CLIENT_ID: ${OIDC_CLIENT_ID:-""}
+ OIDC_TYPE: ${OIDC_TYPE:-""} # google, azure etc.
+ OIDC_CLIENT_SECRET: ${OIDC_CLIENT_SECRET:-""}
+ OIDC_SCOPE: ${OIDC_SCOPE:-"openid email profile"}
+ OIDC_DISCOVERY_URI: ${OIDC_DISCOVERY_URI:-""}
+ OIDC_USE_NONCE: ${OIDC_USE_NONCE:-true}
+ OIDC_PREFERRED_JWS: ${OIDC_PREFERRED_JWS:-"RS256"}
+ OIDC_RESPONSE_TYPE: ${OIDC_RESPONSE_TYPE:-"code"}
+ OIDC_DISABLE_PKCE: ${OIDC_DISABLE_PKCE:-true}
+ OIDC_CALLBACK: ${OIDC_CALLBACK:-"http://localhost:8585/callback"}
+ OIDC_SERVER_URL: ${OIDC_SERVER_URL:-"http://localhost:8585"}
+ OIDC_CLIENT_AUTH_METHOD: ${OIDC_CLIENT_AUTH_METHOD:-"client_secret_post"}
+ OIDC_TENANT: ${OIDC_TENANT:-""}
+ OIDC_MAX_CLOCK_SKEW: ${OIDC_MAX_CLOCK_SKEW:-""}
+ OIDC_CUSTOM_PARAMS: ${OIDC_CUSTOM_PARAMS:-{}}
  # For SAML Authentication
  # SAML_DEBUG_MODE: ${SAML_DEBUG_MODE:-false}
  # SAML_IDP_ENTITY_ID: ${SAML_IDP_ENTITY_ID:-""}

docker/development/docker-compose.yml

@@ -98,6 +98,23 @@ services:
  AUTHENTICATION_CALLBACK_URL: ${AUTHENTICATION_CALLBACK_URL:-""}
  AUTHENTICATION_JWT_PRINCIPAL_CLAIMS: ${AUTHENTICATION_JWT_PRINCIPAL_CLAIMS:-[email,preferred_username,sub]}
  AUTHENTICATION_ENABLE_SELF_SIGNUP: ${AUTHENTICATION_ENABLE_SELF_SIGNUP:-true}
+ AUTHENTICATION_CLIENT_TYPE: ${AUTHENTICATION_CLIENT_TYPE:-public}
+ #For OIDC Authentication, when client is confidential
+ OIDC_CLIENT_ID: ${OIDC_CLIENT_ID:-""}
+ OIDC_TYPE: ${OIDC_TYPE:-""} # google, azure etc.
+ OIDC_CLIENT_SECRET: ${OIDC_CLIENT_SECRET:-""}
+ OIDC_SCOPE: ${OIDC_SCOPE:-"openid email profile"}
+ OIDC_DISCOVERY_URI: ${OIDC_DISCOVERY_URI:-""}
+ OIDC_USE_NONCE: ${OIDC_USE_NONCE:-true}
+ OIDC_PREFERRED_JWS: ${OIDC_PREFERRED_JWS:-"RS256"}
+ OIDC_RESPONSE_TYPE: ${OIDC_RESPONSE_TYPE:-"code"}
+ OIDC_DISABLE_PKCE: ${OIDC_DISABLE_PKCE:-true}
+ OIDC_CALLBACK: ${OIDC_CALLBACK:-"http://localhost:8585/callback"}
+ OIDC_SERVER_URL: ${OIDC_SERVER_URL:-"http://localhost:8585"}
+ OIDC_CLIENT_AUTH_METHOD: ${OIDC_CLIENT_AUTH_METHOD:-"client_secret_post"}
+ OIDC_TENANT: ${OIDC_TENANT:-""}
+ OIDC_MAX_CLOCK_SKEW: ${OIDC_MAX_CLOCK_SKEW:-""}
+ OIDC_CUSTOM_PARAMS: ${OIDC_CUSTOM_PARAMS:-{}}
  # For SAML Authentication
  # SAML_DEBUG_MODE: ${SAML_DEBUG_MODE:-false}
  # SAML_IDP_ENTITY_ID: ${SAML_IDP_ENTITY_ID:-""}
@@ -289,6 +306,23 @@ services:
  AUTHENTICATION_CALLBACK_URL: ${AUTHENTICATION_CALLBACK_URL:-""}
  AUTHENTICATION_JWT_PRINCIPAL_CLAIMS: ${AUTHENTICATION_JWT_PRINCIPAL_CLAIMS:-[email,preferred_username,sub]}
  AUTHENTICATION_ENABLE_SELF_SIGNUP : ${AUTHENTICATION_ENABLE_SELF_SIGNUP:-true}
+ AUTHENTICATION_CLIENT_TYPE: ${AUTHENTICATION_CLIENT_TYPE:-public}
+ #For OIDC Authentication, when client is confidential
+ OIDC_CLIENT_ID: ${OIDC_CLIENT_ID:-""}
+ OIDC_TYPE: ${OIDC_TYPE:-""} # google, azure etc.
+ OIDC_CLIENT_SECRET: ${OIDC_CLIENT_SECRET:-""}
+ OIDC_SCOPE: ${OIDC_SCOPE:-"openid email profile"}
+ OIDC_DISCOVERY_URI: ${OIDC_DISCOVERY_URI:-""}
+ OIDC_USE_NONCE: ${OIDC_USE_NONCE:-true}
+ OIDC_PREFERRED_JWS: ${OIDC_PREFERRED_JWS:-"RS256"}
+ OIDC_RESPONSE_TYPE: ${OIDC_RESPONSE_TYPE:-"code"}
+ OIDC_DISABLE_PKCE: ${OIDC_DISABLE_PKCE:-true}
+ OIDC_CALLBACK: ${OIDC_CALLBACK:-"http://localhost:8585/callback"}
+ OIDC_SERVER_URL: ${OIDC_SERVER_URL:-"http://localhost:8585"}
+ OIDC_CLIENT_AUTH_METHOD: ${OIDC_CLIENT_AUTH_METHOD:-"client_secret_post"}
+ OIDC_TENANT: ${OIDC_TENANT:-""}
+ OIDC_MAX_CLOCK_SKEW: ${OIDC_MAX_CLOCK_SKEW:-""}
+ OIDC_CUSTOM_PARAMS: ${OIDC_CUSTOM_PARAMS:-{}}
  # For SAML Authentication
  # SAML_DEBUG_MODE: ${SAML_DEBUG_MODE:-false}
  # SAML_IDP_ENTITY_ID: ${SAML_IDP_ENTITY_ID:-""}

docker/docker-compose-ingestion/docker-compose-ingestion.yml

@@ -18,7 +18,7 @@ volumes:
 services: 
  ingestion:
  container_name: openmetadata_ingestion
- image: docker.getcollate.io/openmetadata/ingestion:1.4.0-SNAPSHOT
+ image: docker.getcollate.io/openmetadata/ingestion:1.3.4
  environment:
  AIRFLOW__API__AUTH_BACKENDS: "airflow.api.auth.backend.basic_auth,airflow.api.auth.backend.session"
  AIRFLOW__CORE__EXECUTOR: LocalExecutor