diff --git a/.github/workflows/docker-openmetadata-db.yml b/.github/workflows/docker-openmetadata-db.yml index a2277cb0bf33..b03d47627e94 100644 --- a/.github/workflows/docker-openmetadata-db.yml +++ b/.github/workflows/docker-openmetadata-db.yml @@ -31,7 +31,7 @@ jobs: steps: - name: Check trigger type if: ${{ env.input == '' }} - run: echo "input=1.4.0-SNAPSHOT" >> $GITHUB_ENV + run: echo "input=1.3.4" >> $GITHUB_ENV - name: Check out the Repo uses: actions/checkout@v3 diff --git a/.github/workflows/docker-openmetadata-ingestion-base-slim.yml b/.github/workflows/docker-openmetadata-ingestion-base-slim.yml index 0cf5a8cb3096..6aa1dffb00ad 100644 --- a/.github/workflows/docker-openmetadata-ingestion-base-slim.yml +++ b/.github/workflows/docker-openmetadata-ingestion-base-slim.yml @@ -31,7 +31,7 @@ jobs: steps: - name: Check trigger type if: ${{ env.input == '' }} - run: echo "input=1.4.0-SNAPSHOT" >> $GITHUB_ENV + run: echo "input=1.3.4" >> $GITHUB_ENV - name: Check out the Repo uses: actions/checkout@v3 diff --git a/.github/workflows/docker-openmetadata-ingestion-base.yml b/.github/workflows/docker-openmetadata-ingestion-base.yml index cd34badb2482..163d7bd94d43 100644 --- a/.github/workflows/docker-openmetadata-ingestion-base.yml +++ b/.github/workflows/docker-openmetadata-ingestion-base.yml @@ -31,7 +31,7 @@ jobs: steps: - name: Check trigger type if: ${{ env.input == '' }} - run: echo "input=1.4.0-SNAPSHOT" >> $GITHUB_ENV + run: echo "input=1.3.4" >> $GITHUB_ENV - name: Check out the Repo uses: actions/checkout@v3 diff --git a/.github/workflows/docker-openmetadata-ingestion.yml b/.github/workflows/docker-openmetadata-ingestion.yml index d2c97c330d2c..11c86b611796 100644 --- a/.github/workflows/docker-openmetadata-ingestion.yml +++ b/.github/workflows/docker-openmetadata-ingestion.yml @@ -31,7 +31,7 @@ jobs: steps: - name: Check trigger type if: ${{ env.input == '' }} - run: echo "input=1.4.0-SNAPSHOT" >> $GITHUB_ENV + run: echo "input=1.3.4" >> $GITHUB_ENV - name: Check out the Repo uses: actions/checkout@v3 diff --git a/.github/workflows/docker-openmetadata-postgres.yml b/.github/workflows/docker-openmetadata-postgres.yml index 19ccc3f6077d..b8c5672860f9 100644 --- a/.github/workflows/docker-openmetadata-postgres.yml +++ b/.github/workflows/docker-openmetadata-postgres.yml @@ -31,7 +31,7 @@ jobs: steps: - name: Check trigger type if: ${{ env.input == '' }} - run: echo "input=1.4.0-SNAPSHOT" >> $GITHUB_ENV + run: echo "input=1.3.4" >> $GITHUB_ENV - name: Check out the Repo uses: actions/checkout@v3 diff --git a/.github/workflows/docker-openmetadata-server.yml b/.github/workflows/docker-openmetadata-server.yml index 32d36cabfc43..8d7008514748 100644 --- a/.github/workflows/docker-openmetadata-server.yml +++ b/.github/workflows/docker-openmetadata-server.yml @@ -64,7 +64,7 @@ jobs: steps: - name: Check trigger type id: check_trigger - run: echo "DOCKER_RELEASE_TAG=1.4.0-SNAPSHOT" >> $GITHUB_OUTPUT + run: echo "DOCKER_RELEASE_TAG=1.3.4" >> $GITHUB_OUTPUT - name: Download application from Artifiact uses: actions/download-artifact@v2 @@ -129,7 +129,7 @@ jobs: - name: Check trigger type id: check_trigger if: ${{ env.DOCKER_RELEASE_TAG == '' }} - run: echo "DOCKER_RELEASE_TAG=1.4.0-SNAPSHOT" >> $GITHUB_ENV + run: echo "DOCKER_RELEASE_TAG=1.3.4" >> $GITHUB_ENV - name: Check out the Repo uses: actions/checkout@v3 diff --git a/bootstrap/sql/migrations/native/1.3.1/mysql/postDataMigrationSQLScript.sql b/bootstrap/sql/migrations/native/1.3.1/mysql/postDataMigrationSQLScript.sql index e69de29bb2d1..a3693db8f6a8 100644 --- a/bootstrap/sql/migrations/native/1.3.1/mysql/postDataMigrationSQLScript.sql +++ b/bootstrap/sql/migrations/native/1.3.1/mysql/postDataMigrationSQLScript.sql @@ -0,0 +1,13 @@ +-- Update the relation between testDefinition and testCase to 0 (CONTAINS) +UPDATE entity_relationship +SET relation = 0 +WHERE fromEntity = 'testDefinition' AND toEntity = 'testCase' AND relation != 0; + +-- Update the test definition provider +-- If the test definition has OpenMetadata as a test platform, then the provider is system, else it is user +UPDATE test_definition +SET json = CASE + WHEN JSON_CONTAINS(json, '"OpenMetadata"', '$.testPlatforms') THEN JSON_INSERT(json,'$.provider','system') + ELSE JSON_INSERT(json,'$.provider','user') + END +; diff --git a/bootstrap/sql/migrations/native/1.3.1/postgres/postDataMigrationSQLScript.sql b/bootstrap/sql/migrations/native/1.3.1/postgres/postDataMigrationSQLScript.sql index e69de29bb2d1..dca6c0b7e3e3 100644 --- a/bootstrap/sql/migrations/native/1.3.1/postgres/postDataMigrationSQLScript.sql +++ b/bootstrap/sql/migrations/native/1.3.1/postgres/postDataMigrationSQLScript.sql @@ -0,0 +1,13 @@ +-- Update the relation between testDefinition and testCase to 0 (CONTAINS) +UPDATE entity_relationship +SET relation = 0 +WHERE fromEntity = 'testDefinition' AND toEntity = 'testCase' AND relation != 0; + +-- Update the test definition provider +-- If the test definition has OpenMetadata as a test platform, then the provider is system, else it is user +UPDATE test_definition +SET json = + case + when json->'testPlatforms' @> '"OpenMetadata"' then jsonb_set(json,'{provider}','"system"',true) + else jsonb_set(json,'{provider}','"user"', true) + end; diff --git a/bootstrap/sql/migrations/native/1.3.2/mysql/schemaChanges.sql b/bootstrap/sql/migrations/native/1.3.2/mysql/schemaChanges.sql new file mode 100644 index 000000000000..d8d880adcca7 --- /dev/null +++ b/bootstrap/sql/migrations/native/1.3.2/mysql/schemaChanges.sql @@ -0,0 +1,3 @@ +ALTER TABLE test_case ADD COLUMN status VARCHAR(56) GENERATED ALWAYS AS (json ->> '$.testCaseResult.testCaseStatus') STORED NULL; +ALTER TABLE test_case ADD COLUMN entityLink VARCHAR(512) GENERATED ALWAYS AS (json ->> '$.entityLink') STORED NOT NULL; + diff --git a/bootstrap/sql/migrations/native/1.3.2/postgres/schemaChanges.sql b/bootstrap/sql/migrations/native/1.3.2/postgres/schemaChanges.sql new file mode 100644 index 000000000000..957d395251f9 --- /dev/null +++ b/bootstrap/sql/migrations/native/1.3.2/postgres/schemaChanges.sql @@ -0,0 +1,2 @@ +ALTER TABLE test_case ADD COLUMN status VARCHAR(56) GENERATED ALWAYS AS (json -> 'testCaseResult' ->> 'testCaseStatus') STORED NULL; +ALTER TABLE test_case ADD COLUMN entityLink VARCHAR(512) GENERATED ALWAYS AS (json ->> 'entityLink') STORED NOT NULL; diff --git a/bootstrap/sql/migrations/native/1.3.3/mysql/schemaChanges.sql b/bootstrap/sql/migrations/native/1.3.3/mysql/schemaChanges.sql new file mode 100644 index 000000000000..fc06c0e58c4b --- /dev/null +++ b/bootstrap/sql/migrations/native/1.3.3/mysql/schemaChanges.sql @@ -0,0 +1,24 @@ +-- Change scheduleType to scheduleTimeline + +UPDATE installed_apps +SET json = JSON_INSERT( + JSON_REMOVE(json, '$.appSchedule.scheduleType'), + '$.appSchedule.scheduleTimeline', + JSON_EXTRACT(json, '$.appSchedule.scheduleType') + ); +delete from apps_extension_time_series; + + +-- Change systemApp to system +UPDATE installed_apps +SET json = JSON_INSERT( + JSON_REMOVE(json, '$.systemApp'), + '$.system', + JSON_EXTRACT(json, '$.systemApp') + ); +UPDATE apps_marketplace +SET json = JSON_INSERT( + JSON_REMOVE(json, '$.systemApp'), + '$.system', + JSON_EXTRACT(json, '$.systemApp') + ); \ No newline at end of file diff --git a/bootstrap/sql/migrations/native/1.3.3/postgres/schemaChanges.sql b/bootstrap/sql/migrations/native/1.3.3/postgres/schemaChanges.sql new file mode 100644 index 000000000000..4173eb4a9687 --- /dev/null +++ b/bootstrap/sql/migrations/native/1.3.3/postgres/schemaChanges.sql @@ -0,0 +1,32 @@ +-- change scheduleType to scheduleTimeline, this was failing earlier in 1.3.2 so updating it here +UPDATE installed_apps +SET json = jsonb_set( + json::jsonb, + '{appSchedule}', + jsonb_set( + json->'appSchedule', + '{scheduleTimeline}', + json->'appSchedule'->'scheduleType' + ) - 'scheduleType', + true + ) +WHERE json->'appSchedule'->>'scheduleType' IS NOT NULL; + +delete from apps_extension_time_series; + +-- Change systemApp to system, this was failing earlier in 1.3.2 so updating it here +UPDATE installed_apps +SET json = jsonb_set( + json::jsonb, + '{system}', + json->'systemApp' + ) - 'systemApp' +WHERE jsonb_exists(json::jsonb, 'systemApp') = true; + +UPDATE apps_marketplace +SET json = jsonb_set( + json::jsonb, + '{system}', + json->'systemApp' + ) - 'systemApp' +WHERE jsonb_exists(json::jsonb, 'systemApp') = true; diff --git a/bootstrap/sql/migrations/native/1.4.0/mysql/postDataMigrationSQLScript.sql b/bootstrap/sql/migrations/native/1.4.0/mysql/postDataMigrationSQLScript.sql deleted file mode 100644 index a3693db8f6a8..000000000000 --- a/bootstrap/sql/migrations/native/1.4.0/mysql/postDataMigrationSQLScript.sql +++ /dev/null @@ -1,13 +0,0 @@ --- Update the relation between testDefinition and testCase to 0 (CONTAINS) -UPDATE entity_relationship -SET relation = 0 -WHERE fromEntity = 'testDefinition' AND toEntity = 'testCase' AND relation != 0; - --- Update the test definition provider --- If the test definition has OpenMetadata as a test platform, then the provider is system, else it is user -UPDATE test_definition -SET json = CASE - WHEN JSON_CONTAINS(json, '"OpenMetadata"', '$.testPlatforms') THEN JSON_INSERT(json,'$.provider','system') - ELSE JSON_INSERT(json,'$.provider','user') - END -; diff --git a/bootstrap/sql/migrations/native/1.4.0/mysql/schemaChanges.sql b/bootstrap/sql/migrations/native/1.4.0/mysql/schemaChanges.sql deleted file mode 100644 index 06035180588e..000000000000 --- a/bootstrap/sql/migrations/native/1.4.0/mysql/schemaChanges.sql +++ /dev/null @@ -1,4 +0,0 @@ --- Add the supportsProfiler field to the MongoDB connection configuration -UPDATE dbservice_entity -SET json = JSON_INSERT(json, '$.connection.config.supportsProfiler', TRUE) -WHERE serviceType = 'MongoDB'; \ No newline at end of file diff --git a/bootstrap/sql/migrations/native/1.4.0/postgres/postDataMigrationSQLScript.sql b/bootstrap/sql/migrations/native/1.4.0/postgres/postDataMigrationSQLScript.sql deleted file mode 100644 index dca6c0b7e3e3..000000000000 --- a/bootstrap/sql/migrations/native/1.4.0/postgres/postDataMigrationSQLScript.sql +++ /dev/null @@ -1,13 +0,0 @@ --- Update the relation between testDefinition and testCase to 0 (CONTAINS) -UPDATE entity_relationship -SET relation = 0 -WHERE fromEntity = 'testDefinition' AND toEntity = 'testCase' AND relation != 0; - --- Update the test definition provider --- If the test definition has OpenMetadata as a test platform, then the provider is system, else it is user -UPDATE test_definition -SET json = - case - when json->'testPlatforms' @> '"OpenMetadata"' then jsonb_set(json,'{provider}','"system"',true) - else jsonb_set(json,'{provider}','"user"', true) - end; diff --git a/bootstrap/sql/migrations/native/1.4.0/postgres/schemaChanges.sql b/bootstrap/sql/migrations/native/1.4.0/postgres/schemaChanges.sql deleted file mode 100644 index c8b6830c129e..000000000000 --- a/bootstrap/sql/migrations/native/1.4.0/postgres/schemaChanges.sql +++ /dev/null @@ -1,4 +0,0 @@ --- Add the supportsProfiler field to the MongoDB connection configuration -UPDATE dbservice_entity -SET json = jsonb_set(json::jsonb, '{connection,config,supportsProfiler}', 'true'::jsonb) -WHERE serviceType = 'MongoDB'; \ No newline at end of file diff --git a/common/pom.xml b/common/pom.xml index 42f9bd40227f..ef17715284fc 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -18,7 +18,7 @@ platform org.open-metadata - 1.4.0-SNAPSHOT + 1.3.4 4.0.0 diff --git a/common/src/main/java/org/openmetadata/common/utils/CommonUtil.java b/common/src/main/java/org/openmetadata/common/utils/CommonUtil.java index 2d2e0fa4fc91..c4e5c691a9ae 100644 --- a/common/src/main/java/org/openmetadata/common/utils/CommonUtil.java +++ b/common/src/main/java/org/openmetadata/common/utils/CommonUtil.java @@ -17,6 +17,7 @@ import java.io.File; import java.io.IOException; +import java.lang.reflect.Method; import java.net.URI; import java.net.URISyntaxException; import java.nio.charset.StandardCharsets; @@ -207,4 +208,24 @@ public static URI getUri(String uri) { } return null; } + + public static boolean findChildren(List list, String methodName, String fqn) { + if (list == null || list.isEmpty()) return false; + try { + Method getChildren = list.get(0).getClass().getMethod(methodName); + Method getFQN = list.get(0).getClass().getMethod("getFullyQualifiedName"); + return list.stream() + .anyMatch( + o -> { + try { + return getFQN.invoke(o).equals(fqn) + || findChildren((List) getChildren.invoke(o), methodName, fqn); + } catch (Exception e) { + return false; + } + }); + } catch (Exception e) { + return false; + } + } } diff --git a/conf/openmetadata.yaml b/conf/openmetadata.yaml index 28fa143d0fb3..4c54958679f9 100644 --- a/conf/openmetadata.yaml +++ b/conf/openmetadata.yaml @@ -163,6 +163,7 @@ authorizerConfiguration: enableSecureSocketConnection : ${AUTHORIZER_ENABLE_SECURE_SOCKET:-false} authenticationConfiguration: + clientType: ${AUTHENTICATION_CLIENT_TYPE:-public} provider: ${AUTHENTICATION_PROVIDER:-basic} # This is used by auth provider provide response as either id_token or code responseType: ${AUTHENTICATION_RESPONSE_TYPE:-id_token} @@ -174,6 +175,22 @@ authenticationConfiguration: callbackUrl: ${AUTHENTICATION_CALLBACK_URL:-""} jwtPrincipalClaims: ${AUTHENTICATION_JWT_PRINCIPAL_CLAIMS:-[email,preferred_username,sub]} enableSelfSignup : ${AUTHENTICATION_ENABLE_SELF_SIGNUP:-true} + oidcConfiguration: + id: ${OIDC_CLIENT_ID:-""} + type: ${OIDC_TYPE:-""} # google, azure etc. + secret: ${OIDC_CLIENT_SECRET:-""} + scope: ${OIDC_SCOPE:-"openid email profile"} + discoveryUri: ${OIDC_DISCOVERY_URI:-""} + useNonce: ${OIDC_USE_NONCE:-true} + preferredJwsAlgorithm: ${OIDC_PREFERRED_JWS:-"RS256"} + responseType: ${OIDC_RESPONSE_TYPE:-"code"} + disablePkce: ${OIDC_DISABLE_PKCE:-true} + callbackUrl: ${OIDC_CALLBACK:-"http://localhost:8585/callback"} + serverUrl: ${OIDC_SERVER_URL:-"http://localhost:8585"} + clientAuthenticationMethod: ${OIDC_CLIENT_AUTH_METHOD:-"client_secret_post"} + tenant: ${OIDC_TENANT:-""} + maxClockSkew: ${OIDC_MAX_CLOCK_SKEW:-""} + customParams: ${OIDC_CUSTOM_PARAMS:-} samlConfiguration: debugMode: ${SAML_DEBUG_MODE:-false} idp: @@ -263,7 +280,7 @@ eventMonitoringConfiguration: eventMonitor: ${EVENT_MONITOR:-prometheus} # Possible values are "prometheus", "cloudwatch" batchSize: ${EVENT_MONITOR_BATCH_SIZE:-10} pathPattern: ${EVENT_MONITOR_PATH_PATTERN:-["/api/v1/tables/*", "/api/v1/health-check"]} - latency: ${EVENT_MONITOR_LATENCY:-[]} # For value p99=0.99, p90=0.90, p50=0.50 etc. + latency: ${EVENT_MONITOR_LATENCY:-[0.99, 0.90]} # For value p99=0.99, p90=0.90, p50=0.50 etc. # it will use the default auth provider for AWS services if parameters are not set # parameters: # region: ${OM_MONITOR_REGION:-""} diff --git a/ingestion/src/metadata/__init__.py b/docker/__init__.py similarity index 100% rename from ingestion/src/metadata/__init__.py rename to docker/__init__.py diff --git a/docker/development/docker-compose-postgres.yml b/docker/development/docker-compose-postgres.yml index b742b6104a72..50f0dd0a8640 100644 --- a/docker/development/docker-compose-postgres.yml +++ b/docker/development/docker-compose-postgres.yml @@ -98,6 +98,23 @@ services: AUTHENTICATION_CALLBACK_URL: ${AUTHENTICATION_CALLBACK_URL:-""} AUTHENTICATION_JWT_PRINCIPAL_CLAIMS: ${AUTHENTICATION_JWT_PRINCIPAL_CLAIMS:-[email,preferred_username,sub]} AUTHENTICATION_ENABLE_SELF_SIGNUP: ${AUTHENTICATION_ENABLE_SELF_SIGNUP:-true} + AUTHENTICATION_CLIENT_TYPE: ${AUTHENTICATION_CLIENT_TYPE:-public} + #For OIDC Authentication, when client is confidential + OIDC_CLIENT_ID: ${OIDC_CLIENT_ID:-""} + OIDC_TYPE: ${OIDC_TYPE:-""} # google, azure etc. + OIDC_CLIENT_SECRET: ${OIDC_CLIENT_SECRET:-""} + OIDC_SCOPE: ${OIDC_SCOPE:-"openid email profile"} + OIDC_DISCOVERY_URI: ${OIDC_DISCOVERY_URI:-""} + OIDC_USE_NONCE: ${OIDC_USE_NONCE:-true} + OIDC_PREFERRED_JWS: ${OIDC_PREFERRED_JWS:-"RS256"} + OIDC_RESPONSE_TYPE: ${OIDC_RESPONSE_TYPE:-"code"} + OIDC_DISABLE_PKCE: ${OIDC_DISABLE_PKCE:-true} + OIDC_CALLBACK: ${OIDC_CALLBACK:-"http://localhost:8585/callback"} + OIDC_SERVER_URL: ${OIDC_SERVER_URL:-"http://localhost:8585"} + OIDC_CLIENT_AUTH_METHOD: ${OIDC_CLIENT_AUTH_METHOD:-"client_secret_post"} + OIDC_TENANT: ${OIDC_TENANT:-""} + OIDC_MAX_CLOCK_SKEW: ${OIDC_MAX_CLOCK_SKEW:-""} + OIDC_CUSTOM_PARAMS: ${OIDC_CUSTOM_PARAMS:-{}} # For SAML Authentication # SAML_DEBUG_MODE: ${SAML_DEBUG_MODE:-false} # SAML_IDP_ENTITY_ID: ${SAML_IDP_ENTITY_ID:-""} @@ -292,6 +309,23 @@ services: AUTHENTICATION_CALLBACK_URL: ${AUTHENTICATION_CALLBACK_URL:-""} AUTHENTICATION_JWT_PRINCIPAL_CLAIMS: ${AUTHENTICATION_JWT_PRINCIPAL_CLAIMS:-[email,preferred_username,sub]} AUTHENTICATION_ENABLE_SELF_SIGNUP : ${AUTHENTICATION_ENABLE_SELF_SIGNUP:-true} + AUTHENTICATION_CLIENT_TYPE: ${AUTHENTICATION_CLIENT_TYPE:-public} + #For OIDC Authentication, when client is confidential + OIDC_CLIENT_ID: ${OIDC_CLIENT_ID:-""} + OIDC_TYPE: ${OIDC_TYPE:-""} # google, azure etc. + OIDC_CLIENT_SECRET: ${OIDC_CLIENT_SECRET:-""} + OIDC_SCOPE: ${OIDC_SCOPE:-"openid email profile"} + OIDC_DISCOVERY_URI: ${OIDC_DISCOVERY_URI:-""} + OIDC_USE_NONCE: ${OIDC_USE_NONCE:-true} + OIDC_PREFERRED_JWS: ${OIDC_PREFERRED_JWS:-"RS256"} + OIDC_RESPONSE_TYPE: ${OIDC_RESPONSE_TYPE:-"code"} + OIDC_DISABLE_PKCE: ${OIDC_DISABLE_PKCE:-true} + OIDC_CALLBACK: ${OIDC_CALLBACK:-"http://localhost:8585/callback"} + OIDC_SERVER_URL: ${OIDC_SERVER_URL:-"http://localhost:8585"} + OIDC_CLIENT_AUTH_METHOD: ${OIDC_CLIENT_AUTH_METHOD:-"client_secret_post"} + OIDC_TENANT: ${OIDC_TENANT:-""} + OIDC_MAX_CLOCK_SKEW: ${OIDC_MAX_CLOCK_SKEW:-""} + OIDC_CUSTOM_PARAMS: ${OIDC_CUSTOM_PARAMS:-{}} # For SAML Authentication # SAML_DEBUG_MODE: ${SAML_DEBUG_MODE:-false} # SAML_IDP_ENTITY_ID: ${SAML_IDP_ENTITY_ID:-""} diff --git a/docker/development/docker-compose.yml b/docker/development/docker-compose.yml index 09baf7809de8..2a2e91dc149e 100644 --- a/docker/development/docker-compose.yml +++ b/docker/development/docker-compose.yml @@ -98,6 +98,23 @@ services: AUTHENTICATION_CALLBACK_URL: ${AUTHENTICATION_CALLBACK_URL:-""} AUTHENTICATION_JWT_PRINCIPAL_CLAIMS: ${AUTHENTICATION_JWT_PRINCIPAL_CLAIMS:-[email,preferred_username,sub]} AUTHENTICATION_ENABLE_SELF_SIGNUP: ${AUTHENTICATION_ENABLE_SELF_SIGNUP:-true} + AUTHENTICATION_CLIENT_TYPE: ${AUTHENTICATION_CLIENT_TYPE:-public} + #For OIDC Authentication, when client is confidential + OIDC_CLIENT_ID: ${OIDC_CLIENT_ID:-""} + OIDC_TYPE: ${OIDC_TYPE:-""} # google, azure etc. + OIDC_CLIENT_SECRET: ${OIDC_CLIENT_SECRET:-""} + OIDC_SCOPE: ${OIDC_SCOPE:-"openid email profile"} + OIDC_DISCOVERY_URI: ${OIDC_DISCOVERY_URI:-""} + OIDC_USE_NONCE: ${OIDC_USE_NONCE:-true} + OIDC_PREFERRED_JWS: ${OIDC_PREFERRED_JWS:-"RS256"} + OIDC_RESPONSE_TYPE: ${OIDC_RESPONSE_TYPE:-"code"} + OIDC_DISABLE_PKCE: ${OIDC_DISABLE_PKCE:-true} + OIDC_CALLBACK: ${OIDC_CALLBACK:-"http://localhost:8585/callback"} + OIDC_SERVER_URL: ${OIDC_SERVER_URL:-"http://localhost:8585"} + OIDC_CLIENT_AUTH_METHOD: ${OIDC_CLIENT_AUTH_METHOD:-"client_secret_post"} + OIDC_TENANT: ${OIDC_TENANT:-""} + OIDC_MAX_CLOCK_SKEW: ${OIDC_MAX_CLOCK_SKEW:-""} + OIDC_CUSTOM_PARAMS: ${OIDC_CUSTOM_PARAMS:-{}} # For SAML Authentication # SAML_DEBUG_MODE: ${SAML_DEBUG_MODE:-false} # SAML_IDP_ENTITY_ID: ${SAML_IDP_ENTITY_ID:-""} @@ -289,6 +306,23 @@ services: AUTHENTICATION_CALLBACK_URL: ${AUTHENTICATION_CALLBACK_URL:-""} AUTHENTICATION_JWT_PRINCIPAL_CLAIMS: ${AUTHENTICATION_JWT_PRINCIPAL_CLAIMS:-[email,preferred_username,sub]} AUTHENTICATION_ENABLE_SELF_SIGNUP : ${AUTHENTICATION_ENABLE_SELF_SIGNUP:-true} + AUTHENTICATION_CLIENT_TYPE: ${AUTHENTICATION_CLIENT_TYPE:-public} + #For OIDC Authentication, when client is confidential + OIDC_CLIENT_ID: ${OIDC_CLIENT_ID:-""} + OIDC_TYPE: ${OIDC_TYPE:-""} # google, azure etc. + OIDC_CLIENT_SECRET: ${OIDC_CLIENT_SECRET:-""} + OIDC_SCOPE: ${OIDC_SCOPE:-"openid email profile"} + OIDC_DISCOVERY_URI: ${OIDC_DISCOVERY_URI:-""} + OIDC_USE_NONCE: ${OIDC_USE_NONCE:-true} + OIDC_PREFERRED_JWS: ${OIDC_PREFERRED_JWS:-"RS256"} + OIDC_RESPONSE_TYPE: ${OIDC_RESPONSE_TYPE:-"code"} + OIDC_DISABLE_PKCE: ${OIDC_DISABLE_PKCE:-true} + OIDC_CALLBACK: ${OIDC_CALLBACK:-"http://localhost:8585/callback"} + OIDC_SERVER_URL: ${OIDC_SERVER_URL:-"http://localhost:8585"} + OIDC_CLIENT_AUTH_METHOD: ${OIDC_CLIENT_AUTH_METHOD:-"client_secret_post"} + OIDC_TENANT: ${OIDC_TENANT:-""} + OIDC_MAX_CLOCK_SKEW: ${OIDC_MAX_CLOCK_SKEW:-""} + OIDC_CUSTOM_PARAMS: ${OIDC_CUSTOM_PARAMS:-{}} # For SAML Authentication # SAML_DEBUG_MODE: ${SAML_DEBUG_MODE:-false} # SAML_IDP_ENTITY_ID: ${SAML_IDP_ENTITY_ID:-""} diff --git a/docker/docker-compose-ingestion/docker-compose-ingestion.yml b/docker/docker-compose-ingestion/docker-compose-ingestion.yml index 8537698e7d51..a665b001656c 100644 --- a/docker/docker-compose-ingestion/docker-compose-ingestion.yml +++ b/docker/docker-compose-ingestion/docker-compose-ingestion.yml @@ -18,7 +18,7 @@ volumes: services: ingestion: container_name: openmetadata_ingestion - image: docker.getcollate.io/openmetadata/ingestion:1.4.0-SNAPSHOT + image: docker.getcollate.io/openmetadata/ingestion:1.3.4 environment: AIRFLOW__API__AUTH_BACKENDS: "airflow.api.auth.backend.basic_auth,airflow.api.auth.backend.session" AIRFLOW__CORE__EXECUTOR: LocalExecutor diff --git a/docker/docker-compose-openmetadata/docker-compose-openmetadata.yml b/docker/docker-compose-openmetadata/docker-compose-openmetadata.yml index 2ac1cf9bb9d1..94cb770caa1a 100644 --- a/docker/docker-compose-openmetadata/docker-compose-openmetadata.yml +++ b/docker/docker-compose-openmetadata/docker-compose-openmetadata.yml @@ -14,7 +14,7 @@ services: execute-migrate-all: container_name: execute_migrate_all command: "./bootstrap/openmetadata-ops.sh migrate" - image: docker.getcollate.io/openmetadata/server:1.4.0-SNAPSHOT + image: docker.getcollate.io/openmetadata/server:1.3.4 environment: OPENMETADATA_CLUSTER_NAME: ${OPENMETADATA_CLUSTER_NAME:-openmetadata} SERVER_PORT: ${SERVER_PORT:-8585} @@ -42,6 +42,23 @@ services: AUTHENTICATION_CALLBACK_URL: ${AUTHENTICATION_CALLBACK_URL:-""} AUTHENTICATION_JWT_PRINCIPAL_CLAIMS: ${AUTHENTICATION_JWT_PRINCIPAL_CLAIMS:-[email,preferred_username,sub]} AUTHENTICATION_ENABLE_SELF_SIGNUP: ${AUTHENTICATION_ENABLE_SELF_SIGNUP:-true} + AUTHENTICATION_CLIENT_TYPE: ${AUTHENTICATION_CLIENT_TYPE:-public} + #For OIDC Authentication, when client is confidential + OIDC_CLIENT_ID: ${OIDC_CLIENT_ID:-""} + OIDC_TYPE: ${OIDC_TYPE:-""} # google, azure etc. + OIDC_CLIENT_SECRET: ${OIDC_CLIENT_SECRET:-""} + OIDC_SCOPE: ${OIDC_SCOPE:-"openid email profile"} + OIDC_DISCOVERY_URI: ${OIDC_DISCOVERY_URI:-""} + OIDC_USE_NONCE: ${OIDC_USE_NONCE:-true} + OIDC_PREFERRED_JWS: ${OIDC_PREFERRED_JWS:-"RS256"} + OIDC_RESPONSE_TYPE: ${OIDC_RESPONSE_TYPE:-"code"} + OIDC_DISABLE_PKCE: ${OIDC_DISABLE_PKCE:-true} + OIDC_CALLBACK: ${OIDC_CALLBACK:-"http://localhost:8585/callback"} + OIDC_SERVER_URL: ${OIDC_SERVER_URL:-"http://localhost:8585"} + OIDC_CLIENT_AUTH_METHOD: ${OIDC_CLIENT_AUTH_METHOD:-"client_secret_post"} + OIDC_TENANT: ${OIDC_TENANT:-""} + OIDC_MAX_CLOCK_SKEW: ${OIDC_MAX_CLOCK_SKEW:-""} + OIDC_CUSTOM_PARAMS: ${OIDC_CUSTOM_PARAMS:-{}} # For SAML Authentication # SAML_DEBUG_MODE: ${SAML_DEBUG_MODE:-false} # SAML_IDP_ENTITY_ID: ${SAML_IDP_ENTITY_ID:-""} @@ -207,7 +224,7 @@ services: openmetadata-server: container_name: openmetadata_server restart: always - image: docker.getcollate.io/openmetadata/server:1.4.0-SNAPSHOT + image: docker.getcollate.io/openmetadata/server:1.3.4 environment: OPENMETADATA_CLUSTER_NAME: ${OPENMETADATA_CLUSTER_NAME:-openmetadata} SERVER_PORT: ${SERVER_PORT:-8585} @@ -232,6 +249,23 @@ services: AUTHENTICATION_CALLBACK_URL: ${AUTHENTICATION_CALLBACK_URL:-""} AUTHENTICATION_JWT_PRINCIPAL_CLAIMS: ${AUTHENTICATION_JWT_PRINCIPAL_CLAIMS:-[email,preferred_username,sub]} AUTHENTICATION_ENABLE_SELF_SIGNUP: ${AUTHENTICATION_ENABLE_SELF_SIGNUP:-true} + AUTHENTICATION_CLIENT_TYPE: ${AUTHENTICATION_CLIENT_TYPE:-public} + #For OIDC Authentication, when client is confidential + OIDC_CLIENT_ID: ${OIDC_CLIENT_ID:-""} + OIDC_TYPE: ${OIDC_TYPE:-""} # google, azure etc. + OIDC_CLIENT_SECRET: ${OIDC_CLIENT_SECRET:-""} + OIDC_SCOPE: ${OIDC_SCOPE:-"openid email profile"} + OIDC_DISCOVERY_URI: ${OIDC_DISCOVERY_URI:-""} + OIDC_USE_NONCE: ${OIDC_USE_NONCE:-true} + OIDC_PREFERRED_JWS: ${OIDC_PREFERRED_JWS:-"RS256"} + OIDC_RESPONSE_TYPE: ${OIDC_RESPONSE_TYPE:-"code"} + OIDC_DISABLE_PKCE: ${OIDC_DISABLE_PKCE:-true} + OIDC_CALLBACK: ${OIDC_CALLBACK:-"http://localhost:8585/callback"} + OIDC_SERVER_URL: ${OIDC_SERVER_URL:-"http://localhost:8585"} + OIDC_CLIENT_AUTH_METHOD: ${OIDC_CLIENT_AUTH_METHOD:-"client_secret_post"} + OIDC_TENANT: ${OIDC_TENANT:-""} + OIDC_MAX_CLOCK_SKEW: ${OIDC_MAX_CLOCK_SKEW:-""} + OIDC_CUSTOM_PARAMS: ${OIDC_CUSTOM_PARAMS:-{}} # For SAML Authentication # SAML_DEBUG_MODE: ${SAML_DEBUG_MODE:-false} # SAML_IDP_ENTITY_ID: ${SAML_IDP_ENTITY_ID:-""} diff --git a/docker/docker-compose-quickstart/Dockerfile b/docker/docker-compose-quickstart/Dockerfile index 72b2064227e7..14a6d7eef6f7 100644 --- a/docker/docker-compose-quickstart/Dockerfile +++ b/docker/docker-compose-quickstart/Dockerfile @@ -11,7 +11,7 @@ # Build stage FROM alpine:3.19 AS build -ARG RI_VERSION="1.3.0-SNAPSHOT" +ARG RI_VERSION="1.3.4" ENV RELEASE_URL="https://github.com/open-metadata/OpenMetadata/releases/download/${RI_VERSION}-release/openmetadata-${RI_VERSION}.tar.gz" RUN mkdir -p /opt/openmetadata && \ @@ -21,7 +21,7 @@ RUN mkdir -p /opt/openmetadata && \ # Final stage FROM alpine:3.19 -ARG RI_VERSION="1.3.0-SNAPSHOT" +ARG RI_VERSION="1.3.4" ARG BUILD_DATE ARG COMMIT_ID LABEL maintainer="OpenMetadata" diff --git a/docker/docker-compose-quickstart/docker-compose-postgres.yml b/docker/docker-compose-quickstart/docker-compose-postgres.yml index 9cd0d99a546f..73e4fbe9251d 100644 --- a/docker/docker-compose-quickstart/docker-compose-postgres.yml +++ b/docker/docker-compose-quickstart/docker-compose-postgres.yml @@ -18,7 +18,7 @@ volumes: services: postgresql: container_name: openmetadata_postgresql - image: docker.getcollate.io/openmetadata/postgresql:1.4.0-SNAPSHOT + image: docker.getcollate.io/openmetadata/postgresql:1.3.4 restart: always command: "--work_mem=10MB" environment: @@ -61,7 +61,7 @@ services: execute-migrate-all: container_name: execute_migrate_all - image: docker.getcollate.io/openmetadata/server:1.4.0-SNAPSHOT + image: docker.getcollate.io/openmetadata/server:1.3.4 command: "./bootstrap/openmetadata-ops.sh migrate" environment: OPENMETADATA_CLUSTER_NAME: ${OPENMETADATA_CLUSTER_NAME:-openmetadata} @@ -90,6 +90,23 @@ services: AUTHENTICATION_CALLBACK_URL: ${AUTHENTICATION_CALLBACK_URL:-""} AUTHENTICATION_JWT_PRINCIPAL_CLAIMS: ${AUTHENTICATION_JWT_PRINCIPAL_CLAIMS:-[email,preferred_username,sub]} AUTHENTICATION_ENABLE_SELF_SIGNUP: ${AUTHENTICATION_ENABLE_SELF_SIGNUP:-true} + AUTHENTICATION_CLIENT_TYPE: ${AUTHENTICATION_CLIENT_TYPE:-public} + #For OIDC Authentication, when client is confidential + OIDC_CLIENT_ID: ${OIDC_CLIENT_ID:-""} + OIDC_TYPE: ${OIDC_TYPE:-""} # google, azure etc. + OIDC_CLIENT_SECRET: ${OIDC_CLIENT_SECRET:-""} + OIDC_SCOPE: ${OIDC_SCOPE:-"openid email profile"} + OIDC_DISCOVERY_URI: ${OIDC_DISCOVERY_URI:-""} + OIDC_USE_NONCE: ${OIDC_USE_NONCE:-true} + OIDC_PREFERRED_JWS: ${OIDC_PREFERRED_JWS:-"RS256"} + OIDC_RESPONSE_TYPE: ${OIDC_RESPONSE_TYPE:-"code"} + OIDC_DISABLE_PKCE: ${OIDC_DISABLE_PKCE:-true} + OIDC_CALLBACK: ${OIDC_CALLBACK:-"http://localhost:8585/callback"} + OIDC_SERVER_URL: ${OIDC_SERVER_URL:-"http://localhost:8585"} + OIDC_CLIENT_AUTH_METHOD: ${OIDC_CLIENT_AUTH_METHOD:-"client_secret_post"} + OIDC_TENANT: ${OIDC_TENANT:-""} + OIDC_MAX_CLOCK_SKEW: ${OIDC_MAX_CLOCK_SKEW:-""} + OIDC_CUSTOM_PARAMS: ${OIDC_CUSTOM_PARAMS:-{}} # For SAML Authentication # SAML_DEBUG_MODE: ${SAML_DEBUG_MODE:-false} # SAML_IDP_ENTITY_ID: ${SAML_IDP_ENTITY_ID:-""} @@ -255,7 +272,7 @@ services: openmetadata-server: container_name: openmetadata_server restart: always - image: docker.getcollate.io/openmetadata/server:1.4.0-SNAPSHOT + image: docker.getcollate.io/openmetadata/server:1.3.4 environment: OPENMETADATA_CLUSTER_NAME: ${OPENMETADATA_CLUSTER_NAME:-openmetadata} SERVER_PORT: ${SERVER_PORT:-8585} @@ -280,6 +297,23 @@ services: AUTHENTICATION_CALLBACK_URL: ${AUTHENTICATION_CALLBACK_URL:-""} AUTHENTICATION_JWT_PRINCIPAL_CLAIMS: ${AUTHENTICATION_JWT_PRINCIPAL_CLAIMS:-[email,preferred_username,sub]} AUTHENTICATION_ENABLE_SELF_SIGNUP: ${AUTHENTICATION_ENABLE_SELF_SIGNUP:-true} + AUTHENTICATION_CLIENT_TYPE: ${AUTHENTICATION_CLIENT_TYPE:-public} + #For OIDC Authentication, when client is confidential + OIDC_CLIENT_ID: ${OIDC_CLIENT_ID:-""} + OIDC_TYPE: ${OIDC_TYPE:-""} # google, azure etc. + OIDC_CLIENT_SECRET: ${OIDC_CLIENT_SECRET:-""} + OIDC_SCOPE: ${OIDC_SCOPE:-"openid email profile"} + OIDC_DISCOVERY_URI: ${OIDC_DISCOVERY_URI:-""} + OIDC_USE_NONCE: ${OIDC_USE_NONCE:-true} + OIDC_PREFERRED_JWS: ${OIDC_PREFERRED_JWS:-"RS256"} + OIDC_RESPONSE_TYPE: ${OIDC_RESPONSE_TYPE:-"code"} + OIDC_DISABLE_PKCE: ${OIDC_DISABLE_PKCE:-true} + OIDC_CALLBACK: ${OIDC_CALLBACK:-"http://localhost:8585/callback"} + OIDC_SERVER_URL: ${OIDC_SERVER_URL:-"http://localhost:8585"} + OIDC_CLIENT_AUTH_METHOD: ${OIDC_CLIENT_AUTH_METHOD:-"client_secret_post"} + OIDC_TENANT: ${OIDC_TENANT:-""} + OIDC_MAX_CLOCK_SKEW: ${OIDC_MAX_CLOCK_SKEW:-""} + OIDC_CUSTOM_PARAMS: ${OIDC_CUSTOM_PARAMS:-{}} # For SAML Authentication # SAML_DEBUG_MODE: ${SAML_DEBUG_MODE:-false} # SAML_IDP_ENTITY_ID: ${SAML_IDP_ENTITY_ID:-""} @@ -444,7 +478,7 @@ services: ingestion: container_name: openmetadata_ingestion - image: docker.getcollate.io/openmetadata/ingestion:1.4.0-SNAPSHOT + image: docker.getcollate.io/openmetadata/ingestion:1.3.4 depends_on: elasticsearch: condition: service_started diff --git a/docker/docker-compose-quickstart/docker-compose.yml b/docker/docker-compose-quickstart/docker-compose.yml index c0647108af6c..298c9e9155e7 100644 --- a/docker/docker-compose-quickstart/docker-compose.yml +++ b/docker/docker-compose-quickstart/docker-compose.yml @@ -18,7 +18,7 @@ volumes: services: mysql: container_name: openmetadata_mysql - image: docker.getcollate.io/openmetadata/db:1.4.0-SNAPSHOT + image: docker.getcollate.io/openmetadata/db:1.3.4 command: "--sort_buffer_size=10M" restart: always environment: @@ -59,7 +59,7 @@ services: execute-migrate-all: container_name: execute_migrate_all - image: docker.getcollate.io/openmetadata/server:1.4.0-SNAPSHOT + image: docker.getcollate.io/openmetadata/server:1.3.4 command: "./bootstrap/openmetadata-ops.sh migrate" environment: OPENMETADATA_CLUSTER_NAME: ${OPENMETADATA_CLUSTER_NAME:-openmetadata} @@ -88,6 +88,23 @@ services: AUTHENTICATION_CALLBACK_URL: ${AUTHENTICATION_CALLBACK_URL:-""} AUTHENTICATION_JWT_PRINCIPAL_CLAIMS: ${AUTHENTICATION_JWT_PRINCIPAL_CLAIMS:-[email,preferred_username,sub]} AUTHENTICATION_ENABLE_SELF_SIGNUP: ${AUTHENTICATION_ENABLE_SELF_SIGNUP:-true} + AUTHENTICATION_CLIENT_TYPE: ${AUTHENTICATION_CLIENT_TYPE:-public} + #For OIDC Authentication, when client is confidential + OIDC_CLIENT_ID: ${OIDC_CLIENT_ID:-""} + OIDC_TYPE: ${OIDC_TYPE:-""} # google, azure etc. + OIDC_CLIENT_SECRET: ${OIDC_CLIENT_SECRET:-""} + OIDC_SCOPE: ${OIDC_SCOPE:-"openid email profile"} + OIDC_DISCOVERY_URI: ${OIDC_DISCOVERY_URI:-""} + OIDC_USE_NONCE: ${OIDC_USE_NONCE:-true} + OIDC_PREFERRED_JWS: ${OIDC_PREFERRED_JWS:-"RS256"} + OIDC_RESPONSE_TYPE: ${OIDC_RESPONSE_TYPE:-"code"} + OIDC_DISABLE_PKCE: ${OIDC_DISABLE_PKCE:-true} + OIDC_CALLBACK: ${OIDC_CALLBACK:-"http://localhost:8585/callback"} + OIDC_SERVER_URL: ${OIDC_SERVER_URL:-"http://localhost:8585"} + OIDC_CLIENT_AUTH_METHOD: ${OIDC_CLIENT_AUTH_METHOD:-"client_secret_post"} + OIDC_TENANT: ${OIDC_TENANT:-""} + OIDC_MAX_CLOCK_SKEW: ${OIDC_MAX_CLOCK_SKEW:-""} + OIDC_CUSTOM_PARAMS: ${OIDC_CUSTOM_PARAMS:-{}} # For SAML Authentication # SAML_DEBUG_MODE: ${SAML_DEBUG_MODE:-false} # SAML_IDP_ENTITY_ID: ${SAML_IDP_ENTITY_ID:-""} @@ -253,7 +270,7 @@ services: openmetadata-server: container_name: openmetadata_server restart: always - image: docker.getcollate.io/openmetadata/server:1.4.0-SNAPSHOT + image: docker.getcollate.io/openmetadata/server:1.3.4 environment: OPENMETADATA_CLUSTER_NAME: ${OPENMETADATA_CLUSTER_NAME:-openmetadata} SERVER_PORT: ${SERVER_PORT:-8585} @@ -278,6 +295,23 @@ services: AUTHENTICATION_CALLBACK_URL: ${AUTHENTICATION_CALLBACK_URL:-""} AUTHENTICATION_JWT_PRINCIPAL_CLAIMS: ${AUTHENTICATION_JWT_PRINCIPAL_CLAIMS:-[email,preferred_username,sub]} AUTHENTICATION_ENABLE_SELF_SIGNUP: ${AUTHENTICATION_ENABLE_SELF_SIGNUP:-true} + AUTHENTICATION_CLIENT_TYPE: ${AUTHENTICATION_CLIENT_TYPE:-public} + #For OIDC Authentication, when client is confidential + OIDC_CLIENT_ID: ${OIDC_CLIENT_ID:-""} + OIDC_TYPE: ${OIDC_TYPE:-""} # google, azure etc. + OIDC_CLIENT_SECRET: ${OIDC_CLIENT_SECRET:-""} + OIDC_SCOPE: ${OIDC_SCOPE:-"openid email profile"} + OIDC_DISCOVERY_URI: ${OIDC_DISCOVERY_URI:-""} + OIDC_USE_NONCE: ${OIDC_USE_NONCE:-true} + OIDC_PREFERRED_JWS: ${OIDC_PREFERRED_JWS:-"RS256"} + OIDC_RESPONSE_TYPE: ${OIDC_RESPONSE_TYPE:-"code"} + OIDC_DISABLE_PKCE: ${OIDC_DISABLE_PKCE:-true} + OIDC_CALLBACK: ${OIDC_CALLBACK:-"http://localhost:8585/callback"} + OIDC_SERVER_URL: ${OIDC_SERVER_URL:-"http://localhost:8585"} + OIDC_CLIENT_AUTH_METHOD: ${OIDC_CLIENT_AUTH_METHOD:-"client_secret_post"} + OIDC_TENANT: ${OIDC_TENANT:-""} + OIDC_MAX_CLOCK_SKEW: ${OIDC_MAX_CLOCK_SKEW:-""} + OIDC_CUSTOM_PARAMS: ${OIDC_CUSTOM_PARAMS:-{}} # For SAML Authentication # SAML_DEBUG_MODE: ${SAML_DEBUG_MODE:-false} # SAML_IDP_ENTITY_ID: ${SAML_IDP_ENTITY_ID:-""} @@ -442,7 +476,7 @@ services: ingestion: container_name: openmetadata_ingestion - image: docker.getcollate.io/openmetadata/ingestion:1.4.0-SNAPSHOT + image: docker.getcollate.io/openmetadata/ingestion:1.3.4 depends_on: elasticsearch: condition: service_started diff --git a/docker/validate_compose.py b/docker/validate_compose.py index 812d5e49596c..c90b46b311ea 100644 --- a/docker/validate_compose.py +++ b/docker/validate_compose.py @@ -23,11 +23,9 @@ def get_last_run_info() -> Tuple[str, str]: while retries < max_retries: log_ansi_encoded_string(message="Waiting for DAG Run data...") time.sleep(5) - res = requests.get( + runs = requests.get( "http://localhost:8080/api/v1/dags/sample_data/dagRuns", auth=BASIC_AUTH, timeout=REQUESTS_TIMEOUT - ) - res.raise_for_status() - runs = res.json() + ).json() dag_runs = runs.get("dag_runs") if dag_runs[0].get("dag_run_id"): return dag_runs[0].get("dag_run_id"), "success" diff --git a/ingestion/Dockerfile b/ingestion/Dockerfile index a895e68bafc9..134599c6bdd7 100644 --- a/ingestion/Dockerfile +++ b/ingestion/Dockerfile @@ -81,7 +81,7 @@ ARG INGESTION_DEPENDENCY="all" ENV PIP_NO_CACHE_DIR=1 # Make pip silent ENV PIP_QUIET=1 -ARG RI_VERSION="1.3.0.0.dev0" +ARG RI_VERSION="1.3.4.0" RUN pip install --upgrade pip RUN pip install "openmetadata-managed-apis~=${RI_VERSION}" --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-2.7.3/constraints-3.10.txt" RUN pip install "openmetadata-ingestion[${INGESTION_DEPENDENCY}]~=${RI_VERSION}" diff --git a/ingestion/operators/docker/Dockerfile b/ingestion/operators/docker/Dockerfile index 2221394cecf3..eec6233eae38 100644 --- a/ingestion/operators/docker/Dockerfile +++ b/ingestion/operators/docker/Dockerfile @@ -87,7 +87,7 @@ ENV PIP_QUIET=1 RUN pip install --upgrade pip ARG INGESTION_DEPENDENCY="all" -ARG RI_VERSION="1.3.0.0.dev0" +ARG RI_VERSION="1.3.4.0" RUN pip install --upgrade pip RUN pip install "openmetadata-ingestion[airflow]~=${RI_VERSION}" RUN pip install "openmetadata-ingestion[${INGESTION_DEPENDENCY}]~=${RI_VERSION}" diff --git a/ingestion/pyproject.toml b/ingestion/pyproject.toml index 9a9be4820adf..3bd4441bfd6e 100644 --- a/ingestion/pyproject.toml +++ b/ingestion/pyproject.toml @@ -6,7 +6,7 @@ build-backend = "setuptools.build_meta" # since it helps us organize and isolate version management [project] name = "openmetadata-ingestion" -version = "1.4.0.0.dev0" +version = "1.3.4.0" dynamic = ["readme", "dependencies", "optional-dependencies"] authors = [ {name = "OpenMetadata Committers"} diff --git a/ingestion/setup.py b/ingestion/setup.py index 72e9bad4c93f..41f7d1c970b5 100644 --- a/ingestion/setup.py +++ b/ingestion/setup.py @@ -163,7 +163,13 @@ }, "db2": {"ibm-db-sa~=0.3"}, "db2-ibmi": {"sqlalchemy-ibmi~=0.9.3"}, - "databricks": {VERSIONS["sqlalchemy-databricks"], VERSIONS["databricks-sdk"]}, + "databricks": { + VERSIONS["sqlalchemy-databricks"], + VERSIONS["databricks-sdk"], + "ndg-httpsclient~=0.5.1", + "pyOpenSSL~=24.1.0", + "pyasn1~=0.6.0", + }, "datalake-azure": { VERSIONS["azure-storage-blob"], VERSIONS["azure-identity"], @@ -313,7 +319,6 @@ VERSIONS["snowflake"], VERSIONS["elasticsearch8"], VERSIONS["giturlparse"], - "testcontainers==3.7.1", } e2e_test = { diff --git a/ingestion/src/metadata/clients/azure_client.py b/ingestion/src/metadata/clients/azure_client.py new file mode 100644 index 000000000000..f80cc0ad5e6a --- /dev/null +++ b/ingestion/src/metadata/clients/azure_client.py @@ -0,0 +1,85 @@ +# Copyright 2021 Collate +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Module containing Azure Client +""" + +from metadata.generated.schema.security.credentials.azureCredentials import ( + AzureCredentials, +) +from metadata.utils.logger import utils_logger + +logger = utils_logger() + + +class AzureClient: + """ + AzureClient based on AzureCredentials. + """ + + def __init__(self, credentials: "AzureCredentials"): + self.credentials = credentials + if not isinstance(credentials, AzureCredentials): + self.credentials = AzureCredentials.parse_obj(credentials) + + def create_client( + self, + ): + from azure.identity import ClientSecretCredential, DefaultAzureCredential + + try: + if ( + getattr(self.credentials, "tenantId", None) + and getattr(self.credentials, "clientId", None) + and getattr(self.credentials, "clientSecret", None) + ): + logger.info("Using Client Secret Credentials") + return ClientSecretCredential( + tenant_id=self.credentials.tenantId, + client_id=self.credentials.clientId, + client_secret=self.credentials.clientSecret.get_secret_value(), + ) + else: + logger.info("Using Default Azure Credentials") + return DefaultAzureCredential() + except Exception as e: + logger.error(f"Error creating Azure Client: {e}") + raise e + + def create_blob_client(self): + from azure.storage.blob import BlobServiceClient + + try: + logger.info("Creating Blob Service Client") + if self.credentials.accountName: + return BlobServiceClient( + account_url=f"https://{self.credentials.accountName}.blob.core.windows.net/", + credential=self.create_client(), + ) + raise ValueError("Account Name is required to create Blob Service Client") + except Exception as e: + logger.error(f"Error creating Blob Service Client: {e}") + raise e + + def create_secret_client(self): + from azure.keyvault.secrets import SecretClient + + try: + if self.credentials.vaultName: + logger.info("Creating Secret Client") + return SecretClient( + vault_url=f"https://{self.credentials.vaultName}.vault.azure.net/", + credential=self.create_client(), + ) + raise ValueError("Vault Name is required to create a Secret Client") + except Exception as e: + logger.error(f"Error creating Secret Client: {e}") + raise e diff --git a/ingestion/src/metadata/data_quality/source/test_suite.py b/ingestion/src/metadata/data_quality/source/test_suite.py index cdb0bdd37a8b..a60639db569c 100644 --- a/ingestion/src/metadata/data_quality/source/test_suite.py +++ b/ingestion/src/metadata/data_quality/source/test_suite.py @@ -83,11 +83,11 @@ def _get_test_cases_from_test_suite( ) -> Optional[List[TestCase]]: """Return test cases if the test suite exists and has them""" if test_suite: - test_cases = self.metadata.list_entities( + test_cases = self.metadata.list_all_entities( entity=TestCase, fields=["testSuite", "entityLink", "testDefinition"], params={"testSuiteId": test_suite.id.__root__}, - ).entities + ) test_cases = cast(List[TestCase], test_cases) # satisfy type checker return test_cases diff --git a/ingestion/src/metadata/examples/workflows/datalake_azure.yaml b/ingestion/src/metadata/examples/workflows/datalake_azure_client_secret.yaml similarity index 100% rename from ingestion/src/metadata/examples/workflows/datalake_azure.yaml rename to ingestion/src/metadata/examples/workflows/datalake_azure_client_secret.yaml diff --git a/ingestion/src/metadata/examples/workflows/datalake_azure_default.yaml b/ingestion/src/metadata/examples/workflows/datalake_azure_default.yaml new file mode 100644 index 000000000000..2a4f248232e3 --- /dev/null +++ b/ingestion/src/metadata/examples/workflows/datalake_azure_default.yaml @@ -0,0 +1,29 @@ +source: + type: datalake + serviceName: local_datalake4 + serviceConnection: + config: + type: Datalake + configSource: + securityConfig: + clientId: clientId + accountName: accountName + bucketName: bucket name + prefix: prefix + sourceConfig: + config: + type: DatabaseMetadata + tableFilterPattern: + includes: + - '' +sink: + type: metadata-rest + config: {} +workflowConfig: +# loggerLevel: INFO # DEBUG, INFO, WARN or ERROR + openMetadataServerConfig: + hostPort: http://localhost:8585/api + authProvider: openmetadata + securityConfig: + jwtToken: "eyJraWQiOiJHYjM4OWEtOWY3Ni1nZGpzLWE5MmotMDI0MmJrOTQzNTYiLCJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiJ9.eyJzdWIiOiJhZG1pbiIsImlzQm90IjpmYWxzZSwiaXNzIjoib3Blbi1tZXRhZGF0YS5vcmciLCJpYXQiOjE2NjM5Mzg0NjIsImVtYWlsIjoiYWRtaW5Ab3Blbm1ldGFkYXRhLm9yZyJ9.tS8um_5DKu7HgzGBzS1VTA5uUjKWOCU0B_j08WXBiEC0mr0zNREkqVfwFDD-d24HlNEbrqioLsBuFRiwIWKc1m_ZlVQbG7P36RUxhuv2vbSp80FKyNM-Tj93FDzq91jsyNmsQhyNv_fNr3TXfzzSPjHt8Go0FMMP66weoKMgW2PbXlhVKwEuXUHyakLLzewm9UMeQaEiRzhiTMU3UkLXcKbYEJJvfNFcLwSl9W8JCO_l0Yj3ud-qt_nQYEZwqW6u5nfdQllN133iikV4fM5QZsMCnm8Rq1mvLR0y9bmJiD7fwM1tmJ791TUWqmKaTnP49U493VanKpUAfzIiOiIbhg" + \ No newline at end of file diff --git a/ingestion/src/metadata/examples/workflows/dbt.yaml b/ingestion/src/metadata/examples/workflows/dbt.yaml index c5266ec890a6..7d5f1e95ee72 100644 --- a/ingestion/src/metadata/examples/workflows/dbt.yaml +++ b/ingestion/src/metadata/examples/workflows/dbt.yaml @@ -5,21 +5,28 @@ source: config: type: DBT # For DBT, choose one of Cloud, Local, HTTP, S3 or GCS configurations - # dbtConfigSource: - # # For cloud - # dbtCloudAuthToken: token - # dbtCloudAccountId: ID - # dbtCloudJobId: JOB ID - # dbtCloudUrl: https://cloud.getdbt.com + # For cloud + dbtConfigSource: + dbtConfigType: cloud + dbtCloudAuthToken: token + dbtCloudAccountId: ID + dbtCloudJobId: JOB ID + dbtCloudUrl: https://cloud.getdbt.com # # For Local + # dbtConfigSource: + # dbtConfigType: local # dbtCatalogFilePath: path-to-catalog.json # dbtManifestFilePath: path-to-manifest.json # dbtRunResultsFilePath: path-to-run_results.json # # For HTTP + # dbtConfigSource: + # dbtConfigType: http # dbtCatalogHttpPath: http://path-to-catalog.json # dbtManifestHttpPath: http://path-to-manifest.json # dbtRunResultsHttpPath: http://path-to-run_results.json # # For S3 + # dbtConfigSource: + # dbtConfigType: s3 # dbtSecurityConfig: # These are modeled after all AWS credentials # awsAccessKeyId: KEY # awsSecretAccessKey: SECRET @@ -28,6 +35,8 @@ source: # dbtBucketName: bucket_name # dbtObjectPrefix: "main_dir/dbt_files" # # For GCS + # dbtConfigSource: + # dbtConfigType: gcs # dbtSecurityConfig: # These are modeled after all GCS credentials # gcpConfig: # type: My Type @@ -47,6 +56,8 @@ source: # dbtBucketName: bucket_name # dbtObjectPrefix: "main_dir/dbt_files" # # For Azure + # dbtConfigSource: + # dbtConfigType: azure # dbtSecurityConfig: # These are modeled after all Azure credentials # clientId: clientId # clientSecret: clientSecret diff --git a/ingestion/src/metadata/great_expectations/action.py b/ingestion/src/metadata/great_expectations/action.py index 7a475cc6a10c..76fc5797ad8b 100644 --- a/ingestion/src/metadata/great_expectations/action.py +++ b/ingestion/src/metadata/great_expectations/action.py @@ -15,6 +15,7 @@ This subpackage needs to be used in Great Expectations checkpoints actions. """ +import logging import traceback from datetime import datetime, timezone from typing import Dict, List, Optional, Union, cast @@ -72,9 +73,10 @@ from metadata.ingestion.ometa.ometa_api import OpenMetadata from metadata.utils import fqn from metadata.utils.entity_link import get_entity_link -from metadata.utils.logger import great_expectations_logger -logger = great_expectations_logger() +logger = logging.getLogger( + "great_expectations.validation_operators.validation_operators.openmetadata" +) class OpenMetadataValidationAction(ValidationAction): @@ -107,7 +109,7 @@ def __init__( self.config_file_path = config_file_path self.ometa_conn = self._create_ometa_connection() - def _run( # pylint: disable=unused-argument,arguments-renamed + def _run( # pylint: disable=unused-argument self, validation_result_suite: ExpectationSuiteValidationResult, validation_result_suite_identifier: Union[ @@ -124,6 +126,7 @@ def _run( # pylint: disable=unused-argument,arguments-renamed validation_result_suite: result suite returned when checkpoint is ran validation_result_suite_identifier: type of result suite data_asset: + payload: expectation_suite_identifier: type of expectation suite checkpoint_identifier: identifier for the checkpoint """ @@ -428,7 +431,7 @@ def _handle_test_case( test_case_fqn=test_case.fullyQualifiedName.__root__, ) - logger.info( + logger.debug( f"Test case result for {test_case.fullyQualifiedName.__root__} successfully ingested" ) diff --git a/ingestion/src/metadata/ingestion/api/parser.py b/ingestion/src/metadata/ingestion/api/parser.py index c05d4dbdb21d..7808b281efc5 100644 --- a/ingestion/src/metadata/ingestion/api/parser.py +++ b/ingestion/src/metadata/ingestion/api/parser.py @@ -69,6 +69,28 @@ DatabaseServiceQueryUsagePipeline, DatabaseUsageConfigType, ) +from metadata.generated.schema.metadataIngestion.dbtconfig.dbtAzureConfig import ( + DbtAzureConfig, +) +from metadata.generated.schema.metadataIngestion.dbtconfig.dbtCloudConfig import ( + DbtCloudConfig, +) +from metadata.generated.schema.metadataIngestion.dbtconfig.dbtGCSConfig import ( + DbtGcsConfig, +) +from metadata.generated.schema.metadataIngestion.dbtconfig.dbtHttpConfig import ( + DbtHttpConfig, +) +from metadata.generated.schema.metadataIngestion.dbtconfig.dbtLocalConfig import ( + DbtLocalConfig, +) +from metadata.generated.schema.metadataIngestion.dbtconfig.dbtS3Config import ( + DbtS3Config, +) +from metadata.generated.schema.metadataIngestion.dbtPipeline import ( + DbtConfigType, + DbtPipeline, +) from metadata.generated.schema.metadataIngestion.messagingServiceMetadataPipeline import ( MessagingMetadataConfigType, MessagingServiceMetadataPipeline, @@ -125,6 +147,16 @@ DatabaseMetadataConfigType.DatabaseMetadata.value: DatabaseServiceMetadataPipeline, StorageMetadataConfigType.StorageMetadata.value: StorageServiceMetadataPipeline, SearchMetadataConfigType.SearchMetadata.value: SearchServiceMetadataPipeline, + DbtConfigType.DBT.value: DbtPipeline, +} + +DBT_CONFIG_TYPE_MAP = { + "cloud": DbtCloudConfig, + "local": DbtLocalConfig, + "http": DbtHttpConfig, + "s3": DbtS3Config, + "gcs": DbtGcsConfig, + "azure": DbtAzureConfig, } @@ -171,6 +203,7 @@ def get_source_config_class( Type[PipelineServiceMetadataPipeline], Type[MlModelServiceMetadataPipeline], Type[DatabaseServiceMetadataPipeline], + Type[DbtPipeline], ]: """ Return the source config type for a source string @@ -179,7 +212,7 @@ def get_source_config_class( """ source_config_class = SOURCE_CONFIG_CLASS_MAP.get(source_config_type) - if source_config_type: + if source_config_class: return source_config_class raise ValueError(f"Cannot find the service type of {source_config_type}") @@ -266,6 +299,27 @@ def _unsafe_parse_config(config: dict, cls: Type[T], message: str) -> None: raise err +def _unsafe_parse_dbt_config(config: dict, cls: Type[T], message: str) -> None: + """ + Given a config dictionary and the class it should match, + try to parse it or log the given message + """ + logger.debug(f"Parsing message: [{message}]") + try: + # Parse the oneOf config types of dbt to check + dbt_config_type = config["dbtConfigSource"]["dbtConfigType"] + dbt_config_class = DBT_CONFIG_TYPE_MAP.get(dbt_config_type) + dbt_config_class.parse_obj(config["dbtConfigSource"]) + + # Parse the entire dbtPipeline object + cls.parse_obj(config) + except ValidationError as err: + logger.debug( + f"The supported properties for {cls.__name__} are {list(cls.__fields__.keys())}" + ) + raise err + + def _parse_inner_connection(config_dict: dict, source_type: str) -> None: """ Parse the inner connection of the flagged connectors @@ -291,32 +345,35 @@ def parse_service_connection(config_dict: dict) -> None: :param config_dict: JSON configuration """ # Unsafe access to the keys. Allow a KeyError if the config is not well formatted - source_type = config_dict["source"]["serviceConnection"]["config"].get("type") - if source_type is None: - raise InvalidWorkflowException("Missing type in the serviceConnection config") + if config_dict["source"].get("serviceConnection"): + source_type = config_dict["source"]["serviceConnection"]["config"].get("type") + if source_type is None: + raise InvalidWorkflowException( + "Missing type in the serviceConnection config" + ) - logger.debug( - f"Error parsing the Workflow Configuration for {source_type} ingestion" - ) + logger.debug( + f"Error parsing the Workflow Configuration for {source_type} ingestion" + ) - service_type = get_service_type(source_type) - connection_class = get_connection_class(source_type, service_type) + service_type = get_service_type(source_type) + connection_class = get_connection_class(source_type, service_type) - if source_type in HAS_INNER_CONNECTION: - # We will first parse the inner `connection` configuration - _parse_inner_connection( - config_dict["source"]["serviceConnection"]["config"]["connection"][ - "config" - ]["connection"], - source_type, - ) + if source_type in HAS_INNER_CONNECTION: + # We will first parse the inner `connection` configuration + _parse_inner_connection( + config_dict["source"]["serviceConnection"]["config"]["connection"][ + "config" + ]["connection"], + source_type, + ) - # Parse the service connection dictionary with the scoped class - _unsafe_parse_config( - config=config_dict["source"]["serviceConnection"]["config"], - cls=connection_class, - message="Error parsing the service connection", - ) + # Parse the service connection dictionary with the scoped class + _unsafe_parse_config( + config=config_dict["source"]["serviceConnection"]["config"], + cls=connection_class, + message="Error parsing the service connection", + ) def parse_source_config(config_dict: dict) -> None: @@ -334,6 +391,13 @@ def parse_source_config(config_dict: dict) -> None: source_config_class = get_source_config_class(source_config_type) + if source_config_class == DbtPipeline: + _unsafe_parse_dbt_config( + config=config_dict["source"]["sourceConfig"]["config"], + cls=source_config_class, + message="Error parsing the dbt source config", + ) + _unsafe_parse_config( config=config_dict["source"]["sourceConfig"]["config"], cls=source_config_class, diff --git a/ingestion/src/metadata/ingestion/api/topology_runner.py b/ingestion/src/metadata/ingestion/api/topology_runner.py index 2e3c52a94db4..398043cec204 100644 --- a/ingestion/src/metadata/ingestion/api/topology_runner.py +++ b/ingestion/src/metadata/ingestion/api/topology_runner.py @@ -265,11 +265,12 @@ def yield_and_update_context( if entity: same_fingerprint = True - create_entity_request_hash = generate_source_hash( - create_request=entity_request.right, - ) + create_entity_request_hash = None if hasattr(entity_request.right, "sourceHash"): + create_entity_request_hash = generate_source_hash( + create_request=entity_request.right, + ) entity_request.right.sourceHash = create_entity_request_hash if entity is None and stage.use_cache: diff --git a/ingestion/src/metadata/ingestion/lineage/parser.py b/ingestion/src/metadata/ingestion/lineage/parser.py index 5d855784523d..368dd738abab 100644 --- a/ingestion/src/metadata/ingestion/lineage/parser.py +++ b/ingestion/src/metadata/ingestion/lineage/parser.py @@ -217,6 +217,11 @@ def get_comparison_elements( """ aliases = self.table_aliases values = identifier.value.split(".") + + if len(values) > 4: + logger.debug(f"Invalid comparison element from identifier: {identifier}") + return None, None + database_name, schema_name, table_or_alias, column_name = ( [None] * (4 - len(values)) ) + values @@ -307,29 +312,39 @@ def stateful_add_joins_from_statement( comparisons.append(sub) for comparison in comparisons: - if "." not in comparison.left.value or "." not in comparison.right.value: - logger.debug(f"Ignoring comparison {comparison}") - continue - - table_left, column_left = self.get_comparison_elements( - identifier=comparison.left - ) - table_right, column_right = self.get_comparison_elements( - identifier=comparison.right - ) + try: + if ( + "." not in comparison.left.value + or "." not in comparison.right.value + ): + logger.debug(f"Ignoring comparison {comparison}") + continue + + table_left, column_left = self.get_comparison_elements( + identifier=comparison.left + ) + table_right, column_right = self.get_comparison_elements( + identifier=comparison.right + ) - if not table_left or not table_right: - logger.warning(f"Cannot find ingredients from {comparison}") - continue + if not table_left or not table_right: + logger.warning( + f"Can't extract table names when parsing JOIN information from {comparison}" + ) + logger.debug(f"Query: {sql_statement}") + continue - left_table_column = TableColumn(table=table_left, column=column_left) - right_table_column = TableColumn(table=table_right, column=column_right) + left_table_column = TableColumn(table=table_left, column=column_left) + right_table_column = TableColumn(table=table_right, column=column_right) - # We just send the info once, from Left -> Right. - # The backend will prepare the symmetric information. - self.stateful_add_table_joins( - join_data, left_table_column, right_table_column - ) + # We just send the info once, from Left -> Right. + # The backend will prepare the symmetric information. + self.stateful_add_table_joins( + join_data, left_table_column, right_table_column + ) + except Exception as exc: + logger.debug(f"Cannot process comparison {comparison}: {exc}") + logger.debug(traceback.format_exc()) @cached_property def table_joins(self) -> Dict[str, List[TableColumnJoin]]: diff --git a/ingestion/src/metadata/ingestion/models/custom_properties.py b/ingestion/src/metadata/ingestion/models/custom_properties.py index 6a62c1dbc053..c287f64f122c 100644 --- a/ingestion/src/metadata/ingestion/models/custom_properties.py +++ b/ingestion/src/metadata/ingestion/models/custom_properties.py @@ -41,7 +41,6 @@ class CustomPropertyDataTypes(Enum): class OMetaCustomProperties(BaseModel): entity_type: Type[T] - custom_property_type: Optional[CustomPropertyDataTypes] createCustomPropertyRequest: CreateCustomPropertyRequest diff --git a/ingestion/src/metadata/ingestion/models/patch_request.py b/ingestion/src/metadata/ingestion/models/patch_request.py index 513ea5be3ac6..862c412b432e 100644 --- a/ingestion/src/metadata/ingestion/models/patch_request.py +++ b/ingestion/src/metadata/ingestion/models/patch_request.py @@ -19,6 +19,7 @@ from metadata.ingestion.api.models import Entity, T from metadata.ingestion.ometa.mixins.patch_mixin_utils import PatchOperation +from metadata.ingestion.ometa.utils import model_str class PatchRequest(BaseModel): @@ -138,12 +139,15 @@ class PatchedEntity(BaseModel): RESTRICT_UPDATE_LIST = ["description", "tags", "owner"] +ARRAY_ENTITY_FIELDS = ["columns", "tasks", "fields"] + def build_patch( source: T, destination: T, allowed_fields: Optional[Dict] = None, restrict_update_fields: Optional[List] = None, + array_entity_fields: Optional[List] = None, ) -> Optional[jsonpatch.JsonPatch]: """ Given an Entity type and Source entity and Destination entity, @@ -163,6 +167,13 @@ def build_patch( source = _remove_change_description(source) destination = _remove_change_description(destination) + if array_entity_fields: + _sort_array_entity_fields( + source=source, + destination=destination, + array_entity_fields=array_entity_fields, + ) + # Get the difference between source and destination if allowed_fields: patch = jsonpatch.make_patch( @@ -192,20 +203,61 @@ def build_patch( # for a user editable fields like descriptions, tags we only want to support "add" operation in patch # we will remove the other operations for replace, remove from here if restrict_update_fields: - patch.patch = [ - patch_ops - for patch_ops in patch.patch + patch_ops_list = [] + for patch_ops in patch.patch or []: if _determine_restricted_operation( - patch_ops=patch_ops, - restrict_update_fields=restrict_update_fields, - ) - ] - + patch_ops=patch_ops, restrict_update_fields=restrict_update_fields + ): + if ( + patch_ops.get("op") == PatchOperation.REPLACE.value + and patch_ops.get("value") is None + ): + patch_ops["op"] = PatchOperation.REMOVE.value + del patch_ops["value"] + patch_ops_list.append(patch_ops) + patch.patch = patch_ops_list return patch +def _sort_array_entity_fields( + source: T, + destination: T, + array_entity_fields: Optional[List] = None, +): + """ + Sort the array entity fields to make sure the order is consistent + """ + for field in array_entity_fields or []: + if hasattr(destination, field) and hasattr(source, field): + destination_attributes = getattr(destination, field) + source_attributes = getattr(source, field) + + # Create a dictionary of destination attributes for easy lookup + destination_dict = { + model_str(attr.name): attr for attr in destination_attributes + } + + updated_attributes = [] + for source_attr in source_attributes or []: + # Update the destination attribute with the source attribute + destination_attr = destination_dict.get(model_str(source_attr.name)) + if destination_attr: + updated_attributes.append( + source_attr.copy(update=destination_attr.__dict__) + ) + # Remove the updated attribute from the destination dictionary + del destination_dict[model_str(source_attr.name)] + else: + updated_attributes.append(None) + + # Combine the updated attributes with the remaining destination attributes + final_attributes = updated_attributes + list(destination_dict.values()) + setattr(destination, field, final_attributes) + + def _determine_restricted_operation( - patch_ops: Dict, restrict_update_fields: Optional[List] = None + patch_ops: Dict, + restrict_update_fields: Optional[List] = None, ) -> bool: """ Only retain add operation for restrict_update_fields fields diff --git a/ingestion/src/metadata/ingestion/ometa/mixins/custom_property_mixin.py b/ingestion/src/metadata/ingestion/ometa/mixins/custom_property_mixin.py index a60aea192466..5b0fe01115c0 100644 --- a/ingestion/src/metadata/ingestion/ometa/mixins/custom_property_mixin.py +++ b/ingestion/src/metadata/ingestion/ometa/mixins/custom_property_mixin.py @@ -15,7 +15,8 @@ """ from typing import Dict -from metadata.generated.schema.api.data.createCustomProperty import PropertyType +from metadata.generated.schema.type.customProperty import PropertyType +from metadata.generated.schema.type.entityReference import EntityReference from metadata.ingestion.models.custom_properties import ( CustomPropertyDataTypes, CustomPropertyType, @@ -54,16 +55,6 @@ def create_or_update_custom_property( f"/metadata/types/name/{entity_type}?category=field" ) - # Get the data type of the custom property - if not ometa_custom_property.createCustomPropertyRequest.propertyType: - custom_property_type = self.get_custom_property_type( - data_type=ometa_custom_property.custom_property_type - ) - property_type = PropertyType(id=custom_property_type.id, type="type") - ometa_custom_property.createCustomPropertyRequest.propertyType = ( - property_type - ) - resp = self.client.put( f"/metadata/types/{entity_schema.get('id')}", data=ometa_custom_property.createCustomPropertyRequest.json(), @@ -78,3 +69,12 @@ def get_custom_property_type( """ resp = self.client.get(f"/metadata/types/name/{data_type.value}?category=field") return CustomPropertyType(**resp) + + def get_property_type_ref(self, data_type: CustomPropertyDataTypes) -> PropertyType: + """ + Get the PropertyType for custom properties + """ + custom_property_type = self.get_custom_property_type(data_type=data_type) + return PropertyType( + __root__=EntityReference(id=custom_property_type.id, type="type") + ) diff --git a/ingestion/src/metadata/ingestion/ometa/mixins/patch_mixin.py b/ingestion/src/metadata/ingestion/ometa/mixins/patch_mixin.py index 3e627597d8f0..542861294825 100644 --- a/ingestion/src/metadata/ingestion/ometa/mixins/patch_mixin.py +++ b/ingestion/src/metadata/ingestion/ometa/mixins/patch_mixin.py @@ -46,7 +46,7 @@ ) from metadata.ingestion.ometa.utils import model_str from metadata.utils.deprecation import deprecated -from metadata.utils.logger import ometa_logger +from metadata.utils.logger import get_log_name, ometa_logger logger = ometa_logger() @@ -119,6 +119,7 @@ def patch( destination: T, allowed_fields: Optional[Dict] = None, restrict_update_fields: Optional[List] = None, + array_entity_fields: Optional[List] = None, ) -> Optional[T]: """ Given an Entity type and Source entity and Destination entity, @@ -140,6 +141,7 @@ def patch( destination=destination, allowed_fields=allowed_fields, restrict_update_fields=restrict_update_fields, + array_entity_fields=array_entity_fields, ) if not patch: @@ -153,9 +155,7 @@ def patch( except Exception as exc: logger.debug(traceback.format_exc()) - logger.error( - f"Error trying to PATCH {entity.__name__} [{source.id.__root__}]: {exc}" - ) + logger.error(f"Error trying to PATCH {get_log_name(source)}: {exc}") return None diff --git a/ingestion/src/metadata/ingestion/ometa/routes.py b/ingestion/src/metadata/ingestion/ometa/routes.py index f625459a179f..c92dda4fc3d5 100644 --- a/ingestion/src/metadata/ingestion/ometa/routes.py +++ b/ingestion/src/metadata/ingestion/ometa/routes.py @@ -90,6 +90,16 @@ from metadata.generated.schema.api.tests.createTestSuite import CreateTestSuiteRequest from metadata.generated.schema.dataInsight.dataInsightChart import DataInsightChart from metadata.generated.schema.dataInsight.kpi.kpi import Kpi +from metadata.generated.schema.entity.applications.app import App +from metadata.generated.schema.entity.applications.createAppRequest import ( + CreateAppRequest, +) +from metadata.generated.schema.entity.applications.marketplace.appMarketPlaceDefinition import ( + AppMarketPlaceDefinition, +) +from metadata.generated.schema.entity.applications.marketplace.createAppMarketPlaceDefinitionReq import ( + CreateAppMarketPlaceDefinitionRequest, +) from metadata.generated.schema.entity.automations.workflow import Workflow from metadata.generated.schema.entity.bot import Bot from metadata.generated.schema.entity.classification.classification import ( @@ -232,4 +242,9 @@ # Suggestions Suggestion.__name__: "/suggestions", CreateSuggestionRequest.__name__: "/suggestions", + # Apps + App.__name__: "/apps", + CreateAppRequest.__name__: "/apps", + AppMarketPlaceDefinition.__name__: "/apps/marketplace", + CreateAppMarketPlaceDefinitionRequest.__name__: "/apps/marketplace", } diff --git a/ingestion/src/metadata/ingestion/sink/metadata_rest.py b/ingestion/src/metadata/ingestion/sink/metadata_rest.py index cdff7d632dc8..e6d6e3c2014e 100644 --- a/ingestion/src/metadata/ingestion/sink/metadata_rest.py +++ b/ingestion/src/metadata/ingestion/sink/metadata_rest.py @@ -62,6 +62,7 @@ from metadata.ingestion.models.ometa_topic_data import OMetaTopicSampleData from metadata.ingestion.models.patch_request import ( ALLOWED_COMMON_PATCH_FIELDS, + ARRAY_ENTITY_FIELDS, RESTRICT_UPDATE_LIST, PatchedEntity, PatchRequest, @@ -179,6 +180,7 @@ def patch_entity(self, record: PatchRequest) -> Either[Entity]: destination=record.new_entity, allowed_fields=ALLOWED_COMMON_PATCH_FIELDS, restrict_update_fields=RESTRICT_UPDATE_LIST, + array_entity_fields=ARRAY_ENTITY_FIELDS, ) patched_entity = PatchedEntity(new_entity=entity) if entity else None return Either(right=patched_entity) diff --git a/ingestion/src/metadata/ingestion/source/dashboard/looker/utils.py b/ingestion/src/metadata/ingestion/source/dashboard/looker/utils.py index a2e1656992d0..d4b237f75464 100644 --- a/ingestion/src/metadata/ingestion/source/dashboard/looker/utils.py +++ b/ingestion/src/metadata/ingestion/source/dashboard/looker/utils.py @@ -54,14 +54,16 @@ def _clone_repo( return url = None + allow_unsafe_protocols = False if isinstance(credential, GitHubCredentials): url = f"https://x-oauth-basic:{credential.token.__root__.get_secret_value()}@github.com/{repo_name}.git" elif isinstance(credential, BitBucketCredentials): - url = f"https://x-token-auth::{credential.token.__root__.get_secret_value()}@bitbucket.or/{repo_name}.git" + url = f"https://x-token-auth:{credential.token.__root__.get_secret_value()}@bitbucket.org/{repo_name}.git" + allow_unsafe_protocols = True assert url is not None - Repo.clone_from(url, path) + Repo.clone_from(url, path, allow_unsafe_protocols=allow_unsafe_protocols) logger.info(f"repo {repo_name} cloned to {path}") except Exception as exc: diff --git a/ingestion/src/metadata/ingestion/source/dashboard/metabase/client.py b/ingestion/src/metadata/ingestion/source/dashboard/metabase/client.py index dad59df5e4a6..9a0bf81d83fd 100644 --- a/ingestion/src/metadata/ingestion/source/dashboard/metabase/client.py +++ b/ingestion/src/metadata/ingestion/source/dashboard/metabase/client.py @@ -124,6 +124,10 @@ def get_dashboard_details( try: resp_dashboard = self.client.get(f"/dashboard/{dashboard_id}") if resp_dashboard: + # Small hack needed to support Metabase versions older than 0.48 + # https://www.metabase.com/releases/metabase-48#fyi--breaking-changes + if "ordered_cards" in resp_dashboard: + resp_dashboard["dashcards"] = resp_dashboard["ordered_cards"] return MetabaseDashboardDetails(**resp_dashboard) except Exception: logger.debug(traceback.format_exc()) diff --git a/ingestion/src/metadata/ingestion/source/dashboard/metabase/metadata.py b/ingestion/src/metadata/ingestion/source/dashboard/metabase/metadata.py index 677d883acec4..7ed7020bf276 100644 --- a/ingestion/src/metadata/ingestion/source/dashboard/metabase/metadata.py +++ b/ingestion/src/metadata/ingestion/source/dashboard/metabase/metadata.py @@ -178,7 +178,7 @@ def yield_dashboard_chart( Returns: Iterable[CreateChartRequest] """ - charts = dashboard_details.ordered_cards + charts = dashboard_details.dashcards for chart in charts: try: chart_details = chart.card @@ -225,7 +225,7 @@ def yield_dashboard_lineage_details( if not db_service_name: return chart_list, dashboard_name = ( - dashboard_details.ordered_cards, + dashboard_details.dashcards, str(dashboard_details.id), ) for chart in chart_list: diff --git a/ingestion/src/metadata/ingestion/source/dashboard/metabase/models.py b/ingestion/src/metadata/ingestion/source/dashboard/metabase/models.py index 7605ed31574e..fce0d030adc7 100644 --- a/ingestion/src/metadata/ingestion/source/dashboard/metabase/models.py +++ b/ingestion/src/metadata/ingestion/source/dashboard/metabase/models.py @@ -67,7 +67,7 @@ class MetabaseChart(BaseModel): display: Optional[str] -class OrderedCard(BaseModel): +class DashCard(BaseModel): card: MetabaseChart @@ -77,7 +77,7 @@ class MetabaseDashboardDetails(BaseModel): """ description: Optional[str] - ordered_cards: List[OrderedCard] + dashcards: List[DashCard] name: Optional[str] id: int collection_id: Optional[str] diff --git a/ingestion/src/metadata/ingestion/source/dashboard/powerbi/client.py b/ingestion/src/metadata/ingestion/source/dashboard/powerbi/client.py index 900147135924..e72cead1d795 100644 --- a/ingestion/src/metadata/ingestion/source/dashboard/powerbi/client.py +++ b/ingestion/src/metadata/ingestion/source/dashboard/powerbi/client.py @@ -19,6 +19,9 @@ import msal +from metadata.generated.schema.entity.services.connections.dashboard.powerBIConnection import ( + PowerBIConnection, +) from metadata.ingestion.api.steps import InvalidSourceException from metadata.ingestion.ometa.client import REST, ClientConfig from metadata.ingestion.source.dashboard.powerbi.models import ( @@ -52,7 +55,7 @@ class PowerBiApiClient: client: REST - def __init__(self, config): + def __init__(self, config: PowerBIConnection): self.config = config self.msal_client = msal.ConfidentialClientApplication( client_id=self.config.clientId, diff --git a/ingestion/src/metadata/ingestion/source/database/azuresql/connection.py b/ingestion/src/metadata/ingestion/source/database/azuresql/connection.py index 1cb201d7b099..9fd23b2fa19e 100644 --- a/ingestion/src/metadata/ingestion/source/database/azuresql/connection.py +++ b/ingestion/src/metadata/ingestion/source/database/azuresql/connection.py @@ -15,12 +15,13 @@ from typing import Optional, Union from urllib.parse import quote_plus -from sqlalchemy.engine import Engine +from sqlalchemy.engine import URL, Engine from metadata.generated.schema.entity.automations.workflow import ( Workflow as AutomationWorkflow, ) from metadata.generated.schema.entity.services.connections.database.azureSQLConnection import ( + Authentication, AzureSQLConnection, ) from metadata.generated.schema.entity.services.connections.database.mssqlConnection import ( @@ -40,13 +41,29 @@ def get_connection_url(connection: Union[AzureSQLConnection, MssqlConnection]) - Build the connection URL """ + if connection.authenticationMode: + connection_string = f"Driver={connection.driver};Server={connection.hostPort};Database={connection.database};" + connection_string += f"Uid={connection.username};" + if ( + connection.authenticationMode.authentication + == Authentication.ActiveDirectoryPassword + ): + connection_string += f"Pwd={connection.password.get_secret_value()};" + + connection_string += f"Encrypt={'yes' if connection.authenticationMode.encrypt else 'no'};TrustServerCertificate={'yes' if connection.authenticationMode.trustServerCertificate else 'no'};" + connection_string += f"Connection Timeout={connection.authenticationMode.connectionTimeout or 30};Authentication={connection.authenticationMode.authentication.value};" + + connection_url = URL.create( + "mssql+pyodbc", query={"odbc_connect": connection_string} + ) + return connection_url url = f"{connection.scheme.value}://" if connection.username: url += f"{quote_plus(connection.username)}" url += ( f":{quote_plus(connection.password.get_secret_value())}" - if connection + if connection.password else "" ) url += "@" @@ -54,12 +71,13 @@ def get_connection_url(connection: Union[AzureSQLConnection, MssqlConnection]) - url += f"{connection.hostPort}" url += f"/{quote_plus(connection.database)}" if connection.database else "" url += f"?driver={quote_plus(connection.driver)}" + options = get_connection_options_dict(connection) if options: if not connection.database: url += "/" params = "&".join( - f"{key}={quote_plus(value)}" for (key, value) in options.items() if value + f"{key}={quote_plus(value)}" for key, value in options.items() if value ) url = f"{url}?{params}" diff --git a/ingestion/src/metadata/ingestion/source/database/bigquery/metadata.py b/ingestion/src/metadata/ingestion/source/database/bigquery/metadata.py index 28fc26659481..ab090e6e6216 100644 --- a/ingestion/src/metadata/ingestion/source/database/bigquery/metadata.py +++ b/ingestion/src/metadata/ingestion/source/database/bigquery/metadata.py @@ -251,7 +251,9 @@ def _test_connection(self) -> None: test_connection_fn( self.metadata, inspector_details.engine, self.service_connection ) - if os.environ[GOOGLE_CREDENTIALS]: + # GOOGLE_CREDENTIALS may not have been set, + # to avoid key error, we use `get` for dict + if os.environ.get(GOOGLE_CREDENTIALS): self.temp_credentials_file_path.append(os.environ[GOOGLE_CREDENTIALS]) def query_table_names_and_types( @@ -442,7 +444,8 @@ def set_inspector(self, database_name: str): inspector_details = get_inspector_details( database_name=database_name, service_connection=self.service_connection ) - self.temp_credentials_file_path.append(os.environ[GOOGLE_CREDENTIALS]) + if os.environ.get(GOOGLE_CREDENTIALS): + self.temp_credentials_file_path.append(os.environ[GOOGLE_CREDENTIALS]) self.client = inspector_details.client self.engine = inspector_details.engine self.inspector = inspector_details.inspector diff --git a/ingestion/src/metadata/ingestion/source/database/datalake/connection.py b/ingestion/src/metadata/ingestion/source/database/datalake/connection.py index 56e5315da5ba..c5be85eb5a13 100644 --- a/ingestion/src/metadata/ingestion/source/database/datalake/connection.py +++ b/ingestion/src/metadata/ingestion/source/database/datalake/connection.py @@ -20,6 +20,7 @@ from google.cloud import storage +from metadata.clients.azure_client import AzureClient from metadata.generated.schema.entity.automations.workflow import ( Workflow as AutomationWorkflow, ) @@ -88,22 +89,9 @@ def _(config: GCSConfig): @get_datalake_client.register def _(config: AzureConfig): - from azure.identity import ClientSecretCredential - from azure.storage.blob import BlobServiceClient try: - credentials = ClientSecretCredential( - config.securityConfig.tenantId, - config.securityConfig.clientId, - config.securityConfig.clientSecret.get_secret_value(), - ) - - azure_client = BlobServiceClient( - f"https://{config.securityConfig.accountName}.blob.core.windows.net/", - credential=credentials, - ) - return azure_client - + return AzureClient(config.securityConfig).create_blob_client() except Exception as exc: raise RuntimeError( f"Unknown error connecting with {config.securityConfig}: {exc}." diff --git a/ingestion/src/metadata/ingestion/source/database/datalake/metadata.py b/ingestion/src/metadata/ingestion/source/database/datalake/metadata.py index a32a2a9aa7ec..727898801832 100644 --- a/ingestion/src/metadata/ingestion/source/database/datalake/metadata.py +++ b/ingestion/src/metadata/ingestion/source/database/datalake/metadata.py @@ -407,7 +407,7 @@ def yield_table( schema_name = self.context.database_schema try: table_constraints = None - data_frame = fetch_dataframe( + data_frame, raw_data = fetch_dataframe( config_source=self.config_source, client=self.client, file_fqn=DatalakeTableSchemaWrapper( @@ -415,10 +415,11 @@ def yield_table( bucket_name=schema_name, file_extension=table_extension, ), + fetch_raw_data=True, ) if data_frame: column_parser = DataFrameColumnParser.create( - data_frame[0], table_extension + data_frame[0], table_extension, raw_data=raw_data ) columns = column_parser.get_columns() else: diff --git a/ingestion/src/metadata/ingestion/source/database/dbt/dbt_config.py b/ingestion/src/metadata/ingestion/source/database/dbt/dbt_config.py index c130494c73c6..79e9bc1ec2e8 100644 --- a/ingestion/src/metadata/ingestion/source/database/dbt/dbt_config.py +++ b/ingestion/src/metadata/ingestion/source/database/dbt/dbt_config.py @@ -20,6 +20,7 @@ import requests from metadata.clients.aws_client import AWSClient +from metadata.clients.azure_client import AzureClient from metadata.generated.schema.metadataIngestion.dbtconfig.dbtAzureConfig import ( DbtAzureConfig, ) @@ -172,7 +173,7 @@ def _(config: DbtCloudConfig): # pylint: disable=too-many-locals params_data["job_definition_id"] = job_id response = client.get(f"/accounts/{account_id}/runs", data=params_data) - if not response and not response.get("data"): + if not response or not response.get("data"): raise DBTConfigException( "Unable to get the dbt job runs information.\n" "Please check if the auth token is correct and has the necessary scopes to fetch dbt runs" @@ -357,21 +358,8 @@ def _(config: DbtGcsConfig): def _(config: DbtAzureConfig): try: bucket_name, prefix = get_dbt_prefix_config(config) - from azure.identity import ( # pylint: disable=import-outside-toplevel - ClientSecretCredential, - ) - from azure.storage.blob import ( # pylint: disable=import-outside-toplevel - BlobServiceClient, - ) - client = BlobServiceClient( - f"https://{config.dbtSecurityConfig.accountName}.blob.core.windows.net/", - credential=ClientSecretCredential( - config.dbtSecurityConfig.tenantId, - config.dbtSecurityConfig.clientId, - config.dbtSecurityConfig.clientSecret.get_secret_value(), - ), - ) + client = AzureClient(config.dbtSecurityConfig).create_blob_client() if not bucket_name: container_dicts = client.list_containers() diff --git a/ingestion/src/metadata/ingestion/source/database/mssql/lineage.py b/ingestion/src/metadata/ingestion/source/database/mssql/lineage.py index 0f22eddf2993..4a112563d354 100644 --- a/ingestion/src/metadata/ingestion/source/database/mssql/lineage.py +++ b/ingestion/src/metadata/ingestion/source/database/mssql/lineage.py @@ -27,5 +27,6 @@ class MssqlLineageSource(MssqlQueryParserSource, LineageSource): OR lower(t.text) LIKE '%%merge%%' ) AND lower(t.text) NOT LIKE '%%create%%procedure%%' + AND lower(t.text) NOT LIKE '%%create%%function%%' AND lower(t.text) NOT LIKE '%%declare%%' """ diff --git a/ingestion/src/metadata/ingestion/source/database/mssql/usage.py b/ingestion/src/metadata/ingestion/source/database/mssql/usage.py index acb131e931f5..5a9aabee2a95 100644 --- a/ingestion/src/metadata/ingestion/source/database/mssql/usage.py +++ b/ingestion/src/metadata/ingestion/source/database/mssql/usage.py @@ -19,4 +19,8 @@ class MssqlUsageSource(MssqlQueryParserSource, UsageSource): sql_stmt = MSSQL_SQL_STATEMENT - filters = "" # No filtering in the queries + filters = """ + AND lower(t.text) NOT LIKE '%%create%%procedure%%' + AND lower(t.text) NOT LIKE '%%create%%function%%' + AND lower(t.text) NOT LIKE '%%declare%%' + """ diff --git a/ingestion/src/metadata/ingestion/source/database/mysql/connection.py b/ingestion/src/metadata/ingestion/source/database/mysql/connection.py index dca28eefbaef..f5e6e61d5f40 100644 --- a/ingestion/src/metadata/ingestion/source/database/mysql/connection.py +++ b/ingestion/src/metadata/ingestion/source/database/mysql/connection.py @@ -16,9 +16,13 @@ from sqlalchemy.engine import Engine +from metadata.clients.azure_client import AzureClient from metadata.generated.schema.entity.automations.workflow import ( Workflow as AutomationWorkflow, ) +from metadata.generated.schema.entity.services.connections.database.common.basicAuth import ( + BasicAuth, +) from metadata.generated.schema.entity.services.connections.database.mysqlConnection import ( MysqlConnection, ) @@ -38,6 +42,16 @@ def get_connection(connection: MysqlConnection) -> Engine: """ Create connection """ + if hasattr(connection.authType, "azureConfig"): + azure_client = AzureClient(connection.authType.azureConfig).create_client() + if not connection.authType.azureConfig.scopes: + raise ValueError( + "Azure Scopes are missing, please refer https://learn.microsoft.com/en-gb/azure/mysql/flexible-server/how-to-azure-ad#2---retrieve-microsoft-entra-access-token and fetch the resource associated with it, for e.g. https://ossrdbms-aad.database.windows.net/.default" + ) + access_token_obj = azure_client.get_token( + *connection.authType.azureConfig.scopes.split(",") + ) + connection.authType = BasicAuth(password=access_token_obj.token) if connection.sslCA or connection.sslCert or connection.sslKey: if not connection.connectionOptions: connection.connectionOptions = init_empty_connection_options() diff --git a/ingestion/src/metadata/ingestion/source/database/oracle/connection.py b/ingestion/src/metadata/ingestion/source/database/oracle/connection.py index 5a98d7a5197e..324aa5def4f7 100644 --- a/ingestion/src/metadata/ingestion/source/database/oracle/connection.py +++ b/ingestion/src/metadata/ingestion/source/database/oracle/connection.py @@ -38,6 +38,7 @@ ) from metadata.ingestion.connections.test_connections import test_connection_db_common from metadata.ingestion.ometa.ometa_api import OpenMetadata +from metadata.ingestion.source.database.oracle.queries import CHECK_ACCESS_TO_DBA from metadata.utils.logger import ingestion_logger CX_ORACLE_LIB_VERSION = "8.3.0" @@ -136,9 +137,13 @@ def test_connection( Test connection. This can be executed either as part of a metadata workflow or during an Automation Workflow """ + + test_conn_queries = {"CheckAccess": CHECK_ACCESS_TO_DBA} + test_connection_db_common( metadata=metadata, engine=engine, service_connection=service_connection, automation_workflow=automation_workflow, + queries=test_conn_queries, ) diff --git a/ingestion/src/metadata/ingestion/source/database/oracle/queries.py b/ingestion/src/metadata/ingestion/source/database/oracle/queries.py index d69be9a2d137..3540946aa2e2 100644 --- a/ingestion/src/metadata/ingestion/source/database/oracle/queries.py +++ b/ingestion/src/metadata/ingestion/source/database/oracle/queries.py @@ -88,7 +88,7 @@ type = 'PROCEDURE' and owner = '{schema}' """ ) - +CHECK_ACCESS_TO_DBA = "SELECT table_name FROM DBA_TABLES where ROWNUM < 2" ORACLE_GET_STORED_PROCEDURE_QUERIES = textwrap.dedent( """ WITH SP_HISTORY AS (SELECT diff --git a/ingestion/src/metadata/ingestion/source/database/postgres/connection.py b/ingestion/src/metadata/ingestion/source/database/postgres/connection.py index 2b34896cd0e2..3427fdb0e5f1 100644 --- a/ingestion/src/metadata/ingestion/source/database/postgres/connection.py +++ b/ingestion/src/metadata/ingestion/source/database/postgres/connection.py @@ -17,9 +17,13 @@ from sqlalchemy.engine import Engine +from metadata.clients.azure_client import AzureClient from metadata.generated.schema.entity.automations.workflow import ( Workflow as AutomationWorkflow, ) +from metadata.generated.schema.entity.services.connections.database.common.basicAuth import ( + BasicAuth, +) from metadata.generated.schema.entity.services.connections.database.postgresConnection import ( PostgresConnection, SslMode, @@ -46,6 +50,17 @@ def get_connection(connection: PostgresConnection) -> Engine: """ Create connection """ + + if hasattr(connection.authType, "azureConfig"): + azure_client = AzureClient(connection.authType.azureConfig).create_client() + if not connection.authType.azureConfig.scopes: + raise ValueError( + "Azure Scopes are missing, please refer https://learn.microsoft.com/en-gb/azure/postgresql/flexible-server/how-to-configure-sign-in-azure-ad-authentication#retrieve-the-microsoft-entra-access-token and fetch the resource associated with it, for e.g. https://ossrdbms-aad.database.windows.net/.default" + ) + access_token_obj = azure_client.get_token( + *connection.authType.azureConfig.scopes.split(",") + ) + connection.authType = BasicAuth(password=access_token_obj.token) if connection.sslMode: if not connection.connectionArguments: connection.connectionArguments = init_empty_connection_arguments() diff --git a/ingestion/src/metadata/ingestion/source/database/stored_procedures_mixin.py b/ingestion/src/metadata/ingestion/source/database/stored_procedures_mixin.py index 199f01d3efbf..057fdf5aee74 100644 --- a/ingestion/src/metadata/ingestion/source/database/stored_procedures_mixin.py +++ b/ingestion/src/metadata/ingestion/source/database/stored_procedures_mixin.py @@ -138,7 +138,7 @@ def is_lineage_query(query_type: str, query_text: str) -> bool: return True if query_type == "INSERT" and re.search( - "^.*insert.*into.*select.*$", query_text, re.IGNORECASE + "^.*insert.*into.*select.*$", query_text.replace("\n", " "), re.IGNORECASE ): return True diff --git a/ingestion/src/metadata/ingestion/source/database/unitycatalog/connection.py b/ingestion/src/metadata/ingestion/source/database/unitycatalog/connection.py index 1035a0fc398a..2ba24d3ac328 100644 --- a/ingestion/src/metadata/ingestion/source/database/unitycatalog/connection.py +++ b/ingestion/src/metadata/ingestion/source/database/unitycatalog/connection.py @@ -68,16 +68,20 @@ def get_catalogs(connection: WorkspaceClient, table_obj: DatabricksTable): break def get_schemas(connection: WorkspaceClient, table_obj: DatabricksTable): - for schema in connection.schemas.list(catalog_name=table_obj.catalog_name): - table_obj.schema_name = schema.name - break + for catalog in connection.catalogs.list(): + for schema in connection.schemas.list(catalog_name=catalog.name): + if schema.name: + table_obj.schema_name = schema.name + table_obj.catalog_name = catalog.name + return def get_tables(connection: WorkspaceClient, table_obj: DatabricksTable): - for table in connection.tables.list( - catalog_name=table_obj.catalog_name, schema_name=table_obj.schema_name - ): - table_obj.name = table.name - break + if table_obj.catalog_name and table_obj.schema_name: + for table in connection.tables.list( + catalog_name=table_obj.catalog_name, schema_name=table_obj.schema_name + ): + table_obj.name = table.name + break test_fn = { "CheckAccess": connection.catalogs.list, diff --git a/ingestion/src/metadata/ingestion/source/pipeline/dagster/metadata.py b/ingestion/src/metadata/ingestion/source/pipeline/dagster/metadata.py index eeec4d1d5c5f..5fb0d36793d4 100644 --- a/ingestion/src/metadata/ingestion/source/pipeline/dagster/metadata.py +++ b/ingestion/src/metadata/ingestion/source/pipeline/dagster/metadata.py @@ -214,7 +214,9 @@ def yield_pipeline_status( service_name=self.context.pipeline_service, pipeline_name=self.context.pipeline, ) - pipeline_entity = self.metadata.get_by_name(entity=Pipeline, fqn=pipeline_fqn) + pipeline_entity = self.metadata.get_by_name( + entity=Pipeline, fqn=pipeline_fqn, fields=["tasks"] + ) for task in pipeline_entity.tasks or []: try: runs = self.client.get_task_runs( diff --git a/ingestion/src/metadata/ingestion/source/storage/storage_service.py b/ingestion/src/metadata/ingestion/source/storage/storage_service.py index 2c3f9a4dad18..6b3e2eba33df 100644 --- a/ingestion/src/metadata/ingestion/source/storage/storage_service.py +++ b/ingestion/src/metadata/ingestion/source/storage/storage_service.py @@ -260,7 +260,7 @@ def extract_column_definitions( metadata_entry: MetadataEntry, ) -> List[Column]: """Extract Column related metadata from s3""" - data_structure_details = fetch_dataframe( + data_structure_details, raw_data = fetch_dataframe( config_source=config_source, client=client, file_fqn=DatalakeTableSchemaWrapper( @@ -269,10 +269,13 @@ def extract_column_definitions( file_extension=SupportedTypes(metadata_entry.structureFormat), separator=metadata_entry.separator, ), + fetch_raw_data=True, ) columns = [] column_parser = DataFrameColumnParser.create( - data_structure_details, SupportedTypes(metadata_entry.structureFormat) + data_structure_details, + SupportedTypes(metadata_entry.structureFormat), + raw_data=raw_data, ) columns = column_parser.get_columns() return columns diff --git a/ingestion/src/metadata/parsers/json_schema_parser.py b/ingestion/src/metadata/parsers/json_schema_parser.py index 818fc27fbce3..f56edce4fc61 100644 --- a/ingestion/src/metadata/parsers/json_schema_parser.py +++ b/ingestion/src/metadata/parsers/json_schema_parser.py @@ -18,6 +18,8 @@ from enum import Enum from typing import List, Optional +from pydantic.main import ModelMetaclass + from metadata.generated.schema.type.schema import FieldModel from metadata.utils.logger import ingestion_logger @@ -36,20 +38,25 @@ class JsonSchemaDataTypes(Enum): NULL = "null" RECORD = "object" ARRAY = "array" + UNKNOWN = "unknown" -def parse_json_schema(schema_text: str) -> Optional[List[FieldModel]]: +def parse_json_schema( + schema_text: str, cls: ModelMetaclass = FieldModel +) -> Optional[List[FieldModel]]: """ Method to parse the jsonschema """ try: json_schema_data = json.loads(schema_text) field_models = [ - FieldModel( + cls( name=json_schema_data.get("title", "default"), dataType=JsonSchemaDataTypes(json_schema_data.get("type")).name, description=json_schema_data.get("description"), - children=get_json_schema_fields(json_schema_data.get("properties")), + children=get_json_schema_fields( + json_schema_data.get("properties", {}), cls=cls + ), ) ] return field_models @@ -59,7 +66,9 @@ def parse_json_schema(schema_text: str) -> Optional[List[FieldModel]]: return None -def get_json_schema_fields(properties) -> Optional[List[FieldModel]]: +def get_json_schema_fields( + properties, cls: ModelMetaclass = FieldModel +) -> Optional[List[FieldModel]]: """ Recursively convert the parsed schema into required models """ @@ -67,9 +76,10 @@ def get_json_schema_fields(properties) -> Optional[List[FieldModel]]: for key, value in properties.items(): try: field_models.append( - FieldModel( - name=value.get("title", key), - dataType=JsonSchemaDataTypes(value.get("type")).name, + cls( + name=key, + displayName=value.get("title"), + dataType=JsonSchemaDataTypes(value.get("type", "unknown")).name, description=value.get("description"), children=get_json_schema_fields(value.get("properties")) if value.get("type") == "object" diff --git a/ingestion/src/metadata/pii/scanners/ner_scanner.py b/ingestion/src/metadata/pii/scanners/ner_scanner.py index 6ce29a0740fa..c177a0af7673 100644 --- a/ingestion/src/metadata/pii/scanners/ner_scanner.py +++ b/ingestion/src/metadata/pii/scanners/ner_scanner.py @@ -21,7 +21,7 @@ from metadata.generated.schema.entity.classification.tag import Tag from metadata.pii.constants import PII, SPACY_EN_MODEL -from metadata.pii.models import TagAndConfidence, TagType +from metadata.pii.models import TagAndConfidence from metadata.pii.ner import NEREntity from metadata.utils import fqn from metadata.utils.logger import pii_logger @@ -119,13 +119,15 @@ def scan(self, sample_data_rows: List[Any]) -> Optional[TagAndConfidence]: if entities_score: label, score = self.get_highest_score_label(entities_score) - tag_type = NEREntity.__members__.get(label, TagType.NONSENSITIVE).value + tag_type = NEREntity.__members__.get(label) + if not tag_type: + return None return TagAndConfidence( tag_fqn=fqn.build( metadata=None, entity_type=Tag, classification_name=PII, - tag_name=tag_type, + tag_name=tag_type.value, ), confidence=score, ) diff --git a/ingestion/src/metadata/profiler/__init__.py b/ingestion/src/metadata/profiler/__init__.py deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/ingestion/src/metadata/profiler/adaptors/__init__.py b/ingestion/src/metadata/profiler/adaptors/__init__.py deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/ingestion/src/metadata/profiler/adaptors/adaptor_factory.py b/ingestion/src/metadata/profiler/adaptors/adaptor_factory.py deleted file mode 100644 index fb06b9969bb2..000000000000 --- a/ingestion/src/metadata/profiler/adaptors/adaptor_factory.py +++ /dev/null @@ -1,39 +0,0 @@ -# Copyright 2024 Collate -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# http://www.apache.org/licenses/LICENSE-2.0 -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" -factory for NoSQL adaptors that are used in the NoSQLProfiler. -""" - -from metadata.generated.schema.entity.services.connections.database.mongoDBConnection import ( - MongoDBConnection, -) -from metadata.profiler.adaptors.mongodb import MongoDB -from metadata.profiler.factory import Factory -from metadata.utils.logger import profiler_logger - -logger = profiler_logger() - - -class NoSQLAdaptorFactory(Factory): - def create(self, interface_type: str, *args, **kwargs) -> any: - logger.debug(f"Creating NoSQL client for {interface_type}") - client_class = self._interface_type.get(interface_type) - if not client_class: - raise ValueError(f"Unknown NoSQL source: {interface_type}") - logger.debug(f"Using NoSQL client constructor: {client_class.__name__}") - return client_class(*args, **kwargs) - - -adaptors = profilers = { - MongoDBConnection.__name__: MongoDB, -} -factory = NoSQLAdaptorFactory() -factory.register_many(adaptors) diff --git a/ingestion/src/metadata/profiler/adaptors/mongodb.py b/ingestion/src/metadata/profiler/adaptors/mongodb.py deleted file mode 100644 index 22194535bb91..000000000000 --- a/ingestion/src/metadata/profiler/adaptors/mongodb.py +++ /dev/null @@ -1,175 +0,0 @@ -# Copyright 2024 Collate -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# http://www.apache.org/licenses/LICENSE-2.0 -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" -MongoDB adaptor for the NoSQL profiler. -""" -import json -from enum import Enum -from typing import TYPE_CHECKING, Dict, List, Optional, Union - -from pydantic import BaseModel, Field -from pymongo.command_cursor import CommandCursor -from pymongo.cursor import Cursor - -from metadata.generated.schema.entity.data.table import Column, Table -from metadata.profiler.adaptors.nosql_adaptor import NoSQLAdaptor -from metadata.utils.sqa_like_column import SQALikeColumn - -if TYPE_CHECKING: - from pymongo import MongoClient -else: - MongoClient = None # pylint: disable=invalid-name - - -class AggregationFunction(Enum): - SUM = "$sum" - MEAN = "$avg" - COUNT = "$count" - MAX = "$max" - MIN = "$min" - - -class Executable(BaseModel): - def to_executable(self, client: MongoClient) -> Union[CommandCursor, Cursor]: - raise NotImplementedError - - -class Query(Executable): - database: str - collection: str - filter: dict = Field(default_factory=dict) - limit: Optional[int] = None - - def to_executable(self, client: MongoClient) -> Cursor: - db = client[self.database] - collection = db[self.collection] - query = collection.find(self.filter) - if self.limit: - query = query.limit(self.limit) - return query - - -class Aggregation(Executable): - database: str - collection: str - column: str - aggregations: List[AggregationFunction] - - def to_executable(self, client: MongoClient) -> CommandCursor: - db = client[self.database] - collection = db[self.collection] - return collection.aggregate( - [ - { - "$group": { - "_id": None, - **{ - a.name.lower(): {a.value: f"${self.column}"} - for a in self.aggregations - }, - } - } - ] - ) - - -class MongoDB(NoSQLAdaptor): - """A MongoDB client that serves as an adaptor for profiling data assets on MongoDB""" - - def __init__(self, client: MongoClient): - self.client = client - - def item_count(self, table: Table) -> int: - db = self.client[table.databaseSchema.name] - collection = db[table.name.__root__] - return collection.count_documents({}) - - def scan( - self, table: Table, columns: List[Column], limit: int - ) -> List[Dict[str, any]]: - return self.execute( - Query( - database=table.databaseSchema.name, - collection=table.name.__root__, - limit=limit, - ) - ) - - def query( - self, table: Table, columns: List[Column], query: any, limit: int - ) -> List[Dict[str, any]]: - try: - json_query = json.loads(query) - except json.JSONDecodeError: - raise ValueError("Invalid JSON query") - return self.execute( - Query( - database=table.databaseSchema.name, - collection=table.name.__root__, - filter=json_query, - ) - ) - - def get_aggregates( - self, - table: Table, - column: SQALikeColumn, - aggregate_functions: List[AggregationFunction], - ) -> Dict[str, Union[int, float]]: - """ - Get the aggregate functions for a column in a table - Returns: - Dict[str, Union[int, float]]: A dictionary of the aggregate functions - Example: - { - "sum": 100, - "avg": 50, - "count": 2, - "max": 75, - "min": 25 - } - """ - row = self.execute( - Aggregation( - database=table.databaseSchema.name, - collection=table.name.__root__, - column=column.name, - aggregations=aggregate_functions, - ) - )[0] - return {k: v for k, v in row.items() if k != "_id"} - - def sum(self, table: Table, column: SQALikeColumn) -> AggregationFunction: - return AggregationFunction.SUM - - def mean(self, table: Table, column: SQALikeColumn) -> AggregationFunction: - return AggregationFunction.MEAN - - def max(self, table: Table, column: SQALikeColumn) -> AggregationFunction: - return AggregationFunction.MAX - - def min(self, table: Table, column: SQALikeColumn) -> AggregationFunction: - return AggregationFunction.MIN - - def execute(self, query: Executable) -> List[Dict[str, any]]: - records = list(query.to_executable(self.client)) - result = [] - for r in records: - result.append({c: self._json_safe(r.get(c)) for c in r}) - return result - - @staticmethod - def _json_safe(data: any): - try: - json.dumps(data) - return data - except Exception: # noqa - return str(data) diff --git a/ingestion/src/metadata/profiler/adaptors/nosql_adaptor.py b/ingestion/src/metadata/profiler/adaptors/nosql_adaptor.py deleted file mode 100644 index 4a78100c5686..000000000000 --- a/ingestion/src/metadata/profiler/adaptors/nosql_adaptor.py +++ /dev/null @@ -1,65 +0,0 @@ -# Copyright 2024 Collate -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# http://www.apache.org/licenses/LICENSE-2.0 -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" -NoSQL adaptor for the NoSQL profiler. -""" -from abc import ABC, abstractmethod -from typing import Dict, List, Union - -from metadata.generated.schema.entity.data.table import Column, Table -from metadata.utils.sqa_like_column import SQALikeColumn - - -class NoSQLAdaptor(ABC): - """ - NoSQL adaptor for the NoSQL profiler. This class implememts the required methods for retreiving data from a NoSQL - database. - """ - - @abstractmethod - def item_count(self, table: Table) -> int: - raise NotImplementedError - - @abstractmethod - def scan( - self, table: Table, columns: List[Column], limit: int - ) -> List[Dict[str, any]]: - pass - - def query( - self, table: Table, columns: List[Column], query: any, limit: int - ) -> List[Dict[str, any]]: - raise NotImplementedError - - def get_aggregates( - self, table: Table, column: SQALikeColumn, aggregate_functions: List[any] - ) -> Dict[str, Union[int, float]]: - raise NotImplementedError - - def sum( - self, table: Table, column: Column # pylint: disable=unused-argument - ) -> any: - return None - - def mean( - self, table: Table, column: Column # pylint: disable=unused-argument - ) -> any: - return None - - def max( - self, table: Table, column: Column # pylint: disable=unused-argument - ) -> any: - return None - - def min( - self, table: Table, column: Column # pylint: disable=unused-argument - ) -> any: - return None diff --git a/ingestion/src/metadata/profiler/api/models.py b/ingestion/src/metadata/profiler/api/models.py index 499f9b4f149d..961d6fcd1280 100644 --- a/ingestion/src/metadata/profiler/api/models.py +++ b/ingestion/src/metadata/profiler/api/models.py @@ -122,7 +122,7 @@ def __str__(self): class ThreadPoolMetrics(ConfigModel): - """A container for all metrics to be computed on the same thread.""" + """thread pool metric""" metrics: Union[List[Union[Type[Metric], CustomMetric]], Type[Metric]] metric_type: MetricTypes diff --git a/ingestion/src/metadata/profiler/factory.py b/ingestion/src/metadata/profiler/factory.py deleted file mode 100644 index fa89590401b7..000000000000 --- a/ingestion/src/metadata/profiler/factory.py +++ /dev/null @@ -1,41 +0,0 @@ -# Copyright 2021 Collate -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# http://www.apache.org/licenses/LICENSE-2.0 -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -""" -Factory class for creating profiler interface objects -""" -from abc import ABC, abstractmethod - - -class Factory(ABC): - """Creational factory for interface objects""" - - def __init__(self): - self._interface_type = {} - - def register(self, interface_type: str, interface_class): - """Register a new interface""" - self._interface_type[interface_type] = interface_class - - def register_many(self, interface_dict): - """ - Registers multiple profiler interfaces at once. - - Args: - interface_dict: A dictionary mapping connection class names (strings) to their - corresponding profiler interface classes. - """ - for interface_type, interface_class in interface_dict.items(): - self.register(interface_type, interface_class) - - @abstractmethod - def create(self, interface_type: str, *args, **kwargs) -> any: - pass diff --git a/ingestion/src/metadata/profiler/interface/nosql/profiler_interface.py b/ingestion/src/metadata/profiler/interface/nosql/profiler_interface.py deleted file mode 100644 index 852f88a70e29..000000000000 --- a/ingestion/src/metadata/profiler/interface/nosql/profiler_interface.py +++ /dev/null @@ -1,231 +0,0 @@ -# Copyright 2021 Collate -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# http://www.apache.org/licenses/LICENSE-2.0 -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# pylint: disable=arguments-differ - -""" -Interfaces with database for all database engine -supporting sqlalchemy abstraction layer -""" -import traceback -from collections import defaultdict -from datetime import datetime, timezone -from typing import Dict, List, Optional, Type - -from sqlalchemy import Column - -from metadata.generated.schema.entity.data.table import TableData -from metadata.generated.schema.tests.customMetric import CustomMetric -from metadata.profiler.adaptors.adaptor_factory import factory -from metadata.profiler.adaptors.nosql_adaptor import NoSQLAdaptor -from metadata.profiler.api.models import ThreadPoolMetrics -from metadata.profiler.interface.profiler_interface import ProfilerInterface -from metadata.profiler.metrics.core import Metric, MetricTypes -from metadata.profiler.metrics.registry import Metrics -from metadata.profiler.processor.sampler.nosql.sampler import NoSQLSampler -from metadata.utils.logger import profiler_interface_registry_logger -from metadata.utils.sqa_like_column import SQALikeColumn - -logger = profiler_interface_registry_logger() - - -class NoSQLProfilerInterface(ProfilerInterface): - """ - Interface to interact with registry supporting - sqlalchemy. - """ - - # pylint: disable=too-many-arguments - - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - self.sampler = self._get_sampler() - - def _compute_table_metrics( - self, - metrics: List[Type[Metric]], - runner: NoSQLAdaptor, - *args, - **kwargs, - ): - result = {} - for metric in metrics: - try: - fn = metric().nosql_fn(runner) - result[metric.name()] = fn(self.table) - except Exception as exc: - logger.debug( - f"{traceback.format_exc()}\n" - f"Error trying to compute metric {metric} for {self.table.fullyQualifiedName}: {exc}" - ) - raise RuntimeError( - f"Error trying to compute metric {metric.name()} for {self.table.fullyQualifiedName}: {exc}" - ) - return result - - def _compute_static_metrics( - self, - metrics: List[Metrics], - runner: NoSQLAdaptor, - column: SQALikeColumn, - *args, - **kwargs, - ) -> Dict[str, any]: - try: - aggs = [metric(column).nosql_fn(runner)(self.table) for metric in metrics] - filtered = [agg for agg in aggs if agg is not None] - if not filtered: - return {} - row = runner.get_aggregates(self.table, column, filtered) - return dict(row) - except Exception as exc: - logger.debug( - f"{traceback.format_exc()}\n" - f"Error trying to compute metrics for {self.table.fullyQualifiedName}: {exc}" - ) - raise RuntimeError( - f"Error trying to compute metris for {self.table.fullyQualifiedName}: {exc}" - ) - - def _compute_query_metrics( - self, - metric: Metrics, - runner, - *args, - **kwargs, - ): - return None - - def _compute_window_metrics( - self, - metrics: List[Metrics], - runner, - *args, - **kwargs, - ): - return None - - def _compute_system_metrics( - self, - metrics: Metrics, - runner: List, - *args, - **kwargs, - ): - return None - - def _compute_custom_metrics( - self, metrics: List[CustomMetric], runner, *args, **kwargs - ): - return None - - def compute_metrics( - self, - client: NoSQLAdaptor, - metric_func: ThreadPoolMetrics, - ): - """Run metrics in processor worker""" - logger.debug(f"Running profiler for {metric_func.table}") - try: - row = self._get_metric_fn[metric_func.metric_type.value]( - metric_func.metrics, - client, - column=metric_func.column, - ) - except Exception as exc: - name = f"{metric_func.column if metric_func.column is not None else metric_func.table}" - error = f"{name} metric_type.value: {exc}" - logger.error(error) - self.status.failed_profiler(error, traceback.format_exc()) - row = None - if metric_func.column is not None: - column = metric_func.column.name - self.status.scanned(f"{metric_func.table.name.__root__}.{column}") - else: - self.status.scanned(metric_func.table.name.__root__) - column = None - return row, column, metric_func.metric_type.value - - def fetch_sample_data(self, table, columns: List[SQALikeColumn]) -> TableData: - return self.sampler.fetch_sample_data(columns) - - def _get_sampler(self) -> NoSQLSampler: - """Get NoSQL sampler from config""" - from metadata.profiler.processor.sampler.sampler_factory import ( # pylint: disable=import-outside-toplevel - sampler_factory_, - ) - - return sampler_factory_.create( - self.service_connection_config.__class__.__name__, - table=self.table, - client=factory.create( - self.service_connection_config.__class__.__name__, self.connection - ), - profile_sample_config=self.profile_sample_config, - partition_details=self.partition_details, - profile_sample_query=self.profile_query, - ) - - def get_composed_metrics( - self, column: Column, metric: Metrics, column_results: Dict - ): - return None - - def get_hybrid_metrics( - self, column: Column, metric: Metrics, column_results: Dict, **kwargs - ): - return None - - def get_all_metrics( - self, - metric_funcs: List[ThreadPoolMetrics], - ): - """get all profiler metrics""" - profile_results = {"table": {}, "columns": defaultdict(dict)} - runner = factory.create( - self.service_connection_config.__class__.__name__, self.connection - ) - metric_list = [ - self.compute_metrics(runner, metric_func) for metric_func in metric_funcs - ] - for metric_result in metric_list: - profile, column, metric_type = metric_result - if profile: - if metric_type == MetricTypes.Table.value: - profile_results["table"].update(profile) - if metric_type == MetricTypes.System.value: - profile_results["system"] = profile - elif metric_type == MetricTypes.Custom.value and column is None: - profile_results["table"].update(profile) - else: - profile_results["columns"][column].update( - { - "name": column, - "timestamp": int( - datetime.now(tz=timezone.utc).timestamp() * 1000 - ), - **profile, - } - ) - return profile_results - - @property - def table(self): - """OM Table entity""" - return self.table_entity - - def get_columns(self) -> List[Optional[SQALikeColumn]]: - return [ - SQALikeColumn(name=c.name.__root__, type=c.dataType) - for c in self.table.columns - ] - - def close(self): - self.connection.close() diff --git a/ingestion/src/metadata/profiler/interface/profiler_interface.py b/ingestion/src/metadata/profiler/interface/profiler_interface.py index 8dc5f330aba3..e1881d1806ec 100644 --- a/ingestion/src/metadata/profiler/interface/profiler_interface.py +++ b/ingestion/src/metadata/profiler/interface/profiler_interface.py @@ -33,7 +33,7 @@ TableData, ) from metadata.generated.schema.entity.services.connections.connectionBasicType import ( - SampleDataStorageConfig, + DataStorageConfig, ) from metadata.generated.schema.entity.services.connections.database.datalakeConnection import ( DatalakeConnection, @@ -93,7 +93,7 @@ def __init__( service_connection_config: Union[DatabaseConnection, DatalakeConnection], ometa_client: OpenMetadata, entity: Table, - storage_config: SampleDataStorageConfig, + storage_config: DataStorageConfig, profile_sample_config: Optional[ProfileSampleConfig], source_config: DatabaseServiceProfilerPipeline, sample_query: Optional[str], @@ -248,7 +248,7 @@ def _get_sample_storage_config( DatabaseProfilerConfig, DatabaseAndSchemaConfig, ] - ): + ) -> Optional[DataStorageConfig]: if ( config and config.sampleDataStorageConfig @@ -264,7 +264,7 @@ def get_storage_config_for_table( database_profiler_config: Optional[DatabaseProfilerConfig], db_service: Optional[DatabaseService], profiler_config: ProfilerProcessorConfig, - ) -> Optional[SampleDataStorageConfig]: + ) -> Optional[DataStorageConfig]: """Get config for a specific entity Args: @@ -425,12 +425,8 @@ def _compute_static_metrics( runner, *args, **kwargs, - ) -> Dict[str, Any]: - """Get metrics - Return: - Dict[str, Any]: dict of metrics tio be merged into the final column profile. Keys need to be compatible with - the `metadata.generated.schema.entity.data.table.ColumnProfile` schema. - """ + ): + """Get metrics""" raise NotImplementedError @abstractmethod diff --git a/ingestion/src/metadata/profiler/interface/profiler_interface_factory.py b/ingestion/src/metadata/profiler/interface/profiler_interface_factory.py index 3a03a921a9d2..2733d0a8c4cb 100644 --- a/ingestion/src/metadata/profiler/interface/profiler_interface_factory.py +++ b/ingestion/src/metadata/profiler/interface/profiler_interface_factory.py @@ -30,9 +30,6 @@ from metadata.generated.schema.entity.services.connections.database.mariaDBConnection import ( MariaDBConnection, ) -from metadata.generated.schema.entity.services.connections.database.mongoDBConnection import ( - MongoDBConnection, -) from metadata.generated.schema.entity.services.connections.database.singleStoreConnection import ( SingleStoreConnection, ) @@ -46,8 +43,6 @@ UnityCatalogConnection, ) from metadata.generated.schema.entity.services.databaseService import DatabaseConnection -from metadata.profiler.factory import Factory -from metadata.profiler.interface.nosql.profiler_interface import NoSQLProfilerInterface from metadata.profiler.interface.pandas.profiler_interface import ( PandasProfilerInterface, ) @@ -81,7 +76,27 @@ ) -class ProfilerInterfaceFactory(Factory): +class ProfilerInterfaceFactory: + """Creational factory for profiler interface objects""" + + def __init__(self): + self._interface_type = {} + + def register(self, interface_type: str, interface_class): + """Register a new interface""" + self._interface_type[interface_type] = interface_class + + def register_many(self, interface_dict): + """ + Registers multiple profiler interfaces at once. + + Args: + interface_dict: A dictionary mapping connection class names (strings) to their + corresponding profiler interface classes. + """ + for interface_type, interface_class in interface_dict.items(): + self.register(interface_type, interface_class) + def create(self, interface_type: str, *args, **kwargs): """Create interface object based on interface type""" interface_class = self._interface_type.get(interface_type) @@ -103,6 +118,6 @@ def create(self, interface_type: str, *args, **kwargs): UnityCatalogConnection.__name__: UnityCatalogProfilerInterface, DatabricksConnection.__name__: DatabricksProfilerInterface, Db2Connection.__name__: DB2ProfilerInterface, - MongoDBConnection.__name__: NoSQLProfilerInterface, } + profiler_interface_factory.register_many(profilers) diff --git a/ingestion/src/metadata/profiler/interface/sqlalchemy/profiler_interface.py b/ingestion/src/metadata/profiler/interface/sqlalchemy/profiler_interface.py index 0a4441595dfd..d99adbf53896 100644 --- a/ingestion/src/metadata/profiler/interface/sqlalchemy/profiler_interface.py +++ b/ingestion/src/metadata/profiler/interface/sqlalchemy/profiler_interface.py @@ -189,6 +189,7 @@ def _compute_table_metrics( runner=runner, metrics=metrics, conn_config=self.service_connection_config, + entity=self.table_entity, ) row = table_metric_computer.compute() if row: diff --git a/ingestion/src/metadata/profiler/metrics/core.py b/ingestion/src/metadata/profiler/metrics/core.py index 70e387a7daeb..9cc219777a53 100644 --- a/ingestion/src/metadata/profiler/metrics/core.py +++ b/ingestion/src/metadata/profiler/metrics/core.py @@ -18,14 +18,11 @@ from abc import ABC, abstractmethod from enum import Enum from functools import wraps -from typing import Any, Callable, Dict, Optional, Tuple, TypeVar +from typing import Any, Dict, Optional, Tuple, TypeVar from sqlalchemy import Column from sqlalchemy.orm import DeclarativeMeta, Session -from metadata.generated.schema.entity.data.table import Table -from metadata.profiler.adaptors.nosql_adaptor import NoSQLAdaptor - # When creating complex metrics, use inherit_cache = CACHE CACHE = True @@ -90,9 +87,6 @@ def _new_init(self, *args, **kw): return inner -T = TypeVar("T") - - class Metric(ABC): """ Parent class metric @@ -159,13 +153,6 @@ def metric_type(self): """ return self.col.type.python_type if self.col else None - def nosql_fn(self, client: NoSQLAdaptor) -> Callable[[Table], Optional[T]]: - """ - Return the function to be used for NoSQL clients to calculate the metric. - By default, returns a "do nothing" function that returns None. - """ - return lambda table: None - TMetric = TypeVar("TMetric", bound=Metric) diff --git a/ingestion/src/metadata/profiler/metrics/static/max.py b/ingestion/src/metadata/profiler/metrics/static/max.py index 65f9ec7181c7..4eb7b933f4c9 100644 --- a/ingestion/src/metadata/profiler/metrics/static/max.py +++ b/ingestion/src/metadata/profiler/metrics/static/max.py @@ -12,16 +12,14 @@ """ Max Metric definition """ -from functools import partial -from typing import Callable, Optional +# pylint: disable=duplicate-code + from sqlalchemy import TIME, column from sqlalchemy.ext.compiler import compiles from sqlalchemy.sql.functions import GenericFunction -from metadata.generated.schema.entity.data.table import Table -from metadata.profiler.adaptors.nosql_adaptor import NoSQLAdaptor -from metadata.profiler.metrics.core import CACHE, StaticMetric, T, _label +from metadata.profiler.metrics.core import CACHE, StaticMetric, _label from metadata.profiler.orm.functions.length import LenFn from metadata.profiler.orm.registry import ( FLOAT_SET, @@ -31,8 +29,6 @@ is_quantifiable, ) -# pylint: disable=duplicate-code - class MaxFn(GenericFunction): name = __qualname__ @@ -100,9 +96,3 @@ def df_fn(self, dfs=None): max_ = max((df[self.col.name].max() for df in dfs)) return int(max_.timestamp() * 1000) return 0 - - def nosql_fn(self, adaptor: NoSQLAdaptor) -> Callable[[Table], Optional[T]]: - """nosql function""" - if is_quantifiable(self.col.type): - return partial(adaptor.max, column=self.col) - return lambda table: None diff --git a/ingestion/src/metadata/profiler/metrics/static/mean.py b/ingestion/src/metadata/profiler/metrics/static/mean.py index fe53306643d7..aaa5d78783eb 100644 --- a/ingestion/src/metadata/profiler/metrics/static/mean.py +++ b/ingestion/src/metadata/profiler/metrics/static/mean.py @@ -12,16 +12,16 @@ """ AVG Metric definition """ -from functools import partial -from typing import Callable, List, Optional, cast +# pylint: disable=duplicate-code + + +from typing import List, cast from sqlalchemy import column, func from sqlalchemy.ext.compiler import compiles from sqlalchemy.sql.functions import GenericFunction -from metadata.generated.schema.entity.data.table import Table -from metadata.profiler.adaptors.nosql_adaptor import NoSQLAdaptor -from metadata.profiler.metrics.core import CACHE, StaticMetric, T, _label +from metadata.profiler.metrics.core import CACHE, StaticMetric, _label from metadata.profiler.orm.functions.length import LenFn from metadata.profiler.orm.registry import ( FLOAT_SET, @@ -32,9 +32,6 @@ ) from metadata.utils.logger import profiler_logger -# pylint: disable=duplicate-code - - logger = profiler_logger() @@ -145,9 +142,3 @@ def df_fn(self, dfs=None): f"Don't know how to process type {self.col.type} when computing MEAN" ) return None - - def nosql_fn(self, adaptor: NoSQLAdaptor) -> Callable[[Table], Optional[T]]: - """nosql function""" - if is_quantifiable(self.col.type): - return partial(adaptor.mean, column=self.col) - return lambda table: None diff --git a/ingestion/src/metadata/profiler/metrics/static/min.py b/ingestion/src/metadata/profiler/metrics/static/min.py index 5731348708c7..d6e212a34055 100644 --- a/ingestion/src/metadata/profiler/metrics/static/min.py +++ b/ingestion/src/metadata/profiler/metrics/static/min.py @@ -12,16 +12,13 @@ """ Min Metric definition """ -from functools import partial -from typing import Callable, Optional +# pylint: disable=duplicate-code from sqlalchemy import TIME, column from sqlalchemy.ext.compiler import compiles from sqlalchemy.sql.functions import GenericFunction -from metadata.generated.schema.entity.data.table import Table -from metadata.profiler.adaptors.nosql_adaptor import NoSQLAdaptor -from metadata.profiler.metrics.core import CACHE, StaticMetric, T, _label +from metadata.profiler.metrics.core import CACHE, StaticMetric, _label from metadata.profiler.orm.functions.length import LenFn from metadata.profiler.orm.registry import ( FLOAT_SET, @@ -31,8 +28,6 @@ is_quantifiable, ) -# pylint: disable=duplicate-code - class MinFn(GenericFunction): name = __qualname__ @@ -101,9 +96,3 @@ def df_fn(self, dfs=None): min_ = min((df[self.col.name].min() for df in dfs)) return int(min_.timestamp() * 1000) return 0 - - def nosql_fn(self, adaptor: NoSQLAdaptor) -> Callable[[Table], Optional[T]]: - """nosql function""" - if is_quantifiable(self.col.type): - return partial(adaptor.min, column=self.col) - return lambda table: None diff --git a/ingestion/src/metadata/profiler/metrics/static/row_count.py b/ingestion/src/metadata/profiler/metrics/static/row_count.py index c3f70f9d152e..6891ab43b021 100644 --- a/ingestion/src/metadata/profiler/metrics/static/row_count.py +++ b/ingestion/src/metadata/profiler/metrics/static/row_count.py @@ -12,12 +12,11 @@ """ Table Count Metric definition """ -from typing import Callable +# pylint: disable=duplicate-code + from sqlalchemy import func -from metadata.generated.schema.entity.data.table import Table -from metadata.profiler.adaptors.nosql_adaptor import NoSQLAdaptor from metadata.profiler.metrics.core import StaticMetric, _label @@ -51,7 +50,3 @@ def fn(self): def df_fn(self, dfs=None): """pandas function""" return sum(len(df.index) for df in dfs) - - @classmethod - def nosql_fn(cls, client: NoSQLAdaptor) -> Callable[[Table], int]: - return client.item_count diff --git a/ingestion/src/metadata/profiler/metrics/static/sum.py b/ingestion/src/metadata/profiler/metrics/static/sum.py index b118ca1458db..dec3bbbb4b9a 100644 --- a/ingestion/src/metadata/profiler/metrics/static/sum.py +++ b/ingestion/src/metadata/profiler/metrics/static/sum.py @@ -12,20 +12,15 @@ """ SUM Metric definition """ -from functools import partial -from typing import Callable, Optional +# pylint: disable=duplicate-code from sqlalchemy import column -from metadata.generated.schema.entity.data.table import Table -from metadata.profiler.adaptors.nosql_adaptor import NoSQLAdaptor -from metadata.profiler.metrics.core import StaticMetric, T, _label +from metadata.profiler.metrics.core import StaticMetric, _label from metadata.profiler.orm.functions.length import LenFn from metadata.profiler.orm.functions.sum import SumFn from metadata.profiler.orm.registry import is_concatenable, is_quantifiable -# pylint: disable=duplicate-code - class Sum(StaticMetric): """ @@ -57,9 +52,3 @@ def df_fn(self, dfs=None): if is_quantifiable(self.col.type): return sum(df[self.col.name].sum() for df in dfs) return None - - def nosql_fn(self, adaptor: NoSQLAdaptor) -> Callable[[Table], Optional[T]]: - """nosql function""" - if is_quantifiable(self.col.type): - return partial(adaptor.sum, column=self.col) - return lambda table: None diff --git a/ingestion/src/metadata/profiler/orm/functions/sum.py b/ingestion/src/metadata/profiler/orm/functions/sum.py index 545f3bbcbef6..0e8623947a24 100644 --- a/ingestion/src/metadata/profiler/orm/functions/sum.py +++ b/ingestion/src/metadata/profiler/orm/functions/sum.py @@ -40,6 +40,7 @@ def _(element, compiler, **kw): @compiles(SumFn, Dialects.BigQuery) +@compiles(SumFn, Dialects.Postgres) def _(element, compiler, **kw): """Handle case where column type is INTEGER but SUM returns a NUMBER""" proc = compiler.process(element.clauses, **kw) diff --git a/ingestion/src/metadata/profiler/orm/functions/table_metric_computer.py b/ingestion/src/metadata/profiler/orm/functions/table_metric_computer.py index 8d88b0d85d47..f2c8ab0bde3f 100644 --- a/ingestion/src/metadata/profiler/orm/functions/table_metric_computer.py +++ b/ingestion/src/metadata/profiler/orm/functions/table_metric_computer.py @@ -22,6 +22,8 @@ from sqlalchemy.sql.expression import ColumnOperators, and_, cte from sqlalchemy.types import String +from metadata.generated.schema.entity.data.table import Table as OMTable +from metadata.generated.schema.entity.data.table import TableType from metadata.profiler.metrics.registry import Metrics from metadata.profiler.orm.registry import Dialects from metadata.profiler.processor.runner import QueryRunner @@ -31,7 +33,7 @@ COLUMN_COUNT = "columnCount" COLUMN_NAMES = "columnNames" -ROW_COUNT = "rowCount" +ROW_COUNT = Metrics.ROW_COUNT().name() SIZE_IN_BYTES = "sizeInBytes" CREATE_DATETIME = "createDateTime" @@ -43,13 +45,16 @@ class AbstractTableMetricComputer(ABC): """Base table computer""" - def __init__(self, runner: QueryRunner, metrics: List[Metrics], conn_config): + def __init__( + self, runner: QueryRunner, metrics: List[Metrics], conn_config, entity: OMTable + ): """Instantiate base table computer""" self._runner = runner self._metrics = metrics self._conn_config = conn_config self._database = self._runner._session.get_bind().url.database self._table = self._runner.table + self._entity = entity @property def database(self): @@ -141,16 +146,6 @@ def compute(self): class BaseTableMetricComputer(AbstractTableMetricComputer): """Base table computer""" - def _check_and_return(self, res): - """Check if the result is None and return the result or fallback - - Args: - res (object): result - """ - if res.rowCount is None: - return super().compute() - return res - def compute(self): """Default compute behavior for table metrics""" return self.runner.select_first_from_table( @@ -236,7 +231,9 @@ def compute(self): ) res = self.runner._session.execute(query).first() - if res.rowCount is None: + if res.rowCount is None or ( + res.rowCount == 0 and self._entity.tableType == TableType.View + ): # if we don't have any row count, fallback to the base logic return super().compute() return res @@ -263,7 +260,9 @@ def compute(self): ) res = self.runner._session.execute(query).first() - if res.rowCount is None: + if res.rowCount is None or ( + res.rowCount == 0 and self._entity.tableType == TableType.View + ): # if we don't have any row count, fallback to the base logic return super().compute() return res @@ -307,7 +306,9 @@ def table_storage(self): ) res = self.runner._session.execute(query).first() - if res.rowCount is None: + if res.rowCount is None or ( + res.rowCount == 0 and self._entity.tableType == TableType.View + ): # if we don't have any row count, fallback to the base logic return super().compute() return res @@ -336,7 +337,9 @@ def tables(self): where_clause, ) res = self.runner._session.execute(query).first() - if res.rowCount is None: + if res.rowCount is None or ( + res.rowCount == 0 and self._entity.tableType == TableType.View + ): # if we don't have any row count, fallback to the base logic return super().compute() return res @@ -363,9 +366,16 @@ def compute(self): ) res = self.runner._session.execute(query).first() - if res.rowCount is None: + if res.rowCount is None or ( + res.rowCount == 0 and self._entity.tableType == TableType.View + ): # if we don't have any row count, fallback to the base logic return super().compute() + res = res._asdict() + # innodb row count is an estimate we need to patch the row count with COUNT(*) + # https://dev.mysql.com/doc/refman/8.3/en/information-schema-innodb-tablestats-table.html + row_count = self.runner.select_first_from_table(Metrics.ROW_COUNT().fn()) + res.update({ROW_COUNT: row_count.rowCount}) return res @@ -390,7 +400,9 @@ def compute(self): columns, self._build_table("svv_table_info", "pg_catalog"), where_clause ) res = self.runner._session.execute(query).first() - if res.rowCount is None: + if res.rowCount is None or ( + res.rowCount == 0 and self._entity.tableType == TableType.View + ): # if we don't have any row count, fallback to the base logic return super().compute() return res @@ -400,9 +412,15 @@ class TableMetricComputer: """Table Metric Construct""" def __init__( - self, dialect: str, runner: QueryRunner, metrics: List[Metrics], conn_config + self, + dialect: str, + runner: QueryRunner, + metrics: List[Metrics], + conn_config, + entity: OMTable, ): """Instantiate table metric computer with a dialect computer""" + self._entity = entity self._dialect = dialect self._runner = runner self._metrics = metrics @@ -413,6 +431,7 @@ def __init__( runner=self._runner, metrics=self._metrics, conn_config=self._conn_config, + entity=self._entity, ) ) diff --git a/ingestion/src/metadata/profiler/processor/core.py b/ingestion/src/metadata/profiler/processor/core.py index 5081b43304d7..ff0fd7cc0559 100644 --- a/ingestion/src/metadata/profiler/processor/core.py +++ b/ingestion/src/metadata/profiler/processor/core.py @@ -196,10 +196,7 @@ def _check_profile_and_handle( CreateTableProfileRequest: """ for attrs, val in profile.tableProfile: - if ( - attrs not in {"timestamp", "profileSample", "profileSampleType"} - and val is not None - ): + if attrs not in {"timestamp", "profileSample", "profileSampleType"} and val: return for col_element in profile.columnProfile: diff --git a/ingestion/src/metadata/profiler/processor/sample_data_handler.py b/ingestion/src/metadata/profiler/processor/sample_data_handler.py index 33bd65aca60f..f029d2836c98 100644 --- a/ingestion/src/metadata/profiler/processor/sample_data_handler.py +++ b/ingestion/src/metadata/profiler/processor/sample_data_handler.py @@ -17,8 +17,13 @@ from functools import singledispatch from io import BytesIO +from pydantic.json import ENCODERS_BY_TYPE + from metadata.clients.aws_client import AWSClient from metadata.generated.schema.entity.data.table import Table, TableData +from metadata.generated.schema.entity.services.connections.connectionBasicType import ( + DataStorageConfig, +) from metadata.generated.schema.security.credentials.awsCredentials import AWSCredentials from metadata.profiler.interface.profiler_interface import ProfilerInterface from metadata.utils.helpers import clean_uri @@ -27,15 +32,45 @@ logger = profiler_logger() -def _get_object_key(table: Table, prefix: str, overwrite_data: bool) -> str: +class PathPatternException(Exception): + """ + Exception class need to validate the file path pattern + """ + + +def validate_path_pattern(file_path_format: str) -> None: + if not ( + "{service_name}" in file_path_format + and "{database_name}" in file_path_format + and "{database_schema_name}" in file_path_format + and "{table_name}" in file_path_format + and file_path_format.endswith(".parquet") + ): + raise PathPatternException( + "Please provide a valid path pattern, " + "the pattern should include these components {service_name}, " + "{database_name}, {database_schema_name}, {table_name} and " + "it should end with extension .parquet" + ) + + +def _get_object_key( + table: Table, prefix: str, overwrite_data: bool, file_path_format: str +) -> str: + validate_path_pattern(file_path_format) + file_name = file_path_format.format( + service_name=table.service.name, + database_name=table.database.name, + database_schema_name=table.databaseSchema.name, + table_name=table.name.__root__, + ) if not overwrite_data: - file_name = f"sample_data_{datetime.now().strftime('%Y_%m_%d')}.parquet" - else: - file_name = "sample_data.parquet" - path = str(table.fullyQualifiedName.__root__).replace(".", "/") + file_name = file_name.replace( + ".parquet", f"_{datetime.now().strftime('%Y_%m_%d')}.parquet" + ) if prefix: - return f"{clean_uri(prefix)}/{path}/{file_name}" - return f"{path}/{file_name}" + return f"{clean_uri(prefix)}/{file_name}" + return file_name def upload_sample_data(data: TableData, profiler_interface: ProfilerInterface) -> None: @@ -45,9 +80,10 @@ def upload_sample_data(data: TableData, profiler_interface: ProfilerInterface) - import pandas as pd # pylint: disable=import-outside-toplevel try: - sample_storage_config = profiler_interface.storage_config + sample_storage_config: DataStorageConfig = profiler_interface.storage_config if not sample_storage_config: return + ENCODERS_BY_TYPE[bytes] = lambda v: v.decode("utf-8", "ignore") deserialized_data = json.loads(data.json()) df = pd.DataFrame( data=deserialized_data.get("rows", []), @@ -59,6 +95,7 @@ def upload_sample_data(data: TableData, profiler_interface: ProfilerInterface) - table=profiler_interface.table_entity, prefix=sample_storage_config.prefix, overwrite_data=sample_storage_config.overwriteData, + file_path_format=sample_storage_config.filePathPattern, ) upload_to_storage( sample_storage_config.storageConfig, diff --git a/ingestion/src/metadata/profiler/processor/sampler/nosql/sampler.py b/ingestion/src/metadata/profiler/processor/sampler/nosql/sampler.py deleted file mode 100644 index 333d5ae712a8..000000000000 --- a/ingestion/src/metadata/profiler/processor/sampler/nosql/sampler.py +++ /dev/null @@ -1,72 +0,0 @@ -from typing import Dict, List, Optional, Tuple - -from metadata.generated.schema.entity.data.table import ProfileSampleType, TableData -from metadata.profiler.adaptors.nosql_adaptor import NoSQLAdaptor -from metadata.profiler.processor.sampler.sampler_interface import SamplerInterface -from metadata.utils.constants import SAMPLE_DATA_DEFAULT_COUNT -from metadata.utils.sqa_like_column import SQALikeColumn - - -class NoSQLSampler(SamplerInterface): - client: NoSQLAdaptor - - def _rdn_sample_from_user_query(self) -> List[Dict[str, any]]: - """ - Get random sample from user query - """ - limit = self._get_limit() - return self.client.query( - self.table, self.table.columns, self._profile_sample_query, limit - ) - - def _fetch_sample_data_from_user_query(self) -> TableData: - """ - Fetch sample data based on a user query. Assuming the enging has one (example: MongoDB) - If the engine does not support a custom query, an error will be raised. - """ - records = self._rdn_sample_from_user_query() - columns = [ - SQALikeColumn(name=column.name.__root__, type=column.dataType) - for column in self.table.columns - ] - rows, cols = self.transpose_records(records, columns) - return TableData(rows=rows, columns=[c.name for c in cols]) - - def random_sample(self): - pass - - def fetch_sample_data(self, columns: List[SQALikeColumn]) -> TableData: - if self._profile_sample_query: - return self._fetch_sample_data_from_user_query() - return self._fetch_sample_data(columns) - - def _fetch_sample_data(self, columns: List[SQALikeColumn]): - """ - returns sampled ometa dataframes - """ - limit = self._get_limit() - records = self.client.scan(self.table, self.table.columns, limit) - rows, cols = self.transpose_records(records, columns) - return TableData(rows=rows, columns=[col.name for col in cols]) - - def _get_limit(self) -> Optional[int]: - num_rows = self.client.item_count(self.table) - if self.profile_sample_type == ProfileSampleType.PERCENTAGE: - limit = num_rows * (self.profile_sample / 100) - elif self.profile_sample_type == ProfileSampleType.ROWS: - limit = self.profile_sample - else: - limit = SAMPLE_DATA_DEFAULT_COUNT - return limit - - @staticmethod - def transpose_records( - records: List[Dict[str, any]], columns: List[SQALikeColumn] - ) -> Tuple[List[List[any]], List[SQALikeColumn]]: - rows = [] - for record in records: - row = [] - for column in columns: - row.append(record.get(column.name)) - rows.append(row) - return rows, columns diff --git a/ingestion/src/metadata/profiler/processor/sampler/sampler_factory.py b/ingestion/src/metadata/profiler/processor/sampler/sampler_factory.py index 88584f4eb24e..e7c0f25e7e5a 100644 --- a/ingestion/src/metadata/profiler/processor/sampler/sampler_factory.py +++ b/ingestion/src/metadata/profiler/processor/sampler/sampler_factory.py @@ -21,14 +21,10 @@ from metadata.generated.schema.entity.services.connections.database.datalakeConnection import ( DatalakeConnection, ) -from metadata.generated.schema.entity.services.connections.database.mongoDBConnection import ( - MongoDBConnection, -) from metadata.generated.schema.entity.services.connections.database.trinoConnection import ( TrinoConnection, ) from metadata.generated.schema.entity.services.databaseService import DatabaseConnection -from metadata.profiler.processor.sampler.nosql.sampler import NoSQLSampler from metadata.profiler.processor.sampler.pandas.sampler import DatalakeSampler from metadata.profiler.processor.sampler.sqlalchemy.bigquery.sampler import ( BigQuerySampler, @@ -63,4 +59,3 @@ def create( sampler_factory_.register(BigQueryConnection.__name__, BigQuerySampler) sampler_factory_.register(DatalakeConnection.__name__, DatalakeSampler) sampler_factory_.register(TrinoConnection.__name__, TrinoSampler) -sampler_factory_.register(MongoDBConnection.__name__, NoSQLSampler) diff --git a/ingestion/src/metadata/profiler/processor/sampler/sampler_interface.py b/ingestion/src/metadata/profiler/processor/sampler/sampler_interface.py index daba85fcebcc..8711affa2c49 100644 --- a/ingestion/src/metadata/profiler/processor/sampler/sampler_interface.py +++ b/ingestion/src/metadata/profiler/processor/sampler/sampler_interface.py @@ -17,7 +17,7 @@ from sqlalchemy import Column -from metadata.generated.schema.entity.data.table import Table, TableData +from metadata.generated.schema.entity.data.table import TableData from metadata.profiler.api.models import ProfileSampleConfig from metadata.utils.constants import SAMPLE_DATA_DEFAULT_COUNT from metadata.utils.sqa_like_column import SQALikeColumn @@ -29,7 +29,7 @@ class SamplerInterface(ABC): def __init__( self, client, - table: Table, + table, profile_sample_config: Optional[ProfileSampleConfig] = None, partition_details: Optional[Dict] = None, profile_sample_query: Optional[str] = None, diff --git a/ingestion/src/metadata/profiler/source/databricks/profiler_source.py b/ingestion/src/metadata/profiler/source/databricks/profiler_source.py new file mode 100644 index 000000000000..009bd1d6d774 --- /dev/null +++ b/ingestion/src/metadata/profiler/source/databricks/profiler_source.py @@ -0,0 +1,36 @@ +"""Extend the ProfilerSource class to add support for Databricks is_disconnect SQA method""" + +from metadata.generated.schema.entity.services.databaseService import DatabaseService +from metadata.generated.schema.metadataIngestion.workflow import ( + OpenMetadataWorkflowConfig, +) +from metadata.ingestion.ometa.ometa_api import OpenMetadata +from metadata.profiler.source.base.profiler_source import ProfilerSource + + +def is_disconnect(self, e, connection, cursor): + """is_disconnect method for the Databricks dialect""" + if "Invalid SessionHandle: SessionHandle" in str(e): + return True + return False + + +class DataBricksProfilerSource(ProfilerSource): + """Databricks Profiler source""" + + def __init__( + self, + config: OpenMetadataWorkflowConfig, + database: DatabaseService, + ometa_client: OpenMetadata, + ): + super().__init__(config, database, ometa_client) + self.set_is_disconnect() + + def set_is_disconnect(self): + """Set the is_disconnect method for the Databricks dialect""" + from databricks.sqlalchemy import ( + DatabricksDialect, # pylint: disable=import-outside-toplevel + ) + + DatabricksDialect.is_disconnect = is_disconnect diff --git a/ingestion/src/metadata/profiler/source/metadata.py b/ingestion/src/metadata/profiler/source/metadata.py index d14ce2ed4fd2..b266c066ddb6 100644 --- a/ingestion/src/metadata/profiler/source/metadata.py +++ b/ingestion/src/metadata/profiler/source/metadata.py @@ -43,6 +43,10 @@ logger = profiler_logger() +TABLE_FIELDS = ["tableProfilerConfig", "columns", "customMetrics"] +TAGS_FIELD = ["tags"] + + class ProfilerSourceAndEntity(BaseModel): """Return class for the OpenMetadata Profiler Source""" @@ -273,7 +277,9 @@ def get_table_entities(self, database): """ tables = self.metadata.list_all_entities( entity=Table, - fields=["tableProfilerConfig", "columns", "customMetrics"], + fields=TABLE_FIELDS + if not self.source_config.processPiiSensitive + else TABLE_FIELDS + TAGS_FIELD, params={ "service": self.config.source.serviceName, "database": fqn.build( diff --git a/ingestion/src/metadata/profiler/source/profiler_source_factory.py b/ingestion/src/metadata/profiler/source/profiler_source_factory.py index 6fd2973c0bd3..0e616354e8d9 100644 --- a/ingestion/src/metadata/profiler/source/profiler_source_factory.py +++ b/ingestion/src/metadata/profiler/source/profiler_source_factory.py @@ -16,8 +16,12 @@ from metadata.generated.schema.entity.services.connections.database.bigQueryConnection import ( BigqueryType, ) +from metadata.generated.schema.entity.services.connections.database.databricksConnection import ( + DatabricksType, +) from metadata.profiler.source.base.profiler_source import ProfilerSource from metadata.profiler.source.bigquery.profiler_source import BigQueryProfilerSource +from metadata.profiler.source.databricks.profiler_source import DataBricksProfilerSource class ProfilerSourceFactory: @@ -44,3 +48,7 @@ def create(self, source_type: str, *args, **kwargs) -> ProfilerSource: BigqueryType.BigQuery.value.lower(), BigQueryProfilerSource, ) +profiler_source_factory.register_source( + DatabricksType.Databricks.value.lower(), + DataBricksProfilerSource, +) diff --git a/ingestion/src/metadata/readers/dataframe/json.py b/ingestion/src/metadata/readers/dataframe/json.py index c2c16f26bc8f..20be18c0a8b4 100644 --- a/ingestion/src/metadata/readers/dataframe/json.py +++ b/ingestion/src/metadata/readers/dataframe/json.py @@ -16,7 +16,7 @@ import io import json import zipfile -from typing import List, Union +from typing import Any, Dict, List, Optional, Tuple, Union from metadata.readers.dataframe.base import DataFrameReader from metadata.readers.dataframe.common import dataframe_to_chunks @@ -47,7 +47,7 @@ class JSONDataFrameReader(DataFrameReader): @staticmethod def read_from_json( key: str, json_text: bytes, decode: bool = False, **__ - ) -> List["DataFrame"]: + ) -> Tuple[List["DataFrame"], Optional[Dict[str, Any]]]: """ Decompress a JSON file (if needed) and read its contents as a dataframe. @@ -60,20 +60,25 @@ def read_from_json( import pandas as pd json_text = _get_json_text(key=key, text=json_text, decode=decode) + raw_data = None try: data = json.loads(json_text) + if isinstance(data, dict) and data.get("$schema"): + raw_data = json_text except json.decoder.JSONDecodeError: logger.debug("Failed to read as JSON object. Trying to read as JSON Lines") data = [json.loads(json_obj) for json_obj in json_text.strip().split("\n")] # if we get a scalar value (e.g. {"a":"b"}) then we need to specify the index data = data if not isinstance(data, dict) else [data] - return dataframe_to_chunks(pd.DataFrame.from_records(data)) + return dataframe_to_chunks(pd.DataFrame.from_records(data)), raw_data def _read(self, *, key: str, bucket_name: str, **kwargs) -> DatalakeColumnWrapper: text = self.reader.read(key, bucket_name=bucket_name) + dataframes, raw_data = self.read_from_json( + key=key, json_text=text, decode=True, **kwargs + ) return DatalakeColumnWrapper( - dataframes=self.read_from_json( - key=key, json_text=text, decode=True, **kwargs - ) + dataframes=dataframes, + raw_data=raw_data, ) diff --git a/ingestion/src/metadata/readers/dataframe/models.py b/ingestion/src/metadata/readers/dataframe/models.py index 765e6c1ae783..67678b90e4c9 100644 --- a/ingestion/src/metadata/readers/dataframe/models.py +++ b/ingestion/src/metadata/readers/dataframe/models.py @@ -29,6 +29,7 @@ class DatalakeColumnWrapper(BaseModel): columns: Optional[List[Column]] dataframes: Optional[List[Any]] # pandas.Dataframe does not have any validators + raw_data: Any # in special cases like json schema, we need to store the raw data class DatalakeTableSchemaWrapper(BaseModel): diff --git a/ingestion/src/metadata/utils/credentials.py b/ingestion/src/metadata/utils/credentials.py index ca5ab392a887..de2767e71d68 100644 --- a/ingestion/src/metadata/utils/credentials.py +++ b/ingestion/src/metadata/utils/credentials.py @@ -15,7 +15,7 @@ import json import os import tempfile -from typing import Dict, List, Optional +from typing import Dict, List, Optional, Union from cryptography.hazmat.primitives import serialization from google import auth @@ -25,6 +25,9 @@ GCPCredentials, GcpCredentialsPath, ) +from metadata.generated.schema.security.credentials.gcpExternalAccount import ( + GcpExternalAccount, +) from metadata.generated.schema.security.credentials.gcpValues import ( GcpCredentialsValues, ) @@ -85,30 +88,44 @@ def create_credential_tmp_file(credentials: dict) -> str: return temp_file_path -def build_google_credentials_dict(gcp_values: GcpCredentialsValues) -> Dict[str, str]: +def build_google_credentials_dict( + gcp_values: Union[GcpCredentialsValues, GcpExternalAccount] +) -> Dict[str, str]: """ Given GcPCredentialsValues, build a dictionary as the JSON file downloaded from GCP with the service_account :param gcp_values: GCP credentials :return: Dictionary with credentials """ - private_key_str = gcp_values.privateKey.get_secret_value() - # adding the replace string here to escape line break if passed from env - private_key_str = private_key_str.replace("\\n", "\n") - validate_private_key(private_key_str) - - return { - "type": gcp_values.type, - "project_id": gcp_values.projectId.__root__, - "private_key_id": gcp_values.privateKeyId, - "private_key": private_key_str, - "client_email": gcp_values.clientEmail, - "client_id": gcp_values.clientId, - "auth_uri": str(gcp_values.authUri), - "token_uri": str(gcp_values.tokenUri), - "auth_provider_x509_cert_url": str(gcp_values.authProviderX509CertUrl), - "client_x509_cert_url": str(gcp_values.clientX509CertUrl), - } + if isinstance(gcp_values, GcpCredentialsValues): + private_key_str = gcp_values.privateKey.get_secret_value() + # adding the replace string here to escape line break if passed from env + private_key_str = private_key_str.replace("\\n", "\n") + validate_private_key(private_key_str) + + return { + "type": gcp_values.type, + "project_id": gcp_values.projectId.__root__, + "private_key_id": gcp_values.privateKeyId, + "private_key": private_key_str, + "client_email": gcp_values.clientEmail, + "client_id": gcp_values.clientId, + "auth_uri": str(gcp_values.authUri), + "token_uri": str(gcp_values.tokenUri), + "auth_provider_x509_cert_url": str(gcp_values.authProviderX509CertUrl), + "client_x509_cert_url": str(gcp_values.clientX509CertUrl), + } + if isinstance(gcp_values, GcpExternalAccount): + return { + "type": gcp_values.externalType, + "audience": gcp_values.audience, + "subject_token_type": gcp_values.subjectTokenType, + "token_url": gcp_values.tokenURL, + "credential_source": gcp_values.credentialSource, + } + raise InvalidGcpConfigException( + f"Error trying to build GCP credentials dict due to Invalid GCP config {type(gcp_values)}" + ) def set_google_credentials(gcp_credentials: GCPCredentials) -> None: diff --git a/ingestion/src/metadata/utils/datalake/datalake_utils.py b/ingestion/src/metadata/utils/datalake/datalake_utils.py index e067443090f9..3630723eb617 100644 --- a/ingestion/src/metadata/utils/datalake/datalake_utils.py +++ b/ingestion/src/metadata/utils/datalake/datalake_utils.py @@ -17,10 +17,11 @@ import json import random import traceback -from typing import Dict, List, Optional, Union, cast +from typing import Any, Dict, List, Optional, Union, cast from metadata.generated.schema.entity.data.table import Column, DataType from metadata.ingestion.source.database.column_helpers import truncate_column_name +from metadata.parsers.json_schema_parser import parse_json_schema from metadata.readers.dataframe.models import ( DatalakeColumnWrapper, DatalakeTableSchemaWrapper, @@ -35,6 +36,7 @@ def fetch_dataframe( config_source, client, file_fqn: DatalakeTableSchemaWrapper, + fetch_raw_data: bool = False, **kwargs, ) -> Optional[List["DataFrame"]]: """ @@ -60,6 +62,8 @@ def fetch_dataframe( df_wrapper: DatalakeColumnWrapper = df_reader.read( key=key, bucket_name=bucket_name, **kwargs ) + if fetch_raw_data: + return df_wrapper.dataframes, df_wrapper.raw_data return df_wrapper.dataframes except Exception as err: logger.error( @@ -73,6 +77,8 @@ def fetch_dataframe( # Here we need to blow things up. Without the dataframe we cannot move forward raise err + if fetch_raw_data: + return None, None return None @@ -112,6 +118,7 @@ def create( file_type: Optional[SupportedTypes] = None, sample: bool = True, shuffle: bool = False, + raw_data: Any = None, ): """Instantiate a column parser object with the appropriate parser @@ -126,8 +133,14 @@ def create( data_frame = cls._get_data_frame(data_frame, sample, shuffle) if file_type == SupportedTypes.PARQUET: parser = ParquetDataFrameColumnParser(data_frame) - return cls(parser) - parser = GenericDataFrameColumnParser(data_frame) + elif file_type in { + SupportedTypes.JSON, + SupportedTypes.JSONGZ, + SupportedTypes.JSONZIP, + }: + parser = JsonDataFrameColumnParser(data_frame, raw_data=raw_data) + else: + parser = GenericDataFrameColumnParser(data_frame) return cls(parser) @staticmethod @@ -172,8 +185,9 @@ class GenericDataFrameColumnParser: "bytes": DataType.BYTES, } - def __init__(self, data_frame: "DataFrame"): + def __init__(self, data_frame: "DataFrame", raw_data: Any = None): self.data_frame = data_frame + self.raw_data = raw_data def get_columns(self): """ @@ -472,3 +486,19 @@ def _get_pq_data_type(self, column): data_type = self._data_formats.get(str(column.type), DataType.UNKNOWN) return data_type + + +class JsonDataFrameColumnParser(GenericDataFrameColumnParser): + """Given a dataframe object generated from a json file, parse the columns and return a list of Column objects.""" + + def get_columns(self): + """ + method to process column details for json files + """ + if self.raw_data: + try: + return parse_json_schema(schema_text=self.raw_data, cls=Column) + except Exception as exc: + logger.warning(f"Unable to parse the json schema: {exc}") + logger.debug(traceback.format_exc()) + return self._get_columns(self.data_frame) diff --git a/ingestion/src/metadata/utils/secrets/azure_kv_secrets_manager.py b/ingestion/src/metadata/utils/secrets/azure_kv_secrets_manager.py index 4682fc23ace4..566c9154850c 100644 --- a/ingestion/src/metadata/utils/secrets/azure_kv_secrets_manager.py +++ b/ingestion/src/metadata/utils/secrets/azure_kv_secrets_manager.py @@ -17,9 +17,9 @@ from abc import ABC from typing import Optional -from azure.identity import ClientSecretCredential, DefaultAzureCredential -from azure.keyvault.secrets import KeyVaultSecret, SecretClient +from azure.keyvault.secrets import KeyVaultSecret +from metadata.clients.azure_client import AzureClient from metadata.generated.schema.security.secrets.secretsManagerClientLoader import ( SecretsManagerClientLoader, ) @@ -105,23 +105,7 @@ def __init__( ): super().__init__(provider=SecretsManagerProvider.azure_kv, loader=loader) - if ( - self.credentials.tenantId - and self.credentials.clientId - and self.credentials.clientSecret - ): - azure_identity = ClientSecretCredential( - tenant_id=self.credentials.tenantId, - client_id=self.credentials.clientId, - client_secret=self.credentials.clientSecret.get_secret_value(), - ) - else: - azure_identity = DefaultAzureCredential() - - self.client = SecretClient( - vault_url=f"https://{self.credentials.vaultName}.vault.azure.net/", - credential=azure_identity, - ) + self.client = AzureClient(self.credentials).create_secret_client() def get_string_value(self, secret_id: str) -> str: """ diff --git a/ingestion/src/metadata/utils/source_hash.py b/ingestion/src/metadata/utils/source_hash.py index 80599ef7d45e..db76ce42083d 100644 --- a/ingestion/src/metadata/utils/source_hash.py +++ b/ingestion/src/metadata/utils/source_hash.py @@ -14,9 +14,14 @@ """ import hashlib +import traceback from typing import Dict, Optional from metadata.ingestion.ometa.ometa_api import C +from metadata.utils.logger import utils_logger + +logger = utils_logger() + SOURCE_HASH_EXCLUDE_FIELDS = { "sourceHash": True, @@ -25,19 +30,24 @@ def generate_source_hash( create_request: C, exclude_fields: Optional[Dict] = None -) -> str: +) -> Optional[str]: """ Given a create_request model convert it to json string and generate a hash value """ - - # We always want to exclude the sourceHash when generating the fingerprint - exclude_fields = ( - SOURCE_HASH_EXCLUDE_FIELDS.update(exclude_fields) - if exclude_fields - else SOURCE_HASH_EXCLUDE_FIELDS - ) - - create_request_json = create_request.json(exclude=exclude_fields) - - json_bytes = create_request_json.encode("utf-8") - return hashlib.md5(json_bytes).hexdigest() + try: + # We always want to exclude the sourceHash when generating the fingerprint + exclude_fields = ( + SOURCE_HASH_EXCLUDE_FIELDS.update(exclude_fields) + if exclude_fields + else SOURCE_HASH_EXCLUDE_FIELDS + ) + + create_request_json = create_request.json(exclude=exclude_fields) + + json_bytes = create_request_json.encode("utf-8") + return hashlib.md5(json_bytes).hexdigest() + + except Exception as exc: + logger.warning(f"Failed to generate source hash due to - {exc}") + logger.debug(traceback.format_exc()) + return None diff --git a/ingestion/src/metadata/utils/storage_metadata_config.py b/ingestion/src/metadata/utils/storage_metadata_config.py index 3eb12a1670c8..7cfbdd8324e2 100644 --- a/ingestion/src/metadata/utils/storage_metadata_config.py +++ b/ingestion/src/metadata/utils/storage_metadata_config.py @@ -17,6 +17,7 @@ import requests +from metadata.clients.azure_client import AzureClient from metadata.generated.schema.entity.services.connections.database.datalake.azureConfig import ( AzureConfig, ) @@ -153,21 +154,7 @@ def _(config: StorageMetadataAdlsConfig) -> ManifestMetadataConfig: else STORAGE_METADATA_MANIFEST_FILE_NAME ) - from azure.identity import ( # pylint: disable=import-outside-toplevel - ClientSecretCredential, - ) - from azure.storage.blob import ( # pylint: disable=import-outside-toplevel - BlobServiceClient, - ) - - blob_client = BlobServiceClient( - account_url=f"https://{config.securityConfig.accountName}.blob.core.windows.net/", - credential=ClientSecretCredential( - config.securityConfig.tenantId, - config.securityConfig.clientId, - config.securityConfig.clientSecret.get_secret_value(), - ), - ) + blob_client = AzureClient(config.securityConfig).create_blob_client() reader = get_reader( config_source=AzureConfig(securityConfig=config.securityConfig), diff --git a/ingestion/src/metadata/utils/test_utils.py b/ingestion/src/metadata/utils/test_utils.py deleted file mode 100644 index 999f0f30e487..000000000000 --- a/ingestion/src/metadata/utils/test_utils.py +++ /dev/null @@ -1,65 +0,0 @@ -# Copyright 2024 Collate -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# http://www.apache.org/licenses/LICENSE-2.0 -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" -Utility functions for testing -""" -from contextlib import contextmanager - - -class MultipleException(Exception): - def __init__(self, exceptions): - self.exceptions = exceptions - super().__init__(f"Multiple exceptions occurred: {exceptions}") - - -class ErrorHandler: - """ - A context manager that accumulates errors and raises them at the end of the block. - Useful for cleaning up resources and ensuring that all errors are raised at the end of a test. - Example: - ``` - from metadata.utils.test_utils import accumulate_errors - with accumulate_errors() as error_handler: - error_handler.try_execute(lambda : 1 / 0) - error_handler.try_execute(print, "Hello, World!") - ``` - - ``` - > Hello, World! - > Traceback (most recent call last): - > ... - > ZeroDivisionError: division by zero - ``` - """ - - def __init__(self): - self.errors = [] - - def try_execute(self, func, *args, **kwargs): - try: - func(*args, **kwargs) - except Exception as e: - self.errors.append(e) - - def raise_if_errors(self): - if len(self.errors) == 1: - raise self.errors[0] - if len(self.errors) > 1: - raise MultipleException(self.errors) - - -@contextmanager -def accumulate_errors(): - error_handler = ErrorHandler() - try: - yield error_handler - finally: - error_handler.raise_if_errors() diff --git a/ingestion/src/metadata/workflow/base.py b/ingestion/src/metadata/workflow/base.py index 5641b4e253af..1ad9ba63b79c 100644 --- a/ingestion/src/metadata/workflow/base.py +++ b/ingestion/src/metadata/workflow/base.py @@ -108,7 +108,7 @@ def __init__( @property def ingestion_pipeline(self): """Get or create the Ingestion Pipeline from the configuration""" - if not self._ingestion_pipeline: + if not self._ingestion_pipeline and self.config.ingestionPipelineFQN: self._ingestion_pipeline = self.get_or_create_ingestion_pipeline() return self._ingestion_pipeline diff --git a/ingestion/tests/cli_e2e/dbt/redshift/dbt.yaml b/ingestion/tests/cli_e2e/dbt/redshift/dbt.yaml index f551ef127463..fcea4c2177fb 100644 --- a/ingestion/tests/cli_e2e/dbt/redshift/dbt.yaml +++ b/ingestion/tests/cli_e2e/dbt/redshift/dbt.yaml @@ -5,6 +5,7 @@ source: config: type: DBT dbtConfigSource: + dbtConfigType: "http" dbtCatalogHttpPath: $E2E_REDSHIFT_DBT_CATALOG_HTTP_FILE_PATH dbtManifestHttpPath: $E2E_REDSHIFT_DBT_MANIFEST_HTTP_FILE_PATH dbtRunResultsHttpPath: $E2E_REDSHIFT_DBT_RUN_RESULTS_HTTP_FILE_PATH diff --git a/ingestion/tests/integration/ometa/test_ometa_app_api.py b/ingestion/tests/integration/ometa/test_ometa_app_api.py new file mode 100644 index 000000000000..2565526d1b78 --- /dev/null +++ b/ingestion/tests/integration/ometa/test_ometa_app_api.py @@ -0,0 +1,36 @@ +# Copyright 2021 Collate +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +OpenMetadata high-level API App test +""" +from unittest import TestCase + +from metadata.generated.schema.entity.applications.app import App + +from ..integration_base import int_admin_ometa + + +class OMetaTableTest(TestCase): + """ + Run this integration test with the local API available + Install the ingestion package before running the tests + """ + + service_entity_id = None + + metadata = int_admin_ometa() + + def test_get_app(self): + """We can GET an app via the client""" + app = self.metadata.get_by_name(entity=App, fqn="SearchIndexingApplication") + self.assertIsNotNone(app) + self.assertEqual(app.name.__root__, "SearchIndexingApplication") diff --git a/ingestion/tests/integration/ometa/test_ometa_custom_properties_api.py b/ingestion/tests/integration/ometa/test_ometa_custom_properties_api.py index afe57999c705..8834acb121de 100644 --- a/ingestion/tests/integration/ometa/test_ometa_custom_properties_api.py +++ b/ingestion/tests/integration/ometa/test_ometa_custom_properties_api.py @@ -144,9 +144,12 @@ def create_custom_property(self): # Create the table size property ometa_custom_property_request = OMetaCustomProperties( entity_type=Table, - custom_property_type=CustomPropertyDataTypes.STRING, createCustomPropertyRequest=CreateCustomPropertyRequest( - name="TableSize", description="Size of the Table" + name="TableSize", + description="Size of the Table", + propertyType=self.metadata.get_property_type_ref( + CustomPropertyDataTypes.STRING + ), ), ) self.metadata.create_or_update_custom_property( @@ -156,9 +159,12 @@ def create_custom_property(self): # Create the DataQuality property for a table ometa_custom_property_request = OMetaCustomProperties( entity_type=Table, - custom_property_type=CustomPropertyDataTypes.MARKDOWN, createCustomPropertyRequest=CreateCustomPropertyRequest( - name="DataQuality", description="Quality Details of a Table" + name="DataQuality", + description="Quality Details of a Table", + propertyType=self.metadata.get_property_type_ref( + CustomPropertyDataTypes.MARKDOWN + ), ), ) self.metadata.create_or_update_custom_property( @@ -168,9 +174,12 @@ def create_custom_property(self): # Create the SchemaCost property for database schema ometa_custom_property_request = OMetaCustomProperties( entity_type=DatabaseSchema, - custom_property_type=CustomPropertyDataTypes.INTEGER, createCustomPropertyRequest=CreateCustomPropertyRequest( - name="SchemaAge", description="Age in years of a Schema" + name="SchemaAge", + description="Age in years of a Schema", + propertyType=self.metadata.get_property_type_ref( + CustomPropertyDataTypes.INTEGER + ), ), ) self.metadata.create_or_update_custom_property( diff --git a/ingestion/tests/integration/ometa/test_ometa_patch.py b/ingestion/tests/integration/ometa/test_ometa_patch.py index a5eb78932c26..409536091f2c 100644 --- a/ingestion/tests/integration/ometa/test_ometa_patch.py +++ b/ingestion/tests/integration/ometa/test_ometa_patch.py @@ -17,17 +17,6 @@ from datetime import datetime from unittest import TestCase -from ingestion.tests.integration.integration_base import ( - generate_name, - get_create_entity, - get_create_service, - get_create_team_entity, - get_create_test_case, - get_create_test_definition, - get_create_test_suite, - get_create_user_entity, - int_admin_ometa, -) from metadata.generated.schema.entity.data.database import Database from metadata.generated.schema.entity.data.databaseSchema import DatabaseSchema from metadata.generated.schema.entity.data.table import Column, DataType, Table @@ -54,6 +43,18 @@ from metadata.ingestion.models.table_metadata import ColumnTag from metadata.utils.helpers import find_column_in_table +from ..integration_base import ( + generate_name, + get_create_entity, + get_create_service, + get_create_team_entity, + get_create_test_case, + get_create_test_definition, + get_create_test_suite, + get_create_user_entity, + int_admin_ometa, +) + PII_TAG_LABEL = TagLabel( tagFQN="PII.Sensitive", labelType=LabelType.Automated, diff --git a/ingestion/tests/integration/ometa/test_ometa_topology_patch.py b/ingestion/tests/integration/ometa/test_ometa_topology_patch.py new file mode 100644 index 000000000000..7db3fa52ef0f --- /dev/null +++ b/ingestion/tests/integration/ometa/test_ometa_topology_patch.py @@ -0,0 +1,227 @@ +# Copyright 2021 Collate +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Topology Patch Integration Test +""" +from unittest import TestCase + +from metadata.generated.schema.api.data.createDatabase import CreateDatabaseRequest +from metadata.generated.schema.api.data.createDatabaseSchema import ( + CreateDatabaseSchemaRequest, +) +from metadata.generated.schema.api.data.createTable import CreateTableRequest +from metadata.generated.schema.api.services.createDatabaseService import ( + CreateDatabaseServiceRequest, +) +from metadata.generated.schema.entity.data.table import Column, DataType, Table +from metadata.generated.schema.entity.services.connections.database.common.basicAuth import ( + BasicAuth, +) +from metadata.generated.schema.entity.services.connections.database.mysqlConnection import ( + MysqlConnection, +) +from metadata.generated.schema.entity.services.connections.metadata.openMetadataConnection import ( + OpenMetadataConnection, +) +from metadata.generated.schema.entity.services.databaseService import ( + DatabaseConnection, + DatabaseService, + DatabaseServiceType, +) +from metadata.generated.schema.security.client.openMetadataJWTClientConfig import ( + OpenMetadataJWTClientConfig, +) +from metadata.ingestion.models.patch_request import ( + ALLOWED_COMMON_PATCH_FIELDS, + ARRAY_ENTITY_FIELDS, + RESTRICT_UPDATE_LIST, +) +from metadata.ingestion.ometa.ometa_api import OpenMetadata + + +class TopologyPatchTest(TestCase): + """ + Run this integration test with the local API available + Install the ingestion package before running the tests + """ + + server_config = OpenMetadataConnection( + hostPort="http://localhost:8585/api", + authProvider="openmetadata", + securityConfig=OpenMetadataJWTClientConfig( + jwtToken="eyJraWQiOiJHYjM4OWEtOWY3Ni1nZGpzLWE5MmotMDI0MmJrOTQzNTYiLCJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiJ9.eyJzdWIiOiJhZG1pbiIsImlzQm90IjpmYWxzZSwiaXNzIjoib3Blbi1tZXRhZGF0YS5vcmciLCJpYXQiOjE2NjM5Mzg0NjIsImVtYWlsIjoiYWRtaW5Ab3Blbm1ldGFkYXRhLm9yZyJ9.tS8um_5DKu7HgzGBzS1VTA5uUjKWOCU0B_j08WXBiEC0mr0zNREkqVfwFDD-d24HlNEbrqioLsBuFRiwIWKc1m_ZlVQbG7P36RUxhuv2vbSp80FKyNM-Tj93FDzq91jsyNmsQhyNv_fNr3TXfzzSPjHt8Go0FMMP66weoKMgW2PbXlhVKwEuXUHyakLLzewm9UMeQaEiRzhiTMU3UkLXcKbYEJJvfNFcLwSl9W8JCO_l0Yj3ud-qt_nQYEZwqW6u5nfdQllN133iikV4fM5QZsMCnm8Rq1mvLR0y9bmJiD7fwM1tmJ791TUWqmKaTnP49U493VanKpUAfzIiOiIbhg" + ), + ) + metadata = OpenMetadata(server_config) + + assert metadata.health_check() + + service = CreateDatabaseServiceRequest( + name="test-service-topology-patch", + serviceType=DatabaseServiceType.Mysql, + connection=DatabaseConnection( + config=MysqlConnection( + username="username", + authType=BasicAuth( + password="password", + ), + hostPort="http://localhost:1234", + ) + ), + ) + service_type = "databaseService" + + @classmethod + def setUpClass(cls) -> None: + """ + Prepare ingredients + """ + + cls.service_entity = cls.metadata.create_or_update(data=cls.service) + + create_db = CreateDatabaseRequest( + name="test-db-topology-patch", + service=cls.service_entity.fullyQualifiedName, + ) + + cls.create_db_entity = cls.metadata.create_or_update(data=create_db) + + create_schema = CreateDatabaseSchemaRequest( + name="test-schema-topology-patch", + database=cls.create_db_entity.fullyQualifiedName, + ) + + cls.create_schema_entity = cls.metadata.create_or_update(data=create_schema) + + create = CreateTableRequest( + name="test-topology-patch-table-one", + databaseSchema=cls.create_schema_entity.fullyQualifiedName, + columns=[ + Column( + name="column1", dataType=DataType.BIGINT, description="test column1" + ), + Column( + name="column2", dataType=DataType.BIGINT, description="test column2" + ), + Column( + name="column3", dataType=DataType.BIGINT, description="test column3" + ), + Column( + name="column4", dataType=DataType.BIGINT, description="test column4" + ), + Column( + name="column5", dataType=DataType.BIGINT, description="test column5" + ), + ], + ) + cls.table_entity_one = cls.metadata.create_or_update(create) + + create = CreateTableRequest( + name="test-topology-patch-table-two", + databaseSchema=cls.create_schema_entity.fullyQualifiedName, + columns=[ + Column( + name="column1", dataType=DataType.BIGINT, description="test column1" + ), + Column( + name="column2", dataType=DataType.BIGINT, description="test column2" + ), + Column( + name="column3", dataType=DataType.BIGINT, description="test column3" + ), + Column( + name="column4", dataType=DataType.BIGINT, description="test column4" + ), + Column( + name="column5", dataType=DataType.BIGINT, description="test column5" + ), + ], + ) + cls.table_entity_two = cls.metadata.create_or_update(create) + + @classmethod + def tearDownClass(cls) -> None: + """ + Clean up + """ + + service_id = str( + cls.metadata.get_by_name( + entity=DatabaseService, fqn=cls.service.name.__root__ + ).id.__root__ + ) + + cls.metadata.delete( + entity=DatabaseService, + entity_id=service_id, + recursive=True, + hard_delete=True, + ) + + def test_topology_patch_table_columns_with_random_order(self): + """Check if the table columns are patched""" + new_columns_list = [ + Column(name="column3", dataType=DataType.BIGINT), + Column(name="column4", dataType=DataType.BIGINT), + Column(name="column5", dataType=DataType.BIGINT), + Column(name="column1", dataType=DataType.BIGINT), + Column(name="column2", dataType=DataType.BIGINT), + ] + updated_table = self.table_entity_one.copy(deep=True) + updated_table.columns = new_columns_list + self.metadata.patch( + entity=type(self.table_entity_one), + source=self.table_entity_one, + destination=updated_table, + allowed_fields=ALLOWED_COMMON_PATCH_FIELDS, + restrict_update_fields=RESTRICT_UPDATE_LIST, + array_entity_fields=ARRAY_ENTITY_FIELDS, + ) + table_entity = self.metadata.get_by_id( + entity=Table, entity_id=self.table_entity_one.id.__root__ + ) + self.assertEqual(table_entity.columns[0].description.__root__, "test column1") + self.assertEqual(table_entity.columns[1].description.__root__, "test column2") + self.assertEqual(table_entity.columns[2].description.__root__, "test column3") + self.assertEqual(table_entity.columns[3].description.__root__, "test column4") + self.assertEqual(table_entity.columns[4].description.__root__, "test column5") + + def test_topology_patch_table_columns_with_add_del(self): + """Check if the table columns are patched""" + new_columns_list = [ + Column( + name="column7", dataType=DataType.BIGINT, description="test column7" + ), + Column(name="column3", dataType=DataType.BIGINT), + Column(name="column5", dataType=DataType.BIGINT), + Column(name="column1", dataType=DataType.BIGINT), + Column( + name="column6", dataType=DataType.BIGINT, description="test column6" + ), + ] + updated_table = self.table_entity_two.copy(deep=True) + updated_table.columns = new_columns_list + self.metadata.patch( + entity=type(self.table_entity_two), + source=self.table_entity_two, + destination=updated_table, + allowed_fields=ALLOWED_COMMON_PATCH_FIELDS, + restrict_update_fields=RESTRICT_UPDATE_LIST, + array_entity_fields=ARRAY_ENTITY_FIELDS, + ) + table_entity = self.metadata.get_by_id( + entity=Table, entity_id=self.table_entity_two.id.__root__ + ) + self.assertEqual(table_entity.columns[0].description.__root__, "test column1") + self.assertEqual(table_entity.columns[1].description.__root__, "test column3") + self.assertEqual(table_entity.columns[2].description.__root__, "test column5") + self.assertEqual(table_entity.columns[3].description.__root__, "test column7") + self.assertEqual(table_entity.columns[4].description.__root__, "test column6") diff --git a/ingestion/tests/integration/profiler/__init__.py b/ingestion/tests/integration/profiler/__init__.py deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/ingestion/tests/integration/profiler/test_nosql_profiler.py b/ingestion/tests/integration/profiler/test_nosql_profiler.py deleted file mode 100644 index 693ad7ec7c13..000000000000 --- a/ingestion/tests/integration/profiler/test_nosql_profiler.py +++ /dev/null @@ -1,333 +0,0 @@ -# Copyright 2024 Collate -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# http://www.apache.org/licenses/LICENSE-2.0 -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -""" -Test the NoSQL profiler using a MongoDB container -To run this we need OpenMetadata server up and running. -No sample data is required beforehand - -Test Steps: - -1. Start a MongoDB container -2. Ingest data into OpenMetadata -3. Run the profiler workflow -4. Verify the profiler output -5. Tear down the MongoDB container and delete the service from OpenMetadata -""" - -from copy import deepcopy -from datetime import datetime, timedelta -from functools import partial -from pathlib import Path -from random import choice, randint -from unittest import TestCase - -from pymongo import MongoClient, database -from testcontainers.mongodb import MongoDbContainer - -from ingestion.tests.integration.integration_base import int_admin_ometa -from metadata.generated.schema.entity.data.table import ColumnProfile, Table -from metadata.generated.schema.entity.services.databaseService import DatabaseService -from metadata.ingestion.ometa.ometa_api import OpenMetadata -from metadata.profiler.api.models import TableConfig -from metadata.utils.constants import SAMPLE_DATA_DEFAULT_COUNT -from metadata.utils.helpers import datetime_to_ts -from metadata.utils.test_utils import accumulate_errors -from metadata.utils.time_utils import get_end_of_day_timestamp_mill -from metadata.workflow.metadata import MetadataWorkflow -from metadata.workflow.profiler import ProfilerWorkflow -from metadata.workflow.workflow_output_handler import print_status - -SERVICE_NAME = Path(__file__).stem - - -def add_query_config(config, table_config: TableConfig) -> dict: - config_copy = deepcopy(config) - config_copy["processor"]["config"].setdefault("tableConfig", []) - config_copy["processor"]["config"]["tableConfig"].append(table_config) - return config_copy - - -def get_ingestion_config(mongo_port: str, mongo_user: str, mongo_pass: str): - return { - "source": { - "type": "mongodb", - "serviceName": SERVICE_NAME, - "serviceConnection": { - "config": { - "type": "MongoDB", - "hostPort": f"localhost:{mongo_port}", - "username": mongo_user, - "password": mongo_pass, - } - }, - "sourceConfig": {"config": {"type": "DatabaseMetadata"}}, - }, - "sink": {"type": "metadata-rest", "config": {}}, - "workflowConfig": { - "loggerLevel": "DEBUG", - "openMetadataServerConfig": { - "hostPort": "http://localhost:8585/api", - "authProvider": "openmetadata", - "securityConfig": { - "jwtToken": "eyJraWQiOiJHYjM4OWEtOWY3Ni1nZGpzLWE5MmotMDI0MmJrOTQzNTYiLCJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiJ9.eyJzdWIiOiJhZG1pbiIsImlzQm90IjpmYWxzZSwiaXNzIjoib3Blbi1tZXRhZGF0YS5vcmciLCJpYXQiOjE2NjM5Mzg0NjIsImVtYWlsIjoiYWRtaW5Ab3Blbm1ldGFkYXRhLm9yZyJ9.tS8um_5DKu7HgzGBzS1VTA5uUjKWOCU0B_j08WXBiEC0mr0zNREkqVfwFDD-d24HlNEbrqioLsBuFRiwIWKc1m_ZlVQbG7P36RUxhuv2vbSp80FKyNM-Tj93FDzq91jsyNmsQhyNv_fNr3TXfzzSPjHt8Go0FMMP66weoKMgW2PbXlhVKwEuXUHyakLLzewm9UMeQaEiRzhiTMU3UkLXcKbYEJJvfNFcLwSl9W8JCO_l0Yj3ud-qt_nQYEZwqW6u5nfdQllN133iikV4fM5QZsMCnm8Rq1mvLR0y9bmJiD7fwM1tmJ791TUWqmKaTnP49U493VanKpUAfzIiOiIbhg" - }, - }, - }, - } - - -TEST_DATABASE = "test-database" -EMPTY_COLLECTION = "empty-collection" -TEST_COLLECTION = "test-collection" -NUM_ROWS = 200 - - -def random_row(): - return { - "name": choice(["John", "Jane", "Alice", "Bob"]), - "age": randint(20, 60), - "city": choice(["New York", "Chicago", "San Francisco"]), - "nested": {"key": "value" + str(randint(1, 10))}, - } - - -TEST_DATA = [random_row() for _ in range(NUM_ROWS)] + [ - { - "name": "John", - "age": 60, - "city": "New York", - }, - { - "name": "Jane", - "age": 20, - "city": "New York", - }, -] - - -class NoSQLProfiler(TestCase): - """datalake profiler E2E test""" - - mongo_container: MongoDbContainer - client: MongoClient - db: database.Database - collection: database.Collection - ingestion_config: dict - metadata: OpenMetadata - - @classmethod - def setUpClass(cls) -> None: - cls.metadata = int_admin_ometa() - cls.mongo_container = MongoDbContainer("mongo:7.0.5-jammy") - cls.mongo_container.start() - cls.client = MongoClient(cls.mongo_container.get_connection_url()) - cls.db = cls.client[TEST_DATABASE] - cls.collection = cls.db[TEST_COLLECTION] - cls.collection.insert_many(TEST_DATA) - cls.db.create_collection(EMPTY_COLLECTION) - cls.ingestion_config = get_ingestion_config( - cls.mongo_container.get_exposed_port("27017"), "test", "test" - ) - # cls.client["admin"].command("grantRolesToUser", "test", roles=["userAdminAnyDatabase"]) - ingestion_workflow = MetadataWorkflow.create( - cls.ingestion_config, - ) - ingestion_workflow.execute() - ingestion_workflow.raise_from_status() - print_status(ingestion_workflow) - ingestion_workflow.stop() - - @classmethod - def tearDownClass(cls): - with accumulate_errors() as error_handler: - error_handler.try_execute(partial(cls.mongo_container.stop, force=True)) - error_handler.try_execute(cls.delete_service) - - @classmethod - def delete_service(cls): - service_id = str( - cls.metadata.get_by_name( - entity=DatabaseService, fqn=SERVICE_NAME - ).id.__root__ - ) - cls.metadata.delete( - entity=DatabaseService, - entity_id=service_id, - recursive=True, - hard_delete=True, - ) - - def test_setup_teardown(self): - """ - does nothing. useful to check if the setup and teardown methods are working - """ - pass - - def run_profiler_workflow(self, config): - profiler_workflow = ProfilerWorkflow.create(config) - profiler_workflow.execute() - status = profiler_workflow.result_status() - profiler_workflow.stop() - assert status == 0 - - def test_simple(self): - workflow_config = deepcopy(self.ingestion_config) - workflow_config["source"]["sourceConfig"]["config"].update( - { - "type": "Profiler", - } - ) - workflow_config["processor"] = { - "type": "orm-profiler", - "config": {}, - } - self.run_profiler_workflow(workflow_config) - - cases = [ - { - "collection": EMPTY_COLLECTION, - "expected": { - "rowCount": 0, - "columns": [], - }, - }, - { - "collection": TEST_COLLECTION, - "expected": { - "rowCount": len(TEST_DATA), - "columns": [ - ColumnProfile( - name="age", - timestamp=datetime.now().timestamp(), - max=60, - min=20, - ), - ], - }, - }, - ] - - for tc in cases: - collection = tc["collection"] - expected = tc["expected"] - collection_profile = self.metadata.get_profile_data( - f"{SERVICE_NAME}.default.{TEST_DATABASE}.{collection}", - datetime_to_ts(datetime.now() - timedelta(seconds=10)), - get_end_of_day_timestamp_mill(), - ) - assert collection_profile.entities - assert collection_profile.entities[-1].rowCount == expected["rowCount"] - column_profile = self.metadata.get_profile_data( - f"{SERVICE_NAME}.default.{TEST_DATABASE}.{collection}.age", - datetime_to_ts(datetime.now() - timedelta(seconds=10)), - get_end_of_day_timestamp_mill(), - profile_type=ColumnProfile, - ) - assert (len(column_profile.entities) > 0) == ( - len(tc["expected"]["columns"]) > 0 - ) - if len(expected["columns"]) > 0: - for c1, c2 in zip(column_profile.entities, expected["columns"]): - assert c1.name == c2.name - assert c1.max == c2.max - assert c1.min == c2.min - - table = self.metadata.get_by_name( - Table, f"{SERVICE_NAME}.default.{TEST_DATABASE}.{TEST_COLLECTION}" - ) - sample_data = self.metadata.get_sample_data(table) - assert [c.__root__ for c in sample_data.sampleData.columns] == [ - "_id", - "name", - "age", - "city", - "nested", - ] - assert len(sample_data.sampleData.rows) == SAMPLE_DATA_DEFAULT_COUNT - - def test_custom_query(self): - workflow_config = deepcopy(self.ingestion_config) - workflow_config["source"]["sourceConfig"]["config"].update( - { - "type": "Profiler", - } - ) - query_age = TEST_DATA[0]["age"] - workflow_config["processor"] = { - "type": "orm-profiler", - "config": { - "tableConfig": [ - { - "fullyQualifiedName": f"{SERVICE_NAME}.default.{TEST_DATABASE}.{TEST_COLLECTION}", - "profileQuery": '{"age": %s}' % query_age, - } - ], - }, - } - self.run_profiler_workflow(workflow_config) - - cases = [ - { - "collection": EMPTY_COLLECTION, - "expected": { - "rowCount": 0, - "columns": [], - }, - }, - { - "collection": TEST_COLLECTION, - "expected": { - "rowCount": len(TEST_DATA), - "columns": [ - ColumnProfile( - name="age", - timestamp=datetime.now().timestamp(), - max=query_age, - min=query_age, - ), - ], - }, - }, - ] - - for tc in cases: - collection = tc["collection"] - expected_row_count = tc["expected"]["rowCount"] - - collection_profile = self.metadata.get_profile_data( - f"{SERVICE_NAME}.default.{TEST_DATABASE}.{collection}", - datetime_to_ts(datetime.now() - timedelta(seconds=10)), - get_end_of_day_timestamp_mill(), - ) - assert collection_profile.entities, collection - assert ( - collection_profile.entities[-1].rowCount == expected_row_count - ), collection - column_profile = self.metadata.get_profile_data( - f"{SERVICE_NAME}.default.{TEST_DATABASE}.{collection}.age", - datetime_to_ts(datetime.now() - timedelta(seconds=10)), - get_end_of_day_timestamp_mill(), - profile_type=ColumnProfile, - ) - assert (len(column_profile.entities) > 0) == ( - len(tc["expected"]["columns"]) > 0 - ) - table = self.metadata.get_by_name( - Table, f"{SERVICE_NAME}.default.{TEST_DATABASE}.{TEST_COLLECTION}" - ) - sample_data = self.metadata.get_sample_data(table) - age_column_index = [ - col.__root__ for col in sample_data.sampleData.columns - ].index("age") - assert all( - [r[age_column_index] == query_age for r in sample_data.sampleData.rows] - ) diff --git a/ingestion/tests/unit/metadata/cli/resources/profiler_workflow.py b/ingestion/tests/unit/metadata/cli/resources/profiler_workflow.py index 5a9573d50381..b9bfe8e7ed15 100644 --- a/ingestion/tests/unit/metadata/cli/resources/profiler_workflow.py +++ b/ingestion/tests/unit/metadata/cli/resources/profiler_workflow.py @@ -1,7 +1,7 @@ """ This file has been generated from dag_runner.j2 """ -from openmetadata.workflows import workflow_factory +from openmetadata_managed_apis.workflows import workflow_factory workflow = workflow_factory.WorkflowFactory.create( "/airflow/dag_generated_configs/local_redshift_profiler_e9AziRXs.json" diff --git a/ingestion/tests/unit/metadata/cli/resources/profiler_workflow.txt b/ingestion/tests/unit/metadata/cli/resources/profiler_workflow.txt index b3945ba7e255..bdb70bb1fd91 100644 --- a/ingestion/tests/unit/metadata/cli/resources/profiler_workflow.txt +++ b/ingestion/tests/unit/metadata/cli/resources/profiler_workflow.txt @@ -2,7 +2,7 @@ This file has been generated from dag_runner.j2 """ from airflow import DAG -from openmetadata.workflows import workflow_factory +from openmetadata_managed_apis.workflows import workflow_factory workflow = workflow_factory.WorkflowFactory.create("/airflow/dag_generated_configs/local_redshift_profiler_e9AziRXs.json") workflow.generate_dag(globals()) \ No newline at end of file diff --git a/ingestion/tests/unit/test_azure_credentials.py b/ingestion/tests/unit/test_azure_credentials.py new file mode 100644 index 000000000000..bb1f03f96c51 --- /dev/null +++ b/ingestion/tests/unit/test_azure_credentials.py @@ -0,0 +1,63 @@ +import unittest +from unittest.mock import patch + +from metadata.clients.azure_client import AzureClient +from metadata.generated.schema.security.credentials.azureCredentials import ( + AzureCredentials, +) + + +class TestAzureClient(unittest.TestCase): + @patch("azure.identity.ClientSecretCredential") + @patch("azure.identity.DefaultAzureCredential") + def test_create_client( + self, + mock_default_credential, + mock_client_secret_credential, + ): + # Test with ClientSecretCredential + credentials = AzureCredentials( + clientId="clientId", clientSecret="clientSecret", tenantId="tenantId" + ) + instance = AzureClient(credentials) + instance.create_client() + + mock_client_secret_credential.assert_called_once() + mock_client_secret_credential.reset_mock() + + credentials = AzureCredentials( + clientId="clientId", + ) + instance = AzureClient(credentials) + + instance.create_client() + + mock_default_credential.assert_called_once() + + @patch("azure.storage.blob.BlobServiceClient") + def test_create_blob_client(self, mock_blob_service_client): + credentials = AzureCredentials( + clientId="clientId", clientSecret="clientSecret", tenantId="tenantId" + ) + with self.assertRaises(ValueError): + AzureClient(credentials=credentials).create_blob_client() + + credentials.accountName = "accountName" + AzureClient(credentials=credentials).create_blob_client() + mock_blob_service_client.assert_called_once() + + @patch("azure.keyvault.secrets.SecretClient") + def test_create_secret_client(self, mock_secret_client): + credentials = AzureCredentials( + clientId="clientId", clientSecret="clientSecret", tenantId="tenantId" + ) + with self.assertRaises(ValueError): + AzureClient(credentials=credentials).create_secret_client() + + credentials.vaultName = "vaultName" + AzureClient(credentials=credentials).create_secret_client() + mock_secret_client.assert_called_once() + + +if __name__ == "__main__": + unittest.main() diff --git a/ingestion/tests/unit/test_build_connection_url.py b/ingestion/tests/unit/test_build_connection_url.py new file mode 100644 index 000000000000..8cf60dae6774 --- /dev/null +++ b/ingestion/tests/unit/test_build_connection_url.py @@ -0,0 +1,138 @@ +import unittest +from unittest.mock import patch + +from azure.core.credentials import AccessToken +from azure.identity import ClientSecretCredential + +from metadata.generated.schema.entity.services.connections.database.azureSQLConnection import ( + Authentication, + AuthenticationMode, + AzureSQLConnection, +) +from metadata.generated.schema.entity.services.connections.database.common.azureConfig import ( + AzureConfigurationSource, +) +from metadata.generated.schema.entity.services.connections.database.common.basicAuth import ( + BasicAuth, +) +from metadata.generated.schema.entity.services.connections.database.mysqlConnection import ( + MysqlConnection, +) +from metadata.generated.schema.entity.services.connections.database.postgresConnection import ( + PostgresConnection, +) +from metadata.generated.schema.security.credentials.azureCredentials import ( + AzureCredentials, +) +from metadata.ingestion.source.database.azuresql.connection import get_connection_url +from metadata.ingestion.source.database.mysql.connection import ( + get_connection as mysql_get_connection, +) +from metadata.ingestion.source.database.postgres.connection import ( + get_connection as postgres_get_connection, +) + + +class TestGetConnectionURL(unittest.TestCase): + def test_get_connection_url_wo_active_directory_password(self): + connection = AzureSQLConnection( + driver="SQL Server", + hostPort="myserver.database.windows.net", + database="mydb", + username="myuser", + password="mypassword", + authenticationMode=AuthenticationMode( + authentication=Authentication.ActiveDirectoryPassword, + encrypt=True, + trustServerCertificate=False, + connectionTimeout=45, + ), + ) + expected_url = "mssql+pyodbc://?odbc_connect=Driver%3DSQL+Server%3BServer%3Dmyserver.database.windows.net%3BDatabase%3Dmydb%3BUid%3Dmyuser%3BPwd%3Dmypassword%3BEncrypt%3Dyes%3BTrustServerCertificate%3Dno%3BConnection+Timeout%3D45%3BAuthentication%3DActiveDirectoryPassword%3B" + self.assertEqual(str(get_connection_url(connection)), expected_url) + + connection = AzureSQLConnection( + driver="SQL Server", + hostPort="myserver.database.windows.net", + database="mydb", + username="myuser", + password="mypassword", + authenticationMode=AuthenticationMode( + authentication=Authentication.ActiveDirectoryPassword, + ), + ) + + expected_url = "mssql+pyodbc://?odbc_connect=Driver%3DSQL+Server%3BServer%3Dmyserver.database.windows.net%3BDatabase%3Dmydb%3BUid%3Dmyuser%3BPwd%3Dmypassword%3BEncrypt%3Dno%3BTrustServerCertificate%3Dno%3BConnection+Timeout%3D30%3BAuthentication%3DActiveDirectoryPassword%3B" + self.assertEqual(str(get_connection_url(connection)), expected_url) + + def test_get_connection_url_mysql(self): + connection = MysqlConnection( + username="openmetadata_user", + authType=BasicAuth(password="openmetadata_password"), + hostPort="localhost:3306", + databaseSchema="openmetadata_db", + ) + engine_connection = mysql_get_connection(connection) + self.assertEqual( + str(engine_connection.url), + "mysql+pymysql://openmetadata_user:openmetadata_password@localhost:3306/openmetadata_db", + ) + connection = MysqlConnection( + username="openmetadata_user", + authType=AzureConfigurationSource( + azureConfig=AzureCredentials( + clientId="clientid", + tenantId="tenantid", + clientSecret="clientsecret", + scopes="scope1,scope2", + ) + ), + hostPort="localhost:3306", + databaseSchema="openmetadata_db", + ) + with patch.object( + ClientSecretCredential, + "get_token", + return_value=AccessToken(token="mocked_token", expires_on=100), + ): + engine_connection = mysql_get_connection(connection) + self.assertEqual( + str(engine_connection.url), + "mysql+pymysql://openmetadata_user:mocked_token@localhost:3306/openmetadata_db", + ) + + def test_get_connection_url_postgres(self): + connection = PostgresConnection( + username="openmetadata_user", + authType=BasicAuth(password="openmetadata_password"), + hostPort="localhost:3306", + database="openmetadata_db", + ) + engine_connection = postgres_get_connection(connection) + self.assertEqual( + str(engine_connection.url), + "postgresql+psycopg2://openmetadata_user:openmetadata_password@localhost:3306/openmetadata_db", + ) + connection = PostgresConnection( + username="openmetadata_user", + authType=AzureConfigurationSource( + azureConfig=AzureCredentials( + clientId="clientid", + tenantId="tenantid", + clientSecret="clientsecret", + scopes="scope1,scope2", + ) + ), + hostPort="localhost:3306", + database="openmetadata_db", + ) + with patch.object( + ClientSecretCredential, + "get_token", + return_value=AccessToken(token="mocked_token", expires_on=100), + ): + engine_connection = postgres_get_connection(connection) + self.assertEqual( + str(engine_connection.url), + "postgresql+psycopg2://openmetadata_user:mocked_token@localhost:3306/openmetadata_db", + ) diff --git a/ingestion/tests/unit/test_credentials.py b/ingestion/tests/unit/test_credentials.py index 6721da1aa999..ff385d814f5a 100644 --- a/ingestion/tests/unit/test_credentials.py +++ b/ingestion/tests/unit/test_credentials.py @@ -15,6 +15,9 @@ from pydantic import SecretStr +from metadata.generated.schema.security.credentials.gcpExternalAccount import ( + GcpExternalAccount, +) from metadata.generated.schema.security.credentials.gcpValues import ( GcpCredentialsValues, ) @@ -29,7 +32,7 @@ class TestCredentials(TestCase): Validate credentials handling """ - def test_build_google_credentials_dict(self): + def test_build_service_account_google_credentials_dict(self): """ Check how we can validate GCS values """ @@ -52,7 +55,7 @@ def test_build_google_credentials_dict(self): -----END RSA PRIVATE KEY-----""" gcp_values = GcpCredentialsValues( - type="my_type", + type="service_account", projectId=["project_id"], privateKeyId="private_key_id", privateKey=private_key, @@ -62,7 +65,7 @@ def test_build_google_credentials_dict(self): ) expected_dict = { - "type": "my_type", + "type": "service_account", "project_id": ["project_id"], "private_key_id": "private_key_id", "private_key": private_key, @@ -82,3 +85,25 @@ def test_build_google_credentials_dict(self): with self.assertRaises(InvalidPrivateKeyException): build_google_credentials_dict(gcp_values) + + def test_build_external_account_google_credentials_dict(self): + """ + Check how we can validate GCS values + """ + gcp_values = GcpExternalAccount( + externalType="external_account", + audience="audience", + subjectTokenType="subject_token_type", + tokenURL="token_url", + credentialSource={"environmentId": "environment_id"}, + ) + + expected_dict = { + "type": "external_account", + "audience": "audience", + "subject_token_type": "subject_token_type", + "token_url": "token_url", + "credential_source": {"environmentId": "environment_id"}, + } + + self.assertEqual(expected_dict, build_google_credentials_dict(gcp_values)) diff --git a/ingestion/tests/unit/test_dbt.py b/ingestion/tests/unit/test_dbt.py index 072abc5b6615..962332d53a61 100644 --- a/ingestion/tests/unit/test_dbt.py +++ b/ingestion/tests/unit/test_dbt.py @@ -46,6 +46,7 @@ "config": { "type": "DBT", "dbtConfigSource": { + "dbtConfigType": "local", "dbtCatalogFilePath": "sample/dbt_files/catalog.json", "dbtManifestFilePath": "sample/dbt_files/manifest.json", "dbtRunResultsFilePath": "sample/dbt_files/run_results.json", diff --git a/ingestion/tests/unit/test_json_schema_parser.py b/ingestion/tests/unit/test_json_schema_parser.py index 09f5f91d299a..4fd2c9b58636 100644 --- a/ingestion/tests/unit/test_json_schema_parser.py +++ b/ingestion/tests/unit/test_json_schema_parser.py @@ -30,15 +30,18 @@ class JsonSchemaParserTests(TestCase): "properties": { "firstName": { "type": "string", + "title": "First Name", "description": "The person's first name." }, "lastName": { "type": "string", + "title": "Last Name", "description": "The person's last name." }, "age": { "description": "Age in years which must be equal to or greater than zero.", "type": "integer", + "title": "Person Age", "minimum": 0 } } @@ -58,6 +61,12 @@ def test_field_names(self): } self.assertEqual(field_names, {"firstName", "lastName", "age"}) + # validate display names + field_display_names = { + str(field.displayName) for field in self.parsed_schema[0].children + } + self.assertEqual(field_display_names, {"First Name", "Last Name", "Person Age"}) + def test_field_types(self): field_types = { str(field.dataType.name) for field in self.parsed_schema[0].children diff --git a/ingestion/tests/unit/test_workflow_parse.py b/ingestion/tests/unit/test_workflow_parse.py index 106097ecc127..549c7d5a4a1a 100644 --- a/ingestion/tests/unit/test_workflow_parse.py +++ b/ingestion/tests/unit/test_workflow_parse.py @@ -700,3 +700,150 @@ def test_parsing_automation_workflow_athena(self): "1 validation error for AthenaConnection\ns3StagingDir\n invalid or missing URL scheme (type=value_error.url.scheme)", str(err.exception), ) + + def test_parsing_dbt_workflow_ok(self): + """ + Test dbt workflow Config parsing OK + """ + + config_dict = { + "source": { + "type": "dbt", + "serviceName": "dbt_prod", + "sourceConfig": { + "config": { + "type": "DBT", + "dbtConfigSource": { + "dbtConfigType": "local", + "dbtCatalogFilePath": "/path/to/catalog.json", + "dbtManifestFilePath": "/path/to/manifest.json", + "dbtRunResultsFilePath": "/path/to/run_results.json", + }, + "dbtUpdateDescriptions": True, + "includeTags": True, + "dbtClassificationName": "dbtTags", + "databaseFilterPattern": {"includes": ["test"]}, + "schemaFilterPattern": { + "includes": ["test1"], + "excludes": [".*schema.*"], + }, + "tableFilterPattern": { + "includes": ["test3"], + "excludes": [".*table_name.*"], + }, + } + }, + }, + "sink": {"type": "metadata-rest", "config": {}}, + "workflowConfig": { + "loggerLevel": "DEBUG", + "openMetadataServerConfig": { + "hostPort": "http://localhost:8585/api", + "authProvider": "openmetadata", + "securityConfig": {"jwtToken": "jwt_token"}, + }, + }, + } + + self.assertIsNotNone(parse_workflow_config_gracefully(config_dict)) + + def test_parsing_dbt_workflow_ko(self): + """ + Test dbt workflow Config parsing OK + """ + + config_dict_type_error_ko = { + "source": { + "type": "dbt", + "serviceName": "dbt_prod", + "sourceConfig": { + "config": { + "type": "DBT", + "dbtConfigSource": { + "dbtConfigType": "cloud", + "dbtCloudAuthToken": "token", + "dbtCloudAccountId": "ID", + "dbtCloudJobId": "JOB ID", + }, + "dbtUpdateDescriptions": True, + "includeTags": True, + "dbtClassificationName": "dbtTags", + "databaseFilterPattern": {"includes": ["test"]}, + "schemaFilterPattern": { + "includes": ["test1"], + "excludes": [".*schema.*"], + }, + "tableFilterPattern": { + "includes": ["test3"], + "excludes": [".*table_name.*"], + }, + } + }, + }, + "sink": {"type": "metadata-rest", "config": {}}, + "workflowConfig": { + "loggerLevel": "DEBUG", + "openMetadataServerConfig": { + "hostPort": "http://localhost:8585/api", + "authProvider": "openmetadata", + "securityConfig": {"jwtToken": "jwt_token"}, + }, + }, + } + with self.assertRaises(ParsingConfigurationError) as err: + parse_workflow_config_gracefully(config_dict_type_error_ko) + self.assertIn( + "We encountered an error parsing the configuration of your DbtCloudConfig.\nYou might need to review your config based on the original cause of this failure:\n\t - Missing parameter 'dbtCloudUrl'", + str(err.exception), + ) + + def test_parsing_dbt_pipeline_ko(self): + """ + Test dbt workflow Config parsing OK + """ + + config_dict_dbt_pipeline_ko = { + "source": { + "type": "dbt", + "serviceName": "dbt_prod", + "sourceConfig": { + "config": { + "type": "DBT", + "dbtConfigSource": { + "dbtConfigType": "cloud", + "dbtCloudAuthToken": "token", + "dbtCloudAccountId": "ID", + "dbtCloudJobId": "JOB ID", + "dbtCloudUrl": "https://clouddbt.com", + }, + "dbtUpdateDescription": True, + "includeTags": True, + "dbtClassificationName": "dbtTags", + "databaseFilterPattern": {"includes": ["test"]}, + "schemaFilterPattern": { + "includes": ["test1"], + "excludes": [".*schema.*"], + }, + "tableFilterPattern": { + "includes": ["test3"], + "excludes": [".*table_name.*"], + }, + } + }, + }, + "sink": {"type": "metadata-rest", "config": {}}, + "workflowConfig": { + "loggerLevel": "DEBUG", + "openMetadataServerConfig": { + "hostPort": "http://localhost:8585/api", + "authProvider": "openmetadata", + "securityConfig": {"jwtToken": "jwt_token"}, + }, + }, + } + with self.assertRaises(ParsingConfigurationError) as err: + parse_workflow_config_gracefully(config_dict_dbt_pipeline_ko) + self.assertIn( + "We encountered an error parsing the configuration of your DbtPipeline.\nYou might need to review your config based on the original cause of this failure:\n\t - Extra parameter 'dbtUpdateDescription'", + str(err.exception), + ) diff --git a/ingestion/tests/unit/topology/dashboard/test_metabase.py b/ingestion/tests/unit/topology/dashboard/test_metabase.py index 5e5160d5536c..fb5f3783e643 100644 --- a/ingestion/tests/unit/topology/dashboard/test_metabase.py +++ b/ingestion/tests/unit/topology/dashboard/test_metabase.py @@ -47,12 +47,12 @@ from metadata.ingestion.source.dashboard.metabase import metadata as MetabaseMetadata from metadata.ingestion.source.dashboard.metabase.metadata import MetabaseSource from metadata.ingestion.source.dashboard.metabase.models import ( + DashCard, DatasetQuery, MetabaseChart, MetabaseDashboardDetails, MetabaseTable, Native, - OrderedCard, ) from metadata.utils import fqn @@ -127,7 +127,7 @@ MOCK_CHARTS = [ - OrderedCard( + DashCard( card=MetabaseChart( description="Test Chart", table_id=1, @@ -138,7 +138,7 @@ display="chart1", ) ), - OrderedCard( + DashCard( card=MetabaseChart( description="Test Chart", table_id=1, @@ -151,7 +151,7 @@ display="chart2", ) ), - OrderedCard(card=MetabaseChart(name="chart3", id="3")), + DashCard(card=MetabaseChart(name="chart3", id="3")), ] @@ -170,7 +170,7 @@ ) MOCK_DASHBOARD_DETAILS = MetabaseDashboardDetails( - description="SAMPLE DESCRIPTION", name="test_db", id="1", ordered_cards=MOCK_CHARTS + description="SAMPLE DESCRIPTION", name="test_db", id="1", dashcards=MOCK_CHARTS ) @@ -302,21 +302,21 @@ def test_yield_lineage(self, *_): # test out _yield_lineage_from_api mock_dashboard = deepcopy(MOCK_DASHBOARD_DETAILS) - mock_dashboard.ordered_cards = [MOCK_DASHBOARD_DETAILS.ordered_cards[0]] + mock_dashboard.dashcards = [MOCK_DASHBOARD_DETAILS.dashcards[0]] result = self.metabase.yield_dashboard_lineage_details( dashboard_details=mock_dashboard, db_service_name="db.service.name" ) self.assertEqual(next(result).right, EXPECTED_LINEAGE) # test out _yield_lineage_from_query - mock_dashboard.ordered_cards = [MOCK_DASHBOARD_DETAILS.ordered_cards[1]] + mock_dashboard.dashcards = [MOCK_DASHBOARD_DETAILS.dashcards[1]] result = self.metabase.yield_dashboard_lineage_details( dashboard_details=mock_dashboard, db_service_name="db.service.name" ) self.assertEqual(next(result).right, EXPECTED_LINEAGE) # test out if no query type - mock_dashboard.ordered_cards = [MOCK_DASHBOARD_DETAILS.ordered_cards[2]] + mock_dashboard.dashcards = [MOCK_DASHBOARD_DETAILS.dashcards[2]] result = self.metabase.yield_dashboard_lineage_details( dashboard_details=mock_dashboard, db_service_name="db.service.name" ) diff --git a/ingestion/tests/unit/topology/database/test_datalake.py b/ingestion/tests/unit/topology/database/test_datalake.py index 3819f8864557..8579f71cf60e 100644 --- a/ingestion/tests/unit/topology/database/test_datalake.py +++ b/ingestion/tests/unit/topology/database/test_datalake.py @@ -33,7 +33,10 @@ from metadata.ingestion.source.database.datalake.metadata import DatalakeSource from metadata.readers.dataframe.avro import AvroDataFrameReader from metadata.readers.dataframe.json import JSONDataFrameReader -from metadata.utils.datalake.datalake_utils import GenericDataFrameColumnParser +from metadata.utils.datalake.datalake_utils import ( + GenericDataFrameColumnParser, + JsonDataFrameColumnParser, +) mock_datalake_config = { "source": { @@ -231,6 +234,60 @@ EXAMPLE_JSON_COL_4 = deepcopy(EXAMPLE_JSON_COL_3) + +EXAMPLE_JSON_TEST_5 = """ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Person", + "type": "object", + "properties": { + "firstName": { + "type": "string", + "title": "First Name", + "description": "The person's first name." + }, + "lastName": { + "title": "Last Name", + "type": "string", + "description": "The person's last name." + }, + "age": { + "type": "integer", + "description": "Age in years.", + "minimum": 0 + } + }, + "required": ["firstName", "lastName"] +} +""" + +EXAMPLE_JSON_COL_5 = [ + Column( + name="Person", + dataType="RECORD", + children=[ + Column( + name="firstName", + dataType="STRING", + description="The person's first name.", + displayName="First Name", + ), + Column( + name="lastName", + dataType="STRING", + description="The person's last name.", + displayName="Last Name", + ), + Column( + name="age", + dataType="INT", + description="Age in years.", + ), + ], + ) +] + + EXAMPLE_JSON_COL_4[3].children[3].children = [ Column( name="lat", @@ -446,10 +503,10 @@ def test_json_file_parse(self): actual_df_1 = JSONDataFrameReader.read_from_json( key="file.json", json_text=EXAMPLE_JSON_TEST_1, decode=True - )[0] + )[0][0] actual_df_2 = JSONDataFrameReader.read_from_json( key="file.json", json_text=EXAMPLE_JSON_TEST_2, decode=True - )[0] + )[0][0] assert actual_df_1.compare(exp_df_list).empty assert actual_df_2.compare(exp_df_obj).empty @@ -458,7 +515,7 @@ def test_json_file_parse(self): actual_df_3 = JSONDataFrameReader.read_from_json( key="file.json", json_text=EXAMPLE_JSON_TEST_3, decode=True - )[0] + )[0][0] actual_cols_3 = GenericDataFrameColumnParser._get_columns( actual_df_3 ) # pylint: disable=protected-access @@ -466,12 +523,19 @@ def test_json_file_parse(self): actual_df_4 = JSONDataFrameReader.read_from_json( key="file.json", json_text=EXAMPLE_JSON_TEST_4, decode=True - )[0] + )[0][0] actual_cols_4 = GenericDataFrameColumnParser._get_columns( actual_df_4 ) # pylint: disable=protected-access assert actual_cols_4 == EXAMPLE_JSON_COL_4 + actual_df_5, raw_data = JSONDataFrameReader.read_from_json( + key="file.json", json_text=EXAMPLE_JSON_TEST_5, decode=True + ) + json_parser = JsonDataFrameColumnParser(actual_df_5[0], raw_data=raw_data) + actual_cols_5 = json_parser.get_columns() + assert actual_cols_5 == EXAMPLE_JSON_COL_5 + def test_avro_file_parse(self): columns = AvroDataFrameReader.read_from_avro(AVRO_SCHEMA_FILE) Column.__eq__ = custom_column_compare diff --git a/ingestion/tests/unit/topology/storage/test_storage.py b/ingestion/tests/unit/topology/storage/test_storage.py index 3920821dd32b..0cb7a311b6bf 100644 --- a/ingestion/tests/unit/topology/storage/test_storage.py +++ b/ingestion/tests/unit/topology/storage/test_storage.py @@ -298,15 +298,18 @@ def test_generate_structured_container(self): def test_extract_column_definitions(self): with patch( "metadata.ingestion.source.storage.storage_service.fetch_dataframe", - return_value=[ - pd.DataFrame.from_dict( - [ - {"transaction_id": 1, "transaction_value": 100}, - {"transaction_id": 2, "transaction_value": 200}, - {"transaction_id": 3, "transaction_value": 300}, - ] - ) - ], + return_value=( + [ + pd.DataFrame.from_dict( + [ + {"transaction_id": 1, "transaction_value": 100}, + {"transaction_id": 2, "transaction_value": 200}, + {"transaction_id": 3, "transaction_value": 300}, + ] + ) + ], + None, + ), ): Column.__eq__ = custom_column_compare self.assertListEqual( diff --git a/openmetadata-airflow-apis/pyproject.toml b/openmetadata-airflow-apis/pyproject.toml index 38871258438a..8118fceb2e5e 100644 --- a/openmetadata-airflow-apis/pyproject.toml +++ b/openmetadata-airflow-apis/pyproject.toml @@ -6,7 +6,7 @@ build-backend = "setuptools.build_meta" # since it helps us organize and isolate version management [project] name = "openmetadata_managed_apis" -version = "1.4.0.0.dev0" +version = "1.3.4.0" readme = "README.md" authors = [ {name = "OpenMetadata Committers"} diff --git a/openmetadata-clients/openmetadata-java-client/pom.xml b/openmetadata-clients/openmetadata-java-client/pom.xml index c8f2739eeaab..a37e6a2732b2 100644 --- a/openmetadata-clients/openmetadata-java-client/pom.xml +++ b/openmetadata-clients/openmetadata-java-client/pom.xml @@ -5,7 +5,7 @@ openmetadata-clients org.open-metadata - 1.4.0-SNAPSHOT + 1.3.4 4.0.0 diff --git a/openmetadata-clients/pom.xml b/openmetadata-clients/pom.xml index a00e9e611994..22b3245db209 100644 --- a/openmetadata-clients/pom.xml +++ b/openmetadata-clients/pom.xml @@ -5,7 +5,7 @@ platform org.open-metadata - 1.4.0-SNAPSHOT + 1.3.4 4.0.0 diff --git a/openmetadata-dist/pom.xml b/openmetadata-dist/pom.xml index e027cbe69863..a760b5a19f07 100644 --- a/openmetadata-dist/pom.xml +++ b/openmetadata-dist/pom.xml @@ -20,7 +20,7 @@ platform org.open-metadata - 1.4.0-SNAPSHOT + 1.3.4 openmetadata-dist diff --git a/openmetadata-docs/content/partials/v1.3/connectors/metadata/connectors-list.md b/openmetadata-docs/content/partials/v1.3/connectors/metadata/connectors-list.md new file mode 100644 index 000000000000..c70d6f179fd2 --- /dev/null +++ b/openmetadata-docs/content/partials/v1.3/connectors/metadata/connectors-list.md @@ -0,0 +1,7 @@ +{% connectorsListContainer %} + +{% connectorInfoCard name="Amundsen" stage="PROD" href="/connectors/metadata/amundsen" platform="OpenMetadata" / %} +{% connectorInfoCard name="Atlas" stage="PROD" href="/connectors/metadata/atlas" platform="OpenMetadata" / %} +{% connectorInfoCard name="Alation" stage="PROD" href="/connectors/metadata/alation" platform="Collate" / %} + +{% /connectorsListContainer %} \ No newline at end of file diff --git a/openmetadata-docs/content/partials/v1.4/connectors/metadata/connectors-list.md b/openmetadata-docs/content/partials/v1.4/connectors/metadata/connectors-list.md new file mode 100644 index 000000000000..c70d6f179fd2 --- /dev/null +++ b/openmetadata-docs/content/partials/v1.4/connectors/metadata/connectors-list.md @@ -0,0 +1,7 @@ +{% connectorsListContainer %} + +{% connectorInfoCard name="Amundsen" stage="PROD" href="/connectors/metadata/amundsen" platform="OpenMetadata" / %} +{% connectorInfoCard name="Atlas" stage="PROD" href="/connectors/metadata/atlas" platform="OpenMetadata" / %} +{% connectorInfoCard name="Alation" stage="PROD" href="/connectors/metadata/alation" platform="Collate" / %} + +{% /connectorsListContainer %} \ No newline at end of file diff --git a/openmetadata-docs/content/v1.1.x/connectors/database/oracle/index.md b/openmetadata-docs/content/v1.1.x/connectors/database/oracle/index.md index 2b9b62d00a22..98d72291d852 100644 --- a/openmetadata-docs/content/v1.1.x/connectors/database/oracle/index.md +++ b/openmetadata-docs/content/v1.1.x/connectors/database/oracle/index.md @@ -57,6 +57,9 @@ GRANT new_role TO user_name; -- GRANT CREATE SESSION PRIVILEGE TO USER GRANT CREATE SESSION TO new_role; + +-- GRANT SELECT CATALOG ROLE PRIVILEGE TO FETCH METADATA TO ROLE / USER +GRANT SELECT_CATALOG_ROLE TO new_role; ``` With just these permissions, your user should be able to ingest the schemas, but not the tables inside them. To get diff --git a/openmetadata-docs/content/v1.1.x/connectors/database/oracle/yaml.md b/openmetadata-docs/content/v1.1.x/connectors/database/oracle/yaml.md index 1be404f0d675..59390c13835b 100644 --- a/openmetadata-docs/content/v1.1.x/connectors/database/oracle/yaml.md +++ b/openmetadata-docs/content/v1.1.x/connectors/database/oracle/yaml.md @@ -61,8 +61,11 @@ CREATE ROLE new_role; -- GRANT ROLE TO USER GRANT new_role TO user_name; --- GRANT CREATE SESSION PRIVILEGE TO USER +-- GRANT CREATE SESSION PRIVILEGE TO ROLE / USER GRANT CREATE SESSION TO new_role; + +-- GRANT SELECT CATALOG ROLE PRIVILEGE TO FETCH METADATA TO ROLE / USER +GRANT SELECT_CATALOG_ROLE TO new_role; ``` With just these permissions, your user should be able to ingest the schemas, but not the tables inside them. To get diff --git a/openmetadata-docs/content/v1.2.x/connectors/database/bigquery/yaml.md b/openmetadata-docs/content/v1.2.x/connectors/database/bigquery/yaml.md index 308e25001d25..c8cc103b652a 100644 --- a/openmetadata-docs/content/v1.2.x/connectors/database/bigquery/yaml.md +++ b/openmetadata-docs/content/v1.2.x/connectors/database/bigquery/yaml.md @@ -127,7 +127,7 @@ You can checkout [this](https://cloud.google.com/iam/docs/keys-create-delete#iam **1.** Passing the raw credential values provided by BigQuery. This requires us to provide the following information, all provided by BigQuery: - - **type**: Credentials Type is the type of the account, for a service account the value of this field is `service_account`. To fetch this key, look for the value associated with the `type` key in the service account key file. + - **type**: Supported values are service_account and external_account.For service accounts, the value of this field is service_account.For external identities, the value of this field is external_account. - **projectId**: A project ID is a unique string used to differentiate your project from all others in Google Cloud. To fetch this key, look for the value associated with the `project_id` key in the service account key file. You can also pass multiple project id to ingest metadata from different BigQuery projects into one service. - **privateKeyId**: This is a unique identifier for the private key associated with the service account. To fetch this key, look for the value associated with the `private_key_id` key in the service account file. - **privateKey**: This is the private key associated with the service account that is used to authenticate and authorize access to BigQuery. To fetch this key, look for the value associated with the `private_key` key in the service account file. @@ -136,7 +136,11 @@ You can checkout [this](https://cloud.google.com/iam/docs/keys-create-delete#iam - **authUri**: This is the URI for the authorization server. To fetch this key, look for the value associated with the `auth_uri` key in the service account key file. The default value to Auth URI is https://accounts.google.com/o/oauth2/auth. - **tokenUri**: The Google Cloud Token URI is a specific endpoint used to obtain an OAuth 2.0 access token from the Google Cloud IAM service. This token allows you to authenticate and access various Google Cloud resources and APIs that require authorization. To fetch this key, look for the value associated with the `token_uri` key in the service account credentials file. Default Value to Token URI is https://oauth2.googleapis.com/token. - **authProviderX509CertUrl**: This is the URL of the certificate that verifies the authenticity of the authorization server. To fetch this key, look for the value associated with the `auth_provider_x509_cert_url` key in the service account key file. The Default value for Auth Provider X509Cert URL is https://www.googleapis.com/oauth2/v1/certs - - **clientX509CertUrl**: This is the URL of the certificate that verifies the authenticity of the service account. To fetch this key, look for the value associated with the `client_x509_cert_url` key in the service account key file. + - **clientX509CertUrl**: This is the URL of the certificate that verifies the authenticity of the service account. To fetch this key, look for the value associated with the `client_x509_cert_url` key in the service account key file. + - **audience**: This is the Google Security Token Service audience which contains the resource name for the workload identity pool and the provider identifier in that pool. + - **subjectTokenType**: This is Google Security Token Service subject token type based on the OAuth 2.0 token exchange spec.Required when using type external_account. + - **tokenURL**: This is Google Security Token Service token exchange endpoint.Required when using type external_account. + - **credentialSource**: This object defines the mechanism used to retrieve the external credential from the local environment so that it can be exchanged for a GCP access token via the STS endpoint. **2.** Passing a local file path that contains the credentials: - **gcpCredentialsPath** diff --git a/openmetadata-docs/content/v1.2.x/connectors/database/oracle/index.md b/openmetadata-docs/content/v1.2.x/connectors/database/oracle/index.md index 3b91358c84ca..e62a037bdaf4 100644 --- a/openmetadata-docs/content/v1.2.x/connectors/database/oracle/index.md +++ b/openmetadata-docs/content/v1.2.x/connectors/database/oracle/index.md @@ -59,6 +59,9 @@ GRANT new_role TO user_name; -- GRANT CREATE SESSION PRIVILEGE TO USER GRANT CREATE SESSION TO new_role; + +-- GRANT SELECT CATALOG ROLE PRIVILEGE TO FETCH METADATA TO ROLE / USER +GRANT SELECT_CATALOG_ROLE TO new_role; ``` With just these permissions, your user should be able to ingest the schemas, but not the tables inside them. To get diff --git a/openmetadata-docs/content/v1.2.x/connectors/database/oracle/yaml.md b/openmetadata-docs/content/v1.2.x/connectors/database/oracle/yaml.md index 87eef17bca5b..2ff448db4202 100644 --- a/openmetadata-docs/content/v1.2.x/connectors/database/oracle/yaml.md +++ b/openmetadata-docs/content/v1.2.x/connectors/database/oracle/yaml.md @@ -65,6 +65,9 @@ GRANT new_role TO user_name; -- GRANT CREATE SESSION PRIVILEGE TO USER GRANT CREATE SESSION TO new_role; + +-- GRANT SELECT CATALOG ROLE PRIVILEGE TO FETCH METADATA TO ROLE / USER +GRANT SELECT_CATALOG_ROLE TO new_role; ``` With just these permissions, your user should be able to ingest the schemas, but not the tables inside them. To get diff --git a/openmetadata-docs/content/v1.2.x/connectors/ingestion/workflows/profiler/external_workflow.md b/openmetadata-docs/content/v1.2.x/connectors/ingestion/workflows/profiler/external_workflow.md index f994343f820a..2892be3b76e7 100644 --- a/openmetadata-docs/content/v1.2.x/connectors/ingestion/workflows/profiler/external_workflow.md +++ b/openmetadata-docs/content/v1.2.x/connectors/ingestion/workflows/profiler/external_workflow.md @@ -105,6 +105,7 @@ processor: # bucketName: awsdatalake-testing # prefix: data/sales/demo1 # overwriteData: false + # filePathPattern: "{service_name}/{database_name}_{database_schema_name}_{table_name}.parquet" # storageConfig: # awsRegion: us-east-2 # awsAccessKeyId: @@ -123,6 +124,7 @@ processor: # bucketName: awsdatalake-testing # prefix: data/sales/demo1 # overwriteData: false + # filePathPattern: "{service_name}/{database_name}_{database_schema_name}_{table_name}.parquet" # storageConfig: # awsRegion: us-east-2 # awsAccessKeyId: diff --git a/openmetadata-docs/content/v1.2.x/connectors/ingestion/workflows/profiler/sample_data.md b/openmetadata-docs/content/v1.2.x/connectors/ingestion/workflows/profiler/sample_data.md index b16dadc31a78..87c45d220e01 100644 --- a/openmetadata-docs/content/v1.2.x/connectors/ingestion/workflows/profiler/sample_data.md +++ b/openmetadata-docs/content/v1.2.x/connectors/ingestion/workflows/profiler/sample_data.md @@ -93,6 +93,7 @@ The OpenMetadata UI will always show 50 or fewer rows of sample data. *Sample Da - **Bucket Name**: A bucket name is a unique identifier used to organize and store data objects. It's similar to a folder name, but it's used for object storage rather than file storage. - **Prefix**: The prefix of a data source refers to the first part of the data path that identifies the source or origin of the data. The generated sample data parquet file will be uploaded to this prefix path in your bucket. - **Overwrite Sample Data**: If this flag is enabled, only one parquet file will be generated per table to store the sample data. Otherwise, a parquet file will be generated for each day when the profiler workflow runs. +- **File Path Pattern**: You can customize how the file will be stored into your storage bucket, by default the file gets stored at the following path `{service_name}/{database_name}/{database_schema_name}/{table_name}/sample_data.parquet`. For instance you do want all the files to be generated in a single folder then you can provide the path like `{service_name}_{database_name}_{database_schema_name}_{table_name}.parquet` not that the pattern must contain the following elements `{service_name}`, `{database_name}`, `{database_schema_name}` `{table_name}` and the pattern must end with the extension `.parquet` and using these elements you can create your own custom pattern. #### Connection Details for AWS S3 diff --git a/openmetadata-docs/content/v1.2.x/deployment/security/enable-password-masking.md b/openmetadata-docs/content/v1.2.x/deployment/security/enable-password-masking.md deleted file mode 100644 index 541ed54094c5..000000000000 --- a/openmetadata-docs/content/v1.2.x/deployment/security/enable-password-masking.md +++ /dev/null @@ -1,67 +0,0 @@ ---- -title: Enable password masking -slug: /deployment/security/enable-password-masking ---- - -# Enable password masking - -The **1.0.0** version of OpenMetadata now includes a new feature that allows users to activate password masking. -This feature was added in response to feedback from our community of users who expressed concerns about the security of -their passwords when using our application. - -With the password masking feature enabled, all API calls made by your application will replace the password fields with -asterisks (*) before sending the request. This will prevent the password from being sent in plain text. Even though -passwords are replaced by asterisks, it will not affect when editing a connection, saving will update the passwords only -if they are changed. - -{% image -caption="Editing a service connection with masked password." -src="/images/v1.2/deployment/mask-password/edit-connection.png" -alt="mask-password" /%} - -However, note that the `ingestion-bot` user will still send the password in plain text as it needs to access the API -without any obstructions. This is because the `ingestion-bot` user requires full access to the API, and any masking -would hinder its ability to perform its tasks. - -{% note %} - -In future releases, the password masking feature will be activated by default. - -The feature will be automatically enabled to provide an added layer of security for all API calls made. - -{% /note %} - -## How to enable the feature - -To activate the password masking feature in your application, follow the steps below: - -### Docker - -Add the following environment variable to the list: - -```yaml -# openmetadata.prod.env -MASK_PASSWORDS_API=true -``` - -### Bare Metal - -Edit the `openmetadata.yaml` file as it is shown below: - -```yaml -security: - maskPasswordsAPI: true -``` - -### Kubernetes - -Update your helm `maskPasswordsApi` value: - -```yaml -# openmetadata.prod.values.yml -openmetadata: - config: - ... - maskPasswordsApi: true - ... -``` \ No newline at end of file diff --git a/openmetadata-docs/content/v1.2.x/menu.md b/openmetadata-docs/content/v1.2.x/menu.md index 5ae3974d82e7..2bf6eb93b172 100644 --- a/openmetadata-docs/content/v1.2.x/menu.md +++ b/openmetadata-docs/content/v1.2.x/menu.md @@ -154,8 +154,6 @@ site_menu: url: /deployment/security/enable-jwt-tokens - category: Deployment / Enable Security / JWT Troubleshooting url: /deployment/security/jwt-troubleshooting - - category: Deployment / Enable Security / Enable Password Masking - url: /deployment/security/enable-password-masking - category: Deployment / Enable Secrets Manager url: /deployment/secrets-manager diff --git a/openmetadata-docs/content/v1.3.x/connectors/database/oracle/index.md b/openmetadata-docs/content/v1.3.x/connectors/database/oracle/index.md index ad43e7cefeab..73e04c031e88 100644 --- a/openmetadata-docs/content/v1.3.x/connectors/database/oracle/index.md +++ b/openmetadata-docs/content/v1.3.x/connectors/database/oracle/index.md @@ -42,6 +42,9 @@ GRANT new_role TO user_name; -- GRANT CREATE SESSION PRIVILEGE TO USER GRANT CREATE SESSION TO new_role; + +-- GRANT SELECT CATALOG ROLE PRIVILEGE TO FETCH METADATA TO ROLE / USER +GRANT SELECT_CATALOG_ROLE TO new_role; ``` With just these permissions, your user should be able to ingest the schemas, but not the tables inside them. To get diff --git a/openmetadata-docs/content/v1.3.x/connectors/database/oracle/yaml.md b/openmetadata-docs/content/v1.3.x/connectors/database/oracle/yaml.md index 31f909a143a1..86799f4ba57a 100644 --- a/openmetadata-docs/content/v1.3.x/connectors/database/oracle/yaml.md +++ b/openmetadata-docs/content/v1.3.x/connectors/database/oracle/yaml.md @@ -42,6 +42,9 @@ GRANT new_role TO user_name; -- GRANT CREATE SESSION PRIVILEGE TO USER GRANT CREATE SESSION TO new_role; + +-- GRANT SELECT CATALOG ROLE PRIVILEGE TO FETCH METADATA TO ROLE / USER +GRANT SELECT_CATALOG_ROLE TO new_role; ``` With just these permissions, your user should be able to ingest the schemas, but not the tables inside them. To get diff --git a/openmetadata-docs/content/v1.3.x/deployment/ingestion/external/gcs-composer.md b/openmetadata-docs/content/v1.3.x/deployment/ingestion/external/gcs-composer.md index 3bead257c0a8..02e028ba279e 100644 --- a/openmetadata-docs/content/v1.3.x/deployment/ingestion/external/gcs-composer.md +++ b/openmetadata-docs/content/v1.3.x/deployment/ingestion/external/gcs-composer.md @@ -13,7 +13,7 @@ This approach has been last tested against: - Composer version 2.5.4 - Airflow version 2.6.3 -It also requires the ingestion package to be at least `openmetadata-ingestion==1.3.0.0`. +It also requires the ingestion package to be at least `openmetadata-ingestion==1.3.0.1`. ## Using the Python Operator diff --git a/openmetadata-docs/content/v1.3.x/deployment/security/enable-password-masking.md b/openmetadata-docs/content/v1.3.x/deployment/security/enable-password-masking.md deleted file mode 100644 index 43c97bc8e194..000000000000 --- a/openmetadata-docs/content/v1.3.x/deployment/security/enable-password-masking.md +++ /dev/null @@ -1,67 +0,0 @@ ---- -title: Enable password masking -slug: /deployment/security/enable-password-masking ---- - -# Enable password masking - -The **1.0.0** version of OpenMetadata now includes a new feature that allows users to activate password masking. -This feature was added in response to feedback from our community of users who expressed concerns about the security of -their passwords when using our application. - -With the password masking feature enabled, all API calls made by your application will replace the password fields with -asterisks (*) before sending the request. This will prevent the password from being sent in plain text. Even though -passwords are replaced by asterisks, it will not affect when editing a connection, saving will update the passwords only -if they are changed. - -{% image -caption="Editing a service connection with masked password." -src="/images/v1.3/deployment/mask-password/edit-connection.png" -alt="mask-password" /%} - -However, note that the `ingestion-bot` user will still send the password in plain text as it needs to access the API -without any obstructions. This is because the `ingestion-bot` user requires full access to the API, and any masking -would hinder its ability to perform its tasks. - -{% note %} - -In future releases, the password masking feature will be activated by default. - -The feature will be automatically enabled to provide an added layer of security for all API calls made. - -{% /note %} - -## How to enable the feature - -To activate the password masking feature in your application, follow the steps below: - -### Docker - -Add the following environment variable to the list: - -```yaml -# openmetadata.prod.env -MASK_PASSWORDS_API=true -``` - -### Bare Metal - -Edit the `openmetadata.yaml` file as it is shown below: - -```yaml -security: - maskPasswordsAPI: true -``` - -### Kubernetes - -Update your helm `maskPasswordsApi` value: - -```yaml -# openmetadata.prod.values.yml -openmetadata: - config: - ... - maskPasswordsApi: true - ... -``` \ No newline at end of file diff --git a/openmetadata-docs/content/v1.3.x/menu.md b/openmetadata-docs/content/v1.3.x/menu.md index 0166257d5fa4..0275a0e14505 100644 --- a/openmetadata-docs/content/v1.3.x/menu.md +++ b/openmetadata-docs/content/v1.3.x/menu.md @@ -156,8 +156,6 @@ site_menu: url: /deployment/security/enable-jwt-tokens - category: Deployment / Enable Security / JWT Troubleshooting url: /deployment/security/jwt-troubleshooting - - category: Deployment / Enable Security / Enable Password Masking - url: /deployment/security/enable-password-masking - category: Deployment / Enable Secrets Manager url: /deployment/secrets-manager diff --git a/openmetadata-docs/content/v1.4.x-SNAPSHOT/connectors/database/oracle/index.md b/openmetadata-docs/content/v1.4.x-SNAPSHOT/connectors/database/oracle/index.md new file mode 100644 index 000000000000..1241234b75e7 --- /dev/null +++ b/openmetadata-docs/content/v1.4.x-SNAPSHOT/connectors/database/oracle/index.md @@ -0,0 +1,103 @@ +--- +title: Oracle +slug: /connectors/database/oracle +--- + +{% connectorDetailsHeader +name="Oracle" +stage="PROD" +platform="OpenMetadata" +availableFeatures=["Metadata", "Query Usage", "Data Profiler", "Data Quality", "dbt", "Lineage", "Column-level Lineage", "Stored Procedures"] +unavailableFeatures=["Owners", "Tags"] +/ %} + +In this section, we provide guides and references to use the Oracle connector. + +Configure and schedule Oracle metadata and profiler workflows from the OpenMetadata UI: + +- [Requirements](#requirements) +- [Metadata Ingestion](#metadata-ingestion) +- [Data Profiler](/connectors/ingestion/workflows/profiler) +- [Data Quality](/connectors/ingestion/workflows/data-quality) +- [Lineage](/connectors/ingestion/lineage) +- [dbt Integration](/connectors/ingestion/workflows/dbt) + +{% partial file="/v1.4/connectors/ingestion-modes-tiles.md" variables={yamlPath: "/connectors/database/oracle/yaml"} /%} + +## Requirements + +**Note**: To retrieve metadata from an Oracle database, we use the `python-oracledb` library, which provides support for versions 12c, 18c, 19c, and 21c. + +To ingest metadata from oracle user must have `CREATE SESSION` privilege for the user. + +```sql +-- CREATE USER +CREATE USER user_name IDENTIFIED BY admin_password; + +-- CREATE ROLE +CREATE ROLE new_role; + +-- GRANT ROLE TO USER +GRANT new_role TO user_name; + +-- GRANT CREATE SESSION PRIVILEGE TO USER +GRANT CREATE SESSION TO new_role; + +-- GRANT SELECT CATALOG ROLE PRIVILEGE TO FETCH METADATA TO ROLE / USER +GRANT SELECT_CATALOG_ROLE TO new_role; +``` + +With just these permissions, your user should be able to ingest the schemas, but not the tables inside them. To get +the tables, you should grant `SELECT` permissions to the tables you are interested in. E.g., + +```sql +SELECT ON ADMIN.EXAMPLE_TABLE TO new_role; +``` + +You can find further information [here](https://docs.oracle.com/javadb/10.8.3.0/ref/rrefsqljgrant.html). Note that +there is no routine out of the box in Oracle to grant SELECT to a full schema. + +## Metadata Ingestion + +{% partial + file="/v1.4/connectors/metadata-ingestion-ui.md" + variables={ + connector: "Oracle", + selectServicePath: "/images/v1.4/connectors/oracle/select-service.png", + addNewServicePath: "/images/v1.4/connectors/oracle/add-new-service.png", + serviceConnectionPath: "/images/v1.4/connectors/oracle/service-connection.png", +} +/%} + +{% stepsContainer %} +{% extraContent parentTagName="stepsContainer" %} + +#### Connection Details + +- **Username**: Specify the User to connect to Oracle. It should have enough privileges to read all the metadata. +- **Password**: Password to connect to Oracle. +- **Host and Port**: Enter the fully qualified hostname and port number for your Oracle deployment in the Host and Port field. +- **Database Name**: Optional name to give to the database in OpenMetadata. If left blank, we will use default as the database name. It is recommended to use the database name same as the SID, This ensures accurate results and proper identification of tables during profiling, data quality checks and dbt workflow. +- **Oracle Connection Type** : Select the Oracle Connection Type. The type can either be `Oracle Service Name` or `Database Schema` + - **Oracle Service Name**: The Oracle Service name is the TNS alias that you give when you remotely connect to your database and this Service name is recorded in tnsnames. + - **Database Schema**: The name of the database schema available in Oracle that you want to connect with. +- **Oracle instant client directory**: The directory pointing to where the `instantclient` binaries for Oracle are located. In the ingestion Docker image we + provide them by default at `/instantclient`. If this parameter is informed (it is by default), we will run the [thick oracle client](https://python-oracledb.readthedocs.io/en/latest/user_guide/initialization.html#initializing-python-oracledb). + We are shipping the binaries for ARM and AMD architectures from [here](https://www.oracle.com/database/technologies/instant-client/linux-x86-64-downloads.html) + and [here](https://www.oracle.com/database/technologies/instant-client/linux-arm-aarch64-downloads.html) for the instant client version 19. + +{% partial file="/v1.4/connectors/database/advanced-configuration.md" /%} + +{% /extraContent %} + +{% partial file="/v1.4/connectors/test-connection.md" /%} + +{% partial file="/v1.4/connectors/database/configure-ingestion.md" /%} + +{% partial file="/v1.4/connectors/ingestion-schedule-and-deploy.md" /%} + +{% /stepsContainer %} + +{% partial file="/v1.4/connectors/troubleshooting.md" /%} + +{% partial file="/v1.4/connectors/database/related.md" /%} diff --git a/openmetadata-docs/content/v1.4.x-SNAPSHOT/connectors/database/oracle/yaml.md b/openmetadata-docs/content/v1.4.x-SNAPSHOT/connectors/database/oracle/yaml.md new file mode 100644 index 000000000000..907347b2880f --- /dev/null +++ b/openmetadata-docs/content/v1.4.x-SNAPSHOT/connectors/database/oracle/yaml.md @@ -0,0 +1,219 @@ +--- +title: Run the Oracle Connector Externally +slug: /connectors/database/oracle/yaml +--- + +{% connectorDetailsHeader +name="Oracle" +stage="PROD" +platform="OpenMetadata" +availableFeatures=["Metadata", "Query Usage", "Data Profiler", "Data Quality", "dbt", "Lineage", "Column-level Lineage", "Stored Procedures"] +unavailableFeatures=["Owners", "Tags"] +/ %} + +In this section, we provide guides and references to use the Oracle connector. + +Configure and schedule Oracle metadata and profiler workflows from the OpenMetadata UI: + +- [Requirements](#requirements) +- [Metadata Ingestion](#metadata-ingestion) +- [Data Profiler](#data-profiler) +- [Data Quality](#data-quality) +- [Lineage](#lineage) +- [dbt Integration](#dbt-integration) + +{% partial file="/v1.4/connectors/external-ingestion-deployment.md" /%} + +## Requirements + +**Note**: To retrieve metadata from an Oracle database, the python-oracledb library can be utilized, which provides support for versions 12c, 18c, 19c, and 21c. + +To ingest metadata from oracle user must have `CREATE SESSION` privilege for the user. + +```sql +-- CREATE USER +CREATE USER user_name IDENTIFIED BY admin_password; + +-- CREATE ROLE +CREATE ROLE new_role; + +-- GRANT ROLE TO USER +GRANT new_role TO user_name; + +-- GRANT CREATE SESSION PRIVILEGE TO USER +GRANT CREATE SESSION TO new_role; + +-- GRANT SELECT CATALOG ROLE PRIVILEGE TO FETCH METADATA TO ROLE / USER +GRANT SELECT_CATALOG_ROLE TO new_role; +``` + +With just these permissions, your user should be able to ingest the schemas, but not the tables inside them. To get +the tables, you should grant `SELECT` permissions to the tables you are interested in. E.g., + +```sql +SELECT ON ADMIN.EXAMPLE_TABLE TO new_role; +``` + +You can find further information [here](https://docs.oracle.com/javadb/10.8.3.0/ref/rrefsqljgrant.html). Note that +there is no routine out of the box in Oracle to grant SELECT to a full schema. + +### Python Requirements + +To run the Oracle ingestion, you will need to install: + +```bash +pip3 install "openmetadata-ingestion[oracle]" +``` + +## Metadata Ingestion + +All connectors are defined as JSON Schemas. +[Here](https://github.com/open-metadata/OpenMetadata/blob/main/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/database/oracleConnection.json) +you can find the structure to create a connection to Oracle. + +In order to create and run a Metadata Ingestion workflow, we will follow +the steps to create a YAML configuration able to connect to the source, +process the Entities if needed, and reach the OpenMetadata server. + +The workflow is modeled around the following +[JSON Schema](https://github.com/open-metadata/OpenMetadata/blob/main/openmetadata-spec/src/main/resources/json/schema/metadataIngestion/workflow.json) + +### 1. Define the YAML Config + +This is a sample config for Oracle: + +{% codePreview %} + +{% codeInfoContainer %} + +#### Source Configuration - Service Connection + +{% codeInfo srNumber=1 %} + +**username**: Specify the User to connect to Oracle. It should have enough privileges to read all the metadata. + +{% /codeInfo %} + +{% codeInfo srNumber=2 %} + +**password**: Password to connect to Oracle. + +{% /codeInfo %} + +{% codeInfo srNumber=3 %} + +**hostPort**: Enter the fully qualified hostname and port number for your Oracle deployment in the Host and Port field. + +{% /codeInfo %} + +{% codeInfo srNumber=4 %} + +**oracleConnectionType** : +- **oracleServiceName**: The Oracle Service name is the TNS alias that you give when you remotely connect to your database and this Service name is recorded in tnsnames. +- **databaseSchema**: The name of the database schema available in Oracle that you want to connect with. +- **Oracle instant client directory**: The directory pointing to where the `instantclient` binaries for Oracle are located. In the ingestion Docker image we + provide them by default at `/instantclient`. If this parameter is informed (it is by default), we will run the [thick oracle client](https://python-oracledb.readthedocs.io/en/latest/user_guide/initialization.html#initializing-python-oracledb). + We are shipping the binaries for ARM and AMD architectures from [here](https://www.oracle.com/database/technologies/instant-client/linux-x86-64-downloads.html) + and [here](https://www.oracle.com/database/technologies/instant-client/linux-arm-aarch64-downloads.html) for the instant client version 19. + +{% /codeInfo %} + +{% codeInfo srNumber=23 %} + +**databaseName**: Optional name to give to the database in OpenMetadata. If left blank, we will use default as the database name. It is recommended to use the database name same as the SID, This ensures accurate results and proper identification of tables during profiling, data quality checks and dbt workflow. + +{% /codeInfo %} + +{% partial file="/v1.4/connectors/yaml/database/source-config-def.md" /%} + +{% partial file="/v1.4/connectors/yaml/ingestion-sink-def.md" /%} + +{% partial file="/v1.4/connectors/yaml/workflow-config-def.md" /%} + +#### Advanced Configuration + +{% codeInfo srNumber=5 %} + +**Connection Options (Optional)**: Enter the details for any additional connection options that can be sent to Athena during the connection. These details must be added as Key-Value pairs. + +{% /codeInfo %} + +{% codeInfo srNumber=6 %} + +**Connection Arguments (Optional)**: Enter the details for any additional connection arguments such as security or protocol configs that can be sent to Athena during the connection. These details must be added as Key-Value pairs. + +- In case you are using Single-Sign-On (SSO) for authentication, add the `authenticator` details in the Connection Arguments as a Key-Value pair as follows: `"authenticator" : "sso_login_url"` + +{% /codeInfo %} + +{% /codeInfoContainer %} + +{% codeBlock fileName="filename.yaml" %} + +```yaml +source: + type: oracle + serviceName: local_oracle + serviceConnection: + config: + type: Oracle +``` +```yaml {% srNumber=1 %} + hostPort: hostPort +``` +```yaml {% srNumber=2 %} + username: username +``` +```yaml {% srNumber=3 %} + password: password +``` +```yaml {% srNumber=4 %} + # The type can either be oracleServiceName or databaseSchema + oracleConnectionType: + oracleServiceName: serviceName + # databaseSchema: schema +``` +```yaml {% srNumber=23 %} + databaseName: custom_db_display_name +``` +```yaml {% srNumber=5 %} + # connectionOptions: + # key: value +``` +```yaml {% srNumber=6 %} + # connectionArguments: + # key: value +``` + +{% partial file="/v1.4/connectors/yaml/database/source-config.md" /%} + +{% partial file="/v1.4/connectors/yaml/ingestion-sink.md" /%} + +{% partial file="/v1.4/connectors/yaml/workflow-config.md" /%} + +{% /codeBlock %} + +{% /codePreview %} + +{% partial file="/v1.4/connectors/yaml/ingestion-cli.md" /%} + +{% partial file="/v1.4/connectors/yaml/data-profiler.md" variables={connector: "oracle"} /%} + +{% partial file="/v1.4/connectors/yaml/data-quality.md" /%} + +## Lineage + +You can learn more about how to ingest lineage [here](/connectors/ingestion/workflows/lineage). + +## dbt Integration + +{% tilesContainer %} + +{% tile + icon="mediation" + title="dbt Integration" + description="Learn more about how to ingest dbt models' definitions and their lineage." + link="/connectors/ingestion/workflows/dbt" /%} + +{% /tilesContainer %} + diff --git a/openmetadata-service/pom.xml b/openmetadata-service/pom.xml index 45fc311e15cd..0b18742e502f 100644 --- a/openmetadata-service/pom.xml +++ b/openmetadata-service/pom.xml @@ -5,7 +5,7 @@ platform org.open-metadata - 1.4.0-SNAPSHOT + 1.3.4 4.0.0 openmetadata-service @@ -16,13 +16,14 @@ ${project.basedir}/target/site/jacoco-aggregate/jacoco.xml ${project.basedir}/src/test/java 1.19.4 - 2.23.3 + 2.25.21 1.11.2 4.7.3 0.5.11 2.9.0 2.3.4 - 2.3.2 + 2.5.0-rc1 + 5.7.0 @@ -37,6 +38,33 @@ + + org.pac4j + pac4j-core + ${pac4j.version} + + + org.slf4j + slf4j-api + + + + + org.pac4j + pac4j-oidc + ${pac4j.version} + + + net.minidev + json-smart + + + + + net.minidev + json-smart + 2.5.0 + org.open-metadata common @@ -298,7 +326,21 @@ testcontainers ${org.testcontainers.version} test + + + org.apache.commons + commons-compress + + + + + + org.apache.commons + commons-compress + 1.26.1 + test + org.testcontainers junit-jupiter @@ -335,6 +377,12 @@ 2.40 test + + org.assertj + assertj-core + 3.25.3 + test + javax.json @@ -498,6 +546,12 @@ quartz ${quartz.version} + + + com.mchange + c3p0 + 0.10.0 + com.fasterxml.woodstox woodstox-core diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/Entity.java b/openmetadata-service/src/main/java/org/openmetadata/service/Entity.java index 67a75d67d974..ed0a81c9973b 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/Entity.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/Entity.java @@ -579,4 +579,8 @@ public static SearchIndex buildSearchIndex(String entityType, Object entity) { } throw new BadRequestException("searchrepository not initialized"); } + + public static T getDao() { + return (T) collectionDAO; + } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/OpenMetadataApplication.java b/openmetadata-service/src/main/java/org/openmetadata/service/OpenMetadataApplication.java index 418e67086a90..6ffc08bde0d5 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/OpenMetadataApplication.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/OpenMetadataApplication.java @@ -13,7 +13,7 @@ package org.openmetadata.service; -import static org.openmetadata.service.util.MicrometerBundleSingleton.setWebAnalyticsEvents; +import static org.openmetadata.service.security.SecurityUtil.tryCreateOidcClient; import io.dropwizard.Application; import io.dropwizard.configuration.EnvironmentVariableSubstitutor; @@ -25,6 +25,7 @@ import io.dropwizard.jersey.errors.EarlyEofExceptionMapper; import io.dropwizard.jersey.errors.LoggingExceptionMapper; import io.dropwizard.jersey.jackson.JsonProcessingExceptionMapper; +import io.dropwizard.jetty.MutableServletContextHandler; import io.dropwizard.lifecycle.Managed; import io.dropwizard.server.DefaultServerFactory; import io.dropwizard.setup.Bootstrap; @@ -38,7 +39,6 @@ import java.security.KeyStoreException; import java.security.NoSuchAlgorithmException; import java.security.cert.CertificateException; -import java.time.temporal.ChronoUnit; import java.util.EnumSet; import java.util.Optional; import javax.naming.ConfigurationException; @@ -53,6 +53,7 @@ import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.eclipse.jetty.http.pathmap.ServletPathSpec; +import org.eclipse.jetty.server.session.SessionHandler; import org.eclipse.jetty.servlet.FilterHolder; import org.eclipse.jetty.servlet.ServletHolder; import org.eclipse.jetty.websocket.server.NativeWebSocketServletContainerInitializer; @@ -60,12 +61,12 @@ import org.glassfish.jersey.media.multipart.MultiPartFeature; import org.glassfish.jersey.server.ServerProperties; import org.jdbi.v3.core.Jdbi; -import org.jdbi.v3.core.statement.SqlLogger; -import org.jdbi.v3.core.statement.StatementContext; import org.jdbi.v3.sqlobject.SqlObjects; import org.openmetadata.schema.api.security.AuthenticationConfiguration; import org.openmetadata.schema.api.security.AuthorizerConfiguration; +import org.openmetadata.schema.api.security.ClientType; import org.openmetadata.schema.services.connections.metadata.AuthProvider; +import org.openmetadata.service.apps.ApplicationHandler; import org.openmetadata.service.apps.scheduler.AppScheduler; import org.openmetadata.service.config.OMWebBundle; import org.openmetadata.service.config.OMWebConfiguration; @@ -80,21 +81,26 @@ import org.openmetadata.service.fernet.Fernet; import org.openmetadata.service.jdbi3.CollectionDAO; import org.openmetadata.service.jdbi3.EntityRepository; +import org.openmetadata.service.jdbi3.MigrationDAO; import org.openmetadata.service.jdbi3.locator.ConnectionAwareAnnotationSqlLocator; import org.openmetadata.service.jdbi3.locator.ConnectionType; import org.openmetadata.service.migration.Migration; +import org.openmetadata.service.migration.MigrationValidationClient; import org.openmetadata.service.migration.api.MigrationWorkflow; import org.openmetadata.service.monitoring.EventMonitor; +import org.openmetadata.service.monitoring.EventMonitorConfiguration; import org.openmetadata.service.monitoring.EventMonitorFactory; import org.openmetadata.service.monitoring.EventMonitorPublisher; import org.openmetadata.service.resources.CollectionRegistry; import org.openmetadata.service.resources.databases.DatasourceConfig; import org.openmetadata.service.resources.settings.SettingsCache; import org.openmetadata.service.search.SearchRepository; -import org.openmetadata.service.secrets.SecretsManager; import org.openmetadata.service.secrets.SecretsManagerFactory; -import org.openmetadata.service.secrets.SecretsManagerUpdateService; import org.openmetadata.service.secrets.masker.EntityMaskerFactory; +import org.openmetadata.service.security.AuthCallbackServlet; +import org.openmetadata.service.security.AuthLoginServlet; +import org.openmetadata.service.security.AuthLogoutServlet; +import org.openmetadata.service.security.AuthRefreshServlet; import org.openmetadata.service.security.Authorizer; import org.openmetadata.service.security.NoopAuthorizer; import org.openmetadata.service.security.NoopFilter; @@ -115,6 +121,9 @@ import org.openmetadata.service.util.MicrometerBundleSingleton; import org.openmetadata.service.util.incidentSeverityClassifier.IncidentSeverityClassifierInterface; import org.openmetadata.service.util.jdbi.DatabaseAuthenticationProviderFactory; +import org.openmetadata.service.util.jdbi.OMSqlLogger; +import org.pac4j.core.util.CommonHelper; +import org.pac4j.oidc.client.OidcClient; import org.quartz.SchedulerException; /** Main catalog application */ @@ -123,6 +132,8 @@ public class OpenMetadataApplication extends Application() {}); - environment.jersey().register(new JsonProcessingExceptionMapper(true)); - environment.jersey().register(new EarlyEofExceptionMapper()); - environment.jersey().register(JsonMappingExceptionMapper.class); - environment - .healthChecks() - .register("OpenMetadataServerHealthCheck", new OpenMetadataServerHealthCheck()); // start event hub before registering publishers EventPubSub.start(); + ApplicationHandler.initialize(catalogConfig); registerResources(catalogConfig, environment, jdbi); // Register Event Handler @@ -214,10 +223,6 @@ public void run(OpenMetadataApplicationConfig catalogConfig, Environment environ // Register Event publishers registerEventPublisher(catalogConfig); - // update entities secrets if required - new SecretsManagerUpdateService(secretsManager, catalogConfig.getClusterName()) - .updateEntities(); - // start authorizer after event publishers // authorizer creates admin/bot users, ES publisher should start before to index users created // by authorizer @@ -226,31 +231,130 @@ public void run(OpenMetadataApplicationConfig catalogConfig, Environment environ // authenticationHandler Handles auth related activities authenticatorHandler.init(catalogConfig); - setWebAnalyticsEvents(catalogConfig); + registerMicrometerFilter(environment, catalogConfig.getEventMonitorConfiguration()); + + initializeWebsockets(catalogConfig, environment); + registerSamlServlets(catalogConfig, environment); + + // Asset Servlet Registration + registerAssetServlet(catalogConfig.getWebConfiguration(), environment); + + // Handle Pipeline Service Client Status job + PipelineServiceStatusJobHandler pipelineServiceStatusJobHandler = + PipelineServiceStatusJobHandler.create( + catalogConfig.getPipelineServiceClientConfiguration(), catalogConfig.getClusterName()); + pipelineServiceStatusJobHandler.addPipelineServiceStatusJob(); + + // Register Auth Handlers + registerAuthServlets(catalogConfig, environment); + } + + private void registerAuthServlets(OpenMetadataApplicationConfig config, Environment environment) { + if (config.getAuthenticationConfiguration() != null + && config + .getAuthenticationConfiguration() + .getClientType() + .equals(ClientType.CONFIDENTIAL)) { + CommonHelper.assertNotNull( + "OidcConfiguration", config.getAuthenticationConfiguration().getOidcConfiguration()); + + // Set up a Session Manager + MutableServletContextHandler contextHandler = environment.getApplicationContext(); + if (contextHandler.getSessionHandler() == null) { + contextHandler.setSessionHandler(new SessionHandler()); + } + + // Register Servlets + OidcClient oidcClient = + tryCreateOidcClient(config.getAuthenticationConfiguration().getOidcConfiguration()); + oidcClient.setCallbackUrl( + config.getAuthenticationConfiguration().getOidcConfiguration().getCallbackUrl()); + ServletRegistration.Dynamic authLogin = + environment + .servlets() + .addServlet( + "oauth_login", + new AuthLoginServlet( + oidcClient, + config.getAuthenticationConfiguration().getOidcConfiguration().getServerUrl(), + config.getAuthenticationConfiguration().getJwtPrincipalClaims())); + authLogin.addMapping("/api/v1/auth/login"); + ServletRegistration.Dynamic authCallback = + environment + .servlets() + .addServlet( + "auth_callback", + new AuthCallbackServlet( + oidcClient, + config.getAuthenticationConfiguration().getOidcConfiguration().getServerUrl(), + config.getAuthenticationConfiguration().getJwtPrincipalClaims())); + authCallback.addMapping("/callback"); + + ServletRegistration.Dynamic authLogout = + environment + .servlets() + .addServlet( + "auth_logout", + new AuthLogoutServlet( + config + .getAuthenticationConfiguration() + .getOidcConfiguration() + .getServerUrl())); + authLogout.addMapping("/api/v1/auth/logout"); + + ServletRegistration.Dynamic refreshServlet = + environment + .servlets() + .addServlet( + "auth_refresh", + new AuthRefreshServlet( + oidcClient, + config + .getAuthenticationConfiguration() + .getOidcConfiguration() + .getServerUrl())); + refreshServlet.addMapping("/api/v1/auth/refresh"); + } + } + + private void registerHealthCheck(Environment environment) { + environment + .healthChecks() + .register("OpenMetadataServerHealthCheck", new OpenMetadataServerHealthCheck()); + } + + private void registerExceptionMappers(Environment environment) { + environment.jersey().register(CatalogGenericExceptionMapper.class); + // Override constraint violation mapper to catch Json validation errors + environment.jersey().register(new ConstraintViolationExceptionMapper()); + // Restore dropwizard default exception mappers + environment.jersey().register(new LoggingExceptionMapper<>() {}); + environment.jersey().register(new JsonProcessingExceptionMapper(true)); + environment.jersey().register(new EarlyEofExceptionMapper()); + environment.jersey().register(JsonMappingExceptionMapper.class); + } + + private void registerMicrometerFilter( + Environment environment, EventMonitorConfiguration eventMonitorConfiguration) { FilterRegistration.Dynamic micrometerFilter = environment.servlets().addFilter("OMMicrometerHttpFilter", new OMMicrometerHttpFilter()); micrometerFilter.addMappingForUrlPatterns( - EnumSet.allOf(DispatcherType.class), - true, - catalogConfig.getEventMonitorConfiguration().getPathPattern()); - initializeWebsockets(catalogConfig, environment); - registerSamlHandlers(catalogConfig, environment); + EnumSet.allOf(DispatcherType.class), true, eventMonitorConfiguration.getPathPattern()); + } + private void registerAssetServlet(OMWebConfiguration webConfiguration, Environment environment) { // Handle Asset Using Servlet OpenMetadataAssetServlet assetServlet = - new OpenMetadataAssetServlet( - "/assets", "/", "index.html", catalogConfig.getWebConfiguration()); + new OpenMetadataAssetServlet("/assets", "/", "index.html", webConfiguration); String pathPattern = "/" + '*'; environment.servlets().addServlet("static", assetServlet).addMapping(pathPattern); + } - // Handle Pipeline Service Client Status job - PipelineServiceStatusJobHandler pipelineServiceStatusJobHandler = - PipelineServiceStatusJobHandler.create( - catalogConfig.getPipelineServiceClientConfiguration(), catalogConfig.getClusterName()); - pipelineServiceStatusJobHandler.addPipelineServiceStatusJob(); + protected CollectionDAO getDao(Jdbi jdbi) { + return jdbi.onDemand(CollectionDAO.class); } - private void registerSamlHandlers( + private void registerSamlServlets( OpenMetadataApplicationConfig catalogConfig, Environment environment) throws IOException, CertificateException, KeyStoreException, NoSuchAlgorithmException { if (catalogConfig.getAuthenticationConfiguration() != null @@ -280,25 +384,7 @@ private Jdbi createAndSetupJDBI(Environment environment, DataSourceFactory dbFac }); Jdbi jdbi = new JdbiFactory().build(environment, dbFactory, "database"); - SqlLogger sqlLogger = - new SqlLogger() { - @Override - public void logBeforeExecution(StatementContext context) { - LOG.debug("sql {}, parameters {}", context.getRenderedSql(), context.getBinding()); - } - - @Override - public void logAfterExecution(StatementContext context) { - LOG.debug( - "sql {}, parameters {}, timeTaken {} ms", - context.getRenderedSql(), - context.getBinding(), - context.getElapsedTime(ChronoUnit.MILLIS)); - } - }; - if (LOG.isDebugEnabled()) { - jdbi.setSqlLogger(sqlLogger); - } + jdbi.setSqlLogger(new OMSqlLogger()); // Set the Database type for choosing correct queries from annotations jdbi.getConfig(SqlObjects.class) .setSqlLocator(new ConnectionAwareAnnotationSqlLocator(dbFactory.getDriverClass())); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/apps/AbstractNativeApplication.java b/openmetadata-service/src/main/java/org/openmetadata/service/apps/AbstractNativeApplication.java index 02edefe7a56c..f92d546dbde2 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/apps/AbstractNativeApplication.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/apps/AbstractNativeApplication.java @@ -1,7 +1,7 @@ package org.openmetadata.service.apps; import static org.openmetadata.service.apps.scheduler.AbstractOmAppJobListener.JOB_LISTENER_NAME; -import static org.openmetadata.service.apps.scheduler.AppScheduler.APP_INFO_KEY; +import static org.openmetadata.service.apps.scheduler.AppScheduler.APP_NAME; import static org.openmetadata.service.exception.CatalogExceptionMessage.LIVE_APP_SCHEDULE_ERR; import java.util.List; @@ -13,6 +13,7 @@ import org.openmetadata.schema.entity.app.App; import org.openmetadata.schema.entity.app.AppRunRecord; import org.openmetadata.schema.entity.app.AppType; +import org.openmetadata.schema.entity.app.ScheduleTimeline; import org.openmetadata.schema.entity.app.ScheduleType; import org.openmetadata.schema.entity.app.ScheduledExecutionContext; import org.openmetadata.schema.entity.applications.configuration.ApplicationConfig; @@ -61,8 +62,23 @@ public void init(App app) { @Override public void install() { + // If the app does not have any Schedule Return without scheduling + if (app.getAppSchedule() != null + && app.getAppSchedule().getScheduleTimeline().equals(ScheduleTimeline.NONE)) { + return; + } if (app.getAppType() == AppType.Internal && app.getScheduleType().equals(ScheduleType.Scheduled)) { + try { + ApplicationHandler.getInstance().removeOldJobs(app); + ApplicationHandler.getInstance().migrateQuartzConfig(app); + ApplicationHandler.getInstance().fixCorruptedInstallation(app); + } catch (SchedulerException e) { + throw AppException.byMessage( + "ApplicationHandler", + "SchedulerError", + "Error while migrating application configuration: " + app.getName()); + } scheduleInternal(); } else if (app.getAppType() == AppType.External && app.getScheduleType().equals(ScheduleType.Scheduled)) { @@ -197,9 +213,9 @@ protected void validateServerExecutableApp(AppRuntime context) { @Override public void execute(JobExecutionContext jobExecutionContext) { // This is the part of the code that is executed by the scheduler - App jobApp = - JsonUtils.readOrConvertValue( - jobExecutionContext.getJobDetail().getJobDataMap().get(APP_INFO_KEY), App.class); + String appName = (String) jobExecutionContext.getJobDetail().getJobDataMap().get(APP_NAME); + App jobApp = collectionDAO.applicationDAO().findEntityByName(appName); + ApplicationHandler.getInstance().setAppRuntimeProperties(jobApp); // Initialise the Application this.init(jobApp); @@ -212,6 +228,14 @@ public void configure() { /* Not needed by default */ } + @Override + public void raisePreviewMessage(App app) { + throw AppException.byMessage( + app.getName(), + "Preview", + "App is in Preview Mode. Enable it from the server configuration."); + } + public static AppRuntime getAppRuntime(App app) { return JsonUtils.convertValue(app.getRuntime(), ScheduledExecutionContext.class); } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/apps/AppException.java b/openmetadata-service/src/main/java/org/openmetadata/service/apps/AppException.java index dc8dcba82760..7387e59ea867 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/apps/AppException.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/apps/AppException.java @@ -6,7 +6,7 @@ public class AppException extends WebServiceException { private static final String BY_NAME_MESSAGE = "Application [%s] Exception [%s] due to [%s]."; - private static final String ERROR_TYPE = "PIPELINE_SERVICE_ERROR"; + private static final String ERROR_TYPE = "APPLICATION_ERROR"; public AppException(String message) { super(Response.Status.BAD_REQUEST, ERROR_TYPE, message); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/apps/ApplicationHandler.java b/openmetadata-service/src/main/java/org/openmetadata/service/apps/ApplicationHandler.java index c70edba97d81..a9ab04ea0343 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/apps/ApplicationHandler.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/apps/ApplicationHandler.java @@ -1,85 +1,219 @@ package org.openmetadata.service.apps; +import static org.openmetadata.common.utils.CommonUtil.nullOrEmpty; +import static org.openmetadata.service.apps.scheduler.AppScheduler.APPS_JOB_GROUP; +import static org.openmetadata.service.apps.scheduler.AppScheduler.APP_INFO_KEY; +import static org.openmetadata.service.apps.scheduler.AppScheduler.APP_NAME; + import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; -import java.util.HashMap; +import java.util.Collection; +import lombok.Getter; import lombok.extern.slf4j.Slf4j; +import org.openmetadata.schema.api.configuration.apps.AppPrivateConfig; +import org.openmetadata.schema.api.configuration.apps.AppsPrivateConfiguration; import org.openmetadata.schema.entity.app.App; +import org.openmetadata.service.OpenMetadataApplicationConfig; +import org.openmetadata.service.apps.scheduler.AppScheduler; import org.openmetadata.service.exception.UnhandledServerException; +import org.openmetadata.service.jdbi3.AppRepository; import org.openmetadata.service.jdbi3.CollectionDAO; +import org.openmetadata.service.jdbi3.EntityRepository; import org.openmetadata.service.search.SearchRepository; +import org.openmetadata.service.util.JsonUtils; +import org.openmetadata.service.util.OpenMetadataConnectionBuilder; +import org.quartz.JobDataMap; +import org.quartz.JobDetail; +import org.quartz.JobKey; +import org.quartz.SchedulerException; +import org.quartz.impl.matchers.GroupMatcher; @Slf4j public class ApplicationHandler { - private static HashMap instances = new HashMap<>(); + @Getter private static ApplicationHandler instance; + private final OpenMetadataApplicationConfig config; + private final AppsPrivateConfiguration privateConfiguration; + private final AppRepository appRepository; + + private ApplicationHandler(OpenMetadataApplicationConfig config) { + this.config = config; + this.privateConfiguration = config.getAppsPrivateConfiguration(); + this.appRepository = new AppRepository(); + } + + public static void initialize(OpenMetadataApplicationConfig config) { + if (instance != null) { + return; + } + instance = new ApplicationHandler(config); + } + + /** + * Load the apps' OM configuration and private parameters + */ + public void setAppRuntimeProperties(App app) { + app.setOpenMetadataServerConnection( + new OpenMetadataConnectionBuilder(config, app.getBot().getName()).build()); - public static Object getAppInstance(String className) { - return instances.get(className); + if (privateConfiguration != null + && !nullOrEmpty(privateConfiguration.getAppsPrivateConfiguration())) { + for (AppPrivateConfig appPrivateConfig : privateConfiguration.getAppsPrivateConfiguration()) { + if (app.getName().equals(appPrivateConfig.getName())) { + app.setPreview(appPrivateConfig.getPreview()); + app.setPrivateConfiguration(appPrivateConfig.getParameters()); + } + } + } } - private ApplicationHandler() { - /*Helper*/ + public Boolean isPreview(String appName) { + if (privateConfiguration != null + && !nullOrEmpty(privateConfiguration.getAppsPrivateConfiguration())) { + for (AppPrivateConfig appPrivateConfig : privateConfiguration.getAppsPrivateConfiguration()) { + if (appName.equals(appPrivateConfig.getName())) { + return appPrivateConfig.getPreview(); + } + } + } + return false; } - public static void triggerApplicationOnDemand( + public void triggerApplicationOnDemand( App app, CollectionDAO daoCollection, SearchRepository searchRepository) { runMethodFromApplication(app, daoCollection, searchRepository, "triggerOnDemand"); } - public static void installApplication( + public void installApplication( App app, CollectionDAO daoCollection, SearchRepository searchRepository) { runMethodFromApplication(app, daoCollection, searchRepository, "install"); } - public static void configureApplication( + public void configureApplication( App app, CollectionDAO daoCollection, SearchRepository searchRepository) { runMethodFromApplication(app, daoCollection, searchRepository, "configure"); } - public static Object runAppInit( - App app, CollectionDAO daoCollection, SearchRepository searchRepository) + public Object runAppInit(App app, CollectionDAO daoCollection, SearchRepository searchRepository) throws ClassNotFoundException, NoSuchMethodException, InvocationTargetException, InstantiationException, IllegalAccessException { + // add private runtime properties + setAppRuntimeProperties(app); Class clz = Class.forName(app.getClassName()); Object resource = clz.getDeclaredConstructor(CollectionDAO.class, SearchRepository.class) .newInstance(daoCollection, searchRepository); + // Raise preview message if the app is in Preview mode + if (Boolean.TRUE.equals(app.getPreview())) { + Method preview = resource.getClass().getMethod("raisePreviewMessage", App.class); + preview.invoke(resource, app); + } + // Call init Method Method initMethod = resource.getClass().getMethod("init", App.class); initMethod.invoke(resource, app); - instances.put(app.getClassName(), resource); - return resource; } - /** Load an App from its className and call its methods dynamically */ - public static void runMethodFromApplication( + /** + * Load an App from its className and call its methods dynamically + */ + public void runMethodFromApplication( App app, CollectionDAO daoCollection, SearchRepository searchRepository, String methodName) { // Native Application try { - Object resource = getAppInstance(app.getClassName()); - if (resource == null) { - resource = runAppInit(app, daoCollection, searchRepository); - } - + Object resource = runAppInit(app, daoCollection, searchRepository); // Call method on demand Method scheduleMethod = resource.getClass().getMethod(methodName); scheduleMethod.invoke(resource); - } catch (NoSuchMethodException - | InstantiationException - | IllegalAccessException - | InvocationTargetException e) { + } catch (NoSuchMethodException | InstantiationException | IllegalAccessException e) { LOG.error("Exception encountered", e); - throw new UnhandledServerException("Exception encountered", e); + throw new UnhandledServerException(e.getMessage()); } catch (ClassNotFoundException e) { - throw new UnhandledServerException("Exception encountered", e); + throw new UnhandledServerException(e.getMessage()); + } catch (InvocationTargetException e) { + throw AppException.byMessage(app.getName(), methodName, e.getTargetException().getMessage()); + } + } + + public void migrateQuartzConfig(App application) throws SchedulerException { + JobDetail jobDetails = + AppScheduler.getInstance() + .getScheduler() + .getJobDetail(new JobKey(application.getName(), APPS_JOB_GROUP)); + if (jobDetails == null) { + return; + } + JobDataMap jobDataMap = jobDetails.getJobDataMap(); + if (jobDataMap == null) { + return; + } + String appInfo = jobDataMap.getString(APP_INFO_KEY); + if (appInfo == null) { + return; } + LOG.info("migrating app quartz configuration for {}", application.getName()); + App updatedApp = JsonUtils.readOrConvertValue(appInfo, App.class); + App currentApp = appRepository.getDao().findEntityById(application.getId()); + updatedApp.setOpenMetadataServerConnection(null); + updatedApp.setPrivateConfiguration(null); + updatedApp.setScheduleType(currentApp.getScheduleType()); + updatedApp.setAppSchedule(currentApp.getAppSchedule()); + updatedApp.setUpdatedBy(currentApp.getUpdatedBy()); + updatedApp.setFullyQualifiedName(currentApp.getFullyQualifiedName()); + EntityRepository.EntityUpdater updater = + appRepository.getUpdater(currentApp, updatedApp, EntityRepository.Operation.PATCH); + updater.update(); + AppScheduler.getInstance().deleteScheduledApplication(updatedApp); + AppScheduler.getInstance().addApplicationSchedule(updatedApp); + LOG.info("migrated app configuration for {}", application.getName()); + } + + public void fixCorruptedInstallation(App application) throws SchedulerException { + JobDetail jobDetails = + AppScheduler.getInstance() + .getScheduler() + .getJobDetail(new JobKey(application.getName(), APPS_JOB_GROUP)); + if (jobDetails == null) { + return; + } + JobDataMap jobDataMap = jobDetails.getJobDataMap(); + if (jobDataMap == null) { + return; + } + String appName = jobDataMap.getString(APP_NAME); + if (appName == null) { + LOG.info("corrupt entry for app {}, reinstalling", application.getName()); + App app = appRepository.getDao().findEntityByName(application.getName()); + AppScheduler.getInstance().deleteScheduledApplication(app); + AppScheduler.getInstance().addApplicationSchedule(app); + } + } + + public void removeOldJobs(App app) throws SchedulerException { + Collection jobKeys = + AppScheduler.getInstance() + .getScheduler() + .getJobKeys(GroupMatcher.groupContains(APPS_JOB_GROUP)); + jobKeys.forEach( + jobKey -> { + try { + Class clz = + AppScheduler.getInstance().getScheduler().getJobDetail(jobKey).getJobClass(); + if (!jobKey.getName().equals(app.getName()) + && clz.getName().equals(app.getClassName())) { + LOG.info("deleting old job {}", jobKey.getName()); + AppScheduler.getInstance().getScheduler().deleteJob(jobKey); + } + } catch (SchedulerException e) { + LOG.error("Error deleting job {}", jobKey.getName(), e); + } + }); } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/apps/NativeApplication.java b/openmetadata-service/src/main/java/org/openmetadata/service/apps/NativeApplication.java index 58e6edb8d2c4..206a612766d2 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/apps/NativeApplication.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/apps/NativeApplication.java @@ -13,5 +13,7 @@ public interface NativeApplication extends Job { void configure(); + void raisePreviewMessage(App app); + default void startApp(JobExecutionContext jobExecutionContext) {} } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/changeEvent/EventAlertProducer.java b/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/changeEvent/EventAlertProducer.java deleted file mode 100644 index 874473064215..000000000000 --- a/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/changeEvent/EventAlertProducer.java +++ /dev/null @@ -1,3 +0,0 @@ -package org.openmetadata.service.apps.bundles.changeEvent; - -public class EventAlertProducer {} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/changeEvent/gchat/GChatPublisher.java b/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/changeEvent/gchat/GChatPublisher.java index de49397f942b..1aa906896c7d 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/changeEvent/gchat/GChatPublisher.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/changeEvent/gchat/GChatPublisher.java @@ -54,7 +54,7 @@ public GChatPublisher(SubscriptionDestination subscription) { client = getClient(subscription.getTimeout(), subscription.getReadTimeout()); // Build Target - if (webhook.getEndpoint() != null) { + if (webhook != null && webhook.getEndpoint() != null) { String gChatWebhookURL = webhook.getEndpoint().toString(); if (!CommonUtil.nullOrEmpty(gChatWebhookURL)) { target = client.target(gChatWebhookURL).request(); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/changeEvent/msteams/MSTeamsPublisher.java b/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/changeEvent/msteams/MSTeamsPublisher.java index ff2d1cf40823..588bd71d081c 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/changeEvent/msteams/MSTeamsPublisher.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/changeEvent/msteams/MSTeamsPublisher.java @@ -54,7 +54,7 @@ public MSTeamsPublisher(SubscriptionDestination subscription) { client = getClient(subscription.getTimeout(), subscription.getReadTimeout()); // Build Target - if (webhook.getEndpoint() != null) { + if (webhook != null && webhook.getEndpoint() != null) { String msTeamsWebhookURL = webhook.getEndpoint().toString(); if (!CommonUtil.nullOrEmpty(msTeamsWebhookURL)) { target = client.target(msTeamsWebhookURL).request(); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/changeEvent/slack/SlackEventPublisher.java b/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/changeEvent/slack/SlackEventPublisher.java index 9ad9d7ca27c7..36a42d990ce3 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/changeEvent/slack/SlackEventPublisher.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/changeEvent/slack/SlackEventPublisher.java @@ -52,7 +52,7 @@ public SlackEventPublisher(SubscriptionDestination subscription) { client = getClient(subscription.getTimeout(), subscription.getReadTimeout()); // Build Target - if (webhook.getEndpoint() != null) { + if (webhook != null && webhook.getEndpoint() != null) { String slackWebhookURL = webhook.getEndpoint().toString(); if (!CommonUtil.nullOrEmpty(slackWebhookURL)) { target = client.target(slackWebhookURL).request(); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/insights/DataInsightsReportApp.java b/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/insights/DataInsightsReportApp.java index adedbf76f772..56f9ab118533 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/insights/DataInsightsReportApp.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/insights/DataInsightsReportApp.java @@ -8,7 +8,7 @@ import static org.openmetadata.schema.type.DataReportIndex.ENTITY_REPORT_DATA_INDEX; import static org.openmetadata.service.Entity.KPI; import static org.openmetadata.service.Entity.TEAM; -import static org.openmetadata.service.apps.scheduler.AppScheduler.APP_INFO_KEY; +import static org.openmetadata.service.apps.scheduler.AppScheduler.APP_NAME; import static org.openmetadata.service.util.SubscriptionUtil.getAdminsData; import static org.openmetadata.service.util.Utilities.getMonthAndDateFromEpoch; @@ -70,9 +70,8 @@ public DataInsightsReportApp(CollectionDAO collectionDAO, SearchRepository searc @Override public void execute(JobExecutionContext jobExecutionContext) { - App app = - JsonUtils.readOrConvertValue( - jobExecutionContext.getJobDetail().getJobDataMap().get(APP_INFO_KEY), App.class); + String appName = (String) jobExecutionContext.getJobDetail().getJobDataMap().get(APP_NAME); + App app = collectionDAO.applicationDAO().findEntityByName(appName); // Calculate time diff long currentTime = Instant.now().toEpochMilli(); long scheduleTime = currentTime - 604800000L; @@ -295,6 +294,7 @@ private DataInsightDescriptionAndOwnerTemplate createDescriptionTemplate( PERCENTAGE_OF_ENTITIES_WITH_DESCRIPTION_BY_TYPE, currentPercentCompleted, currentPercentCompleted - previousPercentCompleted, + (int) currentCompletedDescription, numberOfDaysChange, dateMap); } @@ -304,6 +304,7 @@ private DataInsightDescriptionAndOwnerTemplate createDescriptionTemplate( PERCENTAGE_OF_ENTITIES_WITH_DESCRIPTION_BY_TYPE, 0D, 0D, + 0, numberOfDaysChange, dateMap); } @@ -363,6 +364,7 @@ private DataInsightDescriptionAndOwnerTemplate createOwnershipTemplate( PERCENTAGE_OF_ENTITIES_WITH_OWNER_BY_TYPE, currentPercentCompleted, currentPercentCompleted - previousPercentCompleted, + (int) currentHasOwner, numberOfDaysChange, dateMap); } @@ -371,6 +373,7 @@ private DataInsightDescriptionAndOwnerTemplate createOwnershipTemplate( PERCENTAGE_OF_ENTITIES_WITH_OWNER_BY_TYPE, 0D, 0D, + 0, numberOfDaysChange, dateMap); } @@ -409,6 +412,7 @@ private DataInsightDescriptionAndOwnerTemplate createTierTemplate( return new DataInsightDescriptionAndOwnerTemplate( DataInsightDescriptionAndOwnerTemplate.MetricType.TIER, null, + "0", 0D, KPI_NOT_SET, 0D, @@ -422,6 +426,7 @@ private DataInsightDescriptionAndOwnerTemplate createTierTemplate( return new DataInsightDescriptionAndOwnerTemplate( DataInsightDescriptionAndOwnerTemplate.MetricType.TIER, null, + "0", 0D, KPI_NOT_SET, 0D, @@ -504,6 +509,7 @@ private DataInsightDescriptionAndOwnerTemplate getTemplate( DataInsightChartResult.DataInsightChartType chartType, Double percentCompleted, Double percentChange, + int totalAssets, int numberOfDaysChange, Map dateMap) { @@ -525,8 +531,8 @@ private DataInsightDescriptionAndOwnerTemplate getTemplate( if (isKpiAvailable) { targetKpi = - String.valueOf( - Double.parseDouble(validKpi.getTargetDefinition().get(0).getValue()) * 100); + String.format( + "%.2f", Double.parseDouble(validKpi.getTargetDefinition().get(0).getValue()) * 100); KpiResult result = getKpiResult(validKpi.getName()); if (result != null) { isTargetMet = result.getTargetResult().get(0).getTargetMet(); @@ -547,6 +553,7 @@ private DataInsightDescriptionAndOwnerTemplate getTemplate( return new DataInsightDescriptionAndOwnerTemplate( metricType, criteria, + String.valueOf(totalAssets), percentCompleted, targetKpi, percentChange, diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/searchIndex/SearchIndexApp.java b/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/searchIndex/SearchIndexApp.java index 9127443e6dbe..1bbe997598f0 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/searchIndex/SearchIndexApp.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/searchIndex/SearchIndexApp.java @@ -1,9 +1,10 @@ package org.openmetadata.service.apps.bundles.searchIndex; +import static org.openmetadata.schema.system.IndexingError.ErrorSource.READER; import static org.openmetadata.service.apps.scheduler.AbstractOmAppJobListener.APP_RUN_STATS; +import static org.openmetadata.service.apps.scheduler.AppScheduler.ON_DEMAND_JOB; import static org.openmetadata.service.workflows.searchIndex.ReindexingUtil.ENTITY_TYPE_KEY; import static org.openmetadata.service.workflows.searchIndex.ReindexingUtil.getTotalRequestToProcess; -import static org.openmetadata.service.workflows.searchIndex.ReindexingUtil.getUpdatedStats; import static org.openmetadata.service.workflows.searchIndex.ReindexingUtil.isDataInsightIndex; import java.util.ArrayList; @@ -19,7 +20,6 @@ import org.openmetadata.schema.analytics.ReportData; import org.openmetadata.schema.entity.app.App; import org.openmetadata.schema.entity.app.AppRunRecord; -import org.openmetadata.schema.entity.app.AppRunType; import org.openmetadata.schema.entity.app.FailureContext; import org.openmetadata.schema.entity.app.SuccessContext; import org.openmetadata.schema.service.configuration.elasticsearch.ElasticSearchConfiguration; @@ -110,64 +110,29 @@ public void init(App app) { if (request.getEntities().contains(ALL)) { request.setEntities(ALL_ENTITIES); } - int totalRecords = getTotalRequestToProcess(request.getEntities(), collectionDAO); - this.jobData = request; - this.jobData.setStats( - new Stats() - .withJobStats( - new StepStats() - .withTotalRecords(totalRecords) - .withFailedRecords(0) - .withSuccessRecords(0))); - request - .getEntities() - .forEach( - entityType -> { - if (!isDataInsightIndex(entityType)) { - List fields = List.of("*"); - PaginatedEntitiesSource source = - new PaginatedEntitiesSource(entityType, jobData.getBatchSize(), fields); - if (!CommonUtil.nullOrEmpty(request.getAfterCursor())) { - source.setCursor(request.getAfterCursor()); - } - paginatedEntitiesSources.add(source); - } else { - paginatedDataInsightSources.add( - new PaginatedDataInsightSource( - collectionDAO, entityType, jobData.getBatchSize())); - } - }); - if (searchRepository.getSearchType().equals(ElasticSearchConfiguration.SearchType.OPENSEARCH)) { - this.entityProcessor = new OpenSearchEntitiesProcessor(totalRecords); - this.dataInsightProcessor = new OpenSearchDataInsightProcessor(totalRecords); - this.searchIndexSink = new OpenSearchIndexSink(searchRepository, totalRecords); - } else { - this.entityProcessor = new ElasticSearchEntitiesProcessor(totalRecords); - this.dataInsightProcessor = new ElasticSearchDataInsightProcessor(totalRecords); - this.searchIndexSink = new ElasticSearchIndexSink(searchRepository, totalRecords); - } + jobData = request; } @Override public void startApp(JobExecutionContext jobExecutionContext) { try { + initializeJob(); LOG.info("Executing Reindexing Job with JobData : {}", jobData); // Update Job Status jobData.setStatus(EventPublisherJob.Status.RUNNING); // Make recreate as false for onDemand - AppRunType runType = - AppRunType.fromValue( - (String) jobExecutionContext.getJobDetail().getJobDataMap().get("triggerType")); + String runType = + (String) jobExecutionContext.getJobDetail().getJobDataMap().get("triggerType"); - // Schedule Run has recreate as false always - if (runType.equals(AppRunType.Scheduled)) { + // Schedule Run has re-create set to false + if (!runType.equals(ON_DEMAND_JOB)) { jobData.setRecreateIndex(false); } // Run ReIndexing - entitiesReIndex(); - dataInsightReindex(); + entitiesReIndex(jobExecutionContext); + dataInsightReindex(jobExecutionContext); // Mark Job as Completed updateJobStatus(); } catch (Exception ex) { @@ -182,12 +147,46 @@ public void startApp(JobExecutionContext jobExecutionContext) { jobData.setStatus(EventPublisherJob.Status.FAILED); jobData.setFailure(indexingError); } finally { - // store job details in Database - jobExecutionContext.getJobDetail().getJobDataMap().put(APP_RUN_STATS, jobData.getStats()); - // Update Record to db - updateRecordToDb(jobExecutionContext); // Send update - sendUpdates(); + sendUpdates(jobExecutionContext); + } + } + + private void initializeJob() { + int totalRecords = getTotalRequestToProcess(jobData.getEntities(), collectionDAO); + this.jobData.setStats( + new Stats() + .withJobStats( + new StepStats() + .withTotalRecords(totalRecords) + .withFailedRecords(0) + .withSuccessRecords(0))); + jobData + .getEntities() + .forEach( + entityType -> { + if (!isDataInsightIndex(entityType)) { + List fields = List.of("*"); + PaginatedEntitiesSource source = + new PaginatedEntitiesSource(entityType, jobData.getBatchSize(), fields); + if (!CommonUtil.nullOrEmpty(jobData.getAfterCursor())) { + source.setCursor(jobData.getAfterCursor()); + } + paginatedEntitiesSources.add(source); + } else { + paginatedDataInsightSources.add( + new PaginatedDataInsightSource( + collectionDAO, entityType, jobData.getBatchSize())); + } + }); + if (searchRepository.getSearchType().equals(ElasticSearchConfiguration.SearchType.OPENSEARCH)) { + this.entityProcessor = new OpenSearchEntitiesProcessor(totalRecords); + this.dataInsightProcessor = new OpenSearchDataInsightProcessor(totalRecords); + this.searchIndexSink = new OpenSearchIndexSink(searchRepository, totalRecords); + } else { + this.entityProcessor = new ElasticSearchEntitiesProcessor(totalRecords); + this.dataInsightProcessor = new ElasticSearchDataInsightProcessor(totalRecords); + this.searchIndexSink = new ElasticSearchIndexSink(searchRepository, totalRecords); } } @@ -212,7 +211,7 @@ public void updateRecordToDb(JobExecutionContext jobExecutionContext) { pushAppStatusUpdates(jobExecutionContext, appRecord, true); } - private void entitiesReIndex() { + private void entitiesReIndex(JobExecutionContext jobExecutionContext) { Map contextData = new HashMap<>(); for (PaginatedEntitiesSource paginatedEntitiesSource : paginatedEntitiesSources) { reCreateIndexes(paginatedEntitiesSource.getEntityType()); @@ -223,17 +222,31 @@ private void entitiesReIndex() { resultList = paginatedEntitiesSource.readNext(null); if (!resultList.getData().isEmpty()) { searchIndexSink.write(entityProcessor.process(resultList, contextData), contextData); + if (!resultList.getErrors().isEmpty()) { + throw new SearchIndexException( + new IndexingError() + .withErrorSource(READER) + .withLastFailedCursor(paginatedEntitiesSource.getLastFailedCursor()) + .withSubmittedCount(paginatedEntitiesSource.getBatchSize()) + .withSuccessCount(resultList.getData().size()) + .withFailedCount(resultList.getErrors().size()) + .withMessage( + "Issues in Reading A Batch For Entities. Check Errors Corresponding to Entities.") + .withFailedEntities(resultList.getErrors())); + } } } catch (SearchIndexException rx) { + jobData.setStatus(EventPublisherJob.Status.FAILED); jobData.setFailure(rx.getIndexingError()); + } finally { + updateStats(paginatedEntitiesSource.getEntityType(), paginatedEntitiesSource.getStats()); + sendUpdates(jobExecutionContext); } } - updateStats(paginatedEntitiesSource.getEntityType(), paginatedEntitiesSource.getStats()); - sendUpdates(); } } - private void dataInsightReindex() { + private void dataInsightReindex(JobExecutionContext jobExecutionContext) { Map contextData = new HashMap<>(); for (PaginatedDataInsightSource paginatedDataInsightSource : paginatedDataInsightSources) { reCreateIndexes(paginatedDataInsightSource.getEntityType()); @@ -247,17 +260,23 @@ private void dataInsightReindex() { dataInsightProcessor.process(resultList, contextData), contextData); } } catch (SearchIndexException ex) { + jobData.setStatus(EventPublisherJob.Status.FAILED); jobData.setFailure(ex.getIndexingError()); + } finally { + updateStats( + paginatedDataInsightSource.getEntityType(), paginatedDataInsightSource.getStats()); + sendUpdates(jobExecutionContext); } } - updateStats( - paginatedDataInsightSource.getEntityType(), paginatedDataInsightSource.getStats()); - sendUpdates(); } } - private void sendUpdates() { + private void sendUpdates(JobExecutionContext jobExecutionContext) { try { + // store job details in Database + jobExecutionContext.getJobDetail().getJobDataMap().put(APP_RUN_STATS, jobData.getStats()); + // Update Record to db + updateRecordToDb(jobExecutionContext); if (WebSocketManager.getInstance() != null) { WebSocketManager.getInstance() .broadCastMessageToAll( @@ -275,7 +294,8 @@ public void updateStats(String entityType, StepStats currentEntityStats) { // Update Entity Level Stats StepStats entityLevelStats = jobDataStats.getEntityStats(); if (entityLevelStats == null) { - entityLevelStats = new StepStats(); + entityLevelStats = + new StepStats().withTotalRecords(null).withFailedRecords(null).withSuccessRecords(null); } entityLevelStats.withAdditionalProperty(entityType, currentEntityStats); @@ -286,8 +306,17 @@ public void updateStats(String entityType, StepStats currentEntityStats) { new StepStats() .withTotalRecords(getTotalRequestToProcess(jobData.getEntities(), collectionDAO)); } - getUpdatedStats( - stats, currentEntityStats.getSuccessRecords(), currentEntityStats.getFailedRecords()); + + stats.setSuccessRecords( + entityLevelStats.getAdditionalProperties().values().stream() + .map(s -> (StepStats) s) + .mapToInt(StepStats::getSuccessRecords) + .sum()); + stats.setFailedRecords( + entityLevelStats.getAdditionalProperties().values().stream() + .map(s -> (StepStats) s) + .mapToInt(StepStats::getFailedRecords) + .sum()); // Update for the Job jobDataStats.setJobStats(stats); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/apps/scheduler/AbstractOmAppJobListener.java b/openmetadata-service/src/main/java/org/openmetadata/service/apps/scheduler/AbstractOmAppJobListener.java index 2d4afc2733b4..cadf140af8f2 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/apps/scheduler/AbstractOmAppJobListener.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/apps/scheduler/AbstractOmAppJobListener.java @@ -1,6 +1,6 @@ package org.openmetadata.service.apps.scheduler; -import static org.openmetadata.service.apps.scheduler.AppScheduler.APP_INFO_KEY; +import static org.openmetadata.service.apps.scheduler.AppScheduler.APP_NAME; import java.util.HashMap; import java.util.Map; @@ -8,9 +8,9 @@ import org.apache.commons.lang.exception.ExceptionUtils; import org.openmetadata.schema.entity.app.App; import org.openmetadata.schema.entity.app.AppRunRecord; -import org.openmetadata.schema.entity.app.AppRunType; import org.openmetadata.schema.entity.app.FailureContext; import org.openmetadata.schema.entity.app.SuccessContext; +import org.openmetadata.service.apps.ApplicationHandler; import org.openmetadata.service.jdbi3.CollectionDAO; import org.openmetadata.service.util.JsonUtils; import org.quartz.JobDataMap; @@ -35,38 +35,37 @@ public String getName() { @Override public void jobToBeExecuted(JobExecutionContext jobExecutionContext) { - AppRunType runType = - AppRunType.fromValue( - (String) jobExecutionContext.getJobDetail().getJobDataMap().get("triggerType")); - App jobApp = - JsonUtils.readOrConvertValue( - jobExecutionContext.getJobDetail().getJobDataMap().get(APP_INFO_KEY), App.class); + String runType = (String) jobExecutionContext.getJobDetail().getJobDataMap().get("triggerType"); + String appName = (String) jobExecutionContext.getJobDetail().getJobDataMap().get(APP_NAME); + App jobApp = collectionDAO.applicationDAO().findEntityByName(appName); + ApplicationHandler.getInstance().setAppRuntimeProperties(jobApp); JobDataMap dataMap = jobExecutionContext.getJobDetail().getJobDataMap(); long jobStartTime = System.currentTimeMillis(); - AppRunRecord runRecord; + AppRunRecord runRecord = + new AppRunRecord() + .withAppId(jobApp.getId()) + .withStartTime(jobStartTime) + .withTimestamp(jobStartTime) + .withRunType(runType) + .withStatus(AppRunRecord.Status.RUNNING) + .withScheduleInfo(jobApp.getAppSchedule()); + ; boolean update = false; if (jobExecutionContext.isRecovering()) { - runRecord = + AppRunRecord latestRunRecord = JsonUtils.readValue( collectionDAO.appExtensionTimeSeriesDao().getLatestAppRun(jobApp.getId()), AppRunRecord.class); + if (latestRunRecord != null) { + runRecord = latestRunRecord; + } update = true; - } else { - runRecord = - new AppRunRecord() - .withAppId(jobApp.getId()) - .withStartTime(jobStartTime) - .withTimestamp(jobStartTime) - .withRunType(runType) - .withStatus(AppRunRecord.Status.RUNNING) - .withScheduleInfo(jobApp.getAppSchedule()); } // Put the Context in the Job Data Map dataMap.put(SCHEDULED_APP_RUN_EXTENSION, JsonUtils.pojoToJson(runRecord)); // Insert new Record Run pushApplicationStatusUpdates(jobExecutionContext, runRecord, update); - this.doJobToBeExecuted(jobExecutionContext); } @@ -125,10 +124,13 @@ public void pushApplicationStatusUpdates( JobExecutionContext context, AppRunRecord runRecord, boolean update) { JobDataMap dataMap = context.getJobDetail().getJobDataMap(); if (dataMap.containsKey(SCHEDULED_APP_RUN_EXTENSION)) { - App jobApp = - JsonUtils.readOrConvertValue( - context.getJobDetail().getJobDataMap().get(APP_INFO_KEY), App.class); - updateStatus(jobApp.getId(), runRecord, update); + // Update the Run Record in Data Map + dataMap.put(SCHEDULED_APP_RUN_EXTENSION, JsonUtils.pojoToJson(runRecord)); + + // Push Updates to the Database + String appName = (String) context.getJobDetail().getJobDataMap().get(APP_NAME); + UUID appId = collectionDAO.applicationDAO().findEntityByName(appName).getId(); + updateStatus(appId, runRecord, update); } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/apps/scheduler/AppScheduler.java b/openmetadata-service/src/main/java/org/openmetadata/service/apps/scheduler/AppScheduler.java index 3aebf1f85e86..98e84ccdf12e 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/apps/scheduler/AppScheduler.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/apps/scheduler/AppScheduler.java @@ -17,15 +17,14 @@ import org.openmetadata.common.utils.CommonUtil; import org.openmetadata.schema.AppRuntime; import org.openmetadata.schema.entity.app.App; -import org.openmetadata.schema.entity.app.AppRunType; import org.openmetadata.schema.entity.app.AppSchedule; +import org.openmetadata.schema.entity.app.ScheduleTimeline; import org.openmetadata.service.OpenMetadataApplicationConfig; import org.openmetadata.service.apps.NativeApplication; import org.openmetadata.service.exception.UnhandledServerException; import org.openmetadata.service.jdbi3.CollectionDAO; import org.openmetadata.service.jdbi3.locator.ConnectionType; import org.openmetadata.service.search.SearchRepository; -import org.openmetadata.service.util.JsonUtils; import org.quartz.CronScheduleBuilder; import org.quartz.JobBuilder; import org.quartz.JobDataMap; @@ -43,6 +42,7 @@ @Slf4j public class AppScheduler { private static final Map defaultAppScheduleConfig = new HashMap<>(); + public static final String ON_DEMAND_JOB = "OnDemandJob"; static { defaultAppScheduleConfig.put("org.quartz.scheduler.instanceName", "AppScheduler"); @@ -66,7 +66,7 @@ public class AppScheduler { public static final String APPS_JOB_GROUP = "OMAppsJobGroup"; public static final String APPS_TRIGGER_GROUP = "OMAppsJobGroup"; public static final String APP_INFO_KEY = "applicationInfoKey"; - public static final String SEARCH_CLIENT_KEY = "searchClientKey"; + public static final String APP_NAME = "appName"; private static AppScheduler instance; private static volatile boolean initialized = false; @Getter private final Scheduler scheduler; @@ -135,16 +135,17 @@ public static AppScheduler getInstance() { public void addApplicationSchedule(App application) { try { - if (scheduler.getJobDetail(new JobKey(application.getId().toString(), APPS_JOB_GROUP)) - != null) { + if (scheduler.getJobDetail(new JobKey(application.getName(), APPS_JOB_GROUP)) != null) { LOG.info("Job already exists for the application, skipping the scheduling"); return; } AppRuntime context = getAppRuntime(application); if (Boolean.TRUE.equals(context.getEnabled())) { - JobDetail jobDetail = jobBuilder(application, application.getId().toString()); - Trigger trigger = trigger(application); - scheduler.scheduleJob(jobDetail, trigger); + JobDetail jobDetail = jobBuilder(application, application.getName()); + if (!application.getAppSchedule().getScheduleTimeline().equals(ScheduleTimeline.NONE)) { + Trigger trigger = trigger(application); + scheduler.scheduleJob(jobDetail, trigger); + } } else { LOG.info("[Applications] App cannot be scheduled since it is disabled"); } @@ -155,14 +156,21 @@ public void addApplicationSchedule(App application) { } public void deleteScheduledApplication(App app) throws SchedulerException { - scheduler.deleteJob(new JobKey(app.getId().toString(), APPS_JOB_GROUP)); - scheduler.unscheduleJob(new TriggerKey(app.getId().toString(), APPS_TRIGGER_GROUP)); + // Scheduled Jobs + scheduler.deleteJob(new JobKey(app.getName(), APPS_JOB_GROUP)); + scheduler.unscheduleJob(new TriggerKey(app.getName(), APPS_TRIGGER_GROUP)); + + // OnDemand Jobs + scheduler.deleteJob( + new JobKey(String.format("%s-%s", app.getName(), ON_DEMAND_JOB), APPS_JOB_GROUP)); + scheduler.unscheduleJob( + new TriggerKey(String.format("%s-%s", app.getName(), ON_DEMAND_JOB), APPS_TRIGGER_GROUP)); } private JobDetail jobBuilder(App app, String jobIdentity) throws ClassNotFoundException { JobDataMap dataMap = new JobDataMap(); - dataMap.put(APP_INFO_KEY, JsonUtils.pojoToJson(app)); - dataMap.put("triggerType", AppRunType.Scheduled.value()); + dataMap.put(APP_NAME, app.getName()); + dataMap.put("triggerType", app.getAppSchedule().getScheduleTimeline().value()); Class clz = (Class) Class.forName(app.getClassName()); JobBuilder jobBuilder = @@ -175,7 +183,7 @@ private JobDetail jobBuilder(App app, String jobIdentity) throws ClassNotFoundEx private Trigger trigger(App app) { return TriggerBuilder.newTrigger() - .withIdentity(app.getId().toString(), APPS_TRIGGER_GROUP) + .withIdentity(app.getName(), APPS_TRIGGER_GROUP) .withSchedule(getCronSchedule(app.getAppSchedule())) .build(); } @@ -187,7 +195,7 @@ public static void shutDown() throws SchedulerException { } public static CronScheduleBuilder getCronSchedule(AppSchedule scheduleInfo) { - switch (scheduleInfo.getScheduleType()) { + switch (scheduleInfo.getScheduleTimeline()) { case HOURLY: return CronScheduleBuilder.cronSchedule("0 0 * ? * *"); case DAILY: @@ -210,12 +218,11 @@ public static CronScheduleBuilder getCronSchedule(AppSchedule scheduleInfo) { public void triggerOnDemandApplication(App application) { try { JobDetail jobDetailScheduled = - scheduler.getJobDetail(new JobKey(application.getId().toString(), APPS_JOB_GROUP)); + scheduler.getJobDetail(new JobKey(application.getName(), APPS_JOB_GROUP)); JobDetail jobDetailOnDemand = scheduler.getJobDetail( new JobKey( - String.format("%s-%s", application.getId(), AppRunType.OnDemand.value()), - APPS_JOB_GROUP)); + String.format("%s-%s", application.getName(), ON_DEMAND_JOB), APPS_JOB_GROUP)); // Check if the job is already running List currentJobs = scheduler.getCurrentlyExecutingJobs(); for (JobExecutionContext context : currentJobs) { @@ -231,14 +238,13 @@ public void triggerOnDemandApplication(App application) { AppRuntime context = getAppRuntime(application); if (Boolean.TRUE.equals(context.getEnabled())) { JobDetail newJobDetail = - jobBuilder( - application, - String.format("%s-%s", application.getId(), AppRunType.OnDemand.value())); - newJobDetail.getJobDataMap().put("triggerType", AppRunType.OnDemand.value()); + jobBuilder(application, String.format("%s-%s", application.getName(), ON_DEMAND_JOB)); + newJobDetail.getJobDataMap().put("triggerType", ON_DEMAND_JOB); + newJobDetail.getJobDataMap().put(APP_NAME, application.getFullyQualifiedName()); Trigger trigger = TriggerBuilder.newTrigger() .withIdentity( - String.format("%s-%s", application.getId(), AppRunType.OnDemand.value()), + String.format("%s-%s", application.getName(), ON_DEMAND_JOB), APPS_TRIGGER_GROUP) .startNow() .build(); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/events/AuditEventHandler.java b/openmetadata-service/src/main/java/org/openmetadata/service/events/AuditEventHandler.java index 75fe9cc29d06..0622997c0fd2 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/events/AuditEventHandler.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/events/AuditEventHandler.java @@ -37,13 +37,6 @@ public void init(OpenMetadataApplicationConfig config) { public Void process( ContainerRequestContext requestContext, ContainerResponseContext responseContext) { - if (requestContext - .getUriInfo() - .getPath() - .contains(WebAnalyticEventHandler.WEB_ANALYTIC_ENDPOINT)) { - // we don't want to send web analytic event to the audit log - return null; - } int responseCode = responseContext.getStatus(); String method = requestContext.getMethod(); if (responseContext.getEntity() != null) { diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/events/ChangeEventHandler.java b/openmetadata-service/src/main/java/org/openmetadata/service/events/ChangeEventHandler.java index fd9fcb299b59..612c4051f8d3 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/events/ChangeEventHandler.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/events/ChangeEventHandler.java @@ -39,11 +39,15 @@ import org.openmetadata.service.socket.WebSocketManager; import org.openmetadata.service.util.FeedUtils; import org.openmetadata.service.util.JsonUtils; +import org.openmetadata.service.util.WebsocketNotificationHandler; @Slf4j public class ChangeEventHandler implements EventHandler { private ObjectMapper mapper; private FeedMessageDecorator feedMessageDecorator = new FeedMessageDecorator(); + private final FeedRepository feedRepository = new FeedRepository(); + private final WebsocketNotificationHandler websocketNotificationHandler = + new WebsocketNotificationHandler(); public void init(OpenMetadataApplicationConfig config) { this.mapper = new ObjectMapper(); @@ -52,17 +56,20 @@ public void init(OpenMetadataApplicationConfig config) { @SneakyThrows public Void process( ContainerRequestContext requestContext, ContainerResponseContext responseContext) { + websocketNotificationHandler.processNotifications(responseContext); String method = requestContext.getMethod(); SecurityContext securityContext = requestContext.getSecurityContext(); String loggedInUserName = securityContext.getUserPrincipal().getName(); try { CollectionDAO collectionDAO = Entity.getCollectionDAO(); CollectionDAO.ChangeEventDAO changeEventDAO = collectionDAO.changeEventDAO(); - FeedRepository feedRepository = new FeedRepository(); Optional optionalChangeEvent = getChangeEventFromResponseContext(responseContext, loggedInUserName, method); if (optionalChangeEvent.isPresent()) { ChangeEvent changeEvent = optionalChangeEvent.get(); + if (changeEvent.getEntityType().equals(Entity.QUERY)) { + return null; + } // Always set the Change Event Username as context Principal, the one creating the CE changeEvent.setUserName(loggedInUserName); LOG.info( diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/events/WebAnalyticEventHandler.java b/openmetadata-service/src/main/java/org/openmetadata/service/events/WebAnalyticEventHandler.java deleted file mode 100644 index bfa933d325ac..000000000000 --- a/openmetadata-service/src/main/java/org/openmetadata/service/events/WebAnalyticEventHandler.java +++ /dev/null @@ -1,47 +0,0 @@ -package org.openmetadata.service.events; - -import io.micrometer.core.instrument.Counter; -import io.micrometer.prometheus.PrometheusMeterRegistry; -import javax.ws.rs.container.ContainerRequestContext; -import javax.ws.rs.container.ContainerResponseContext; -import javax.ws.rs.core.UriInfo; -import lombok.extern.slf4j.Slf4j; -import org.openmetadata.service.OpenMetadataApplicationConfig; -import org.openmetadata.service.util.MicrometerBundleSingleton; - -@Slf4j -public class WebAnalyticEventHandler implements EventHandler { - private PrometheusMeterRegistry prometheusMeterRegistry; - private String clusterName; - public static final String WEB_ANALYTIC_ENDPOINT = "v1/analytics/web/events/collect"; - private static final String COUNTER_NAME = "web.analytics.events"; - - public void init(OpenMetadataApplicationConfig config) { - this.prometheusMeterRegistry = MicrometerBundleSingleton.prometheusMeterRegistry; - this.clusterName = config.getClusterName(); - } - - public Void process( - ContainerRequestContext requestContext, ContainerResponseContext responseContext) { - UriInfo uriInfo = requestContext.getUriInfo(); - if (uriInfo.getPath().contains(WEB_ANALYTIC_ENDPOINT)) { - String username = "anonymous"; - if (requestContext.getSecurityContext().getUserPrincipal() != null) { - username = requestContext.getSecurityContext().getUserPrincipal().getName(); - } - incrementMetric(username); - } - return null; - } - - private void incrementMetric(String username) { - Counter.builder(COUNTER_NAME) - .tags("clusterName", clusterName, "username", username) - .register(prometheusMeterRegistry) - .increment(); - } - - public void close() { - prometheusMeterRegistry.close(); - } -} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/events/scheduled/EventSubscriptionScheduler.java b/openmetadata-service/src/main/java/org/openmetadata/service/events/scheduled/EventSubscriptionScheduler.java index d3ccf1d9b24d..78257ddca415 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/events/scheduled/EventSubscriptionScheduler.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/events/scheduled/EventSubscriptionScheduler.java @@ -132,7 +132,8 @@ private JobDetail jobBuilder( private Trigger trigger(EventSubscription eventSubscription) { return TriggerBuilder.newTrigger() .withIdentity(eventSubscription.getId().toString(), ALERT_TRIGGER_GROUP) - .withSchedule(SimpleScheduleBuilder.repeatSecondlyForever(3)) + .withSchedule( + SimpleScheduleBuilder.repeatSecondlyForever(eventSubscription.getPollInterval())) .startNow() .build(); } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/events/scheduled/template/DataInsightDescriptionAndOwnerTemplate.java b/openmetadata-service/src/main/java/org/openmetadata/service/events/scheduled/template/DataInsightDescriptionAndOwnerTemplate.java index 3925de0895df..fb34e89ac60e 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/events/scheduled/template/DataInsightDescriptionAndOwnerTemplate.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/events/scheduled/template/DataInsightDescriptionAndOwnerTemplate.java @@ -29,9 +29,11 @@ public enum KpiCriteria { NOT_MET } + private String totalAssets; private final String percentCompleted; private boolean kpiAvailable; private String percentChange; + private String percentChangeMessage; private String targetKpi; private String numberOfDaysLeft; private String completeMessage; @@ -42,6 +44,7 @@ public enum KpiCriteria { public DataInsightDescriptionAndOwnerTemplate( MetricType metricType, KpiCriteria criteria, + String totalAssets, Double percentCompleted, String targetKpi, Double percentChange, @@ -53,6 +56,8 @@ public DataInsightDescriptionAndOwnerTemplate( this.percentCompleted = String.format("%.2f", percentCompleted); this.targetKpi = targetKpi; this.percentChange = String.format("%.2f", percentChange); + this.percentChangeMessage = getFormattedPercentChangeMessage(percentChange); + this.totalAssets = totalAssets; this.kpiAvailable = isKpiAvailable; this.numberOfDaysLeft = numberOfDaysLeft; this.tierMap = tierMap; @@ -131,6 +136,22 @@ public void setNumberOfDaysLeft(String numberOfDaysLeft) { this.numberOfDaysLeft = numberOfDaysLeft; } + public String getTotalAssets() { + return totalAssets; + } + + public void setTotalAssets(String totalAssets) { + this.totalAssets = totalAssets; + } + + public String getPercentChangeMessage() { + return percentChangeMessage; + } + + public void setPercentChangeMessage(String message) { + this.percentChangeMessage = message; + } + public String getCompleteMessage() { return completeMessage; } @@ -162,4 +183,18 @@ public Map getDateMap() { public void setDateMap(Map dateMap) { this.dateMap = dateMap; } + + public static String getFormattedPercentChangeMessage(Double percent) { + String symbol = ""; + String color = "#BF0000"; + if (percent > 0) { + symbol = "+"; + color = "#008611"; + } else if (percent < 0) { + symbol = "-"; + } + + return String.format( + "%s%.2f", color, symbol, percent); + } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/events/scheduled/template/DataInsightTotalAssetTemplate.java b/openmetadata-service/src/main/java/org/openmetadata/service/events/scheduled/template/DataInsightTotalAssetTemplate.java index 6e194fa7ea5d..5f7b85749a4c 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/events/scheduled/template/DataInsightTotalAssetTemplate.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/events/scheduled/template/DataInsightTotalAssetTemplate.java @@ -13,12 +13,15 @@ package org.openmetadata.service.events.scheduled.template; +import static org.openmetadata.service.events.scheduled.template.DataInsightDescriptionAndOwnerTemplate.getFormattedPercentChangeMessage; + import java.util.Map; @SuppressWarnings("unused") public class DataInsightTotalAssetTemplate { private String totalDataAssets; private String percentChangeTotalAssets; + private String percentChangeMessage; private String completeMessage; private int numberOfDaysChange; private Map dateMap; @@ -30,6 +33,7 @@ public DataInsightTotalAssetTemplate( Map dateMap) { this.totalDataAssets = String.format("%.2f", totalDataAssets); this.percentChangeTotalAssets = String.format("%.2f", percentChangeTotalAssets); + this.percentChangeMessage = getFormattedPercentChangeMessage(percentChangeTotalAssets); this.numberOfDaysChange = numberOfDaysChange; this.dateMap = dateMap; String color = "#BF0000"; @@ -66,6 +70,14 @@ public void setCompleteMessage(String completeMessage) { this.completeMessage = completeMessage; } + public String getPercentChangeMessage() { + return percentChangeMessage; + } + + public void setPercentChangeMessage(String message) { + this.percentChangeMessage = message; + } + public int getNumberOfDaysChange() { return numberOfDaysChange; } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/events/subscription/AlertUtil.java b/openmetadata-service/src/main/java/org/openmetadata/service/events/subscription/AlertUtil.java index 33f06644c67d..220c05b00743 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/events/subscription/AlertUtil.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/events/subscription/AlertUtil.java @@ -45,6 +45,7 @@ import org.openmetadata.service.exception.CatalogExceptionMessage; import org.openmetadata.service.util.JsonUtils; import org.springframework.expression.Expression; +import org.springframework.expression.spel.support.SimpleEvaluationContext; @Slf4j public final class AlertUtil { @@ -56,8 +57,13 @@ public static void validateExpression(String condition, Class clz) { } Expression expression = parseExpression(condition); AlertsRuleEvaluator ruleEvaluator = new AlertsRuleEvaluator(null); + SimpleEvaluationContext context = + SimpleEvaluationContext.forReadOnlyDataBinding() + .withInstanceMethods() + .withRootObject(ruleEvaluator) + .build(); try { - expression.getValue(ruleEvaluator, clz); + expression.getValue(context, clz); } catch (Exception exception) { // Remove unnecessary class details in the exception message String message = @@ -73,7 +79,12 @@ public static boolean evaluateAlertConditions( String completeCondition = buildCompleteCondition(alertFilterRules); AlertsRuleEvaluator ruleEvaluator = new AlertsRuleEvaluator(changeEvent); Expression expression = parseExpression(completeCondition); - result = Boolean.TRUE.equals(expression.getValue(ruleEvaluator, Boolean.class)); + SimpleEvaluationContext context = + SimpleEvaluationContext.forReadOnlyDataBinding() + .withInstanceMethods() + .withRootObject(ruleEvaluator) + .build(); + result = Boolean.TRUE.equals(expression.getValue(context, Boolean.class)); LOG.debug("Alert evaluated as Result : {}", result); return result; } else { diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/exception/CatalogExceptionMessage.java b/openmetadata-service/src/main/java/org/openmetadata/service/exception/CatalogExceptionMessage.java index ea2792f346cc..ed8d27831827 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/exception/CatalogExceptionMessage.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/exception/CatalogExceptionMessage.java @@ -158,6 +158,10 @@ public static String invalidFieldName(String fieldType, String fieldName) { return String.format("Invalid %s name %s", fieldType, fieldName); } + public static String invalidFieldFQN(String fqn) { + return String.format("Invalid fully qualified field name %s", fqn); + } + public static String entityVersionNotFound(String entityType, UUID id, Double version) { return String.format("%s instance for %s and version %s not found", entityType, id, version); } @@ -268,6 +272,10 @@ public static String systemEntityRenameNotAllowed(String name, String entityType return String.format("System entity [%s] of type %s can not be renamed.", name, entityType); } + public static String systemEntityModifyNotAllowed(String name, String entityType) { + return String.format("System entity [%s] of type %s can not be modified.", name, entityType); + } + public static String mutuallyExclusiveLabels(TagLabel tag1, TagLabel tag2) { return String.format( "Tag labels %s and %s are mutually exclusive and can't be assigned together", diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/formatter/decorators/EmailMessageDecorator.java b/openmetadata-service/src/main/java/org/openmetadata/service/formatter/decorators/EmailMessageDecorator.java index e1933998c7c2..f9c3bb9070cb 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/formatter/decorators/EmailMessageDecorator.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/formatter/decorators/EmailMessageDecorator.java @@ -53,11 +53,11 @@ public String getRemoveMarkerClose() { } @Override - public String getEntityUrl(String entityType, String fqn, String additionalParams) { + public String getEntityUrl(String prefix, String fqn, String additionalParams) { return String.format( "%s", getSmtpSettings().getOpenMetadataUrl(), - entityType, + prefix, fqn.trim(), nullOrEmpty(additionalParams) ? "" : String.format("/%s", additionalParams), fqn.trim()); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/formatter/decorators/FeedMessageDecorator.java b/openmetadata-service/src/main/java/org/openmetadata/service/formatter/decorators/FeedMessageDecorator.java index 05e580a9e4ba..8b193e295ca6 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/formatter/decorators/FeedMessageDecorator.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/formatter/decorators/FeedMessageDecorator.java @@ -51,11 +51,11 @@ public String getRemoveMarkerClose() { } @Override - public String getEntityUrl(String entityType, String fqn, String additionalParams) { + public String getEntityUrl(String prefix, String fqn, String additionalParams) { return String.format( "[%s](/%s/%s%s)", fqn, - entityType, + prefix, fqn.trim(), nullOrEmpty(additionalParams) ? "" : String.format("/%s", additionalParams)); } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/formatter/decorators/GChatMessageDecorator.java b/openmetadata-service/src/main/java/org/openmetadata/service/formatter/decorators/GChatMessageDecorator.java index 93156f94efca..7a2374c6826f 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/formatter/decorators/GChatMessageDecorator.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/formatter/decorators/GChatMessageDecorator.java @@ -55,11 +55,11 @@ public String getRemoveMarkerClose() { } @Override - public String getEntityUrl(String entityType, String fqn, String additionalParams) { + public String getEntityUrl(String prefix, String fqn, String additionalParams) { return String.format( "<%s/%s/%s%s|%s>", getSmtpSettings().getOpenMetadataUrl(), - entityType, + prefix, fqn.trim().replace(" ", "%20"), nullOrEmpty(additionalParams) ? "" : String.format("/%s", additionalParams), fqn.trim()); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/formatter/decorators/MSTeamsMessageDecorator.java b/openmetadata-service/src/main/java/org/openmetadata/service/formatter/decorators/MSTeamsMessageDecorator.java index 36c83d5e295e..d76fcf8cee37 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/formatter/decorators/MSTeamsMessageDecorator.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/formatter/decorators/MSTeamsMessageDecorator.java @@ -55,12 +55,12 @@ public String getRemoveMarkerClose() { } @Override - public String getEntityUrl(String entityType, String fqn, String additionalParams) { + public String getEntityUrl(String prefix, String fqn, String additionalParams) { return String.format( "[%s](/%s/%s%s)", fqn.trim(), getSmtpSettings().getOpenMetadataUrl(), - entityType, + prefix, nullOrEmpty(additionalParams) ? "" : String.format("/%s", additionalParams)); } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/formatter/decorators/MessageDecorator.java b/openmetadata-service/src/main/java/org/openmetadata/service/formatter/decorators/MessageDecorator.java index ad1c6d884737..e08d932d7a12 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/formatter/decorators/MessageDecorator.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/formatter/decorators/MessageDecorator.java @@ -34,6 +34,7 @@ import org.openmetadata.schema.tests.TestCase; import org.openmetadata.schema.type.ChangeEvent; import org.openmetadata.schema.type.Include; +import org.openmetadata.schema.type.ThreadType; import org.openmetadata.service.Entity; import org.openmetadata.service.exception.UnhandledServerException; import org.openmetadata.service.resources.feeds.MessageParser; @@ -60,7 +61,7 @@ default String httpRemoveMarker() { String getRemoveMarkerClose(); - String getEntityUrl(String entityType, String fqn, String additionalInput); + String getEntityUrl(String prefix, String fqn, String additionalInput); T buildEntityMessage(ChangeEvent event); @@ -77,15 +78,57 @@ default String buildEntityUrl(String entityType, EntityInterface entityInterface // Hande Test Case if (entityType.equals(Entity.TEST_CASE)) { TestCase testCase = (TestCase) entityInterface; - MessageParser.EntityLink link = MessageParser.EntityLink.parse(testCase.getEntityLink()); - // TODO: this needs to be fixed no way to know the UI redirection return getEntityUrl( - link.getEntityType(), link.getEntityFQN(), "profiler?activeTab=Data%20Quality"); + "incident-manager", testCase.getFullyQualifiedName(), "test-case-results"); + } + + // Glossary Term + if (entityType.equals(Entity.GLOSSARY_TERM)) { + // Glossary Term is a special case where the URL is different + return getEntityUrl(Entity.GLOSSARY, fqn, ""); + } + + // Tag + if (entityType.equals(Entity.TAG)) { + // Tags need to be redirected to Classification Page + return getEntityUrl("tags", fqn.split("\\.")[0], ""); } return getEntityUrl(entityType, fqn, ""); } + default String buildThreadUrl( + ThreadType threadType, String entityType, EntityInterface entityInterface) { + String activeTab = + threadType.equals(ThreadType.Task) ? "activity_feed/tasks" : "activity_feed/all"; + String fqn = entityInterface.getFullyQualifiedName(); + if (CommonUtil.nullOrEmpty(fqn)) { + EntityInterface result = + Entity.getEntity(entityType, entityInterface.getId(), "id", Include.NON_DELETED); + fqn = result.getFullyQualifiedName(); + } + + // Hande Test Case + if (entityType.equals(Entity.TEST_CASE)) { + TestCase testCase = (TestCase) entityInterface; + return getEntityUrl("incident-manager", testCase.getFullyQualifiedName(), "issues"); + } + + // Glossary Term + if (entityType.equals(Entity.GLOSSARY_TERM)) { + // Glossary Term is a special case where the URL is different + return getEntityUrl(Entity.GLOSSARY, fqn, activeTab); + } + + // Tag + if (entityType.equals(Entity.TAG)) { + // Tags need to be redirected to Classification Page + return getEntityUrl("tags", fqn.split("\\.")[0], ""); + } + + return getEntityUrl(entityType, fqn, activeTab); + } + default T buildOutgoingMessage(ChangeEvent event) { if (event.getEntityType().equals(Entity.THREAD)) { return buildThreadMessage(event); @@ -176,20 +219,28 @@ default OutgoingMessage createThreadMessage(ChangeEvent event) { OutgoingMessage message = new OutgoingMessage(); message.setUserName(event.getUserName()); Thread thread = getThread(event); + + MessageParser.EntityLink entityLink = MessageParser.EntityLink.parse(thread.getAbout()); + EntityInterface entityInterface = Entity.getEntity(entityLink, "", Include.ALL); + String entityUrl = buildEntityUrl(entityLink.getEntityType(), entityInterface); + String headerMessage = ""; List attachmentList = new ArrayList<>(); + + String assetUrl = + getThreadAssetsUrl(thread.getType(), MessageParser.EntityLink.parse(thread.getAbout())); switch (thread.getType()) { case Conversation -> { switch (event.getEventType()) { case THREAD_CREATED -> { headerMessage = String.format( - "@%s started a conversation for asset %s", - thread.getCreatedBy(), thread.getAbout()); + "@%s started a conversation for asset %s", thread.getCreatedBy(), assetUrl); attachmentList.add(replaceEntityLinks(thread.getMessage())); } case POST_CREATED -> { - headerMessage = String.format("@%s posted a message", thread.getCreatedBy()); + headerMessage = + String.format("@%s posted a message on asset %s", thread.getCreatedBy(), assetUrl); attachmentList.add( String.format( "@%s : %s", thread.getCreatedBy(), replaceEntityLinks(thread.getMessage()))); @@ -204,7 +255,9 @@ default OutgoingMessage createThreadMessage(ChangeEvent event) { } case THREAD_UPDATED -> { headerMessage = - String.format("@%s posted update on Conversation", thread.getUpdatedBy()); + String.format( + "@%s posted update on Conversation for asset %s", + thread.getUpdatedBy(), assetUrl); attachmentList.add(replaceEntityLinks(thread.getMessage())); } } @@ -214,8 +267,8 @@ default OutgoingMessage createThreadMessage(ChangeEvent event) { case THREAD_CREATED -> { headerMessage = String.format( - "@%s created a Task with Id : %s", - thread.getCreatedBy(), thread.getTask().getId()); + "@%s created a Task for %s %s", + thread.getCreatedBy(), entityLink.getEntityType(), assetUrl); attachmentList.add(String.format("Task Type : %s", thread.getTask().getType().value())); attachmentList.add( String.format( @@ -229,8 +282,8 @@ default OutgoingMessage createThreadMessage(ChangeEvent event) { case POST_CREATED -> { headerMessage = String.format( - "@%s posted a message on the Task with Id : %s", - thread.getCreatedBy(), thread.getTask().getId()); + "@%s posted a message on the Task with Id : %s for Asset %s", + thread.getCreatedBy(), thread.getTask().getId(), assetUrl); thread .getPosts() .forEach( @@ -243,8 +296,8 @@ default OutgoingMessage createThreadMessage(ChangeEvent event) { case THREAD_UPDATED -> { headerMessage = String.format( - "@%s posted update on the Task with Id : %s", - thread.getUpdatedBy(), thread.getTask().getId()); + "@%s posted update on the Task with Id : %s for Asset %s", + thread.getUpdatedBy(), thread.getTask().getId(), assetUrl); attachmentList.add(String.format("Task Type : %s", thread.getTask().getType().value())); attachmentList.add( String.format( @@ -258,15 +311,15 @@ default OutgoingMessage createThreadMessage(ChangeEvent event) { case TASK_CLOSED -> { headerMessage = String.format( - "@%s closed Task with Id : %s", - thread.getCreatedBy(), thread.getTask().getId()); + "@%s closed Task with Id : %s for Asset %s", + thread.getCreatedBy(), thread.getTask().getId(), assetUrl); attachmentList.add(String.format("Current Status : %s", thread.getTask().getStatus())); } case TASK_RESOLVED -> { headerMessage = String.format( - "@%s resolved Task with Id : %s", - thread.getCreatedBy(), thread.getTask().getId()); + "@%s resolved Task with Id : %s for Asset %s", + thread.getCreatedBy(), thread.getTask().getId(), assetUrl); attachmentList.add(String.format("Current Status : %s", thread.getTask().getStatus())); } } @@ -319,9 +372,23 @@ default OutgoingMessage createThreadMessage(ChangeEvent event) { } message.setHeader(headerMessage); message.setMessages(attachmentList); + + message.setEntityUrl(entityUrl); return message; } + default String getThreadAssetsUrl( + ThreadType threadType, MessageParser.EntityLink aboutEntityLink) { + try { + return this.buildThreadUrl( + threadType, + aboutEntityLink.getEntityType(), + Entity.getEntity(aboutEntityLink, "id", Include.ALL)); + } catch (Exception ex) { + return ""; + } + } + private String getDateString(long epochTimestamp) { Instant instant = Instant.ofEpochSecond(epochTimestamp); LocalDateTime localDateTime = LocalDateTime.ofInstant(instant, ZoneId.systemDefault()); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/formatter/decorators/SlackMessageDecorator.java b/openmetadata-service/src/main/java/org/openmetadata/service/formatter/decorators/SlackMessageDecorator.java index 2075eb236a69..b18446a08ac9 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/formatter/decorators/SlackMessageDecorator.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/formatter/decorators/SlackMessageDecorator.java @@ -55,11 +55,11 @@ public String getRemoveMarkerClose() { return "~"; } - public String getEntityUrl(String entityType, String fqn, String additionalParams) { + public String getEntityUrl(String prefix, String fqn, String additionalParams) { return String.format( "<%s/%s/%s%s|%s>", getSmtpSettings().getOpenMetadataUrl(), - entityType, + prefix, fqn.trim().replaceAll(" ", "%20"), nullOrEmpty(additionalParams) ? "" : String.format("/%s", additionalParams), fqn.trim()); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/AppRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/AppRepository.java index 23d5d0ab7666..7789d3b25728 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/AppRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/AppRepository.java @@ -20,6 +20,7 @@ import org.openmetadata.schema.type.Relationship; import org.openmetadata.service.Entity; import org.openmetadata.service.exception.EntityNotFoundException; +import org.openmetadata.service.exception.UnhandledServerException; import org.openmetadata.service.resources.apps.AppResource; import org.openmetadata.service.security.jwt.JWTTokenGenerator; import org.openmetadata.service.util.EntityUtil; @@ -75,7 +76,7 @@ public EntityReference createNewAppBot(App application) { User botUser; Bot bot; try { - botUser = userRepository.findByName(botName, Include.NON_DELETED); + botUser = userRepository.getByName(null, botName, userRepository.getFields("id")); } catch (EntityNotFoundException ex) { // Get Bot Role EntityReference roleRef = @@ -88,6 +89,7 @@ public EntityReference createNewAppBot(App application) { CreateUser createUser = new CreateUser() .withName(botName) + .withDisplayName(application.getDisplayName()) .withEmail(String.format("%s@openmetadata.org", botName)) .withIsAdmin(false) .withIsBot(true) @@ -135,15 +137,14 @@ public EntityReference createNewAppBot(App application) { @Override public void storeEntity(App entity, boolean update) { - EntityReference botUserRef = entity.getBot(); EntityReference ownerRef = entity.getOwner(); - entity.withBot(null).withOwner(null); + entity.withOwner(null); // Store store(entity, update); // Restore entity fields - entity.withBot(botUserRef).withOwner(ownerRef); + entity.withOwner(ownerRef); } public EntityReference getBotUser(App application) { @@ -209,6 +210,9 @@ protected void cleanup(App app) { public AppRunRecord getLatestAppRuns(UUID appId) { String json = daoCollection.appExtensionTimeSeriesDao().getLatestAppRun(appId); + if (json == null) { + throw new UnhandledServerException("No Available Application Run Records."); + } return JsonUtils.readValue(json, AppRunRecord.class); } @@ -227,6 +231,7 @@ public void entitySpecificUpdate() { recordChange( "appConfiguration", original.getAppConfiguration(), updated.getAppConfiguration()); recordChange("appSchedule", original.getAppSchedule(), updated.getAppSchedule()); + recordChange("bot", original.getBot(), updated.getBot()); } } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java index bdd5e042120f..9110a0a9694b 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java @@ -122,7 +122,6 @@ import org.openmetadata.schema.util.ServicesCount; import org.openmetadata.schema.utils.EntityInterfaceUtil; import org.openmetadata.service.Entity; -import org.openmetadata.service.exception.UnhandledServerException; import org.openmetadata.service.jdbi3.CollectionDAO.TagUsageDAO.TagLabelMapper; import org.openmetadata.service.jdbi3.CollectionDAO.UsageDAO.UsageDetailsMapper; import org.openmetadata.service.jdbi3.FeedRepository.FilterType; @@ -3222,7 +3221,8 @@ List listWithEntityFilter( List listWithoutEntityFilter( @Bind("eventType") String eventType, @Bind("timestamp") long timestamp); - @SqlQuery("SELECT json FROM change_event ORDER BY eventTime ASC LIMIT :limit OFFSET :offset") + @SqlQuery( + "SELECT json FROM change_event ce where ce.offset > :offset ORDER BY ce.eventTime ASC LIMIT :limit") List list(@Bind("limit") long limit, @Bind("offset") long offset); @SqlQuery("SELECT count(*) FROM change_event") @@ -3698,7 +3698,7 @@ default String getLatestAppRun(UUID appId) { if (!nullOrEmpty(result)) { return result.get(0); } - throw new UnhandledServerException("No Available Application Run Records."); + return null; } } @@ -3943,6 +3943,9 @@ interface SystemDAO { @SqlUpdate(value = "DELETE from openmetadata_settings WHERE configType = :configType") void delete(@Bind("configType") String configType); + + @SqlQuery("SELECT 42") + Integer testConnection() throws StatementException; } class SettingsRowMapper implements RowMapper { diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/ColumnUtil.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/ColumnUtil.java index b1654d6b47f1..e57f30f73bb5 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/ColumnUtil.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/ColumnUtil.java @@ -1,5 +1,6 @@ package org.openmetadata.service.jdbi3; +import static org.openmetadata.common.utils.CommonUtil.findChildren; import static org.openmetadata.common.utils.CommonUtil.listOrEmpty; import static org.openmetadata.common.utils.CommonUtil.nullOrEmpty; @@ -9,6 +10,7 @@ import java.util.Set; import java.util.stream.Collectors; import org.openmetadata.schema.type.Column; +import org.openmetadata.schema.type.Field; import org.openmetadata.schema.type.TagLabel; import org.openmetadata.service.exception.CatalogExceptionMessage; import org.openmetadata.service.util.FullyQualifiedName; @@ -56,18 +58,21 @@ public static void setColumnFQN(String parentFQN, List columns) { // Validate if a given column exists in the table public static void validateColumnFQN(List columns, String columnFQN) { - boolean validColumn = false; - for (Column column : columns) { - if (column.getFullyQualifiedName().equals(columnFQN)) { - validColumn = true; - break; - } - } - if (!validColumn) { + boolean exists = findChildren(columns, "getChildren", columnFQN); + if (!exists) { throw new IllegalArgumentException(CatalogExceptionMessage.invalidColumnFQN(columnFQN)); } } + // validate if a given field exists in the topic + public static void validateFieldFQN(List fields, String fieldFQN) { + boolean exists = findChildren(fields, "getChildren", fieldFQN); + if (!exists) { + throw new IllegalArgumentException( + CatalogExceptionMessage.invalidFieldName("field", fieldFQN)); + } + } + public static Set getAllTags(Column column) { Set tags = new HashSet<>(); if (!listOrEmpty(column.getTags()).isEmpty()) { diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityRepository.java index 279824467dd2..21ec38fc5ae9 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityRepository.java @@ -124,6 +124,7 @@ import org.openmetadata.schema.type.LifeCycle; import org.openmetadata.schema.type.ProviderType; import org.openmetadata.schema.type.Relationship; +import org.openmetadata.schema.type.SuggestionType; import org.openmetadata.schema.type.TagLabel; import org.openmetadata.schema.type.TaskType; import org.openmetadata.schema.type.ThreadType; @@ -1907,6 +1908,25 @@ public final EntityReference validateDomain(String domainFqn) { return Entity.getEntityReferenceByName(Entity.DOMAIN, domainFqn, NON_DELETED); } + public final void validateDomain(EntityReference domain) { + if (!supportsDomain) { + throw new IllegalArgumentException(CatalogExceptionMessage.invalidField(FIELD_DOMAIN)); + } + Entity.getEntityReferenceById(Entity.DOMAIN, domain.getId(), NON_DELETED); + } + + public final void validateDataProducts(List dataProducts) { + if (!supportsDataProducts) { + throw new IllegalArgumentException(CatalogExceptionMessage.invalidField(FIELD_DATA_PRODUCTS)); + } + + if (!nullOrEmpty(dataProducts)) { + for (EntityReference dataProduct : dataProducts) { + Entity.getEntityReferenceById(Entity.DATA_PRODUCT, dataProduct.getId(), NON_DELETED); + } + } + } + /** Override this method to support downloading CSV functionality */ public String exportToCsv(String name, String user) throws IOException { throw new IllegalArgumentException(csvNotSupported(entityType)); @@ -1934,8 +1954,8 @@ public TaskWorkflow getTaskWorkflow(ThreadContext threadContext) { } } - public SuggestionRepository.SuggestionWorkflow getSuggestionWorkflow(Suggestion suggestion) { - return new SuggestionRepository.SuggestionWorkflow(suggestion); + public SuggestionRepository.SuggestionWorkflow getSuggestionWorkflow(EntityInterface entity) { + return new SuggestionRepository.SuggestionWorkflow(entity); } public EntityInterface applySuggestion( @@ -1943,6 +1963,13 @@ public EntityInterface applySuggestion( return entity; } + /** + * Bring in the necessary fields required to have all the information before applying a suggestion + */ + public String getSuggestionFields(Suggestion suggestion) { + return suggestion.getType() == SuggestionType.SuggestTagLabel ? "tags" : ""; + } + public final void validateTaskThread(ThreadContext threadContext) { ThreadType threadType = threadContext.getThread().getType(); if (threadType != ThreadType.Task) { @@ -2263,6 +2290,7 @@ && recordChange(FIELD_DOMAIN, origDomain, updatedDomain, true, entityReferenceMa origDomain.getId(), Entity.DOMAIN, original.getId(), entityType, Relationship.HAS); } if (updatedDomain != null) { + validateDomain(updatedDomain); // Add relationship owner --- owns ---> ownedEntity LOG.info( "Adding domain {} for entity {}", @@ -2283,6 +2311,7 @@ private void updateDataProducts() { } List origDataProducts = listOrEmpty(original.getDataProducts()); List updatedDataProducts = listOrEmpty(updated.getDataProducts()); + validateDataProducts(updatedDataProducts); updateFromRelationships( FIELD_DATA_PRODUCTS, DATA_PRODUCT, @@ -2299,6 +2328,7 @@ private void updateExperts() { } List origExperts = getEntityReferences(original.getExperts()); List updatedExperts = getEntityReferences(updated.getExperts()); + validateUsers(updatedExperts); updateToRelationships( FIELD_EXPERTS, entityType, @@ -2317,6 +2347,7 @@ private void updateReviewers() { } List origReviewers = getEntityReferences(original.getReviewers()); List updatedReviewers = getEntityReferences(updated.getReviewers()); + validateUsers(updatedReviewers); updateFromRelationships( "reviewers", Entity.USER, diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/FeedRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/FeedRepository.java index e9fdc6bd1fbb..08a41c610456 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/FeedRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/FeedRepository.java @@ -64,6 +64,7 @@ import org.openmetadata.schema.type.EntityReference; import org.openmetadata.schema.type.EventType; import org.openmetadata.schema.type.Include; +import org.openmetadata.schema.type.MetadataOperation; import org.openmetadata.schema.type.Post; import org.openmetadata.schema.type.Reaction; import org.openmetadata.schema.type.Relationship; @@ -85,6 +86,9 @@ import org.openmetadata.service.resources.feeds.MessageParser; import org.openmetadata.service.resources.feeds.MessageParser.EntityLink; import org.openmetadata.service.security.AuthorizationException; +import org.openmetadata.service.security.Authorizer; +import org.openmetadata.service.security.policyevaluator.OperationContext; +import org.openmetadata.service.security.policyevaluator.ResourceContext; import org.openmetadata.service.util.EntityUtil; import org.openmetadata.service.util.FullyQualifiedName; import org.openmetadata.service.util.JsonUtils; @@ -785,11 +789,12 @@ public final PatchResponse patchThread( } public void checkPermissionsForResolveTask( - Thread thread, boolean closeTask, SecurityContext securityContext) { + Authorizer authorizer, Thread thread, boolean closeTask, SecurityContext securityContext) { String userName = securityContext.getUserPrincipal().getName(); User user = Entity.getEntityByName(USER, userName, TEAMS_FIELD, NON_DELETED); EntityLink about = EntityLink.parse(thread.getAbout()); EntityReference aboutRef = EntityUtil.validateEntityLink(about); + ThreadContext threadContext = getThreadContext(thread); if (Boolean.TRUE.equals(user.getIsAdmin())) { return; // Allow admin resolve/close task } @@ -799,9 +804,25 @@ public void checkPermissionsForResolveTask( // Allow if user created the task to close task (and not resolve task) EntityReference owner = Entity.getOwner(aboutRef); List assignees = thread.getTask().getAssignees(); - if (assignees.stream().anyMatch(assignee -> assignee.getName().equals(userName)) - || owner.getName().equals(userName) - || closeTask && thread.getCreatedBy().equals(userName)) { + if (owner.getName().equals(userName) || closeTask && thread.getCreatedBy().equals(userName)) { + return; + } + + // Allow if user is an assignee of the task and if the assignee has permissions to update the + // entity + if (assignees.stream().anyMatch(assignee -> assignee.getName().equals(userName))) { + // If entity does not exist, this is a create operation, else update operation + ResourceContext resourceContext = + new ResourceContext<>(aboutRef.getType(), aboutRef.getId(), null); + if (EntityUtil.isDescriptionTask(threadContext.getTaskWorkflow().getTaskType())) { + OperationContext operationContext = + new OperationContext(aboutRef.getType(), MetadataOperation.EDIT_DESCRIPTION); + authorizer.authorize(securityContext, operationContext, resourceContext); + } else if (EntityUtil.isTagTask(threadContext.getTaskWorkflow().getTaskType())) { + OperationContext operationContext = + new OperationContext(aboutRef.getType(), MetadataOperation.EDIT_TAGS); + authorizer.authorize(securityContext, operationContext, resourceContext); + } return; } @@ -913,7 +934,7 @@ private boolean fieldsChanged(Post original, Post updated) { } private boolean fieldsChanged(Thread original, Thread updated) { - // Patch supports isResolved, message, task assignees, reactions, and announcements for now + // Patch supports isResolved, message, task assignees, reactions, announcements and AI for now return !original.getResolved().equals(updated.getResolved()) || !original.getMessage().equals(updated.getMessage()) || (Collections.isEmpty(original.getReactions()) @@ -935,6 +956,10 @@ private boolean fieldsChanged(Thread original, Thread updated) { || !Objects.equals( original.getAnnouncement().getEndTime(), updated.getAnnouncement().getEndTime()))) + || (original.getChatbot() == null && updated.getChatbot() != null) + || (original.getChatbot() != null + && updated.getChatbot() != null + && !original.getChatbot().getQuery().equals(updated.getChatbot().getQuery())) || (original.getTask() != null && (original.getTask().getAssignees().size() != updated.getTask().getAssignees().size() || !original diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/GlossaryTermRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/GlossaryTermRepository.java index fadb55a78ff2..3846818b9cd5 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/GlossaryTermRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/GlossaryTermRepository.java @@ -90,7 +90,7 @@ import org.openmetadata.service.util.EntityUtil.Fields; import org.openmetadata.service.util.FullyQualifiedName; import org.openmetadata.service.util.JsonUtils; -import org.openmetadata.service.util.NotificationHandler; +import org.openmetadata.service.util.WebsocketNotificationHandler; @Slf4j public class GlossaryTermRepository extends EntityRepository { @@ -655,7 +655,7 @@ private void createApprovalTask(GlossaryTerm entity, List paren feedRepository.create(thread); // Send WebSocket Notification - NotificationHandler.handleTaskNotification(thread); + WebsocketNotificationHandler.handleTaskNotification(thread); } private void closeApprovalTask(GlossaryTerm entity, String comment) { diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/LineageRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/LineageRepository.java index dbf68dd939ba..09defe6a2725 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/LineageRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/LineageRepository.java @@ -13,6 +13,12 @@ package org.openmetadata.service.jdbi3; +import static org.openmetadata.service.Entity.CONTAINER; +import static org.openmetadata.service.Entity.DASHBOARD; +import static org.openmetadata.service.Entity.DASHBOARD_DATA_MODEL; +import static org.openmetadata.service.Entity.MLMODEL; +import static org.openmetadata.service.Entity.TABLE; +import static org.openmetadata.service.Entity.TOPIC; import static org.openmetadata.service.search.SearchClient.GLOBAL_SEARCH_ALIAS; import static org.openmetadata.service.search.SearchClient.REMOVE_LINEAGE_SCRIPT; @@ -21,14 +27,17 @@ import java.util.List; import java.util.Map; import java.util.UUID; -import java.util.stream.Collectors; import org.apache.commons.lang3.tuple.ImmutablePair; import org.apache.commons.lang3.tuple.Pair; import org.jdbi.v3.sqlobject.transaction.Transaction; import org.openmetadata.common.utils.CommonUtil; -import org.openmetadata.schema.ColumnsEntityInterface; import org.openmetadata.schema.api.lineage.AddLineage; +import org.openmetadata.schema.entity.data.Container; +import org.openmetadata.schema.entity.data.Dashboard; +import org.openmetadata.schema.entity.data.DashboardDataModel; +import org.openmetadata.schema.entity.data.MlModel; import org.openmetadata.schema.entity.data.Table; +import org.openmetadata.schema.entity.data.Topic; import org.openmetadata.schema.type.ColumnLineage; import org.openmetadata.schema.type.Edge; import org.openmetadata.schema.type.EntityLineage; @@ -37,17 +46,17 @@ import org.openmetadata.schema.type.LineageDetails; import org.openmetadata.schema.type.Relationship; import org.openmetadata.service.Entity; +import org.openmetadata.service.exception.CatalogExceptionMessage; import org.openmetadata.service.jdbi3.CollectionDAO.EntityRelationshipRecord; import org.openmetadata.service.search.SearchClient; import org.openmetadata.service.search.models.IndexMapping; -import org.openmetadata.service.util.FullyQualifiedName; import org.openmetadata.service.util.JsonUtils; @Repository public class LineageRepository { private final CollectionDAO dao; - public SearchClient searchClient = Entity.getSearchRepository().getSearchClient(); + private static final SearchClient searchClient = Entity.getSearchRepository().getSearchClient(); public LineageRepository() { this.dao = Entity.getCollectionDAO(); @@ -173,41 +182,86 @@ private String validateLineageDetails( if (details == null) { return null; } - List columnsLineage = details.getColumnsLineage(); if (columnsLineage != null && !columnsLineage.isEmpty()) { - if (areValidEntities(from, to)) { - throw new IllegalArgumentException( - "Column level lineage is only allowed between two tables or from table to dashboard."); - } - Table fromTable = dao.tableDAO().findEntityById(from.getId()); - ColumnsEntityInterface toTable = getToEntity(to); for (ColumnLineage columnLineage : columnsLineage) { for (String fromColumn : columnLineage.getFromColumns()) { - // From column belongs to the fromNode - if (fromColumn.startsWith(fromTable.getFullyQualifiedName())) { - ColumnUtil.validateColumnFQN(fromTable.getColumns(), fromColumn); - } else { - Table otherTable = - dao.tableDAO().findEntityByName(FullyQualifiedName.getTableFQN(fromColumn)); - ColumnUtil.validateColumnFQN(otherTable.getColumns(), fromColumn); - } + validateChildren(fromColumn, from); } - ColumnUtil.validateColumnFQN(toTable.getColumns(), columnLineage.getToColumn()); + validateChildren(columnLineage.getToColumn(), to); } } return JsonUtils.pojoToJson(details); } - private ColumnsEntityInterface getToEntity(EntityReference from) { - return from.getType().equals(Entity.TABLE) - ? dao.tableDAO().findEntityById(from.getId()) - : dao.dashboardDataModelDAO().findEntityById(from.getId()); + private void validateChildren(String columnFQN, EntityReference entityReference) { + switch (entityReference.getType()) { + case TABLE -> { + Table table = + Entity.getEntity(TABLE, entityReference.getId(), "columns", Include.NON_DELETED); + ColumnUtil.validateColumnFQN(table.getColumns(), columnFQN); + } + case TOPIC -> { + Topic topic = + Entity.getEntity(TOPIC, entityReference.getId(), "messageSchema", Include.NON_DELETED); + ColumnUtil.validateFieldFQN(topic.getMessageSchema().getSchemaFields(), columnFQN); + } + case CONTAINER -> { + Container container = + Entity.getEntity(CONTAINER, entityReference.getId(), "dataModel", Include.NON_DELETED); + ColumnUtil.validateColumnFQN(container.getDataModel().getColumns(), columnFQN); + } + case DASHBOARD_DATA_MODEL -> { + DashboardDataModel dashboardDataModel = + Entity.getEntity( + DASHBOARD_DATA_MODEL, entityReference.getId(), "columns", Include.NON_DELETED); + ColumnUtil.validateColumnFQN(dashboardDataModel.getColumns(), columnFQN); + } + case DASHBOARD -> { + Dashboard dashboard = + Entity.getEntity(DASHBOARD, entityReference.getId(), "charts", Include.NON_DELETED); + dashboard.getCharts().stream() + .filter(c -> c.getFullyQualifiedName().equals(columnFQN)) + .findAny() + .orElseThrow( + () -> + new IllegalArgumentException( + CatalogExceptionMessage.invalidFieldName("chart", columnFQN))); + } + case MLMODEL -> { + MlModel mlModel = + Entity.getEntity(MLMODEL, entityReference.getId(), "", Include.NON_DELETED); + mlModel.getMlFeatures().stream() + .filter(f -> f.getFullyQualifiedName().equals(columnFQN)) + .findAny() + .orElseThrow( + () -> + new IllegalArgumentException( + CatalogExceptionMessage.invalidFieldName("feature", columnFQN))); + } + default -> throw new IllegalArgumentException( + String.format("Unsupported Entity Type %s for lineage", entityReference.getType())); + } } - private boolean areValidEntities(EntityReference from, EntityReference to) { - return !from.getType().equals(Entity.TABLE) - || !(to.getType().equals(Entity.TABLE) || to.getType().equals(Entity.DASHBOARD_DATA_MODEL)); + @Transaction + public boolean deleteLineageByFQN( + String fromEntity, String fromFQN, String toEntity, String toFQN) { + EntityReference from = + Entity.getEntityReferenceByName(fromEntity, fromFQN, Include.NON_DELETED); + EntityReference to = Entity.getEntityReferenceByName(toEntity, toFQN, Include.NON_DELETED); + // Finally, delete lineage relationship + boolean result = + dao.relationshipDAO() + .delete( + from.getId(), + from.getType(), + to.getId(), + to.getType(), + Relationship.UPSTREAM.ordinal()) + > 0; + deleteLineageFromSearch(from, to); + return result; } @Transaction @@ -260,7 +314,7 @@ private EntityLineage getLineage( getDownstreamLineage(primary.getId(), primary.getType(), lineage, downstreamDepth); // Remove duplicate nodes - lineage.withNodes(lineage.getNodes().stream().distinct().collect(Collectors.toList())); + lineage.withNodes(lineage.getNodes().stream().distinct().toList()); return lineage; } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/ListFilter.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/ListFilter.java index ab71d19a0565..11c81f97c3bc 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/ListFilter.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/ListFilter.java @@ -49,24 +49,26 @@ public String getCondition() { } public String getCondition(String tableName) { - String condition = getIncludeCondition(tableName); - condition = addCondition(condition, getDatabaseCondition(tableName)); - condition = addCondition(condition, getDatabaseSchemaCondition(tableName)); - condition = addCondition(condition, getServiceCondition(tableName)); - condition = addCondition(condition, getPipelineTypeCondition(tableName)); - condition = addCondition(condition, getParentCondition(tableName)); - condition = addCondition(condition, getDisabledCondition()); - condition = addCondition(condition, getCategoryCondition(tableName)); - condition = addCondition(condition, getWebhookCondition(tableName)); - condition = addCondition(condition, getWebhookTypeCondition(tableName)); - condition = addCondition(condition, getTestCaseCondition()); - condition = addCondition(condition, getTestSuiteTypeCondition(tableName)); - condition = addCondition(condition, getTestSuiteFQNCondition()); - condition = addCondition(condition, getDomainCondition()); - condition = addCondition(condition, getEntityFQNHashCondition()); - condition = addCondition(condition, getTestCaseResolutionStatusType()); - condition = addCondition(condition, getAssignee()); - condition = addCondition(condition, getEventSubscriptionAlertType()); + ArrayList conditions = new ArrayList<>(); + conditions.add(getIncludeCondition(tableName)); + conditions.add(getDatabaseCondition(tableName)); + conditions.add(getDatabaseSchemaCondition(tableName)); + conditions.add(getServiceCondition(tableName)); + conditions.add(getPipelineTypeCondition(tableName)); + conditions.add(getParentCondition(tableName)); + conditions.add(getDisabledCondition()); + conditions.add(getCategoryCondition(tableName)); + conditions.add(getWebhookCondition(tableName)); + conditions.add(getWebhookTypeCondition(tableName)); + conditions.add(getTestCaseCondition()); + conditions.add(getTestSuiteTypeCondition(tableName)); + conditions.add(getTestSuiteFQNCondition()); + conditions.add(getDomainCondition()); + conditions.add(getEntityFQNHashCondition()); + conditions.add(getTestCaseResolutionStatusType()); + conditions.add(getAssignee()); + conditions.add(getEventSubscriptionAlertType()); + String condition = addCondition(conditions); return condition.isEmpty() ? "WHERE TRUE" : "WHERE " + condition; } @@ -199,27 +201,44 @@ public String getPipelineTypeCondition(String tableName) { } private String getTestCaseCondition() { - String condition1 = ""; + ArrayList conditions = new ArrayList<>(); + String entityFQN = getQueryParam("entityFQN"); boolean includeAllTests = Boolean.parseBoolean(getQueryParam("includeAllTests")); + String status = getQueryParam("testCaseStatus"); + String testSuiteId = getQueryParam("testSuiteId"); + String type = getQueryParam("testCaseType"); + if (entityFQN != null) { - condition1 = + conditions.add( includeAllTests ? String.format( - "entityFQN LIKE '%s%s%%' OR entityFQN = '%s'", + "(entityFQN LIKE '%s%s%%' OR entityFQN = '%s')", escape(entityFQN), Entity.SEPARATOR, escapeApostrophe(entityFQN)) - : String.format("entityFQN = '%s'", escapeApostrophe(entityFQN)); + : String.format("entityFQN = '%s'", escapeApostrophe(entityFQN))); } - String condition2 = ""; - String testSuiteId = getQueryParam("testSuiteId"); if (testSuiteId != null) { - condition2 = + conditions.add( String.format( "id IN (SELECT toId FROM entity_relationship WHERE fromId='%s' AND toEntity='%s' AND relation=%d AND fromEntity='%s')", - testSuiteId, Entity.TEST_CASE, Relationship.CONTAINS.ordinal(), Entity.TEST_SUITE); + testSuiteId, Entity.TEST_CASE, Relationship.CONTAINS.ordinal(), Entity.TEST_SUITE)); } - return addCondition(condition1, condition2); + + if (status != null) { + conditions.add(String.format("status = '%s'", status)); + } + + if (type != null) { + conditions.add( + switch (type) { + case "table" -> "entityLink NOT LIKE '%::columns::%'"; + case "column" -> "entityLink LIKE '%::columns::%'"; + default -> ""; + }); + } + + return addCondition(conditions); } private String getTestSuiteTypeCondition(String tableName) { @@ -312,14 +331,19 @@ private String getStatusPrefixCondition(String tableName, String statusPrefix) { : String.format("%s.status LIKE '%s%s%%'", tableName, statusPrefix, ""); } - protected String addCondition(String condition1, String condition2) { - if (condition1.isEmpty()) { - return condition2; - } - if (condition2.isEmpty()) { - return condition1; + protected String addCondition(List conditions) { + StringBuffer condition = new StringBuffer(); + + for (String c : conditions) { + if (!c.isEmpty()) { + if (!condition.isEmpty()) { + // Add `AND` between conditions + condition.append(" AND "); + } + condition.append(c); + } } - return condition1 + " AND " + condition2; + return condition.toString(); } public static String escapeApostrophe(String name) { diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/MigrationDAO.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/MigrationDAO.java index 3fdf445973b2..e47b7417ebbd 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/MigrationDAO.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/MigrationDAO.java @@ -129,6 +129,9 @@ void upsertServerMigrationSQL( @RegisterRowMapper(FromServerChangeLogMapper.class) List listMetricsFromDBMigrations(); + @SqlQuery("SELECT version FROM SERVER_CHANGE_LOG") + List getMigrationVersions(); + @Getter @Setter class ServerMigrationSQLTable { diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/PolicyRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/PolicyRepository.java index bd19c2377085..d7f8ecdac079 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/PolicyRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/PolicyRepository.java @@ -132,7 +132,9 @@ public void validateRules(Policy policy) { public static List filterRedundantResources(List resources) { // If ALL_RESOURCES are in the resource list, remove redundant resources specifically mentioned boolean containsAllResources = resources.stream().anyMatch(ALL_RESOURCES::equalsIgnoreCase); - return containsAllResources ? new ArrayList<>(List.of(ALL_RESOURCES)) : resources; + return containsAllResources + ? new ArrayList<>(List.of(ALL_RESOURCES)) + : new ArrayList<>(resources); } public static List filterRedundantOperations( @@ -142,9 +144,7 @@ public static List filterRedundantOperations( boolean containsViewAll = operations.stream().anyMatch(o -> o.equals(VIEW_ALL)); if (containsViewAll) { operations = - operations.stream() - .filter(o -> o.equals(VIEW_ALL) || !isViewOperation(o)) - .collect(Collectors.toList()); + operations.stream().filter(o -> o.equals(VIEW_ALL) || !isViewOperation(o)).toList(); } // If EDIT_ALL is in the operation list, remove all the other specific edit operations that are @@ -152,11 +152,9 @@ public static List filterRedundantOperations( boolean containsEditAll = operations.stream().anyMatch(o -> o.equals(EDIT_ALL)); if (containsEditAll) { operations = - operations.stream() - .filter(o -> o.equals(EDIT_ALL) || !isEditOperation(o)) - .collect(Collectors.toList()); + operations.stream().filter(o -> o.equals(EDIT_ALL) || !isEditOperation(o)).toList(); } - return operations; + return new ArrayList<>(operations); } /** Handles entity updated from PUT and POST operation. */ diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/SuggestionFilter.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/SuggestionFilter.java index 3bfd327433c2..99ef71072874 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/SuggestionFilter.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/SuggestionFilter.java @@ -24,7 +24,7 @@ public String getCondition(boolean includePagination) { StringBuilder condition = new StringBuilder(); condition.append("WHERE TRUE "); if (suggestionType != null) { - condition.append(String.format(" AND type = '%s' ", suggestionType.value())); + condition.append(String.format(" AND suggestionType = '%s' ", suggestionType.value())); } if (suggestionStatus != null) { condition.append(String.format(" AND status = '%s' ", suggestionStatus.value())); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/SuggestionRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/SuggestionRepository.java index 7b442f655416..e9061c04964b 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/SuggestionRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/SuggestionRepository.java @@ -16,7 +16,6 @@ import java.util.List; import java.util.UUID; import javax.json.JsonPatch; -import javax.ws.rs.WebApplicationException; import javax.ws.rs.core.Response; import javax.ws.rs.core.SecurityContext; import javax.ws.rs.core.UriInfo; @@ -28,9 +27,11 @@ import org.openmetadata.schema.entity.teams.Team; import org.openmetadata.schema.entity.teams.User; import org.openmetadata.schema.type.EntityReference; +import org.openmetadata.schema.type.MetadataOperation; import org.openmetadata.schema.type.SuggestionStatus; import org.openmetadata.schema.type.SuggestionType; import org.openmetadata.schema.type.TagLabel; +import org.openmetadata.sdk.exception.SuggestionException; import org.openmetadata.service.Entity; import org.openmetadata.service.ResourceRegistry; import org.openmetadata.service.exception.CatalogExceptionMessage; @@ -154,32 +155,31 @@ public void deleteSuggestionInternalForAnEntity(EntityInterface entity) { @Getter public static class SuggestionWorkflow { - protected final Suggestion suggestion; - protected final MessageParser.EntityLink entityLink; + // The workflow is applied to a specific entity at a time + protected final EntityInterface entity; - SuggestionWorkflow(Suggestion suggestion) { - this.suggestion = suggestion; - this.entityLink = MessageParser.EntityLink.parse(suggestion.getEntityLink()); + SuggestionWorkflow(EntityInterface entity) { + this.entity = entity; } - public EntityInterface acceptSuggestions( - EntityRepository repository, EntityInterface entityInterface) { + public EntityInterface acceptSuggestion(Suggestion suggestion, EntityInterface entity) { + MessageParser.EntityLink entityLink = + MessageParser.EntityLink.parse(suggestion.getEntityLink()); if (entityLink.getFieldName() != null) { - entityInterface = - repository.applySuggestion( - entityInterface, entityLink.getFullyQualifiedFieldValue(), suggestion); - return entityInterface; + EntityRepository repository = Entity.getEntityRepository(entityLink.getEntityType()); + return repository.applySuggestion( + entity, entityLink.getFullyQualifiedFieldValue(), suggestion); } else { if (suggestion.getType().equals(SuggestionType.SuggestTagLabel)) { - List tags = new ArrayList<>(entityInterface.getTags()); + List tags = new ArrayList<>(entity.getTags()); tags.addAll(suggestion.getTagLabels()); - entityInterface.setTags(tags); - return entityInterface; + entity.setTags(tags); + return entity; } else if (suggestion.getType().equals(SuggestionType.SuggestDescription)) { - entityInterface.setDescription(suggestion.getDescription()); - return entityInterface; + entity.setDescription(suggestion.getDescription()); + return entity; } else { - throw new WebApplicationException("Invalid suggestion Type"); + throw new SuggestionException("Invalid suggestion Type"); } } } @@ -190,26 +190,43 @@ public RestUtil.PutResponse acceptSuggestion( Suggestion suggestion, SecurityContext securityContext, Authorizer authorizer) { - suggestion.setStatus(SuggestionStatus.Accepted); acceptSuggestion(suggestion, securityContext, authorizer); Suggestion updatedHref = SuggestionsResource.addHref(uriInfo, suggestion); return new RestUtil.PutResponse<>(Response.Status.OK, updatedHref, SUGGESTION_ACCEPTED); } + public RestUtil.PutResponse> acceptSuggestionList( + UriInfo uriInfo, + List suggestions, + SuggestionType suggestionType, + SecurityContext securityContext, + Authorizer authorizer) { + acceptSuggestionList(suggestions, suggestionType, securityContext, authorizer); + List updatedHref = + suggestions.stream() + .map(suggestion -> SuggestionsResource.addHref(uriInfo, suggestion)) + .toList(); + return new RestUtil.PutResponse<>(Response.Status.OK, updatedHref, SUGGESTION_ACCEPTED); + } + protected void acceptSuggestion( Suggestion suggestion, SecurityContext securityContext, Authorizer authorizer) { String user = securityContext.getUserPrincipal().getName(); MessageParser.EntityLink entityLink = MessageParser.EntityLink.parse(suggestion.getEntityLink()); + EntityRepository repository = Entity.getEntityRepository(entityLink.getEntityType()); EntityInterface entity = - Entity.getEntity( - entityLink, suggestion.getType() == SuggestionType.SuggestTagLabel ? "tags" : "", ALL); + Entity.getEntity(entityLink, repository.getSuggestionFields(suggestion), ALL); + // Prepare the original JSON before updating the Entity, otherwise we get an empty patch String origJson = JsonUtils.pojoToJson(entity); - SuggestionWorkflow suggestionWorkflow = getSuggestionWorkflow(suggestion); - EntityRepository repository = Entity.getEntityRepository(entityLink.getEntityType()); - EntityInterface updatedEntity = suggestionWorkflow.acceptSuggestions(repository, entity); + SuggestionWorkflow suggestionWorkflow = repository.getSuggestionWorkflow(entity); + + EntityInterface updatedEntity = suggestionWorkflow.acceptSuggestion(suggestion, entity); String updatedEntityJson = JsonUtils.pojoToJson(updatedEntity); + + // Patch the entity with the updated suggestions JsonPatch patch = JsonUtils.getJsonPatch(origJson, updatedEntityJson); + OperationContext operationContext = new OperationContext(entityLink.getEntityType(), patch); authorizer.authorize( securityContext, @@ -220,6 +237,60 @@ protected void acceptSuggestion( update(suggestion, user); } + @Transaction + protected void acceptSuggestionList( + List suggestions, + SuggestionType suggestionType, + SecurityContext securityContext, + Authorizer authorizer) { + String user = securityContext.getUserPrincipal().getName(); + + // Entity being updated + EntityInterface entity = null; + EntityRepository repository = null; + String origJson = null; + SuggestionWorkflow suggestionWorkflow = null; + + for (Suggestion suggestion : suggestions) { + MessageParser.EntityLink entityLink = + MessageParser.EntityLink.parse(suggestion.getEntityLink()); + + // Validate all suggestions indeed talk about the same entity + if (entity == null) { + // Initialize the Entity and the Repository + entity = + Entity.getEntity( + entityLink, + suggestionType == SuggestionType.SuggestTagLabel ? "tags" : "", + NON_DELETED); + repository = Entity.getEntityRepository(entityLink.getEntityType()); + origJson = JsonUtils.pojoToJson(entity); + suggestionWorkflow = repository.getSuggestionWorkflow(entity); + } else if (!entity.getFullyQualifiedName().equals(entityLink.getEntityFQN())) { + throw new SuggestionException("All suggestions must be for the same entity"); + } + // update entity with the suggestion + entity = suggestionWorkflow.acceptSuggestion(suggestion, entity); + } + + // Patch the entity with the updated suggestions + String updatedEntityJson = JsonUtils.pojoToJson(entity); + JsonPatch patch = JsonUtils.getJsonPatch(origJson, updatedEntityJson); + + OperationContext operationContext = new OperationContext(repository.getEntityType(), patch); + authorizer.authorize( + securityContext, + operationContext, + new ResourceContext<>(repository.getEntityType(), entity.getId(), null)); + repository.patch(null, entity.getId(), user, patch); + + // Only mark the suggestions as accepted after the entity has been successfully updated + for (Suggestion suggestion : suggestions) { + suggestion.setStatus(SuggestionStatus.Accepted); + update(suggestion, user); + } + } + public RestUtil.PutResponse rejectSuggestion( UriInfo uriInfo, Suggestion suggestion, String user) { suggestion.setStatus(SuggestionStatus.Rejected); @@ -228,6 +299,17 @@ public RestUtil.PutResponse rejectSuggestion( return new RestUtil.PutResponse<>(Response.Status.OK, updatedHref, SUGGESTION_REJECTED); } + @Transaction + public RestUtil.PutResponse> rejectSuggestionList( + UriInfo uriInfo, List suggestions, String user) { + for (Suggestion suggestion : suggestions) { + suggestion.setStatus(SuggestionStatus.Rejected); + update(suggestion, user); + SuggestionsResource.addHref(uriInfo, suggestion); + } + return new RestUtil.PutResponse<>(Response.Status.OK, suggestions, SUGGESTION_REJECTED); + } + public void checkPermissionsForUpdateSuggestion( Suggestion suggestion, SecurityContext securityContext) { String userName = securityContext.getUserPrincipal().getName(); @@ -272,11 +354,23 @@ public void checkPermissionsForAcceptOrRejectSuggestion( } } - public SuggestionWorkflow getSuggestionWorkflow(Suggestion suggestion) { + public void checkPermissionsForEditEntity( + Suggestion suggestion, + SuggestionType suggestionType, + SecurityContext securityContext, + Authorizer authorizer) { MessageParser.EntityLink entityLink = MessageParser.EntityLink.parse(suggestion.getEntityLink()); - EntityRepository repository = Entity.getEntityRepository(entityLink.getEntityType()); - return repository.getSuggestionWorkflow(suggestion); + EntityInterface entity = Entity.getEntity(entityLink, "", NON_DELETED); + // Check that the user has the right permissions to update the entity + authorizer.authorize( + securityContext, + new OperationContext( + entityLink.getEntityType(), + suggestionType == SuggestionType.SuggestTagLabel + ? MetadataOperation.EDIT_TAGS + : MetadataOperation.EDIT_DESCRIPTION), + new ResourceContext<>(entityLink.getEntityType(), entity.getId(), null)); } public int listCount(SuggestionFilter filter) { @@ -333,4 +427,9 @@ private List getSuggestionList(List jsons) { } return suggestions; } + + public final List listAll(SuggestionFilter filter) { + ResultList suggestionList = listAfter(filter, Integer.MAX_VALUE - 1, ""); + return suggestionList.getData(); + } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/SystemRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/SystemRepository.java index 94c92c91908d..28f308d8dcec 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/SystemRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/SystemRepository.java @@ -13,16 +13,26 @@ import org.jdbi.v3.sqlobject.transaction.Transaction; import org.openmetadata.schema.api.configuration.SlackAppConfiguration; import org.openmetadata.schema.email.SmtpSettings; +import org.openmetadata.schema.entity.services.ingestionPipelines.PipelineServiceClientResponse; +import org.openmetadata.schema.services.connections.metadata.OpenMetadataConnection; import org.openmetadata.schema.settings.Settings; import org.openmetadata.schema.settings.SettingsType; +import org.openmetadata.schema.system.StepValidation; +import org.openmetadata.schema.system.ValidationResponse; import org.openmetadata.schema.util.EntitiesCount; import org.openmetadata.schema.util.ServicesCount; +import org.openmetadata.sdk.PipelineServiceClient; import org.openmetadata.service.Entity; +import org.openmetadata.service.OpenMetadataApplicationConfig; import org.openmetadata.service.exception.CustomExceptionMessage; import org.openmetadata.service.fernet.Fernet; import org.openmetadata.service.jdbi3.CollectionDAO.SystemDAO; +import org.openmetadata.service.migration.MigrationValidationClient; import org.openmetadata.service.resources.settings.SettingsCache; +import org.openmetadata.service.search.SearchRepository; +import org.openmetadata.service.security.JwtFilter; import org.openmetadata.service.util.JsonUtils; +import org.openmetadata.service.util.OpenMetadataConnectionBuilder; import org.openmetadata.service.util.RestUtil; import org.openmetadata.service.util.ResultList; @@ -32,10 +42,28 @@ public class SystemRepository { private static final String FAILED_TO_UPDATE_SETTINGS = "Failed to Update Settings"; public static final String INTERNAL_SERVER_ERROR_WITH_REASON = "Internal Server Error. Reason :"; private final SystemDAO dao; + private final MigrationValidationClient migrationValidationClient; + + private enum ValidationStepDescription { + DATABASE("Validate that we can properly run a query against the configured database."), + SEARCH("Validate that the search client is available."), + PIPELINE_SERVICE_CLIENT("Validate that the pipeline service client is available."), + JWT_TOKEN("Validate that the ingestion-bot JWT token can be properly decoded."), + MIGRATION("Validate that all the necessary migrations have been properly executed."); + + public final String key; + + ValidationStepDescription(String param) { + this.key = param; + } + } + + private static final String INDEX_NAME = "table_search_index"; public SystemRepository() { this.dao = Entity.getCollectionDAO().systemDAO(); Entity.setSystemRepository(this); + migrationValidationClient = MigrationValidationClient.getInstance(); } public EntitiesCount getAllEntitiesCount(ListFilter filter) { @@ -210,4 +238,98 @@ public static SlackAppConfiguration decryptSlackAppSetting(String encryptedSetti } return JsonUtils.readValue(encryptedSetting, SlackAppConfiguration.class); } + + public ValidationResponse validateSystem( + OpenMetadataApplicationConfig applicationConfig, + PipelineServiceClient pipelineServiceClient, + JwtFilter jwtFilter) { + ValidationResponse validation = new ValidationResponse(); + + validation.setDatabase(getDatabaseValidation()); + validation.setSearchInstance(getSearchValidation()); + validation.setPipelineServiceClient(getPipelineServiceClientValidation(pipelineServiceClient)); + validation.setJwks(getJWKsValidation(applicationConfig, jwtFilter)); + validation.setMigrations(getMigrationValidation(migrationValidationClient)); + + return validation; + } + + private StepValidation getDatabaseValidation() { + try { + dao.testConnection(); + return new StepValidation() + .withDescription(ValidationStepDescription.DATABASE.key) + .withPassed(Boolean.TRUE); + } catch (Exception exc) { + return new StepValidation() + .withDescription(ValidationStepDescription.DATABASE.key) + .withPassed(Boolean.FALSE) + .withMessage(exc.getMessage()); + } + } + + private StepValidation getSearchValidation() { + SearchRepository searchRepository = Entity.getSearchRepository(); + if (Boolean.TRUE.equals(searchRepository.getSearchClient().isClientAvailable()) + && searchRepository.getSearchClient().indexExists(INDEX_NAME)) { + return new StepValidation() + .withDescription(ValidationStepDescription.SEARCH.key) + .withPassed(Boolean.TRUE); + } else { + return new StepValidation() + .withDescription(ValidationStepDescription.SEARCH.key) + .withPassed(Boolean.FALSE) + .withMessage("Search instance is not reachable or available"); + } + } + + private StepValidation getPipelineServiceClientValidation( + PipelineServiceClient pipelineServiceClient) { + PipelineServiceClientResponse pipelineResponse = pipelineServiceClient.getServiceStatus(); + if (pipelineResponse.getCode() == 200) { + return new StepValidation() + .withDescription(ValidationStepDescription.PIPELINE_SERVICE_CLIENT.key) + .withPassed(Boolean.TRUE); + } else { + return new StepValidation() + .withDescription(ValidationStepDescription.PIPELINE_SERVICE_CLIENT.key) + .withPassed(Boolean.FALSE) + .withMessage(pipelineResponse.getReason()); + } + } + + private StepValidation getJWKsValidation( + OpenMetadataApplicationConfig applicationConfig, JwtFilter jwtFilter) { + OpenMetadataConnection openMetadataServerConnection = + new OpenMetadataConnectionBuilder(applicationConfig).build(); + try { + jwtFilter.validateAndReturnDecodedJwtToken( + openMetadataServerConnection.getSecurityConfig().getJwtToken()); + return new StepValidation() + .withDescription(ValidationStepDescription.JWT_TOKEN.key) + .withPassed(Boolean.TRUE); + } catch (Exception e) { + return new StepValidation() + .withDescription(ValidationStepDescription.JWT_TOKEN.key) + .withPassed(Boolean.FALSE) + .withMessage(e.getMessage()); + } + } + + private StepValidation getMigrationValidation( + MigrationValidationClient migrationValidationClient) { + List currentVersions = migrationValidationClient.getCurrentVersions(); + if (currentVersions.equals(migrationValidationClient.getExpectedMigrationList())) { + return new StepValidation() + .withDescription(ValidationStepDescription.MIGRATION.key) + .withPassed(Boolean.TRUE); + } + return new StepValidation() + .withDescription(ValidationStepDescription.MIGRATION.key) + .withPassed(Boolean.FALSE) + .withMessage( + String.format( + "Found the versions [%s], but expected [%s]", + currentVersions, migrationValidationClient.getExpectedMigrationList())); + } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TableRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TableRepository.java index 85c64e05b8c3..3c2a457bb100 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TableRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TableRepository.java @@ -45,7 +45,6 @@ import java.util.function.Predicate; import java.util.stream.Collectors; import java.util.stream.Stream; -import javax.ws.rs.WebApplicationException; import lombok.extern.slf4j.Slf4j; import org.apache.commons.csv.CSVPrinter; import org.apache.commons.csv.CSVRecord; @@ -85,6 +84,7 @@ import org.openmetadata.schema.type.csv.CsvFile; import org.openmetadata.schema.type.csv.CsvHeader; import org.openmetadata.schema.type.csv.CsvImportResult; +import org.openmetadata.sdk.exception.SuggestionException; import org.openmetadata.service.Entity; import org.openmetadata.service.exception.CatalogExceptionMessage; import org.openmetadata.service.exception.EntityNotFoundException; @@ -739,9 +739,14 @@ public TaskWorkflow getTaskWorkflow(ThreadContext threadContext) { return super.getTaskWorkflow(threadContext); } + @Override + public String getSuggestionFields(Suggestion suggestion) { + return suggestion.getType() == SuggestionType.SuggestTagLabel ? "columns,tags" : ""; + } + @Override public Table applySuggestion(EntityInterface entity, String columnFQN, Suggestion suggestion) { - Table table = Entity.getEntity(TABLE, entity.getId(), "columns,tags", ALL); + Table table = (Table) entity; for (Column col : table.getColumns()) { if (col.getFullyQualifiedName().equals(columnFQN)) { if (suggestion.getType().equals(SuggestionType.SuggestTagLabel)) { @@ -751,7 +756,7 @@ public Table applySuggestion(EntityInterface entity, String columnFQN, Suggestio } else if (suggestion.getType().equals(SuggestionType.SuggestDescription)) { col.setDescription(suggestion.getDescription()); } else { - throw new WebApplicationException("Invalid suggestion Type"); + throw new SuggestionException("Invalid suggestion Type"); } } } @@ -1231,7 +1236,7 @@ protected void addRecord(CsvFile csvFile, Table entity) { addRecord(csvFile, recordList, table.getColumns().get(0), false); for (int i = 1; i < entity.getColumns().size(); i++) { - addRecord(csvFile, new ArrayList<>(), table.getColumns().get(1), true); + addRecord(csvFile, new ArrayList<>(), table.getColumns().get(i), true); } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TestCaseRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TestCaseRepository.java index ae0d2fdcaa44..11cb5f60741d 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TestCaseRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TestCaseRepository.java @@ -409,13 +409,15 @@ private void setTestSuiteSummary( updateResultSummaries(testCase, isDeleted, resultSummaries, resultSummary); // Update test case result summary attribute for the test suite + TestSuiteRepository testSuiteRepository = + (TestSuiteRepository) Entity.getEntityRepository(Entity.TEST_SUITE); + TestSuite original = + TestSuiteRepository.copyTestSuite( + testSuite); // we'll need the original state to update the test suite testSuite.setTestCaseResultSummary(resultSummaries); - daoCollection - .testSuiteDAO() - .update( - testSuite.getId(), - testSuite.getFullyQualifiedName(), - JsonUtils.pojoToJson(testSuite)); + EntityRepository.EntityUpdater testSuiteUpdater = + testSuiteRepository.getUpdater(original, testSuite, Operation.PUT); + testSuiteUpdater.update(); } } @@ -652,11 +654,16 @@ private void removeTestCaseFromTestSuiteResultSummary(UUID testSuiteId, String t testSuite.setSummary(null); // we don't want to store the summary in the database List resultSummaries = testSuite.getTestCaseResultSummary(); resultSummaries.removeIf(summary -> summary.getTestCaseName().equals(testCaseFqn)); + + TestSuiteRepository testSuiteRepository = + (TestSuiteRepository) Entity.getEntityRepository(Entity.TEST_SUITE); + TestSuite original = + TestSuiteRepository.copyTestSuite( + testSuite); // we'll need the original state to update the test suite testSuite.setTestCaseResultSummary(resultSummaries); - daoCollection - .testSuiteDAO() - .update( - testSuite.getId(), testSuite.getFullyQualifiedName(), JsonUtils.pojoToJson(testSuite)); + EntityRepository.EntityUpdater testSuiteUpdater = + testSuiteRepository.getUpdater(original, testSuite, Operation.PUT); + testSuiteUpdater.update(); } @Override diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TestSuiteRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TestSuiteRepository.java index bbb507705427..c85e618a4056 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TestSuiteRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TestSuiteRepository.java @@ -9,10 +9,14 @@ import static org.openmetadata.service.Entity.TEST_SUITE; import static org.openmetadata.service.util.FullyQualifiedName.quoteName; +import java.io.IOException; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.UUID; +import javax.json.JsonArray; +import javax.json.JsonObject; +import javax.json.JsonValue; import javax.ws.rs.core.SecurityContext; import lombok.extern.slf4j.Slf4j; import org.jdbi.v3.sqlobject.transaction.Transaction; @@ -23,7 +27,6 @@ import org.openmetadata.schema.tests.type.TestSummary; import org.openmetadata.schema.type.EntityReference; import org.openmetadata.schema.type.EventType; -import org.openmetadata.schema.type.Include; import org.openmetadata.schema.type.Relationship; import org.openmetadata.service.Entity; import org.openmetadata.service.resources.dqtests.TestSuiteResource; @@ -55,7 +58,7 @@ public void setFields(TestSuite entity, EntityUtil.Fields fields) { fields.contains("pipelines") ? getIngestionPipelines(entity) : entity.getPipelines()); entity.setSummary( fields.contains("summary") ? getTestCasesExecutionSummary(entity) : entity.getSummary()); - entity.withTests(fields.contains("tests") ? getTestCases(entity) : entity.getTests()); + entity.withTests(fields.contains(UPDATE_FIELDS) ? getTestCases(entity) : entity.getTests()); } @Override @@ -71,7 +74,7 @@ public void setInheritedFields(TestSuite testSuite, EntityUtil.Fields fields) { public void clearFields(TestSuite entity, EntityUtil.Fields fields) { entity.setPipelines(fields.contains("pipelines") ? entity.getPipelines() : null); entity.setSummary(fields.contains("summary") ? entity.getSummary() : null); - entity.withTests(fields.contains("tests") ? entity.getTests() : null); + entity.withTests(fields.contains(UPDATE_FIELDS) ? entity.getTests() : null); } private TestSummary buildTestSummary(Map testCaseSummary) { @@ -117,31 +120,81 @@ private TestSummary getTestCasesExecutionSummary(TestSuite entity) { return buildTestSummary(testCaseSummary); } - private TestSummary getTestCasesExecutionSummary(List entities) { - if (entities.isEmpty()) return new TestSummary(); - Map testsSummary = new HashMap<>(); - for (TestSuite testSuite : entities) { - Map testSummary = getResultSummary(testSuite); - for (Map.Entry entry : testSummary.entrySet()) { - testsSummary.put( - entry.getKey(), testsSummary.getOrDefault(entry.getKey(), 0) + entry.getValue()); + private TestSummary getTestCasesExecutionSummary(JsonObject aggregation) { + // Initialize the test summary with 0 values + TestSummary testSummary = + new TestSummary().withAborted(0).withFailed(0).withSuccess(0).withQueued(0).withTotal(0); + JsonObject summary = aggregation.getJsonObject("nested#testCaseResultSummary"); + testSummary.setTotal(summary.getJsonNumber("doc_count").intValue()); + + JsonObject statusCount = summary.getJsonObject("sterms#status_counts"); + JsonArray buckets = statusCount.getJsonArray("buckets"); + + for (JsonValue bucket : buckets) { + String key = ((JsonObject) bucket).getString("key"); + Integer count = ((JsonObject) bucket).getJsonNumber("doc_count").intValue(); + switch (key) { + case "Success": + testSummary.setSuccess(count); + break; + case "Failed": + testSummary.setFailed(count); + break; + case "Aborted": + testSummary.setAborted(count); + break; + case "Queued": + testSummary.setQueued(count); + break; } - testSuite.getTestCaseResultSummary().size(); } - return buildTestSummary(testsSummary); + return testSummary; } - public TestSummary getTestSummary(UUID testSuiteId) { + public TestSummary getTestSummary(UUID testSuiteId) throws IOException { + String aggregationQuery = + """ + { + "aggregations": { + "test_case_results": { + "nested": { + "path": "testCaseResultSummary" + }, + "aggs": { + "status_counts": { + "terms": { + "field": "testCaseResultSummary.status" + } + } + } + } + } + } + """; + JsonObject aggregationJson = JsonUtils.readJson(aggregationQuery).asJsonObject(); TestSummary testSummary; if (testSuiteId == null) { - ListFilter filter = new ListFilter(); - filter.addQueryParam("testSuiteType", "executable"); - List testSuites = listAll(EntityUtil.Fields.EMPTY_FIELDS, filter); - testSummary = getTestCasesExecutionSummary(testSuites); + JsonObject testCaseResultSummary = + searchRepository.aggregate(null, TEST_SUITE, aggregationJson); + testSummary = getTestCasesExecutionSummary(testCaseResultSummary); } else { + String query = + """ + { + "query": { + "bool": { + "must": { + "term": {"id": "%s"} + } + } + } + } + """ + .formatted(testSuiteId); // don't want to get it from the cache as test results summary may be stale - TestSuite testSuite = Entity.getEntity(TEST_SUITE, testSuiteId, "", Include.ALL, false); - testSummary = getTestCasesExecutionSummary(testSuite); + JsonObject testCaseResultSummary = + searchRepository.aggregate(query, TEST_SUITE, aggregationJson); + testSummary = getTestCasesExecutionSummary(testCaseResultSummary); } return testSummary; } @@ -211,6 +264,26 @@ public RestUtil.DeleteResponse deleteLogicalTestSuite( return new RestUtil.DeleteResponse<>(updated, changeType); } + public static TestSuite copyTestSuite(TestSuite testSuite) { + return new TestSuite() + .withConnection(testSuite.getConnection()) + .withDescription(testSuite.getDescription()) + .withChangeDescription(testSuite.getChangeDescription()) + .withDeleted(testSuite.getDeleted()) + .withDisplayName(testSuite.getDisplayName()) + .withFullyQualifiedName(testSuite.getFullyQualifiedName()) + .withHref(testSuite.getHref()) + .withId(testSuite.getId()) + .withName(testSuite.getName()) + .withExecutable(testSuite.getExecutable()) + .withExecutableEntityReference(testSuite.getExecutableEntityReference()) + .withServiceType(testSuite.getServiceType()) + .withOwner(testSuite.getOwner()) + .withUpdatedBy(testSuite.getUpdatedBy()) + .withUpdatedAt(testSuite.getUpdatedAt()) + .withVersion(testSuite.getVersion()); + } + public class TestSuiteUpdater extends EntityUpdater { public TestSuiteUpdater(TestSuite original, TestSuite updated, Operation operation) { super(original, updated, operation); @@ -221,7 +294,13 @@ public TestSuiteUpdater(TestSuite original, TestSuite updated, Operation operati public void entitySpecificUpdate() { List origTests = listOrEmpty(original.getTests()); List updatedTests = listOrEmpty(updated.getTests()); - recordChange("tests", origTests, updatedTests); + List origTestCaseResultSummary = + listOrEmpty(original.getTestCaseResultSummary()); + List updatedTestCaseResultSummary = + listOrEmpty(updated.getTestCaseResultSummary()); + recordChange(UPDATE_FIELDS, origTests, updatedTests); + recordChange( + "testCaseResultSummary", origTestCaseResultSummary, updatedTestCaseResultSummary); } } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TypeRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TypeRepository.java index 68ad87df8f2b..bce5765a243b 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TypeRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TypeRepository.java @@ -23,6 +23,7 @@ import static org.openmetadata.service.util.EntityUtil.getCustomField; import java.util.ArrayList; +import java.util.HashSet; import java.util.List; import java.util.UUID; import javax.ws.rs.core.UriInfo; @@ -32,9 +33,11 @@ import org.openmetadata.schema.entity.Type; import org.openmetadata.schema.entity.type.Category; import org.openmetadata.schema.entity.type.CustomProperty; +import org.openmetadata.schema.type.CustomPropertyConfig; import org.openmetadata.schema.type.EntityReference; import org.openmetadata.schema.type.Include; import org.openmetadata.schema.type.Relationship; +import org.openmetadata.schema.type.customproperties.EnumConfig; import org.openmetadata.service.Entity; import org.openmetadata.service.TypeRegistry; import org.openmetadata.service.resources.types.TypeResource; @@ -117,6 +120,7 @@ public PutResponse addCustomProperty( property.setPropertyType( Entity.getEntityReferenceById( Entity.TYPE, property.getPropertyType().getId(), NON_DELETED)); + validateProperty(property); if (type.getCategory().equals(Category.Field)) { throw new IllegalArgumentException( "Only entity types can be extended and field types can't be extended"); @@ -161,6 +165,30 @@ private List getCustomProperties(Type type) { return customProperties; } + private void validateProperty(CustomProperty customProperty) { + switch (customProperty.getPropertyType().getName()) { + case "enum" -> { + CustomPropertyConfig config = customProperty.getCustomPropertyConfig(); + if (config != null) { + EnumConfig enumConfig = JsonUtils.convertValue(config.getConfig(), EnumConfig.class); + if (enumConfig == null + || (enumConfig.getValues() != null && enumConfig.getValues().isEmpty())) { + throw new IllegalArgumentException( + "Enum Custom Property Type must have EnumConfig populated with values."); + } else if (enumConfig.getValues() != null + && enumConfig.getValues().stream().distinct().count() + != enumConfig.getValues().size()) { + throw new IllegalArgumentException( + "Enum Custom Property values cannot have duplicates."); + } + } else { + throw new IllegalArgumentException("Enum Custom Property Type must have EnumConfig."); + } + } + case "int", "string" -> {} + } + } + /** Handles entity updated from PUT and POST operation. */ public class TypeUpdater extends EntityUpdater { public TypeUpdater(Type original, Type updated, Operation operation) { @@ -199,6 +227,7 @@ private void updateCustomProperties() { continue; } updateCustomPropertyDescription(updated, storedProperty, updateProperty); + updateCustomPropertyConfig(updated, storedProperty, updateProperty); } } @@ -270,5 +299,55 @@ private void updateCustomPropertyDescription( customPropertyJson); } } + + private void updateCustomPropertyConfig( + Type entity, CustomProperty origProperty, CustomProperty updatedProperty) { + String fieldName = getCustomField(origProperty, "customPropertyConfig"); + if (previous == null || !previous.getVersion().equals(updated.getVersion())) { + validatePropertyConfigUpdate(entity, origProperty, updatedProperty); + } + if (recordChange( + fieldName, + origProperty.getCustomPropertyConfig(), + updatedProperty.getCustomPropertyConfig())) { + String customPropertyFQN = + getCustomPropertyFQN(entity.getName(), updatedProperty.getName()); + EntityReference propertyType = + updatedProperty.getPropertyType(); // Don't store entity reference + String customPropertyJson = JsonUtils.pojoToJson(updatedProperty.withPropertyType(null)); + updatedProperty.withPropertyType(propertyType); // Restore entity reference + daoCollection + .fieldRelationshipDAO() + .upsert( + customPropertyFQN, + updatedProperty.getPropertyType().getName(), + customPropertyFQN, + updatedProperty.getPropertyType().getName(), + Entity.TYPE, + Entity.TYPE, + Relationship.HAS.ordinal(), + "customProperty", + customPropertyJson); + } + } + + private void validatePropertyConfigUpdate( + Type entity, CustomProperty origProperty, CustomProperty updatedProperty) { + if (origProperty.getPropertyType().getName().equals("enum")) { + EnumConfig origConfig = + JsonUtils.convertValue( + origProperty.getCustomPropertyConfig().getConfig(), EnumConfig.class); + EnumConfig updatedConfig = + JsonUtils.convertValue( + updatedProperty.getCustomPropertyConfig().getConfig(), EnumConfig.class); + HashSet updatedValues = new HashSet<>(updatedConfig.getValues()); + if (updatedValues.size() != updatedConfig.getValues().size()) { + throw new IllegalArgumentException("Enum Custom Property values cannot have duplicates."); + } else if (!updatedValues.containsAll(origConfig.getValues())) { + throw new IllegalArgumentException( + "Existing Enum Custom Property values cannot be removed."); + } + } + } } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/UserRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/UserRepository.java index 84f20bf36e8e..7389866ce275 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/UserRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/UserRepository.java @@ -63,6 +63,7 @@ import org.openmetadata.service.secrets.SecretsManager; import org.openmetadata.service.secrets.SecretsManagerFactory; import org.openmetadata.service.security.SecurityUtil; +import org.openmetadata.service.security.auth.BotTokenCache; import org.openmetadata.service.security.policyevaluator.SubjectContext; import org.openmetadata.service.util.EntityUtil; import org.openmetadata.service.util.EntityUtil.Fields; @@ -509,6 +510,14 @@ public static String invalidTeam(int field, String team, String user, String use } } + @Override + protected void postDelete(User entity) { + // If the User is bot it's token needs to be invalidated + if (Boolean.TRUE.equals(entity.getIsBot())) { + BotTokenCache.invalidateToken(entity.getName()); + } + } + /** Handles entity updated from PUT and POST operation. */ public class UserUpdater extends EntityUpdater { public UserUpdater(User original, User updated, Operation operation) { diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/migration/MigrationValidationClient.java b/openmetadata-service/src/main/java/org/openmetadata/service/migration/MigrationValidationClient.java new file mode 100644 index 000000000000..d803ebc7dd9e --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/migration/MigrationValidationClient.java @@ -0,0 +1,72 @@ +package org.openmetadata.service.migration; + +import java.io.File; +import java.util.Arrays; +import java.util.List; +import java.util.Objects; +import java.util.stream.Stream; +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; +import org.openmetadata.service.OpenMetadataApplicationConfig; +import org.openmetadata.service.jdbi3.MigrationDAO; + +@Slf4j +public class MigrationValidationClient { + @Getter public static MigrationValidationClient instance; + + private final MigrationDAO migrationDAO; + private final OpenMetadataApplicationConfig config; + @Getter private final List expectedMigrationList; + + private MigrationValidationClient( + MigrationDAO migrationDAO, OpenMetadataApplicationConfig config) { + this.migrationDAO = migrationDAO; + this.config = config; + this.expectedMigrationList = loadExpectedMigrationList(); + } + + public static MigrationValidationClient initialize( + MigrationDAO migrationDAO, OpenMetadataApplicationConfig config) { + + if (instance == null) { + instance = new MigrationValidationClient(migrationDAO, config); + } + return instance; + } + + public List getCurrentVersions() { + return migrationDAO.getMigrationVersions(); + } + + private List loadExpectedMigrationList() { + try { + String nativePath = config.getMigrationConfiguration().getNativePath(); + String extensionPath = config.getMigrationConfiguration().getExtensionPath(); + + List availableOMNativeMigrations = getMigrationFilesFromPath(nativePath); + + // If we only have OM migrations, return them + if (extensionPath == null || extensionPath.isEmpty()) { + return availableOMNativeMigrations; + } + + // Otherwise, fetch the extension migration and sort the results + List availableOMExtensionMigrations = getMigrationFilesFromPath(extensionPath); + + return Stream.concat( + availableOMNativeMigrations.stream(), availableOMExtensionMigrations.stream()) + .sorted() + .toList(); + } catch (Exception e) { + LOG.error("Error loading expected migration list", e); + return List.of(); + } + } + + private List getMigrationFilesFromPath(String path) { + return Arrays.stream(Objects.requireNonNull(new File(path).listFiles(File::isDirectory))) + .map(File::getName) + .sorted() + .toList(); + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/migration/MigrationValidationClientException.java b/openmetadata-service/src/main/java/org/openmetadata/service/migration/MigrationValidationClientException.java new file mode 100644 index 000000000000..2636d2839f83 --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/migration/MigrationValidationClientException.java @@ -0,0 +1,44 @@ +/* + * Copyright 2021 Collate + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.openmetadata.service.migration; + +import javax.ws.rs.core.Response; +import org.openmetadata.sdk.exception.WebServiceException; + +public class MigrationValidationClientException extends WebServiceException { + private static final String BY_NAME_MESSAGE = "Migration Validation Exception [%s] due to [%s]."; + private static final String ERROR_TYPE = "MIGRATION_VALIDATION"; + + public MigrationValidationClientException(String message) { + super(Response.Status.BAD_REQUEST, ERROR_TYPE, message); + } + + private MigrationValidationClientException(Response.Status status, String message) { + super(status, ERROR_TYPE, message); + } + + public static MigrationValidationClientException byMessage( + String name, String errorMessage, Response.Status status) { + return new MigrationValidationClientException(status, buildMessageByName(name, errorMessage)); + } + + public static MigrationValidationClientException byMessage(String name, String errorMessage) { + return new MigrationValidationClientException( + Response.Status.BAD_REQUEST, buildMessageByName(name, errorMessage)); + } + + private static String buildMessageByName(String name, String errorMessage) { + return String.format(BY_NAME_MESSAGE, name, errorMessage); + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/migration/mysql/v132/Migration.java b/openmetadata-service/src/main/java/org/openmetadata/service/migration/mysql/v132/Migration.java new file mode 100644 index 000000000000..a863138e504f --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/migration/mysql/v132/Migration.java @@ -0,0 +1,37 @@ +package org.openmetadata.service.migration.mysql.v132; + +import static org.openmetadata.service.migration.utils.v132.MigrationUtil.migrateDbtConfigType; + +import lombok.SneakyThrows; +import org.jdbi.v3.core.Handle; +import org.openmetadata.service.jdbi3.CollectionDAO; +import org.openmetadata.service.migration.api.MigrationProcessImpl; +import org.openmetadata.service.migration.utils.MigrationFile; + +public class Migration extends MigrationProcessImpl { + private CollectionDAO collectionDAO; + private Handle handle; + + public Migration(MigrationFile migrationFile) { + super(migrationFile); + } + + @Override + public void initialize(Handle handle) { + super.initialize(handle); + this.handle = handle; + this.collectionDAO = handle.attach(CollectionDAO.class); + } + + @Override + @SneakyThrows + public void runDataMigration() { + String getDbtPipelinesQuery = + "SELECT * from ingestion_pipeline_entity ipe WHERE JSON_EXTRACT(json, '$.pipelineType') = 'dbt'"; + String updateSqlQuery = + "UPDATE ingestion_pipeline_entity ipe SET json = :json " + + "WHERE JSON_EXTRACT(json, '$.pipelineType') = 'dbt'" + + "AND id = :id"; + migrateDbtConfigType(handle, updateSqlQuery, getDbtPipelinesQuery); + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/migration/mysql/v131/Migration.java b/openmetadata-service/src/main/java/org/openmetadata/service/migration/mysql/v133/Migration.java similarity index 93% rename from openmetadata-service/src/main/java/org/openmetadata/service/migration/mysql/v131/Migration.java rename to openmetadata-service/src/main/java/org/openmetadata/service/migration/mysql/v133/Migration.java index 04a27b28dec1..8192db1a66f0 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/migration/mysql/v131/Migration.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/migration/mysql/v133/Migration.java @@ -1,4 +1,4 @@ -package org.openmetadata.service.migration.mysql.v131; +package org.openmetadata.service.migration.mysql.v133; import static org.openmetadata.service.migration.utils.v131.MigrationUtil.migrateCronExpression; diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/migration/postgres/v132/Migration.java b/openmetadata-service/src/main/java/org/openmetadata/service/migration/postgres/v132/Migration.java new file mode 100644 index 000000000000..f7f5950e830b --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/migration/postgres/v132/Migration.java @@ -0,0 +1,37 @@ +package org.openmetadata.service.migration.postgres.v132; + +import static org.openmetadata.service.migration.utils.v132.MigrationUtil.migrateDbtConfigType; + +import lombok.SneakyThrows; +import org.jdbi.v3.core.Handle; +import org.openmetadata.service.jdbi3.CollectionDAO; +import org.openmetadata.service.migration.api.MigrationProcessImpl; +import org.openmetadata.service.migration.utils.MigrationFile; + +public class Migration extends MigrationProcessImpl { + private CollectionDAO collectionDAO; + private Handle handle; + + public Migration(MigrationFile migrationFile) { + super(migrationFile); + } + + @Override + public void initialize(Handle handle) { + super.initialize(handle); + this.handle = handle; + this.collectionDAO = handle.attach(CollectionDAO.class); + } + + @Override + @SneakyThrows + public void runDataMigration() { + String getDbtPipelinesQuery = + "SELECT * from ingestion_pipeline_entity ipe WHERE json #>> '{pipelineType}' = 'dbt'"; + String updateSqlQuery = + "UPDATE ingestion_pipeline_entity ipe SET json = :json::jsonb " + + "WHERE json #>> '{pipelineType}' = 'dbt'" + + "AND id = :id"; + migrateDbtConfigType(handle, updateSqlQuery, getDbtPipelinesQuery); + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/migration/postgres/v131/Migration.java b/openmetadata-service/src/main/java/org/openmetadata/service/migration/postgres/v133/Migration.java similarity index 93% rename from openmetadata-service/src/main/java/org/openmetadata/service/migration/postgres/v131/Migration.java rename to openmetadata-service/src/main/java/org/openmetadata/service/migration/postgres/v133/Migration.java index b5570372ba37..82325342ebd4 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/migration/postgres/v131/Migration.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/migration/postgres/v133/Migration.java @@ -1,4 +1,4 @@ -package org.openmetadata.service.migration.postgres.v131; +package org.openmetadata.service.migration.postgres.v133; import static org.openmetadata.service.migration.utils.v131.MigrationUtil.migrateCronExpression; diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/migration/utils/v132/MigrationUtil.java b/openmetadata-service/src/main/java/org/openmetadata/service/migration/utils/v132/MigrationUtil.java new file mode 100644 index 000000000000..27fe3e7c62c0 --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/migration/utils/v132/MigrationUtil.java @@ -0,0 +1,102 @@ +package org.openmetadata.service.migration.utils.v132; + +import java.util.LinkedHashMap; +import lombok.extern.slf4j.Slf4j; +import org.jdbi.v3.core.Handle; +import org.json.JSONObject; +import org.openmetadata.schema.entity.services.ingestionPipelines.IngestionPipeline; +import org.openmetadata.schema.metadataIngestion.dbtconfig.DbtAzureConfig; +import org.openmetadata.schema.metadataIngestion.dbtconfig.DbtCloudConfig; +import org.openmetadata.schema.metadataIngestion.dbtconfig.DbtGCSConfig; +import org.openmetadata.schema.metadataIngestion.dbtconfig.DbtHttpConfig; +import org.openmetadata.schema.metadataIngestion.dbtconfig.DbtLocalConfig; +import org.openmetadata.schema.metadataIngestion.dbtconfig.DbtS3Config; +import org.openmetadata.service.exception.UnhandledServerException; +import org.openmetadata.service.util.JsonUtils; + +@Slf4j +public class MigrationUtil { + + private MigrationUtil() { + /* Cannot create object util class*/ + } + + public static void migrateDbtConfigType( + Handle handle, String updateSqlQuery, String dbtGetDbtPipelinesQuery) { + handle + .createQuery(dbtGetDbtPipelinesQuery) + .mapToMap() + .forEach( + row -> { + try { + IngestionPipeline ingestionPipeline = + JsonUtils.readValue(row.get("json").toString(), IngestionPipeline.class); + String id = row.get("id").toString(); + LinkedHashMap sourceConfig = + (LinkedHashMap) ingestionPipeline.getSourceConfig().getConfig(); + LinkedHashMap dbtConfigSource = (LinkedHashMap) sourceConfig.get("dbtConfigSource"); + + sourceConfig.put("dbtConfigSource", addDbtConfigType(dbtConfigSource)); + String json = JsonUtils.pojoToJson(ingestionPipeline); + + handle.createUpdate(updateSqlQuery).bind("json", json).bind("id", id).execute(); + + } catch (Exception ex) { + LOG.warn("Error during the dbt type migration due to ", ex); + } + }); + } + + public static Object addDbtConfigType(LinkedHashMap dbtConfigSource) { + String jsonString = new JSONObject(dbtConfigSource).toString(); + + // For adding s3 type + try { + DbtS3Config dbtS3Config = JsonUtils.readValue(jsonString, DbtS3Config.class); + dbtS3Config.setDbtConfigType(DbtS3Config.DbtConfigType.S_3); + return dbtS3Config; + } catch (UnhandledServerException ex) { + } + + // For adding GCS type + try { + DbtGCSConfig dbtGCSConfig = JsonUtils.readValue(jsonString, DbtGCSConfig.class); + dbtGCSConfig.setDbtConfigType(DbtGCSConfig.DbtConfigType.GCS); + return dbtGCSConfig; + } catch (UnhandledServerException ex) { + } + + // For adding Azure type + try { + DbtAzureConfig dbtAzureConfig = JsonUtils.readValue(jsonString, DbtAzureConfig.class); + dbtAzureConfig.setDbtConfigType(DbtAzureConfig.DbtConfigType.AZURE); + return dbtAzureConfig; + } catch (UnhandledServerException ex) { + } + + // For adding cloud type + try { + DbtCloudConfig dbtCloudConfig = JsonUtils.readValue(jsonString, DbtCloudConfig.class); + dbtCloudConfig.setDbtConfigType(DbtCloudConfig.DbtConfigType.CLOUD); + return dbtCloudConfig; + } catch (UnhandledServerException ex) { + } + + // For adding local type + try { + DbtLocalConfig dbtLocalConfig = JsonUtils.readValue(jsonString, DbtLocalConfig.class); + dbtLocalConfig.setDbtConfigType(DbtLocalConfig.DbtConfigType.LOCAL); + return dbtLocalConfig; + } catch (UnhandledServerException ex) { + } + + // For adding http type + try { + DbtHttpConfig dbtHttpConfig = JsonUtils.readValue(jsonString, DbtHttpConfig.class); + dbtHttpConfig.setDbtConfigType(DbtHttpConfig.DbtConfigType.HTTP); + return dbtHttpConfig; + } catch (UnhandledServerException ex) { + } + return null; + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/apps/AppMarketPlaceResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/apps/AppMarketPlaceResource.java index 423b0b6360e3..6970ef4fa2e5 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/apps/AppMarketPlaceResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/apps/AppMarketPlaceResource.java @@ -47,6 +47,7 @@ import org.openmetadata.sdk.PipelineServiceClient; import org.openmetadata.service.Entity; import org.openmetadata.service.OpenMetadataApplicationConfig; +import org.openmetadata.service.apps.ApplicationHandler; import org.openmetadata.service.clients.pipeline.PipelineServiceClientFactory; import org.openmetadata.service.jdbi3.AppMarketPlaceRepository; import org.openmetadata.service.jdbi3.ListFilter; @@ -211,7 +212,10 @@ public AppMarketPlaceDefinition get( @QueryParam("include") @DefaultValue("non-deleted") Include include) { - return getInternal(uriInfo, securityContext, id, fieldsParam, include); + AppMarketPlaceDefinition definition = + getInternal(uriInfo, securityContext, id, fieldsParam, include); + definition.setPreview(ApplicationHandler.getInstance().isPreview(definition.getName())); + return definition; } @GET @@ -247,7 +251,10 @@ public AppMarketPlaceDefinition getByName( @QueryParam("include") @DefaultValue("non-deleted") Include include) { - return getByNameInternal(uriInfo, securityContext, name, fieldsParam, include); + AppMarketPlaceDefinition definition = + getByNameInternal(uriInfo, securityContext, name, fieldsParam, include); + definition.setPreview(ApplicationHandler.getInstance().isPreview(definition.getName())); + return definition; } @GET @@ -442,7 +449,8 @@ private AppMarketPlaceDefinition getApplicationDefinition( .withAppScreenshots(create.getAppScreenshots()) .withFeatures(create.getFeatures()) .withSourcePythonClass(create.getSourcePythonClass()) - .withAllowConfiguration(create.getAllowConfiguration()); + .withAllowConfiguration(create.getAllowConfiguration()) + .withSystem(create.getSystem()); // Validate App validateApplication(app); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/apps/AppResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/apps/AppResource.java index 3cf18bb2d130..c4c600ef806f 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/apps/AppResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/apps/AppResource.java @@ -45,8 +45,6 @@ import lombok.extern.slf4j.Slf4j; import org.openmetadata.common.utils.CommonUtil; import org.openmetadata.schema.ServiceEntityInterface; -import org.openmetadata.schema.api.configuration.apps.AppPrivateConfig; -import org.openmetadata.schema.api.configuration.apps.AppsPrivateConfiguration; import org.openmetadata.schema.api.data.RestoreEntity; import org.openmetadata.schema.entity.app.App; import org.openmetadata.schema.entity.app.AppMarketPlaceDefinition; @@ -68,6 +66,7 @@ import org.openmetadata.service.apps.ApplicationHandler; import org.openmetadata.service.apps.scheduler.AppScheduler; import org.openmetadata.service.clients.pipeline.PipelineServiceClientFactory; +import org.openmetadata.service.exception.CatalogExceptionMessage; import org.openmetadata.service.exception.EntityNotFoundException; import org.openmetadata.service.jdbi3.AppRepository; import org.openmetadata.service.jdbi3.CollectionDAO; @@ -98,7 +97,6 @@ public class AppResource extends EntityResource { public static final String COLLECTION_PATH = "v1/apps/"; private OpenMetadataApplicationConfig openMetadataApplicationConfig; - private AppsPrivateConfiguration privateConfiguration; private PipelineServiceClient pipelineServiceClient; static final String FIELDS = "owner"; private SearchRepository searchRepository; @@ -107,7 +105,6 @@ public class AppResource extends EntityResource { public void initialize(OpenMetadataApplicationConfig config) { try { this.openMetadataApplicationConfig = config; - this.privateConfiguration = config.getAppsPrivateConfiguration(); this.pipelineServiceClient = PipelineServiceClientFactory.createPipelineServiceClient( config.getPipelineServiceClientConfiguration()); @@ -139,23 +136,8 @@ public void initialize(OpenMetadataApplicationConfig config) { // Schedule if (app.getScheduleType().equals(ScheduleType.Scheduled)) { - setAppRuntimeProperties(app); - ApplicationHandler.installApplication(app, Entity.getCollectionDAO(), searchRepository); - } - } - - // Initialize installed applications - for (App installedApp : repository.listAll()) { - App appWithBot = getAppForInit(installedApp.getName()); - if (appWithBot == null) { - LOG.error( - String.format( - "Failed to init app [%s]. GET should return the installed app", - installedApp.getName())); - } else { - setAppRuntimeProperties(appWithBot); - ApplicationHandler.runAppInit(appWithBot, dao, searchRepository); - LOG.info(String.format("Initialized installed app [%s]", installedApp.getName())); + ApplicationHandler.getInstance() + .installApplication(app, Entity.getCollectionDAO(), searchRepository); } } } catch (Exception ex) { @@ -183,24 +165,6 @@ public static class AppRunList extends ResultList { /* Required for serde */ } - /** - * Load the apps' OM configuration and private parameters - */ - private void setAppRuntimeProperties(App app) { - app.setOpenMetadataServerConnection( - new OpenMetadataConnectionBuilder(openMetadataApplicationConfig, app.getBot().getName()) - .build()); - - if (privateConfiguration != null - && !nullOrEmpty(privateConfiguration.getAppsPrivateConfiguration())) { - for (AppPrivateConfig appPrivateConfig : privateConfiguration.getAppsPrivateConfiguration()) { - if (app.getName().equals(appPrivateConfig.getName())) { - app.setPrivateConfiguration(appPrivateConfig.getParameters()); - } - } - } - } - /** * We don't want to store runtime information into the DB */ @@ -580,10 +544,11 @@ public Response create( create.getName(), new EntityUtil.Fields(repository.getMarketPlace().getAllowedFields())); App app = getApplication(definition, create, securityContext.getUserPrincipal().getName()); - setAppRuntimeProperties(app); if (app.getScheduleType().equals(ScheduleType.Scheduled)) { - ApplicationHandler.installApplication(app, Entity.getCollectionDAO(), searchRepository); - ApplicationHandler.configureApplication(app, Entity.getCollectionDAO(), searchRepository); + ApplicationHandler.getInstance() + .installApplication(app, Entity.getCollectionDAO(), searchRepository); + ApplicationHandler.getInstance() + .configureApplication(app, Entity.getCollectionDAO(), searchRepository); } // We don't want to store this information unsetAppRuntimeProperties(app); @@ -617,13 +582,16 @@ public Response patchApplication( JsonPatch patch) throws SchedulerException { App app = repository.get(null, id, repository.getFields("bot,pipelines")); + if (app.getSystem()) { + throw new IllegalArgumentException( + CatalogExceptionMessage.systemEntityModifyNotAllowed(app.getName(), "SystemApp")); + } AppScheduler.getInstance().deleteScheduledApplication(app); Response response = patchInternal(uriInfo, securityContext, id, patch); App updatedApp = (App) response.getEntity(); - setAppRuntimeProperties(updatedApp); if (app.getScheduleType().equals(ScheduleType.Scheduled)) { - ApplicationHandler.installApplication( - updatedApp, Entity.getCollectionDAO(), searchRepository); + ApplicationHandler.getInstance() + .installApplication(updatedApp, Entity.getCollectionDAO(), searchRepository); } // We don't want to store this information unsetAppRuntimeProperties(updatedApp); @@ -656,9 +624,9 @@ public Response createOrUpdate( new EntityUtil.Fields(repository.getMarketPlace().getAllowedFields())); App app = getApplication(definition, create, securityContext.getUserPrincipal().getName()); AppScheduler.getInstance().deleteScheduledApplication(app); - setAppRuntimeProperties(app); if (app.getScheduleType().equals(ScheduleType.Scheduled)) { - ApplicationHandler.installApplication(app, Entity.getCollectionDAO(), searchRepository); + ApplicationHandler.getInstance() + .installApplication(app, Entity.getCollectionDAO(), searchRepository); } // We don't want to store this information unsetAppRuntimeProperties(app); @@ -673,6 +641,9 @@ public Response createOrUpdate( description = "Delete a App by `name`.", responses = { @ApiResponse(responseCode = "200", description = "OK"), + @ApiResponse( + responseCode = "400", + description = "System entity {name} of type SystemApp can not be deleted."), @ApiResponse(responseCode = "404", description = "App for instance {name} is not found") }) public Response delete( @@ -686,6 +657,10 @@ public Response delete( @PathParam("name") String name) { App app = repository.getByName(null, name, repository.getFields("bot,pipelines")); + if (app.getSystem()) { + throw new IllegalArgumentException( + CatalogExceptionMessage.systemEntityDeleteNotAllowed(app.getName(), "SystemApp")); + } // Remove from Pipeline Service deleteApp(securityContext, app, hardDelete); return deleteByName(uriInfo, securityContext, name, true, hardDelete); @@ -699,6 +674,9 @@ public Response delete( description = "Delete a App by `Id`.", responses = { @ApiResponse(responseCode = "200", description = "OK"), + @ApiResponse( + responseCode = "400", + description = "System entity {name} of type SystemApp can not be deleted."), @ApiResponse(responseCode = "404", description = "App for instance {id} is not found") }) public Response delete( @@ -711,6 +689,10 @@ public Response delete( @Parameter(description = "Id of the App", schema = @Schema(type = "UUID")) @PathParam("id") UUID id) { App app = repository.get(null, id, repository.getFields("bot,pipelines")); + if (app.getSystem()) { + throw new IllegalArgumentException( + CatalogExceptionMessage.systemEntityDeleteNotAllowed(app.getName(), "SystemApp")); + } // Remove from Pipeline Service deleteApp(securityContext, app, hardDelete); // Remove from repository @@ -739,9 +721,9 @@ public Response restoreApp( Response response = restoreEntity(uriInfo, securityContext, restore.getId()); if (response.getStatus() == Response.Status.OK.getStatusCode()) { App app = (App) response.getEntity(); - setAppRuntimeProperties(app); if (app.getScheduleType().equals(ScheduleType.Scheduled)) { - ApplicationHandler.installApplication(app, Entity.getCollectionDAO(), searchRepository); + ApplicationHandler.getInstance() + .installApplication(app, Entity.getCollectionDAO(), searchRepository); } // We don't want to store this information unsetAppRuntimeProperties(app); @@ -775,9 +757,9 @@ public Response scheduleApplication( @Context SecurityContext securityContext) { App app = repository.getByName(uriInfo, name, new EntityUtil.Fields(repository.getAllowedFields())); - setAppRuntimeProperties(app); if (app.getScheduleType().equals(ScheduleType.Scheduled)) { - ApplicationHandler.installApplication(app, repository.getDaoCollection(), searchRepository); + ApplicationHandler.getInstance() + .installApplication(app, repository.getDaoCollection(), searchRepository); return Response.status(Response.Status.OK).entity("App is Scheduled.").build(); } throw new IllegalArgumentException("App is not of schedule type Scheduled."); @@ -811,9 +793,9 @@ public Response configureApplication( repository.getByName(uriInfo, name, new EntityUtil.Fields(repository.getAllowedFields())); // The application will have the updated appConfiguration we can use to run the `configure` // logic - setAppRuntimeProperties(app); try { - ApplicationHandler.configureApplication(app, repository.getDaoCollection(), searchRepository); + ApplicationHandler.getInstance() + .configureApplication(app, repository.getDaoCollection(), searchRepository); return Response.status(Response.Status.OK).entity("App has been configured.").build(); } catch (RuntimeException e) { return Response.status(Response.Status.INTERNAL_SERVER_ERROR) @@ -845,10 +827,9 @@ public Response triggerApplicationRun( String name) { EntityUtil.Fields fields = getFields(String.format("%s,bot,pipelines", FIELD_OWNER)); App app = repository.getByName(uriInfo, name, fields); - setAppRuntimeProperties(app); if (app.getAppType().equals(AppType.Internal)) { - ApplicationHandler.triggerApplicationOnDemand( - app, Entity.getCollectionDAO(), searchRepository); + ApplicationHandler.getInstance() + .triggerApplicationOnDemand(app, Entity.getCollectionDAO(), searchRepository); return Response.status(Response.Status.OK).entity("Application Triggered").build(); } else { if (!app.getPipelines().isEmpty()) { @@ -894,9 +875,9 @@ public Response deployApplicationFlow( String name) { EntityUtil.Fields fields = getFields(String.format("%s,bot,pipelines", FIELD_OWNER)); App app = repository.getByName(uriInfo, name, fields); - setAppRuntimeProperties(app); if (app.getAppType().equals(AppType.Internal)) { - ApplicationHandler.installApplication(app, Entity.getCollectionDAO(), searchRepository); + ApplicationHandler.getInstance() + .installApplication(app, Entity.getCollectionDAO(), searchRepository); return Response.status(Response.Status.OK).entity("Application Deployed").build(); } else { if (!app.getPipelines().isEmpty()) { @@ -978,7 +959,8 @@ private App getApplication( .withAppScreenshots(marketPlaceDefinition.getAppScreenshots()) .withFeatures(marketPlaceDefinition.getFeatures()) .withSourcePythonClass(marketPlaceDefinition.getSourcePythonClass()) - .withAllowConfiguration(marketPlaceDefinition.getAllowConfiguration()); + .withAllowConfiguration(marketPlaceDefinition.getAllowConfiguration()) + .withSystem(marketPlaceDefinition.getSystem()); // validate Bot if provided validateAndAddBot(app, createAppRequest.getBot()); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/dqtests/TestCaseResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/dqtests/TestCaseResource.java index a448fd16db65..4ad718b6c70a 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/dqtests/TestCaseResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/dqtests/TestCaseResource.java @@ -174,12 +174,31 @@ public ResultList list( schema = @Schema(implementation = Include.class)) @QueryParam("include") @DefaultValue("non-deleted") - Include include) { + Include include, + @Parameter( + description = "Filter test case by status", + schema = + @Schema( + type = "string", + allowableValues = {"Success", "Failed", "Aborted", "Queued"})) + @QueryParam("testCaseStatus") + String status, + @Parameter( + description = "Filter for test case type (e.g. column, table, all", + schema = + @Schema( + type = "string", + allowableValues = {"column", "table", "all"})) + @QueryParam("testCaseType") + @DefaultValue("all") + String type) { ListFilter filter = new ListFilter(include) .addQueryParam("testSuiteId", testSuiteId) .addQueryParam("includeAllTests", includeAllTests.toString()) - .addQueryParam("orderByLastExecutionDate", orderByLastExecutionDate.toString()); + .addQueryParam("orderByLastExecutionDate", orderByLastExecutionDate.toString()) + .addQueryParam("testCaseStatus", status) + .addQueryParam("testCaseType", type); ResourceContextInterface resourceContext; if (entityLink != null) { EntityLink entityLinkParsed = EntityLink.parse(entityLink); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/dqtests/TestSuiteResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/dqtests/TestSuiteResource.java index 7bf403ae1fe9..553577e9b4c7 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/dqtests/TestSuiteResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/dqtests/TestSuiteResource.java @@ -11,6 +11,7 @@ import io.swagger.v3.oas.annotations.parameters.RequestBody; import io.swagger.v3.oas.annotations.responses.ApiResponse; import io.swagger.v3.oas.annotations.tags.Tag; +import java.io.IOException; import java.util.List; import java.util.UUID; import javax.json.JsonPatch; @@ -321,7 +322,8 @@ public TestSummary getTestsExecutionSummary( description = "get summary for a specific test suite", schema = @Schema(type = "String", format = "uuid")) @QueryParam("testSuiteId") - UUID testSuiteId) { + UUID testSuiteId) + throws IOException { ResourceContext resourceContext = getResourceContext(); OperationContext operationContext = new OperationContext(Entity.TABLE, MetadataOperation.VIEW_TESTS); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/feeds/FeedResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/feeds/FeedResource.java index be0649a6e0b7..976799e1d0f1 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/feeds/FeedResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/feeds/FeedResource.java @@ -297,7 +297,7 @@ public Response resolveTask( String id, @Valid ResolveTask resolveTask) { Thread task = dao.getTask(Integer.parseInt(id)); - dao.checkPermissionsForResolveTask(task, false, securityContext); + dao.checkPermissionsForResolveTask(authorizer, task, false, securityContext); return dao.resolveTask(uriInfo, task, securityContext.getUserPrincipal().getName(), resolveTask) .toResponse(); } @@ -326,7 +326,7 @@ public Response closeTask( String id, @Valid CloseTask closeTask) { Thread task = dao.getTask(Integer.parseInt(id)); - dao.checkPermissionsForResolveTask(task, true, securityContext); + dao.checkPermissionsForResolveTask(authorizer, task, true, securityContext); return dao.closeTask(uriInfo, task, securityContext.getUserPrincipal().getName(), closeTask) .toResponse(); } @@ -590,6 +590,7 @@ private Thread getThread(SecurityContext securityContext, CreateThread create) { .withType(create.getType()) .withTask(getTaskDetails(create.getTaskDetails())) .withAnnouncement(create.getAnnouncementDetails()) + .withChatbot(create.getChatbotDetails()) .withUpdatedBy(securityContext.getUserPrincipal().getName()) .withUpdatedAt(System.currentTimeMillis()); } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/feeds/SuggestionsResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/feeds/SuggestionsResource.java index 6e74397708d8..29056eb367db 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/feeds/SuggestionsResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/feeds/SuggestionsResource.java @@ -14,7 +14,9 @@ package org.openmetadata.service.resources.feeds; import static org.openmetadata.common.utils.CommonUtil.listOrEmpty; +import static org.openmetadata.common.utils.CommonUtil.nullOrEmpty; import static org.openmetadata.schema.type.EventType.SUGGESTION_CREATED; +import static org.openmetadata.schema.type.EventType.SUGGESTION_REJECTED; import static org.openmetadata.schema.type.EventType.SUGGESTION_UPDATED; import static org.openmetadata.service.util.RestUtil.CHANGE_CUSTOM_HEADER; @@ -206,12 +208,12 @@ public Suggestion get( @Path("/{id}/accept") @Operation( operationId = "acceptSuggestion", - summary = "Close a task", - description = "Close a task without making any changes to the entity.", + summary = "Accept a Suggestion", + description = "Accept a Suggestion and apply the changes to the entity.", responses = { @ApiResponse( responseCode = "200", - description = "The task thread.", + description = "The suggestion.", content = @Content( mediaType = "application/json", @@ -259,6 +261,107 @@ public Response rejectSuggestion( .toResponse(); } + @PUT + @Path("accept-all") + @Operation( + operationId = "acceptAllSuggestion", + summary = "Accept all Suggestions from a user and an Entity", + description = "Accept a Suggestion and apply the changes to the entity.", + responses = { + @ApiResponse( + responseCode = "200", + description = "The suggestion.", + content = + @Content( + mediaType = "application/json", + schema = @Schema(implementation = Suggestion.class))), + @ApiResponse(responseCode = "400", description = "Bad request") + }) + public RestUtil.PutResponse> acceptAllSuggestions( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @Parameter(description = "user id", schema = @Schema(type = "string")) @QueryParam("userId") + UUID userId, + @Parameter(description = "fullyQualifiedName of entity", schema = @Schema(type = "string")) + @QueryParam("entityFQN") + String entityFQN, + @Parameter(description = "Suggestion type being accepted", schema = @Schema(type = "string")) + @QueryParam("suggestionType") + @DefaultValue("SuggestDescription") + SuggestionType suggestionType) { + SuggestionFilter filter = + SuggestionFilter.builder() + .suggestionStatus(SuggestionStatus.Open) + .entityFQN(entityFQN) + .createdBy(userId) + .suggestionType(suggestionType) + .build(); + List suggestions = dao.listAll(filter); + if (!nullOrEmpty(suggestions)) { + // Validate the permissions for one suggestion + Suggestion suggestion = dao.get(suggestions.get(0).getId()); + dao.checkPermissionsForAcceptOrRejectSuggestion( + suggestion, SuggestionStatus.Rejected, securityContext); + dao.checkPermissionsForEditEntity(suggestion, suggestionType, securityContext, authorizer); + return dao.acceptSuggestionList( + uriInfo, suggestions, suggestionType, securityContext, authorizer); + } else { + // No suggestions found + return new RestUtil.PutResponse<>( + Response.Status.BAD_REQUEST, List.of(), SUGGESTION_REJECTED); + } + } + + @PUT + @Path("reject-all") + @Operation( + operationId = "rejectAllSuggestion", + summary = "Reject all Suggestions from a user and an Entity", + description = "Reject all Suggestions from a user and an Entity", + responses = { + @ApiResponse( + responseCode = "200", + description = "The suggestion.", + content = + @Content( + mediaType = "application/json", + schema = @Schema(implementation = Suggestion.class))), + @ApiResponse(responseCode = "400", description = "Bad request") + }) + public RestUtil.PutResponse> rejectAllSuggestions( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @Parameter(description = "user id", schema = @Schema(type = "string")) @QueryParam("userId") + UUID userId, + @Parameter(description = "fullyQualifiedName of entity", schema = @Schema(type = "string")) + @QueryParam("entityFQN") + String entityFQN, + @Parameter(description = "Suggestion type being rejected", schema = @Schema(type = "string")) + @QueryParam("suggestionType") + @DefaultValue("SuggestDescription") + SuggestionType suggestionType) { + SuggestionFilter filter = + SuggestionFilter.builder() + .suggestionStatus(SuggestionStatus.Open) + .entityFQN(entityFQN) + .createdBy(userId) + .suggestionType(suggestionType) + .build(); + List suggestions = dao.listAll(filter); + if (!nullOrEmpty(suggestions)) { + // Validate the permissions for one suggestion + Suggestion suggestion = dao.get(suggestions.get(0).getId()); + dao.checkPermissionsForAcceptOrRejectSuggestion( + suggestion, SuggestionStatus.Rejected, securityContext); + return dao.rejectSuggestionList( + uriInfo, suggestions, securityContext.getUserPrincipal().getName()); + } else { + // No suggestions found + return new RestUtil.PutResponse<>( + Response.Status.BAD_REQUEST, List.of(), SUGGESTION_REJECTED); + } + } + @PUT @Path("/{id}") @Operation( diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/lineage/LineageResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/lineage/LineageResource.java index 5472de2d7102..5d81fc924352 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/lineage/LineageResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/lineage/LineageResource.java @@ -273,6 +273,53 @@ public Response deleteLineage( return Response.status(Status.OK).build(); } + @DELETE + @Path("/{fromEntity}/name/{fromFQN}/{toEntity}/name/{toFQN}") + @Operation( + operationId = "deleteLineageEdgeByName", + summary = "Delete a lineage edge by FQNs", + description = + "Delete a lineage edge with from entity as upstream node and to entity as downstream node.", + responses = { + @ApiResponse(responseCode = "200"), + @ApiResponse( + responseCode = "404", + description = "Entity for instance {fromFQN} is not found") + }) + public Response deleteLineageByName( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @Parameter( + description = "Entity type of upstream entity of the edge", + required = true, + schema = @Schema(type = "string", example = "table, report, metrics, or dashboard")) + @PathParam("fromEntity") + String fromEntity, + @Parameter(description = "Entity FQN", required = true, schema = @Schema(type = "string")) + @PathParam("fromFQN") + String fromFQN, + @Parameter( + description = "Entity type for downstream entity of the edge", + required = true, + schema = @Schema(type = "string", example = "table, report, metrics, or dashboard")) + @PathParam("toEntity") + String toEntity, + @Parameter(description = "Entity FQN", required = true, schema = @Schema(type = "string")) + @PathParam("toFQN") + String toFQN) { + authorizer.authorize( + securityContext, + new OperationContext(LINEAGE_FIELD, MetadataOperation.EDIT_LINEAGE), + new LineageResourceContext()); + boolean deleted = dao.deleteLineageByFQN(fromEntity, fromFQN, toEntity, toFQN); + if (!deleted) { + return Response.status(NOT_FOUND) + .entity(new ErrorMessage(NOT_FOUND.getStatusCode(), "Lineage edge not found")) + .build(); + } + return Response.status(Status.OK).build(); + } + private EntityLineage addHref(UriInfo uriInfo, EntityLineage lineage) { Entity.withHref(uriInfo, lineage.getEntity()); Entity.withHref(uriInfo, lineage.getNodes()); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/policies/PolicyResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/policies/PolicyResource.java index 0734bd2b2a7f..879e5ecf028d 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/policies/PolicyResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/policies/PolicyResource.java @@ -498,6 +498,7 @@ public void validateCondition( @Parameter(description = "Expression of validating rule", schema = @Schema(type = "string")) @PathParam("expression") String expression) { + authorizer.authorizeAdmin(securityContext); CompiledRule.validateExpression(expression, Boolean.class); } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/system/ConfigResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/system/ConfigResource.java index 4b7785a11cd5..79e9a76fbb70 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/system/ConfigResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/system/ConfigResource.java @@ -84,6 +84,8 @@ public AuthenticationConfiguration getAuthConfig() { authenticationConfiguration.getSamlConfiguration().getIdp().getAuthorityUrl())); authenticationConfiguration.setSamlConfiguration(ssoClientConfig); } + + authenticationConfiguration.setOidcConfiguration(null); } return authenticationConfiguration; } @@ -150,12 +152,12 @@ public LoginConfiguration getLoginConfiguration() { @GET @Path(("/pipeline-service-client")) @Operation( - operationId = "getAirflowConfiguration", - summary = "Get airflow configuration", + operationId = "getPipelineServiceConfiguration", + summary = "Get Pipeline Service Client configuration", responses = { @ApiResponse( responseCode = "200", - description = "Airflow configuration", + description = "Pipeline Service Client configuration", content = @Content( mediaType = "application/json", diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/system/SystemResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/system/SystemResource.java index 93cfd5d89b51..d7193b16d0cf 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/system/SystemResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/system/SystemResource.java @@ -34,16 +34,20 @@ import org.openmetadata.schema.auth.EmailRequest; import org.openmetadata.schema.settings.Settings; import org.openmetadata.schema.settings.SettingsType; +import org.openmetadata.schema.system.ValidationResponse; import org.openmetadata.schema.type.Include; import org.openmetadata.schema.util.EntitiesCount; import org.openmetadata.schema.util.ServicesCount; +import org.openmetadata.sdk.PipelineServiceClient; import org.openmetadata.service.Entity; import org.openmetadata.service.OpenMetadataApplicationConfig; +import org.openmetadata.service.clients.pipeline.PipelineServiceClientFactory; import org.openmetadata.service.exception.UnhandledServerException; import org.openmetadata.service.jdbi3.ListFilter; import org.openmetadata.service.jdbi3.SystemRepository; import org.openmetadata.service.resources.Collection; import org.openmetadata.service.security.Authorizer; +import org.openmetadata.service.security.JwtFilter; import org.openmetadata.service.util.EmailUtil; import org.openmetadata.service.util.ResultList; @@ -55,19 +59,26 @@ @Collection(name = "system") @Slf4j public class SystemResource { - public static final String COLLECTION_PATH = "/v1/util"; + public static final String COLLECTION_PATH = "/v1/system"; private final SystemRepository systemRepository; private final Authorizer authorizer; private OpenMetadataApplicationConfig applicationConfig; + private PipelineServiceClient pipelineServiceClient; + private JwtFilter jwtFilter; public SystemResource(Authorizer authorizer) { this.systemRepository = Entity.getSystemRepository(); this.authorizer = authorizer; } - @SuppressWarnings("unused") // Method used for reflection public void initialize(OpenMetadataApplicationConfig config) { this.applicationConfig = config; + this.pipelineServiceClient = + PipelineServiceClientFactory.createPipelineServiceClient( + config.getPipelineServiceClientConfiguration()); + + this.jwtFilter = + new JwtFilter(config.getAuthenticationConfiguration(), config.getAuthorizerConfiguration()); } public static class SettingsList extends ResultList { @@ -287,4 +298,24 @@ public ServicesCount listServicesCount( ListFilter filter = new ListFilter(include); return systemRepository.getAllServicesCount(filter); } + + @GET + @Path("/status") + @Operation( + operationId = "validateDeployment", + summary = "Validate the OpenMetadata deployment", + description = + "Check connectivity against your database, elasticsearch/opensearch, migrations,...", + responses = { + @ApiResponse( + responseCode = "200", + description = "validation OK", + content = + @Content( + mediaType = "application/json", + schema = @Schema(implementation = ServicesCount.class))) + }) + public ValidationResponse validate() { + return systemRepository.validateSystem(applicationConfig, pipelineServiceClient, jwtFilter); + } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchClient.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchClient.java index d67843c3f5ad..b6ffee7c7adb 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchClient.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchClient.java @@ -8,6 +8,7 @@ import java.util.List; import java.util.Map; import java.util.TreeMap; +import javax.json.JsonObject; import javax.net.ssl.SSLContext; import javax.ws.rs.core.Response; import org.apache.commons.lang3.tuple.Pair; @@ -87,6 +88,8 @@ Response searchLineage( Response aggregate(String index, String fieldName, String value, String query) throws IOException; + JsonObject aggregate(String query, String index, JsonObject aggregationJson) throws IOException; + Response suggest(SearchRequest request) throws IOException; void createEntity(String indexName, String docId, String doc); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchIndexUtils.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchIndexUtils.java index f8fc750ebbc8..cffaa6a1f916 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchIndexUtils.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchIndexUtils.java @@ -3,7 +3,7 @@ import java.util.Collections; import java.util.List; import java.util.Map; -import java.util.stream.Collectors; +import java.util.Set; import org.openmetadata.schema.type.EntityReference; import org.openmetadata.schema.type.TagLabel; @@ -15,15 +15,39 @@ public static List parseFollowers(List followersRef) { if (followersRef == null) { return Collections.emptyList(); } - return followersRef.stream().map(item -> item.getId().toString()).collect(Collectors.toList()); + return followersRef.stream().map(item -> item.getId().toString()).toList(); } - public static void removeNonIndexableFields(Map doc, List fields) { + public static void removeNonIndexableFields(Map doc, Set fields) { for (String key : fields) { - doc.remove(key); + if (key.contains(".")) { + removeFieldByPath(doc, key); + } else { + doc.remove(key); + } } } + public static void removeFieldByPath(Map jsonMap, String path) { + String[] pathElements = path.split("\\."); + Map currentMap = jsonMap; + + for (int i = 0; i < pathElements.length - 1; i++) { + String key = pathElements[i]; + Object value = currentMap.get(key); + if (value instanceof Map) { + currentMap = (Map) value; + } else { + // Path Not Found + return; + } + } + + // Remove the field at the last path element + String lastKey = pathElements[pathElements.length - 1]; + currentMap.remove(lastKey); + } + public static List parseTags(List tags) { if (tags == null) { return Collections.emptyList(); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchRepository.java index 36d22d87787f..220bcc788cf3 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchRepository.java @@ -252,7 +252,7 @@ public void createEntity(EntityInterface entity) { try { IndexMapping indexMapping = entityIndexMap.get(entityType); SearchIndex index = searchIndexFactory.buildIndex(entityType, entity); - String doc = JsonUtils.pojoToJson(index.buildESDoc()); + String doc = JsonUtils.pojoToJson(index.buildSearchIndexDoc()); searchClient.createEntity(indexMapping.getIndexName(clusterAlias), entityId, doc); } catch (Exception ie) { LOG.error( @@ -280,7 +280,7 @@ public void createTimeSeriesEntity(EntityTimeSeriesInterface entity) { try { IndexMapping indexMapping = entityIndexMap.get(entityType); SearchIndex index = searchIndexFactory.buildIndex(entityType, entity); - String doc = JsonUtils.pojoToJson(index.buildESDoc()); + String doc = JsonUtils.pojoToJson(index.buildSearchIndexDoc()); searchClient.createTimeSeriesEntity(indexMapping.getIndexName(clusterAlias), entityId, doc); } catch (Exception ie) { LOG.error( @@ -309,7 +309,7 @@ public void updateEntity(EntityInterface entity) { scriptTxt = getScriptWithParams(entity, doc); } else { SearchIndex elasticSearchIndex = searchIndexFactory.buildIndex(entityType, entity); - doc = elasticSearchIndex.buildESDoc(); + doc = elasticSearchIndex.buildSearchIndexDoc(); } searchClient.updateEntity( indexMapping.getIndexName(clusterAlias), entityId, doc, scriptTxt); @@ -669,6 +669,11 @@ public Response aggregate(String index, String fieldName, String value, String q return searchClient.aggregate(index, fieldName, value, query); } + public JsonObject aggregate(String query, String index, JsonObject aggregationJson) + throws IOException { + return searchClient.aggregate(query, index, aggregationJson); + } + public Response suggest(SearchRequest request) throws IOException { return searchClient.suggest(request); } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchClient.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchClient.java index 4d7bc8cb68c2..ed704139686b 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchClient.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchClient.java @@ -105,6 +105,7 @@ import java.util.TreeMap; import java.util.concurrent.TimeUnit; import java.util.stream.Stream; +import javax.json.JsonObject; import javax.net.ssl.SSLContext; import javax.ws.rs.core.Response; import lombok.extern.slf4j.Slf4j; @@ -704,6 +705,82 @@ public Response aggregate(String index, String fieldName, String value, String q return Response.status(OK).entity(response).build(); } + /* + Build dynamic aggregation from elasticsearch JSON like aggregation query. + See TestSuiteResourceTest for example usage (ln. 506) for tested aggregation query. + + @param aggregations - JsonObject containing the aggregation query + */ + public static List buildAggregation(JsonObject aggregations) { + List aggregationBuilders = new ArrayList<>(); + for (String key : aggregations.keySet()) { + JsonObject aggregation = aggregations.getJsonObject(key); + for (String aggregationType : aggregation.keySet()) { + switch (aggregationType) { + case "terms": + JsonObject termAggregation = aggregation.getJsonObject(aggregationType); + TermsAggregationBuilder termsAggregationBuilder = + AggregationBuilders.terms(key).field(termAggregation.getString("field")); + aggregationBuilders.add(termsAggregationBuilder); + break; + case "nested": + JsonObject nestedAggregation = aggregation.getJsonObject("nested"); + AggregationBuilder nestedAggregationBuilder = + AggregationBuilders.nested( + nestedAggregation.getString("path"), nestedAggregation.getString("path")); + JsonObject nestedAggregations = aggregation.getJsonObject("aggs"); + + List nestedAggregationBuilders = + buildAggregation(nestedAggregations); + for (AggregationBuilder nestedAggregationBuilder1 : nestedAggregationBuilders) { + nestedAggregationBuilder.subAggregation(nestedAggregationBuilder1); + } + aggregationBuilders.add(nestedAggregationBuilder); + break; + default: + break; + } + } + } + return aggregationBuilders; + } + + @Override + public JsonObject aggregate(String query, String index, JsonObject aggregationJson) + throws IOException { + JsonObject aggregations = aggregationJson.getJsonObject("aggregations"); + if (aggregations == null) { + return null; + } + + List aggregationBuilder = buildAggregation(aggregations); + es.org.elasticsearch.action.search.SearchRequest searchRequest = + new es.org.elasticsearch.action.search.SearchRequest( + Entity.getSearchRepository().getIndexOrAliasName(index)); + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + if (query != null) { + XContentParser queryParser = + XContentType.JSON + .xContent() + .createParser(xContentRegistry, LoggingDeprecationHandler.INSTANCE, query); + QueryBuilder parsedQuery = SearchSourceBuilder.fromXContent(queryParser).query(); + BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery().must(parsedQuery); + searchSourceBuilder.query(boolQueryBuilder); + } + + searchSourceBuilder.size(0).timeout(new TimeValue(30, TimeUnit.SECONDS)); + + for (AggregationBuilder aggregation : aggregationBuilder) { + searchSourceBuilder.aggregation(aggregation); + } + + searchRequest.source(searchSourceBuilder); + + String response = client.search(searchRequest, RequestOptions.DEFAULT).toString(); + JsonObject jsonResponse = JsonUtils.readJson(response).asJsonObject(); + return jsonResponse.getJsonObject("aggregations"); + } + private static ScriptScoreFunctionBuilder boostScore() { return ScoreFunctionBuilders.scriptFunction( "double score = _score;" @@ -819,6 +896,7 @@ private static SearchSourceBuilder buildDashboardSearchBuilder(String query, int .aggregation( AggregationBuilders.terms("dataModels.displayName.keyword") .field("dataModels.displayName.keyword")) + .aggregation(AggregationBuilders.terms("project.keyword").field("project.keyword")) .aggregation( AggregationBuilders.terms("charts.displayName.keyword") .field("charts.displayName.keyword")); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchDataInsightProcessor.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchDataInsightProcessor.java index c4fe74ee6ef7..d10afe6a18bd 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchDataInsightProcessor.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchDataInsightProcessor.java @@ -86,7 +86,8 @@ private UpdateRequest getUpdateRequest(String entityType, ReportData reportData) indexMapping.getIndexName(Entity.getSearchRepository().getClusterAlias()), reportData.getId().toString()); updateRequest.doc( - JsonUtils.pojoToJson(new ReportDataIndexes(reportData).buildESDoc()), XContentType.JSON); + JsonUtils.pojoToJson(new ReportDataIndexes(reportData).buildSearchIndexDoc()), + XContentType.JSON); updateRequest.docAsUpsert(true); return updateRequest; } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchEntitiesProcessor.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchEntitiesProcessor.java index 7d07972c8900..13313aeb034e 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchEntitiesProcessor.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchEntitiesProcessor.java @@ -89,7 +89,8 @@ public static UpdateRequest getUpdateRequest(String entityType, EntityInterface entity.getId().toString()); updateRequest.doc( JsonUtils.pojoToJson( - Objects.requireNonNull(Entity.buildSearchIndex(entityType, entity)).buildESDoc()), + Objects.requireNonNull(Entity.buildSearchIndex(entityType, entity)) + .buildSearchIndexDoc()), XContentType.JSON); updateRequest.docAsUpsert(true); return updateRequest; diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/AggregatedCostAnalysisReportDataIndex.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/AggregatedCostAnalysisReportDataIndex.java index 8341391d20fc..354105c19406 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/AggregatedCostAnalysisReportDataIndex.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/AggregatedCostAnalysisReportDataIndex.java @@ -2,13 +2,16 @@ import java.util.Map; import org.openmetadata.schema.analytics.ReportData; -import org.openmetadata.service.util.JsonUtils; public record AggregatedCostAnalysisReportDataIndex(ReportData reportData) implements SearchIndex { @Override - public Map buildESDoc() { - Map doc = JsonUtils.getMap(reportData); + public Object getEntity() { + return reportData; + } + + @Override + public Map buildSearchIndexDocInternal(Map doc) { doc.put("entityType", "aggregatedCostAnalysisReportData"); return doc; } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/ChartIndex.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/ChartIndex.java index fca43e42386e..721f57e4b83a 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/ChartIndex.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/ChartIndex.java @@ -8,14 +8,11 @@ import org.openmetadata.service.Entity; import org.openmetadata.service.search.SearchIndexUtils; import org.openmetadata.service.search.models.SearchSuggest; -import org.openmetadata.service.util.JsonUtils; public record ChartIndex(Chart chart) implements SearchIndex { - private static final List excludeFields = List.of("changeDescription"); - public Map buildESDoc() { - Map doc = JsonUtils.getMap(chart); - SearchIndexUtils.removeNonIndexableFields(doc, excludeFields); + @Override + public Map buildSearchIndexDocInternal(Map doc) { List suggest = new ArrayList<>(); suggest.add(SearchSuggest.builder().input(chart.getName()).weight(10).build()); suggest.add(SearchSuggest.builder().input(chart.getFullyQualifiedName()).weight(5).build()); @@ -27,6 +24,7 @@ public Map buildESDoc() { doc.put("entityType", Entity.CHART); doc.put("owner", getEntityWithDisplayName(chart.getOwner())); doc.put("domain", getEntityWithDisplayName(chart.getDomain())); + doc.put("followers", SearchIndexUtils.parseFollowers(chart.getFollowers())); doc.put( "totalVotes", CommonUtil.nullOrEmpty(chart.getVotes()) @@ -34,4 +32,9 @@ public Map buildESDoc() { : chart.getVotes().getUpVotes() - chart.getVotes().getDownVotes()); return doc; } + + @Override + public Object getEntity() { + return chart; + } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/ClassificationIndex.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/ClassificationIndex.java index ce6a80712929..b2215af369ef 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/ClassificationIndex.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/ClassificationIndex.java @@ -3,19 +3,15 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; -import java.util.stream.Collectors; import org.openmetadata.schema.entity.classification.Classification; import org.openmetadata.service.Entity; import org.openmetadata.service.search.SearchIndexUtils; import org.openmetadata.service.search.models.SearchSuggest; -import org.openmetadata.service.util.JsonUtils; public record ClassificationIndex(Classification classification) implements SearchIndex { - private static final List excludeFields = List.of("changeDescription"); - public Map buildESDoc() { - Map doc = JsonUtils.getMap(classification); - SearchIndexUtils.removeNonIndexableFields(doc, excludeFields); + @Override + public Map buildSearchIndexDocInternal(Map doc) { List suggest = new ArrayList<>(); suggest.add(SearchSuggest.builder().input(classification.getName()).weight(10).build()); suggest.add( @@ -24,10 +20,16 @@ public Map buildESDoc() { "fqnParts", getFQNParts( classification.getFullyQualifiedName(), - suggest.stream().map(SearchSuggest::getInput).collect(Collectors.toList()))); + suggest.stream().map(SearchSuggest::getInput).toList())); doc.put("suggest", suggest); doc.put("entityType", Entity.CLASSIFICATION); doc.put("owner", getEntityWithDisplayName(classification.getOwner())); + doc.put("followers", SearchIndexUtils.parseFollowers(classification.getFollowers())); return doc; } + + @Override + public Object getEntity() { + return classification; + } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/ContainerIndex.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/ContainerIndex.java index 97b200caf9dc..a3694ea8655a 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/ContainerIndex.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/ContainerIndex.java @@ -15,19 +15,21 @@ import org.openmetadata.service.search.SearchIndexUtils; import org.openmetadata.service.search.models.FlattenColumn; import org.openmetadata.service.search.models.SearchSuggest; -import org.openmetadata.service.util.JsonUtils; public record ContainerIndex(Container container) implements ColumnIndex { - private static final List excludeFields = List.of("changeDescription"); - public Map buildESDoc() { - Map doc = JsonUtils.getMap(container); + @Override + public Object getEntity() { + return container; + } + + @Override + public Map buildSearchIndexDocInternal(Map doc) { List suggest = new ArrayList<>(); List columnSuggest = new ArrayList<>(); List serviceSuggest = new ArrayList<>(); Set> tagsWithChildren = new HashSet<>(); List columnsWithChildrenName = new ArrayList<>(); - SearchIndexUtils.removeNonIndexableFields(doc, excludeFields); suggest.add(SearchSuggest.builder().input(container.getFullyQualifiedName()).weight(5).build()); suggest.add(SearchSuggest.builder().input(container.getName()).weight(10).build()); if (container.getDataModel() != null && container.getDataModel().getColumns() != null) { diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/DashboardDataModelIndex.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/DashboardDataModelIndex.java index 21f136c0602c..538688f89b64 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/DashboardDataModelIndex.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/DashboardDataModelIndex.java @@ -15,15 +15,17 @@ import org.openmetadata.service.search.SearchIndexUtils; import org.openmetadata.service.search.models.FlattenColumn; import org.openmetadata.service.search.models.SearchSuggest; -import org.openmetadata.service.util.JsonUtils; public record DashboardDataModelIndex(DashboardDataModel dashboardDataModel) implements ColumnIndex { - private static final List excludeFields = List.of("changeDescription"); - public Map buildESDoc() { - Map doc = JsonUtils.getMap(dashboardDataModel); - SearchIndexUtils.removeNonIndexableFields(doc, excludeFields); + @Override + public Object getEntity() { + return dashboardDataModel; + } + + @Override + public Map buildSearchIndexDocInternal(Map doc) { List suggest = new ArrayList<>(); List columnSuggest = new ArrayList<>(); suggest.add(SearchSuggest.builder().input(dashboardDataModel.getName()).weight(10).build()); @@ -34,7 +36,6 @@ public Map buildESDoc() { .build()); Set> tagsWithChildren = new HashSet<>(); List columnsWithChildrenName = new ArrayList<>(); - SearchIndexUtils.removeNonIndexableFields(doc, excludeFields); if (dashboardDataModel.getColumns() != null) { List cols = new ArrayList<>(); parseColumns(dashboardDataModel.getColumns(), cols, null); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/DashboardIndex.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/DashboardIndex.java index 635767c1c675..5ce774c99651 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/DashboardIndex.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/DashboardIndex.java @@ -12,19 +12,21 @@ import org.openmetadata.service.search.ParseTags; import org.openmetadata.service.search.SearchIndexUtils; import org.openmetadata.service.search.models.SearchSuggest; -import org.openmetadata.service.util.JsonUtils; public class DashboardIndex implements SearchIndex { final Dashboard dashboard; - final List excludeFields = List.of("changeDescription"); public DashboardIndex(Dashboard dashboard) { this.dashboard = dashboard; } - public Map buildESDoc() { - Map doc = JsonUtils.getMap(dashboard); - SearchIndexUtils.removeNonIndexableFields(doc, excludeFields); + @Override + public Object getEntity() { + return dashboard; + } + + @Override + public Map buildSearchIndexDocInternal(Map doc) { List suggest = new ArrayList<>(); List serviceSuggest = new ArrayList<>(); List chartSuggest = new ArrayList<>(); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/DashboardServiceIndex.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/DashboardServiceIndex.java index 3c0e0729da51..cdabefd7d96a 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/DashboardServiceIndex.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/DashboardServiceIndex.java @@ -7,15 +7,16 @@ import org.openmetadata.service.Entity; import org.openmetadata.service.search.SearchIndexUtils; import org.openmetadata.service.search.models.SearchSuggest; -import org.openmetadata.service.util.JsonUtils; public record DashboardServiceIndex(DashboardService dashboardService) implements SearchIndex { - private static final List excludeFields = List.of("changeDescription"); + @Override + public Object getEntity() { + return dashboardService; + } - public Map buildESDoc() { - Map doc = JsonUtils.getMap(dashboardService); - SearchIndexUtils.removeNonIndexableFields(doc, excludeFields); + @Override + public Map buildSearchIndexDocInternal(Map doc) { List suggest = new ArrayList<>(); suggest.add(SearchSuggest.builder().input(dashboardService.getName()).weight(5).build()); suggest.add( @@ -29,6 +30,7 @@ public Map buildESDoc() { doc.put("entityType", Entity.DASHBOARD_SERVICE); doc.put("owner", getEntityWithDisplayName(dashboardService.getOwner())); doc.put("domain", getEntityWithDisplayName(dashboardService.getDomain())); + doc.put("followers", SearchIndexUtils.parseFollowers(dashboardService.getFollowers())); return doc; } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/DataProductIndex.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/DataProductIndex.java index 044e7aeeab72..310d7ca3907e 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/DataProductIndex.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/DataProductIndex.java @@ -7,14 +7,16 @@ import org.openmetadata.service.Entity; import org.openmetadata.service.search.SearchIndexUtils; import org.openmetadata.service.search.models.SearchSuggest; -import org.openmetadata.service.util.JsonUtils; public record DataProductIndex(DataProduct dataProduct) implements SearchIndex { - private static final List excludeFields = List.of("changeDescription"); - public Map buildESDoc() { - Map doc = JsonUtils.getMap(dataProduct); - SearchIndexUtils.removeNonIndexableFields(doc, excludeFields); + @Override + public Object getEntity() { + return dataProduct; + } + + @Override + public Map buildSearchIndexDocInternal(Map doc) { List suggest = new ArrayList<>(); suggest.add(SearchSuggest.builder().input(dataProduct.getName()).weight(5).build()); suggest.add( @@ -27,6 +29,7 @@ public Map buildESDoc() { doc.put("entityType", Entity.DATA_PRODUCT); doc.put("owner", getEntityWithDisplayName(dataProduct.getOwner())); doc.put("domain", getEntityWithDisplayName(dataProduct.getDomain())); + doc.put("followers", SearchIndexUtils.parseFollowers(dataProduct.getFollowers())); return doc; } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/DatabaseIndex.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/DatabaseIndex.java index 5c8623c111d0..0ed613b05493 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/DatabaseIndex.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/DatabaseIndex.java @@ -8,14 +8,16 @@ import org.openmetadata.service.Entity; import org.openmetadata.service.search.SearchIndexUtils; import org.openmetadata.service.search.models.SearchSuggest; -import org.openmetadata.service.util.JsonUtils; public record DatabaseIndex(Database database) implements SearchIndex { - private static final List excludeFields = List.of("changeDescription"); - public Map buildESDoc() { - Map doc = JsonUtils.getMap(database); - SearchIndexUtils.removeNonIndexableFields(doc, excludeFields); + @Override + public Object getEntity() { + return database; + } + + @Override + public Map buildSearchIndexDocInternal(Map doc) { List suggest = new ArrayList<>(); suggest.add(SearchSuggest.builder().input(database.getName()).weight(5).build()); suggest.add(SearchSuggest.builder().input(database.getFullyQualifiedName()).weight(5).build()); @@ -33,6 +35,7 @@ public Map buildESDoc() { : database.getVotes().getUpVotes() - database.getVotes().getDownVotes()); doc.put("owner", getEntityWithDisplayName(database.getOwner())); doc.put("domain", getEntityWithDisplayName(database.getDomain())); + doc.put("followers", SearchIndexUtils.parseFollowers(database.getFollowers())); return doc; } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/DatabaseSchemaIndex.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/DatabaseSchemaIndex.java index 0dde36a0546b..39984999f845 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/DatabaseSchemaIndex.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/DatabaseSchemaIndex.java @@ -8,14 +8,16 @@ import org.openmetadata.service.Entity; import org.openmetadata.service.search.SearchIndexUtils; import org.openmetadata.service.search.models.SearchSuggest; -import org.openmetadata.service.util.JsonUtils; public record DatabaseSchemaIndex(DatabaseSchema databaseSchema) implements SearchIndex { - private static final List excludeFields = List.of("changeDescription"); - public Map buildESDoc() { - Map doc = JsonUtils.getMap(databaseSchema); - SearchIndexUtils.removeNonIndexableFields(doc, excludeFields); + @Override + public Object getEntity() { + return databaseSchema; + } + + @Override + public Map buildSearchIndexDocInternal(Map doc) { List suggest = new ArrayList<>(); suggest.add(SearchSuggest.builder().input(databaseSchema.getName()).weight(5).build()); suggest.add( @@ -34,6 +36,7 @@ public Map buildESDoc() { ? 0 : databaseSchema.getVotes().getUpVotes() - databaseSchema.getVotes().getDownVotes()); doc.put("domain", getEntityWithDisplayName(databaseSchema.getDomain())); + doc.put("followers", SearchIndexUtils.parseFollowers(databaseSchema.getFollowers())); return doc; } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/DatabaseServiceIndex.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/DatabaseServiceIndex.java index 428d5b7e7405..7297d5f4ec13 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/DatabaseServiceIndex.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/DatabaseServiceIndex.java @@ -3,19 +3,20 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; -import java.util.stream.Collectors; import org.openmetadata.schema.entity.services.DatabaseService; import org.openmetadata.service.Entity; import org.openmetadata.service.search.SearchIndexUtils; import org.openmetadata.service.search.models.SearchSuggest; -import org.openmetadata.service.util.JsonUtils; public record DatabaseServiceIndex(DatabaseService databaseService) implements SearchIndex { - private static final List excludeFields = List.of("changeDescription"); - public Map buildESDoc() { - Map doc = JsonUtils.getMap(databaseService); - SearchIndexUtils.removeNonIndexableFields(doc, excludeFields); + @Override + public Object getEntity() { + return databaseService; + } + + @Override + public Map buildSearchIndexDocInternal(Map doc) { List suggest = new ArrayList<>(); suggest.add(SearchSuggest.builder().input(databaseService.getName()).weight(5).build()); suggest.add( @@ -24,11 +25,12 @@ public Map buildESDoc() { "fqnParts", getFQNParts( databaseService.getFullyQualifiedName(), - suggest.stream().map(SearchSuggest::getInput).collect(Collectors.toList()))); + suggest.stream().map(SearchSuggest::getInput).toList())); doc.put("suggest", suggest); doc.put("entityType", Entity.DATABASE_SERVICE); doc.put("owner", getEntityWithDisplayName(databaseService.getOwner())); doc.put("domain", getEntityWithDisplayName(databaseService.getDomain())); + doc.put("followers", SearchIndexUtils.parseFollowers(databaseService.getFollowers())); return doc; } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/DomainIndex.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/DomainIndex.java index 88ce31931988..bac0c7e6c616 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/DomainIndex.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/DomainIndex.java @@ -7,14 +7,16 @@ import org.openmetadata.service.Entity; import org.openmetadata.service.search.SearchIndexUtils; import org.openmetadata.service.search.models.SearchSuggest; -import org.openmetadata.service.util.JsonUtils; public record DomainIndex(Domain domain) implements SearchIndex { - private static final List excludeFields = List.of("changeDescription"); - public Map buildESDoc() { - Map doc = JsonUtils.getMap(domain); - SearchIndexUtils.removeNonIndexableFields(doc, excludeFields); + @Override + public Object getEntity() { + return domain; + } + + @Override + public Map buildSearchIndexDocInternal(Map doc) { List suggest = new ArrayList<>(); suggest.add(SearchSuggest.builder().input(domain.getName()).weight(5).build()); suggest.add(SearchSuggest.builder().input(domain.getFullyQualifiedName()).weight(5).build()); @@ -25,6 +27,7 @@ public Map buildESDoc() { suggest.stream().map(SearchSuggest::getInput).toList())); doc.put("suggest", suggest); doc.put("entityType", Entity.DOMAIN); + doc.put("followers", SearchIndexUtils.parseFollowers(domain.getFollowers())); return doc; } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/EntityReportDataIndex.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/EntityReportDataIndex.java index 857323ee299d..384c4479497a 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/EntityReportDataIndex.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/EntityReportDataIndex.java @@ -2,13 +2,16 @@ import java.util.Map; import org.openmetadata.schema.analytics.ReportData; -import org.openmetadata.service.util.JsonUtils; public record EntityReportDataIndex(ReportData reportData) implements SearchIndex { @Override - public Map buildESDoc() { - Map doc = JsonUtils.getMap(reportData); + public Object getEntity() { + return reportData; + } + + @Override + public Map buildSearchIndexDocInternal(Map doc) { doc.put("entityType", "entityReportData"); return doc; } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/GlossaryIndex.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/GlossaryIndex.java index 960c05cf73ad..b773c34bffa6 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/GlossaryIndex.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/GlossaryIndex.java @@ -8,19 +8,21 @@ import org.openmetadata.service.Entity; import org.openmetadata.service.search.SearchIndexUtils; import org.openmetadata.service.search.models.SearchSuggest; -import org.openmetadata.service.util.JsonUtils; public class GlossaryIndex implements SearchIndex { final Glossary glossary; - final List excludeFields = List.of("changeDescription"); public GlossaryIndex(Glossary glossary) { this.glossary = glossary; } - public Map buildESDoc() { - Map doc = JsonUtils.getMap(glossary); - SearchIndexUtils.removeNonIndexableFields(doc, excludeFields); + @Override + public Object getEntity() { + return glossary; + } + + @Override + public Map buildSearchIndexDocInternal(Map doc) { List suggest = new ArrayList<>(); suggest.add(SearchSuggest.builder().input(glossary.getName()).weight(5).build()); if (glossary.getDisplayName() != null && !glossary.getDisplayName().isEmpty()) { @@ -40,6 +42,7 @@ public Map buildESDoc() { ? 0 : glossary.getVotes().getUpVotes() - glossary.getVotes().getDownVotes()); doc.put("domain", getEntityWithDisplayName(glossary.getDomain())); + doc.put("followers", SearchIndexUtils.parseFollowers(glossary.getFollowers())); return doc; } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/GlossaryTermIndex.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/GlossaryTermIndex.java index 244c2eeb5c89..7a1d48dc1c50 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/GlossaryTermIndex.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/GlossaryTermIndex.java @@ -8,19 +8,21 @@ import org.openmetadata.service.Entity; import org.openmetadata.service.search.SearchIndexUtils; import org.openmetadata.service.search.models.SearchSuggest; -import org.openmetadata.service.util.JsonUtils; public class GlossaryTermIndex implements SearchIndex { final GlossaryTerm glossaryTerm; - final List excludeFields = List.of("changeDescription"); public GlossaryTermIndex(GlossaryTerm glossaryTerm) { this.glossaryTerm = glossaryTerm; } - public Map buildESDoc() { - Map doc = JsonUtils.getMap(glossaryTerm); - SearchIndexUtils.removeNonIndexableFields(doc, excludeFields); + @Override + public Object getEntity() { + return glossaryTerm; + } + + @Override + public Map buildSearchIndexDocInternal(Map doc) { List suggest = new ArrayList<>(); suggest.add(SearchSuggest.builder().input(glossaryTerm.getName()).weight(5).build()); if (glossaryTerm.getDisplayName() != null && !glossaryTerm.getDisplayName().isEmpty()) { @@ -40,6 +42,7 @@ public Map buildESDoc() { : glossaryTerm.getVotes().getUpVotes() - glossaryTerm.getVotes().getDownVotes()); doc.put("owner", getEntityWithDisplayName(glossaryTerm.getOwner())); doc.put("domain", getEntityWithDisplayName(glossaryTerm.getDomain())); + doc.put("followers", SearchIndexUtils.parseFollowers(glossaryTerm.getFollowers())); return doc; } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/IngestionPipelineIndex.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/IngestionPipelineIndex.java index 2432cfe82714..c69a40962f9d 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/IngestionPipelineIndex.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/IngestionPipelineIndex.java @@ -3,26 +3,34 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.Set; import org.openmetadata.common.utils.CommonUtil; import org.openmetadata.schema.entity.services.ingestionPipelines.IngestionPipeline; import org.openmetadata.service.Entity; import org.openmetadata.service.search.ParseTags; import org.openmetadata.service.search.SearchIndexUtils; import org.openmetadata.service.search.models.SearchSuggest; -import org.openmetadata.service.util.JsonUtils; public class IngestionPipelineIndex implements SearchIndex { final IngestionPipeline ingestionPipeline; - final List excludeFields = - List.of("changeDescription", "sourceConfig", "openMetadataServerConnection", "airflowConfig"); + final Set excludeFields = + Set.of("sourceConfig", "openMetadataServerConnection", "airflowConfig"); public IngestionPipelineIndex(IngestionPipeline ingestionPipeline) { this.ingestionPipeline = ingestionPipeline; } - public Map buildESDoc() { - Map doc = JsonUtils.getMap(ingestionPipeline); - SearchIndexUtils.removeNonIndexableFields(doc, excludeFields); + @Override + public Object getEntity() { + return ingestionPipeline; + } + + public Set getExcludedFields() { + return excludeFields; + } + + @Override + public Map buildSearchIndexDocInternal(Map doc) { List suggest = new ArrayList<>(); List serviceSuggest = new ArrayList<>(); suggest.add( diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/MessagingServiceIndex.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/MessagingServiceIndex.java index 11f26bb28341..adcc4c360f51 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/MessagingServiceIndex.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/MessagingServiceIndex.java @@ -3,19 +3,20 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; -import java.util.stream.Collectors; import org.openmetadata.schema.entity.services.MessagingService; import org.openmetadata.service.Entity; import org.openmetadata.service.search.SearchIndexUtils; import org.openmetadata.service.search.models.SearchSuggest; -import org.openmetadata.service.util.JsonUtils; public record MessagingServiceIndex(MessagingService messagingService) implements SearchIndex { - private static final List excludeFields = List.of("changeDescription"); - public Map buildESDoc() { - Map doc = JsonUtils.getMap(messagingService); - SearchIndexUtils.removeNonIndexableFields(doc, excludeFields); + @Override + public Object getEntity() { + return messagingService; + } + + @Override + public Map buildSearchIndexDocInternal(Map doc) { List suggest = new ArrayList<>(); suggest.add(SearchSuggest.builder().input(messagingService.getName()).weight(5).build()); suggest.add( @@ -24,11 +25,12 @@ public Map buildESDoc() { "fqnParts", getFQNParts( messagingService.getFullyQualifiedName(), - suggest.stream().map(SearchSuggest::getInput).collect(Collectors.toList()))); + suggest.stream().map(SearchSuggest::getInput).toList())); doc.put("suggest", suggest); doc.put("entityType", Entity.MESSAGING_SERVICE); doc.put("owner", getEntityWithDisplayName(messagingService.getOwner())); doc.put("domain", getEntityWithDisplayName(messagingService.getDomain())); + doc.put("followers", SearchIndexUtils.parseFollowers(messagingService.getFollowers())); return doc; } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/MetadataServiceIndex.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/MetadataServiceIndex.java index 817e7c4bd865..7c3042d82831 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/MetadataServiceIndex.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/MetadataServiceIndex.java @@ -3,19 +3,20 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; -import java.util.stream.Collectors; import org.openmetadata.schema.entity.services.MetadataService; import org.openmetadata.service.Entity; import org.openmetadata.service.search.SearchIndexUtils; import org.openmetadata.service.search.models.SearchSuggest; -import org.openmetadata.service.util.JsonUtils; public record MetadataServiceIndex(MetadataService metadataService) implements SearchIndex { - private static final List excludeFields = List.of("changeDescription"); - public Map buildESDoc() { - Map doc = JsonUtils.getMap(metadataService); - SearchIndexUtils.removeNonIndexableFields(doc, excludeFields); + @Override + public Object getEntity() { + return metadataService; + } + + @Override + public Map buildSearchIndexDocInternal(Map doc) { List suggest = new ArrayList<>(); suggest.add(SearchSuggest.builder().input(metadataService.getName()).weight(5).build()); suggest.add( @@ -24,10 +25,11 @@ public Map buildESDoc() { "fqnParts", getFQNParts( metadataService.getFullyQualifiedName(), - suggest.stream().map(SearchSuggest::getInput).collect(Collectors.toList()))); + suggest.stream().map(SearchSuggest::getInput).toList())); doc.put("suggest", suggest); doc.put("entityType", Entity.METADATA_SERVICE); doc.put("owner", getEntityWithDisplayName(metadataService.getOwner())); + doc.put("followers", SearchIndexUtils.parseFollowers(metadataService.getFollowers())); return doc; } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/MlModelIndex.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/MlModelIndex.java index f9d890e51aff..095d57af6c3f 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/MlModelIndex.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/MlModelIndex.java @@ -9,20 +9,22 @@ import org.openmetadata.service.search.ParseTags; import org.openmetadata.service.search.SearchIndexUtils; import org.openmetadata.service.search.models.SearchSuggest; -import org.openmetadata.service.util.JsonUtils; public class MlModelIndex implements SearchIndex { final MlModel mlModel; - final List excludeFields = List.of("changeDescription"); public MlModelIndex(MlModel mlModel) { this.mlModel = mlModel; } - public Map buildESDoc() { - Map doc = JsonUtils.getMap(mlModel); + @Override + public Object getEntity() { + return mlModel; + } + + @Override + public Map buildSearchIndexDocInternal(Map doc) { List suggest = new ArrayList<>(); - SearchIndexUtils.removeNonIndexableFields(doc, excludeFields); suggest.add(SearchSuggest.builder().input(mlModel.getFullyQualifiedName()).weight(5).build()); suggest.add(SearchSuggest.builder().input(mlModel.getName()).weight(10).build()); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/MlModelServiceIndex.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/MlModelServiceIndex.java index 71db73140d27..d618de74f18f 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/MlModelServiceIndex.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/MlModelServiceIndex.java @@ -3,19 +3,20 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; -import java.util.stream.Collectors; import org.openmetadata.schema.entity.services.MlModelService; import org.openmetadata.service.Entity; import org.openmetadata.service.search.SearchIndexUtils; import org.openmetadata.service.search.models.SearchSuggest; -import org.openmetadata.service.util.JsonUtils; public record MlModelServiceIndex(MlModelService mlModelService) implements SearchIndex { - private static final List excludeFields = List.of("changeDescription"); - public Map buildESDoc() { - Map doc = JsonUtils.getMap(mlModelService); - SearchIndexUtils.removeNonIndexableFields(doc, excludeFields); + @Override + public Object getEntity() { + return mlModelService; + } + + @Override + public Map buildSearchIndexDocInternal(Map doc) { List suggest = new ArrayList<>(); suggest.add(SearchSuggest.builder().input(mlModelService.getName()).weight(5).build()); suggest.add( @@ -24,10 +25,11 @@ public Map buildESDoc() { "fqnParts", getFQNParts( mlModelService.getFullyQualifiedName(), - suggest.stream().map(SearchSuggest::getInput).collect(Collectors.toList()))); + suggest.stream().map(SearchSuggest::getInput).toList())); doc.put("suggest", suggest); doc.put("entityType", Entity.MLMODEL_SERVICE); doc.put("owner", getEntityWithDisplayName(mlModelService.getOwner())); + doc.put("followers", SearchIndexUtils.parseFollowers(mlModelService.getFollowers())); return doc; } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/PipelineIndex.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/PipelineIndex.java index 7d44114491e5..0ac6c5521e53 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/PipelineIndex.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/PipelineIndex.java @@ -10,19 +10,21 @@ import org.openmetadata.service.search.ParseTags; import org.openmetadata.service.search.SearchIndexUtils; import org.openmetadata.service.search.models.SearchSuggest; -import org.openmetadata.service.util.JsonUtils; public class PipelineIndex implements SearchIndex { final Pipeline pipeline; - final List excludeFields = List.of("changeDescription"); public PipelineIndex(Pipeline pipeline) { this.pipeline = pipeline; } - public Map buildESDoc() { - Map doc = JsonUtils.getMap(pipeline); - SearchIndexUtils.removeNonIndexableFields(doc, excludeFields); + @Override + public Object getEntity() { + return pipeline; + } + + @Override + public Map buildSearchIndexDocInternal(Map doc) { List suggest = new ArrayList<>(); List serviceSuggest = new ArrayList<>(); List taskSuggest = new ArrayList<>(); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/PipelineServiceIndex.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/PipelineServiceIndex.java index 856bc9e444cf..0c84ca16a21d 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/PipelineServiceIndex.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/PipelineServiceIndex.java @@ -3,19 +3,20 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; -import java.util.stream.Collectors; import org.openmetadata.schema.entity.services.PipelineService; import org.openmetadata.service.Entity; import org.openmetadata.service.search.SearchIndexUtils; import org.openmetadata.service.search.models.SearchSuggest; -import org.openmetadata.service.util.JsonUtils; public record PipelineServiceIndex(PipelineService pipelineService) implements SearchIndex { - private static final List excludeFields = List.of("changeDescription"); - public Map buildESDoc() { - Map doc = JsonUtils.getMap(pipelineService); - SearchIndexUtils.removeNonIndexableFields(doc, excludeFields); + @Override + public Object getEntity() { + return pipelineService; + } + + @Override + public Map buildSearchIndexDocInternal(Map doc) { List suggest = new ArrayList<>(); suggest.add(SearchSuggest.builder().input(pipelineService.getName()).weight(5).build()); suggest.add( @@ -26,8 +27,9 @@ public Map buildESDoc() { "fqnParts", getFQNParts( pipelineService.getFullyQualifiedName(), - suggest.stream().map(SearchSuggest::getInput).collect(Collectors.toList()))); + suggest.stream().map(SearchSuggest::getInput).toList())); doc.put("owner", getEntityWithDisplayName(pipelineService.getOwner())); + doc.put("followers", SearchIndexUtils.parseFollowers(pipelineService.getFollowers())); return doc; } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/QueryIndex.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/QueryIndex.java index e19964a21fcc..8bf73d06b681 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/QueryIndex.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/QueryIndex.java @@ -12,24 +12,25 @@ import org.openmetadata.service.search.ParseTags; import org.openmetadata.service.search.SearchIndexUtils; import org.openmetadata.service.search.models.SearchSuggest; -import org.openmetadata.service.util.JsonUtils; public class QueryIndex implements SearchIndex { - final List excludeTopicFields = List.of("changeDescription"); final Query query; public QueryIndex(Query query) { this.query = query; } - public Map buildESDoc() { - Map doc = JsonUtils.getMap(query); + @Override + public Object getEntity() { + return query; + } + + @Override + public Map buildSearchIndexDocInternal(Map doc) { List suggest = new ArrayList<>(); if (query.getDisplayName() != null) { suggest.add(SearchSuggest.builder().input(query.getName()).weight(10).build()); } - SearchIndexUtils.removeNonIndexableFields(doc, excludeTopicFields); - ParseTags parseTags = new ParseTags(Entity.getEntityTags(Entity.QUERY, query)); doc.put("displayName", query.getDisplayName() != null ? query.getDisplayName() : ""); doc.put("tags", parseTags.getTags()); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/RawCostAnalysisReportDataIndex.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/RawCostAnalysisReportDataIndex.java index 440ef9e6590d..3af6cc7df4b9 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/RawCostAnalysisReportDataIndex.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/RawCostAnalysisReportDataIndex.java @@ -2,12 +2,16 @@ import java.util.Map; import org.openmetadata.schema.analytics.ReportData; -import org.openmetadata.service.util.JsonUtils; public record RawCostAnalysisReportDataIndex(ReportData reportData) implements SearchIndex { + + @Override + public Object getEntity() { + return reportData; + } + @Override - public Map buildESDoc() { - Map doc = JsonUtils.getMap(reportData); + public Map buildSearchIndexDocInternal(Map doc) { doc.put("entityType", "rawCostAnalysisReportData"); return doc; } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/ReportDataIndexes.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/ReportDataIndexes.java index 3ae8e1197605..8345a3d8d8e8 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/ReportDataIndexes.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/ReportDataIndexes.java @@ -2,12 +2,16 @@ import java.util.Map; import org.openmetadata.schema.analytics.ReportData; -import org.openmetadata.service.util.JsonUtils; public record ReportDataIndexes(ReportData reportData) implements SearchIndex { + + @Override + public Object getEntity() { + return reportData; + } + @Override - public Map buildESDoc() { - Map doc = JsonUtils.getMap(reportData); + public Map buildSearchIndexDocInternal(Map doc) { doc.put("id", null); doc.put("timestamp", reportData.getTimestamp()); doc.put("reportDataType", reportData.getReportDataType()); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/SearchEntityIndex.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/SearchEntityIndex.java index 06261885d8ed..f29d96fa50b2 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/SearchEntityIndex.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/SearchEntityIndex.java @@ -8,15 +8,17 @@ import org.openmetadata.service.search.ParseTags; import org.openmetadata.service.search.SearchIndexUtils; import org.openmetadata.service.search.models.SearchSuggest; -import org.openmetadata.service.util.JsonUtils; public record SearchEntityIndex(org.openmetadata.schema.entity.data.SearchIndex searchIndex) implements SearchIndex { - private static final List excludeFields = List.of("changeDescription"); - public Map buildESDoc() { - Map doc = JsonUtils.getMap(searchIndex); - SearchIndexUtils.removeNonIndexableFields(doc, excludeFields); + @Override + public Object getEntity() { + return searchIndex; + } + + @Override + public Map buildSearchIndexDocInternal(Map doc) { List suggest = new ArrayList<>(); suggest.add(SearchSuggest.builder().input(searchIndex.getName()).weight(5).build()); suggest.add( diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/SearchIndex.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/SearchIndex.java index a20174ee9b32..abe104404598 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/SearchIndex.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/SearchIndex.java @@ -11,6 +11,7 @@ import static org.openmetadata.service.search.EntityBuilderConstant.NAME_KEYWORD; import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -24,11 +25,33 @@ import org.openmetadata.schema.type.Relationship; import org.openmetadata.service.Entity; import org.openmetadata.service.jdbi3.CollectionDAO; +import org.openmetadata.service.search.SearchIndexUtils; import org.openmetadata.service.util.FullyQualifiedName; import org.openmetadata.service.util.JsonUtils; public interface SearchIndex { - Map buildESDoc(); + Set DEFAULT_EXCLUDED_FIELDS = Set.of("changeDescription"); + + default Map buildSearchIndexDoc() { + Map esDoc = JsonUtils.getMap(getEntity()); + + // Remove non indexable fields + SearchIndexUtils.removeNonIndexableFields(esDoc, DEFAULT_EXCLUDED_FIELDS); + + // Remove Entity Specific Field + SearchIndexUtils.removeNonIndexableFields(esDoc, getExcludedFields()); + + // Build Index Doc + return this.buildSearchIndexDocInternal(esDoc); + } + + Object getEntity(); + + default Set getExcludedFields() { + return Collections.emptySet(); + } + + Map buildSearchIndexDocInternal(Map esDoc); default Set getFQNParts(String fqn, List fqnSplits) { Set fqnParts = new HashSet<>(); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/SearchServiceIndex.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/SearchServiceIndex.java index 98515c994508..5c994b42eafd 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/SearchServiceIndex.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/SearchServiceIndex.java @@ -7,14 +7,16 @@ import org.openmetadata.service.Entity; import org.openmetadata.service.search.SearchIndexUtils; import org.openmetadata.service.search.models.SearchSuggest; -import org.openmetadata.service.util.JsonUtils; public record SearchServiceIndex(SearchService searchService) implements SearchIndex { - private static final List excludeFields = List.of("changeDescription"); - public Map buildESDoc() { - Map doc = JsonUtils.getMap(searchService); - SearchIndexUtils.removeNonIndexableFields(doc, excludeFields); + @Override + public Object getEntity() { + return searchService; + } + + @Override + public Map buildSearchIndexDocInternal(Map doc) { List suggest = new ArrayList<>(); suggest.add(SearchSuggest.builder().input(searchService.getName()).weight(5).build()); suggest.add( @@ -27,6 +29,7 @@ public Map buildESDoc() { searchService.getFullyQualifiedName(), suggest.stream().map(SearchSuggest::getInput).toList())); doc.put("owner", getEntityWithDisplayName(searchService.getOwner())); + doc.put("followers", SearchIndexUtils.parseFollowers(searchService.getFollowers())); return doc; } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/StorageServiceIndex.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/StorageServiceIndex.java index cdb682a90bce..f34ac67c5f9b 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/StorageServiceIndex.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/StorageServiceIndex.java @@ -7,14 +7,16 @@ import org.openmetadata.service.Entity; import org.openmetadata.service.search.SearchIndexUtils; import org.openmetadata.service.search.models.SearchSuggest; -import org.openmetadata.service.util.JsonUtils; public record StorageServiceIndex(StorageService storageService) implements SearchIndex { - private static final List excludeFields = List.of("changeDescription"); - public Map buildESDoc() { - Map doc = JsonUtils.getMap(storageService); - SearchIndexUtils.removeNonIndexableFields(doc, excludeFields); + @Override + public Object getEntity() { + return storageService; + } + + @Override + public Map buildSearchIndexDocInternal(Map doc) { List suggest = new ArrayList<>(); suggest.add(SearchSuggest.builder().input(storageService.getName()).weight(5).build()); suggest.add( @@ -27,6 +29,7 @@ public Map buildESDoc() { doc.put("suggest", suggest); doc.put("entityType", Entity.STORAGE_SERVICE); doc.put("owner", getEntityWithDisplayName(storageService.getOwner())); + doc.put("followers", SearchIndexUtils.parseFollowers(storageService.getFollowers())); return doc; } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/StoredProcedureIndex.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/StoredProcedureIndex.java index cd1910e20e31..0e5c8e1724e3 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/StoredProcedureIndex.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/StoredProcedureIndex.java @@ -9,14 +9,16 @@ import org.openmetadata.service.search.ParseTags; import org.openmetadata.service.search.SearchIndexUtils; import org.openmetadata.service.search.models.SearchSuggest; -import org.openmetadata.service.util.JsonUtils; public record StoredProcedureIndex(StoredProcedure storedProcedure) implements SearchIndex { - private static final List excludeFields = List.of("changeDescription"); - public Map buildESDoc() { - Map doc = JsonUtils.getMap(storedProcedure); - SearchIndexUtils.removeNonIndexableFields(doc, excludeFields); + @Override + public Object getEntity() { + return storedProcedure; + } + + @Override + public Map buildSearchIndexDocInternal(Map doc) { List suggest = new ArrayList<>(); suggest.add( SearchSuggest.builder().input(storedProcedure.getFullyQualifiedName()).weight(5).build()); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/TableIndex.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/TableIndex.java index c1a029e0783d..56d7a293e8ff 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/TableIndex.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/TableIndex.java @@ -16,19 +16,28 @@ import org.openmetadata.service.search.SearchIndexUtils; import org.openmetadata.service.search.models.FlattenColumn; import org.openmetadata.service.search.models.SearchSuggest; -import org.openmetadata.service.util.JsonUtils; public record TableIndex(Table table) implements ColumnIndex { - private static final List excludeFields = - List.of( + private static final Set excludeFields = + Set.of( "sampleData", "tableProfile", "joins", - "changeDescription", + "testSuite.changeDescription", "viewDefinition, tableProfilerConfig, profile, location, tableQueries, tests, dataModel"); - public Map buildESDoc() { - Map doc = JsonUtils.getMap(table); + @Override + public Object getEntity() { + return table; + } + + @Override + public Set getExcludedFields() { + return excludeFields; + } + + @Override + public Map buildSearchIndexDocInternal(Map doc) { List suggest = new ArrayList<>(); List columnSuggest = new ArrayList<>(); List schemaSuggest = new ArrayList<>(); @@ -36,7 +45,6 @@ public Map buildESDoc() { List serviceSuggest = new ArrayList<>(); Set> tagsWithChildren = new HashSet<>(); List columnsWithChildrenName = new ArrayList<>(); - SearchIndexUtils.removeNonIndexableFields(doc, excludeFields); if (table.getColumns() != null) { List cols = new ArrayList<>(); parseColumns(table.getColumns(), cols, null); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/TagIndex.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/TagIndex.java index 6d9ca6725074..d6394e0de980 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/TagIndex.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/TagIndex.java @@ -7,14 +7,16 @@ import org.openmetadata.service.Entity; import org.openmetadata.service.search.SearchIndexUtils; import org.openmetadata.service.search.models.SearchSuggest; -import org.openmetadata.service.util.JsonUtils; public record TagIndex(Tag tag) implements SearchIndex { - private static final List excludeFields = List.of("changeDescription"); - public Map buildESDoc() { - Map doc = JsonUtils.getMap(tag); - SearchIndexUtils.removeNonIndexableFields(doc, excludeFields); + @Override + public Object getEntity() { + return tag; + } + + @Override + public Map buildSearchIndexDocInternal(Map doc) { List suggest = new ArrayList<>(); suggest.add(SearchSuggest.builder().input(tag.getFullyQualifiedName()).weight(5).build()); suggest.add(SearchSuggest.builder().input(tag.getName()).weight(10).build()); @@ -29,6 +31,7 @@ public Map buildESDoc() { } doc.put("suggest", suggest); doc.put("entityType", Entity.TAG); + doc.put("followers", SearchIndexUtils.parseFollowers(tag.getFollowers())); return doc; } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/TeamIndex.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/TeamIndex.java index e9c787603ede..5045747ee245 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/TeamIndex.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/TeamIndex.java @@ -3,24 +3,33 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.Set; import org.openmetadata.common.utils.CommonUtil; import org.openmetadata.schema.entity.teams.Team; import org.openmetadata.service.Entity; import org.openmetadata.service.search.SearchIndexUtils; import org.openmetadata.service.search.models.SearchSuggest; -import org.openmetadata.service.util.JsonUtils; public class TeamIndex implements SearchIndex { final Team team; - final List excludeFields = List.of("owns", "changeDescription"); + final Set excludeFields = Set.of("owns"); public TeamIndex(Team team) { this.team = team; } - public Map buildESDoc() { - Map doc = JsonUtils.getMap(team); - SearchIndexUtils.removeNonIndexableFields(doc, excludeFields); + @Override + public Object getEntity() { + return team; + } + + @Override + public Set getExcludedFields() { + return excludeFields; + } + + @Override + public Map buildSearchIndexDocInternal(Map doc) { List suggest = new ArrayList<>(); suggest.add(SearchSuggest.builder().input(team.getName()).weight(5).build()); suggest.add(SearchSuggest.builder().input(team.getDisplayName()).weight(10).build()); @@ -34,6 +43,7 @@ public Map buildESDoc() { doc.put( "displayName", CommonUtil.nullOrEmpty(team.getDisplayName()) ? team.getName() : team.getDisplayName()); + doc.put("followers", SearchIndexUtils.parseFollowers(team.getFollowers())); return doc; } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/TestCaseIndex.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/TestCaseIndex.java index dc7bac870bfc..6f73dca464c3 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/TestCaseIndex.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/TestCaseIndex.java @@ -3,22 +3,22 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; -import java.util.UUID; -import lombok.SneakyThrows; import org.openmetadata.schema.tests.TestCase; import org.openmetadata.schema.tests.TestSuite; -import org.openmetadata.schema.type.EntityReference; -import org.openmetadata.schema.type.Include; import org.openmetadata.service.Entity; import org.openmetadata.service.search.SearchIndexUtils; import org.openmetadata.service.search.models.SearchSuggest; import org.openmetadata.service.util.JsonUtils; public record TestCaseIndex(TestCase testCase) implements SearchIndex { - private static final List excludeFields = List.of("changeDescription"); - @SneakyThrows - public Map buildESDoc() { + @Override + public Object getEntity() { + return testCase; + } + + @Override + public Map buildSearchIndexDocInternal(Map esDoc) { List testSuiteArray = new ArrayList<>(); if (testCase.getTestSuites() != null) { for (TestSuite suite : testCase.getTestSuites()) { @@ -28,7 +28,7 @@ public Map buildESDoc() { } testCase.setTestSuites(testSuiteArray); Map doc = JsonUtils.getMap(testCase); - SearchIndexUtils.removeNonIndexableFields(doc, excludeFields); + SearchIndexUtils.removeNonIndexableFields(doc, DEFAULT_EXCLUDED_FIELDS); List suggest = new ArrayList<>(); suggest.add(SearchSuggest.builder().input(testCase.getFullyQualifiedName()).weight(5).build()); suggest.add(SearchSuggest.builder().input(testCase.getName()).weight(10).build()); @@ -40,34 +40,10 @@ public Map buildESDoc() { doc.put("suggest", suggest); doc.put("entityType", Entity.TEST_CASE); doc.put("owner", getEntityWithDisplayName(testCase.getOwner())); + doc.put("followers", SearchIndexUtils.parseFollowers(testCase.getFollowers())); return doc; } - public Map buildESDocForCreate() { - EntityReference testSuiteEntityReference = testCase.getTestSuite(); - TestSuite testSuite = getTestSuite(testSuiteEntityReference.getId()); - List testSuiteArray = new ArrayList<>(); - testSuiteArray.add(testSuite); - Map doc = JsonUtils.getMap(testCase); - SearchIndexUtils.removeNonIndexableFields(doc, excludeFields); - doc.put("testSuites", testSuiteArray); - return doc; - } - - private TestSuite getTestSuite(UUID testSuiteId) { - TestSuite testSuite = Entity.getEntity(Entity.TEST_SUITE, testSuiteId, "", Include.ALL); - return new TestSuite() - .withId(testSuite.getId()) - .withName(testSuite.getName()) - .withDisplayName(testSuite.getDisplayName()) - .withDescription(testSuite.getDescription()) - .withFullyQualifiedName(testSuite.getFullyQualifiedName()) - .withDeleted(testSuite.getDeleted()) - .withHref(testSuite.getHref()) - .withExecutable(testSuite.getExecutable()) - .withChangeDescription(null); - } - public static Map getFields() { Map fields = SearchIndex.getDefaultFields(); fields.put("testSuite.fullyQualifiedName", 10.0f); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/TestCaseResolutionStatusIndex.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/TestCaseResolutionStatusIndex.java index ccbd1bac32e2..f6b030156a3f 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/TestCaseResolutionStatusIndex.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/TestCaseResolutionStatusIndex.java @@ -8,7 +8,12 @@ public record TestCaseResolutionStatusIndex(TestCaseResolutionStatus testCaseResolutionStatus) implements SearchIndex { @Override - public Map buildESDoc() { + public Object getEntity() { + return testCaseResolutionStatus; + } + + @Override + public Map buildSearchIndexDocInternal(Map doc) { return JsonUtils.getMap(testCaseResolutionStatus); } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/TestSuiteIndex.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/TestSuiteIndex.java index bbb50678ba83..9df00d160ac1 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/TestSuiteIndex.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/TestSuiteIndex.java @@ -7,14 +7,16 @@ import org.openmetadata.service.Entity; import org.openmetadata.service.search.SearchIndexUtils; import org.openmetadata.service.search.models.SearchSuggest; -import org.openmetadata.service.util.JsonUtils; public record TestSuiteIndex(TestSuite testSuite) implements SearchIndex { - private static final List excludeFields = List.of("changeDescription"); - public Map buildESDoc() { - Map doc = JsonUtils.getMap(testSuite); - SearchIndexUtils.removeNonIndexableFields(doc, excludeFields); + @Override + public Object getEntity() { + return testSuite; + } + + @Override + public Map buildSearchIndexDocInternal(Map doc) { List suggest = new ArrayList<>(); suggest.add(SearchSuggest.builder().input(testSuite.getFullyQualifiedName()).weight(5).build()); suggest.add(SearchSuggest.builder().input(testSuite.getName()).weight(10).build()); @@ -26,6 +28,7 @@ public Map buildESDoc() { doc.put("suggest", suggest); doc.put("entityType", Entity.TEST_SUITE); doc.put("owner", getEntityWithDisplayName(testSuite.getOwner())); + doc.put("followers", SearchIndexUtils.parseFollowers(testSuite.getFollowers())); return doc; } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/TopicIndex.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/TopicIndex.java index 5cda05e2797e..b9ced2a03e9a 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/TopicIndex.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/TopicIndex.java @@ -19,19 +19,27 @@ import org.openmetadata.service.search.models.FlattenSchemaField; import org.openmetadata.service.search.models.SearchSuggest; import org.openmetadata.service.util.FullyQualifiedName; -import org.openmetadata.service.util.JsonUtils; public class TopicIndex implements SearchIndex { - final List excludeTopicFields = - List.of("sampleData", "changeDescription", "messageSchema"); + final Set excludeTopicFields = Set.of("sampleData", "messageSchema"); final Topic topic; public TopicIndex(Topic topic) { this.topic = topic; } - public Map buildESDoc() { - Map doc = JsonUtils.getMap(topic); + @Override + public Object getEntity() { + return topic; + } + + @Override + public Set getExcludedFields() { + return excludeTopicFields; + } + + @Override + public Map buildSearchIndexDocInternal(Map doc) { List suggest = new ArrayList<>(); List fieldSuggest = new ArrayList<>(); List serviceSuggest = new ArrayList<>(); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/UserIndex.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/UserIndex.java index 88d505982df6..643d4ae985b3 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/UserIndex.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/UserIndex.java @@ -3,25 +3,33 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.Set; import org.openmetadata.common.utils.CommonUtil; import org.openmetadata.schema.entity.teams.User; import org.openmetadata.service.Entity; import org.openmetadata.service.search.SearchIndexUtils; import org.openmetadata.service.search.models.SearchSuggest; -import org.openmetadata.service.util.JsonUtils; public class UserIndex implements SearchIndex { final User user; - final List excludeFields = - List.of("owns", "changeDescription", "follows", "authenticationMechanism"); + final Set excludeFields = Set.of("owns", "follows", "authenticationMechanism"); public UserIndex(User user) { this.user = user; } - public Map buildESDoc() { - Map doc = JsonUtils.getMap(user); - SearchIndexUtils.removeNonIndexableFields(doc, excludeFields); + @Override + public Object getEntity() { + return user; + } + + @Override + public Set getExcludedFields() { + return excludeFields; + } + + @Override + public Map buildSearchIndexDocInternal(Map doc) { List suggest = new ArrayList<>(); suggest.add(SearchSuggest.builder().input(user.getName()).weight(5).build()); suggest.add(SearchSuggest.builder().input(user.getDisplayName()).weight(10).build()); @@ -37,6 +45,7 @@ public Map buildESDoc() { if (user.getIsBot() == null) { doc.put("isBot", false); } + doc.put("followers", SearchIndexUtils.parseFollowers(user.getFollowers())); return doc; } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/WebAnalyticEntityViewReportDataIndex.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/WebAnalyticEntityViewReportDataIndex.java index 7f43a9467030..a06f809da678 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/WebAnalyticEntityViewReportDataIndex.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/WebAnalyticEntityViewReportDataIndex.java @@ -2,12 +2,15 @@ import java.util.Map; import org.openmetadata.schema.analytics.ReportData; -import org.openmetadata.service.util.JsonUtils; public record WebAnalyticEntityViewReportDataIndex(ReportData reportData) implements SearchIndex { @Override - public Map buildESDoc() { - Map doc = JsonUtils.getMap(reportData); + public Object getEntity() { + return reportData; + } + + @Override + public Map buildSearchIndexDocInternal(Map doc) { doc.put("entityType", "webAnalyticEntityViewReportData"); return doc; } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/WebAnalyticUserActivityReportDataIndex.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/WebAnalyticUserActivityReportDataIndex.java index 17c3caf0ff64..d4f3b47d134d 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/WebAnalyticUserActivityReportDataIndex.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/WebAnalyticUserActivityReportDataIndex.java @@ -2,12 +2,15 @@ import java.util.Map; import org.openmetadata.schema.analytics.ReportData; -import org.openmetadata.service.util.JsonUtils; public record WebAnalyticUserActivityReportDataIndex(ReportData reportData) implements SearchIndex { @Override - public Map buildESDoc() { - Map doc = JsonUtils.getMap(reportData); + public Object getEntity() { + return reportData; + } + + @Override + public Map buildSearchIndexDocInternal(Map doc) { doc.put("entityType", "webAnalyticUserActivityReportData"); return doc; } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OpenSearchClient.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OpenSearchClient.java index 6cb2fa66a0b5..fee7e9e7f226 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OpenSearchClient.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OpenSearchClient.java @@ -37,6 +37,7 @@ import java.util.TreeMap; import java.util.concurrent.TimeUnit; import java.util.stream.Stream; +import javax.json.JsonObject; import javax.net.ssl.SSLContext; import javax.ws.rs.core.Response; import lombok.extern.slf4j.Slf4j; @@ -653,7 +654,7 @@ private Response searchPipelineLineage( private static ScriptScoreFunctionBuilder boostScore() { return ScoreFunctionBuilders.scriptFunction( "double score = _score;" - + "if (doc.containsKey('totalVotes') && doc['totalVotes'].value != null) { score = score + doc['totalVotes'].value; }" + + "if (doc.containsKey('totalVotes') && doc['totalVotes'].size() != 0) { score = score + doc['totalVotes'].value; }" + "if (doc.containsKey('usageSummary') && doc['usageSummary.weeklyStats.count'].value != null) { score = score + doc['usageSummary.weeklyStats.count'].value; }" + "if (doc.containsKey('tier.tagFQN') && !doc['tier.tagFQN'].empty) { if (doc['tier.tagFQN'].value == 'Tier.Tier2') { score = score + 10; }" + " else if (doc['tier.tagFQN'].value == 'Tier.Tier1') { score = score + 20; }}" @@ -724,6 +725,82 @@ public Response aggregate(String index, String fieldName, String value, String q return Response.status(OK).entity(response).build(); } + /* + Build dynamic aggregation from elasticsearch JSON like aggregation query. + See TestSuiteResourceTest for example usage (ln. 506) for tested aggregation query. + + @param aggregations - JsonObject containing the aggregation query + */ + public static List buildAggregation(JsonObject aggregations) { + List aggregationBuilders = new ArrayList<>(); + for (String key : aggregations.keySet()) { + JsonObject aggregation = aggregations.getJsonObject(key); + for (String aggregationType : aggregation.keySet()) { + switch (aggregationType) { + case "terms": + JsonObject termAggregation = aggregation.getJsonObject(aggregationType); + TermsAggregationBuilder termsAggregationBuilder = + AggregationBuilders.terms(key).field(termAggregation.getString("field")); + aggregationBuilders.add(termsAggregationBuilder); + break; + case "nested": + JsonObject nestedAggregation = aggregation.getJsonObject("nested"); + AggregationBuilder nestedAggregationBuilder = + AggregationBuilders.nested( + nestedAggregation.getString("path"), nestedAggregation.getString("path")); + JsonObject nestedAggregations = aggregation.getJsonObject("aggs"); + + List nestedAggregationBuilders = + buildAggregation(nestedAggregations); + for (AggregationBuilder nestedAggregationBuilder1 : nestedAggregationBuilders) { + nestedAggregationBuilder.subAggregation(nestedAggregationBuilder1); + } + aggregationBuilders.add(nestedAggregationBuilder); + break; + default: + break; + } + } + } + return aggregationBuilders; + } + + @Override + public JsonObject aggregate(String query, String index, JsonObject aggregationJson) + throws IOException { + JsonObject aggregations = aggregationJson.getJsonObject("aggregations"); + if (aggregations == null) { + return null; + } + + List aggregationBuilder = buildAggregation(aggregations); + os.org.opensearch.action.search.SearchRequest searchRequest = + new os.org.opensearch.action.search.SearchRequest( + Entity.getSearchRepository().getIndexOrAliasName(index)); + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + if (query != null) { + XContentParser queryParser = + XContentType.JSON + .xContent() + .createParser(X_CONTENT_REGISTRY, LoggingDeprecationHandler.INSTANCE, query); + QueryBuilder parsedQuery = SearchSourceBuilder.fromXContent(queryParser).query(); + BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery().must(parsedQuery); + searchSourceBuilder.query(boolQueryBuilder); + } + + searchSourceBuilder.size(0).timeout(new TimeValue(30, TimeUnit.SECONDS)); + + for (AggregationBuilder aggregation : aggregationBuilder) { + searchSourceBuilder.aggregation(aggregation); + } + + searchRequest.source(searchSourceBuilder); + + String response = client.search(searchRequest, RequestOptions.DEFAULT).toString(); + JsonObject jsonResponse = JsonUtils.readJson(response).asJsonObject(); + return jsonResponse.getJsonObject("aggregations"); + } + public void updateSearch(UpdateRequest updateRequest) { if (updateRequest != null) { updateRequest.docAsUpsert(true); @@ -833,6 +910,7 @@ private static SearchSourceBuilder buildDashboardSearchBuilder(String query, int .aggregation( AggregationBuilders.terms("dataModels.displayName.keyword") .field("dataModels.displayName.keyword")) + .aggregation(AggregationBuilders.terms("project.keyword").field("project.keyword")) .aggregation( AggregationBuilders.terms("charts.displayName.keyword") .field("charts.displayName.keyword")); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OpenSearchDataInsightProcessor.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OpenSearchDataInsightProcessor.java index 7a353c67ce6a..d6dc8195832e 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OpenSearchDataInsightProcessor.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OpenSearchDataInsightProcessor.java @@ -86,7 +86,8 @@ private UpdateRequest getUpdateRequest(String entityType, ReportData reportData) indexMapping.getIndexName(Entity.getSearchRepository().getClusterAlias()), reportData.getId().toString()); updateRequest.doc( - JsonUtils.pojoToJson(new ReportDataIndexes(reportData).buildESDoc()), XContentType.JSON); + JsonUtils.pojoToJson(new ReportDataIndexes(reportData).buildSearchIndexDoc()), + XContentType.JSON); updateRequest.docAsUpsert(true); return updateRequest; } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OpenSearchEntitiesProcessor.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OpenSearchEntitiesProcessor.java index 38bf872e3ebb..be66b53eeed4 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OpenSearchEntitiesProcessor.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OpenSearchEntitiesProcessor.java @@ -89,7 +89,8 @@ public static UpdateRequest getUpdateRequest(String entityType, EntityInterface entity.getId().toString()); updateRequest.doc( JsonUtils.pojoToJson( - Objects.requireNonNull(Entity.buildSearchIndex(entityType, entity)).buildESDoc()), + Objects.requireNonNull(Entity.buildSearchIndex(entityType, entity)) + .buildSearchIndexDoc()), XContentType.JSON); updateRequest.docAsUpsert(true); return updateRequest; diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/secrets/SecretsManagerUpdateService.java b/openmetadata-service/src/main/java/org/openmetadata/service/secrets/SecretsManagerUpdateService.java index f9270932ef89..30ce91325981 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/secrets/SecretsManagerUpdateService.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/secrets/SecretsManagerUpdateService.java @@ -21,21 +21,21 @@ import java.util.function.Function; import java.util.stream.Collectors; import lombok.extern.slf4j.Slf4j; +import org.openmetadata.schema.EntityInterface; import org.openmetadata.schema.ServiceConnectionEntityInterface; import org.openmetadata.schema.ServiceEntityInterface; import org.openmetadata.schema.entity.automations.Workflow; import org.openmetadata.schema.entity.services.ingestionPipelines.IngestionPipeline; import org.openmetadata.schema.entity.teams.User; import org.openmetadata.service.Entity; +import org.openmetadata.service.exception.EntityNotFoundException; import org.openmetadata.service.exception.SecretsManagerUpdateException; +import org.openmetadata.service.jdbi3.EntityRepository; import org.openmetadata.service.jdbi3.IngestionPipelineRepository; import org.openmetadata.service.jdbi3.ListFilter; import org.openmetadata.service.jdbi3.ServiceEntityRepository; import org.openmetadata.service.jdbi3.UserRepository; import org.openmetadata.service.jdbi3.WorkflowRepository; -import org.openmetadata.service.resources.CollectionRegistry; -import org.openmetadata.service.resources.CollectionRegistry.CollectionDetails; -import org.openmetadata.service.resources.services.ServiceEntityResource; import org.openmetadata.service.util.EntityUtil; import org.openmetadata.service.util.EntityUtil.Fields; @@ -173,37 +173,31 @@ private List retrieveServices( retrieveConnectionTypeRepositoriesMap() { Map, ServiceEntityRepository> connTypeRepositoriesMap = - CollectionRegistry.getInstance().getCollectionMap().values().stream() + Entity.getEntityList().stream() .map(this::retrieveServiceRepository) .filter(Optional::isPresent) .map(Optional::get) .collect( Collectors.toMap( ServiceEntityRepository::getServiceConnectionClass, Function.identity())); + if (connTypeRepositoriesMap.isEmpty()) { throw new SecretsManagerUpdateException("Unexpected error: ServiceRepository not found."); } return connTypeRepositoriesMap; } - private Optional> retrieveServiceRepository( - CollectionDetails collectionDetails) { - Class collectionDetailsClass = extractCollectionDetailsClass(collectionDetails); - if (ServiceEntityResource.class.isAssignableFrom(collectionDetailsClass)) { - return Optional.of( - ((ServiceEntityResource) collectionDetails.getResource()).getRepository()); - } - return Optional.empty(); - } - - private Class extractCollectionDetailsClass(CollectionDetails collectionDetails) { - Class collectionDetailsClass; + private Optional> retrieveServiceRepository(String entityType) { try { - collectionDetailsClass = Class.forName(collectionDetails.getResourceClass()); - } catch (ClassNotFoundException e) { - throw new SecretsManagerUpdateException(e.getMessage(), e.getCause()); + EntityRepository repository = + Entity.getEntityRepository(entityType); + if (ServiceEntityRepository.class.isAssignableFrom(repository.getClass())) { + return Optional.of(((ServiceEntityRepository) repository)); + } + return Optional.empty(); + } catch (EntityNotFoundException e) { + return Optional.empty(); } - return collectionDetailsClass; } private List retrieveBotUsers() { diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/secrets/converter/ClassConverterFactory.java b/openmetadata-service/src/main/java/org/openmetadata/service/secrets/converter/ClassConverterFactory.java index 5f59733c3457..81270bff61cb 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/secrets/converter/ClassConverterFactory.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/secrets/converter/ClassConverterFactory.java @@ -46,25 +46,26 @@ private ClassConverterFactory() { Map.entry(BigQueryConnection.class, new BigQueryConnectionClassConverter()), Map.entry(BigTableConnection.class, new BigTableConnectionClassConverter()), Map.entry(DatalakeConnection.class, new DatalakeConnectionClassConverter()), - Map.entry(MysqlConnection.class, new MysqlConnectionClassConverter()), - Map.entry(HiveConnection.class, new HiveConnectionClassConverter()), - Map.entry(TrinoConnection.class, new TrinoConnectionClassConverter()), - Map.entry(PostgresConnection.class, new PostgresConnectionClassConverter()), Map.entry(DbtGCSConfig.class, new DbtGCSConfigClassConverter()), Map.entry(DbtPipeline.class, new DbtPipelineClassConverter()), + Map.entry(ElasticSearchConnection.class, new ElasticSearchConnectionClassConverter()), Map.entry(GCSConfig.class, new GCPConfigClassConverter()), Map.entry(GCPCredentials.class, new GcpCredentialsClassConverter()), Map.entry(GCSConnection.class, new GcpConnectionClassConverter()), - Map.entry(ElasticSearchConnection.class, new ElasticSearchConnectionClassConverter()), + Map.entry(HiveConnection.class, new HiveConnectionClassConverter()), + Map.entry(IcebergConnection.class, new IcebergConnectionClassConverter()), + Map.entry(IcebergFileSystem.class, new IcebergFileSystemClassConverter()), Map.entry(LookerConnection.class, new LookerConnectionClassConverter()), - Map.entry(SSOAuthMechanism.class, new SSOAuthMechanismClassConverter()), + Map.entry(MysqlConnection.class, new MysqlConnectionClassConverter()), + Map.entry(PostgresConnection.class, new PostgresConnectionClassConverter()), + Map.entry(SapHanaConnection.class, new SapHanaConnectionClassConverter()), Map.entry(SupersetConnection.class, new SupersetConnectionClassConverter()), + Map.entry(SSOAuthMechanism.class, new SSOAuthMechanismClassConverter()), Map.entry(TableauConnection.class, new TableauConnectionClassConverter()), Map.entry( TestServiceConnectionRequest.class, new TestServiceConnectionRequestClassConverter()), - Map.entry(IcebergConnection.class, new IcebergConnectionClassConverter()), - Map.entry(IcebergFileSystem.class, new IcebergFileSystemClassConverter()), + Map.entry(TrinoConnection.class, new TrinoConnectionClassConverter()), Map.entry(Workflow.class, new WorkflowClassConverter())); } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/secrets/converter/SapHanaConnectionClassConverter.java b/openmetadata-service/src/main/java/org/openmetadata/service/secrets/converter/SapHanaConnectionClassConverter.java new file mode 100644 index 000000000000..287f944d60aa --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/secrets/converter/SapHanaConnectionClassConverter.java @@ -0,0 +1,42 @@ +/* + * Copyright 2021 Collate + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.openmetadata.service.secrets.converter; + +import java.util.List; +import org.openmetadata.schema.services.connections.database.SapHanaConnection; +import org.openmetadata.schema.services.connections.database.sapHana.SapHanaHDBConnection; +import org.openmetadata.schema.services.connections.database.sapHana.SapHanaSQLConnection; +import org.openmetadata.service.util.JsonUtils; + +/** Converter class to get an `Sap Hana` object. */ +public class SapHanaConnectionClassConverter extends ClassConverter { + + private static final List> CONNECTION_CLASSES = + List.of(SapHanaSQLConnection.class, SapHanaHDBConnection.class); + + public SapHanaConnectionClassConverter() { + super(SapHanaConnection.class); + } + + @Override + public Object convert(Object object) { + SapHanaConnection sapHanaConnection = + (SapHanaConnection) JsonUtils.convertValue(object, this.clazz); + + tryToConvertOrFail(sapHanaConnection.getConnection(), CONNECTION_CLASSES) + .ifPresent(sapHanaConnection::setConnection); + + return sapHanaConnection; + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/security/AuthCallbackServlet.java b/openmetadata-service/src/main/java/org/openmetadata/service/security/AuthCallbackServlet.java new file mode 100644 index 000000000000..045fe3fc45ac --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/security/AuthCallbackServlet.java @@ -0,0 +1,263 @@ +package org.openmetadata.service.security; + +import static org.openmetadata.service.security.AuthLoginServlet.OIDC_CREDENTIAL_PROFILE; +import static org.openmetadata.service.security.SecurityUtil.getClientAuthentication; +import static org.openmetadata.service.security.SecurityUtil.getErrorMessage; +import static org.openmetadata.service.security.SecurityUtil.sendRedirectWithToken; + +import com.nimbusds.jose.proc.BadJOSEException; +import com.nimbusds.jwt.JWT; +import com.nimbusds.jwt.JWTClaimsSet; +import com.nimbusds.jwt.proc.BadJWTException; +import com.nimbusds.oauth2.sdk.AuthorizationCode; +import com.nimbusds.oauth2.sdk.AuthorizationCodeGrant; +import com.nimbusds.oauth2.sdk.AuthorizationGrant; +import com.nimbusds.oauth2.sdk.ErrorObject; +import com.nimbusds.oauth2.sdk.ParseException; +import com.nimbusds.oauth2.sdk.TokenErrorResponse; +import com.nimbusds.oauth2.sdk.TokenRequest; +import com.nimbusds.oauth2.sdk.TokenResponse; +import com.nimbusds.oauth2.sdk.auth.ClientAuthentication; +import com.nimbusds.oauth2.sdk.http.HTTPRequest; +import com.nimbusds.oauth2.sdk.http.HTTPResponse; +import com.nimbusds.oauth2.sdk.id.ClientID; +import com.nimbusds.oauth2.sdk.id.State; +import com.nimbusds.oauth2.sdk.pkce.CodeVerifier; +import com.nimbusds.oauth2.sdk.token.AccessToken; +import com.nimbusds.openid.connect.sdk.AuthenticationErrorResponse; +import com.nimbusds.openid.connect.sdk.AuthenticationResponse; +import com.nimbusds.openid.connect.sdk.AuthenticationResponseParser; +import com.nimbusds.openid.connect.sdk.AuthenticationSuccessResponse; +import com.nimbusds.openid.connect.sdk.OIDCTokenResponse; +import com.nimbusds.openid.connect.sdk.OIDCTokenResponseParser; +import com.nimbusds.openid.connect.sdk.op.OIDCProviderMetadata; +import com.nimbusds.openid.connect.sdk.token.OIDCTokens; +import com.nimbusds.openid.connect.sdk.validators.BadJWTExceptions; +import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import javax.servlet.annotation.WebServlet; +import javax.servlet.http.HttpServlet; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import lombok.extern.slf4j.Slf4j; +import org.pac4j.core.exception.TechnicalException; +import org.pac4j.core.util.CommonHelper; +import org.pac4j.oidc.client.OidcClient; +import org.pac4j.oidc.credentials.OidcCredentials; + +@WebServlet("/callback") +@Slf4j +public class AuthCallbackServlet extends HttpServlet { + private final OidcClient client; + private final ClientAuthentication clientAuthentication; + private final List claimsOrder; + private final String serverUrl; + + public AuthCallbackServlet(OidcClient oidcClient, String serverUrl, List claimsOrder) { + CommonHelper.assertNotBlank("ServerUrl", serverUrl); + this.client = oidcClient; + this.claimsOrder = claimsOrder; + this.serverUrl = serverUrl; + this.clientAuthentication = getClientAuthentication(client.getConfiguration()); + } + + @Override + protected void doGet(HttpServletRequest req, HttpServletResponse resp) { + try { + LOG.debug("Performing Auth Callback For User Session: {} ", req.getSession().getId()); + String computedCallbackUrl = client.getCallbackUrl(); + Map> parameters = retrieveParameters(req); + AuthenticationResponse response = + AuthenticationResponseParser.parse(new URI(computedCallbackUrl), parameters); + + if (response instanceof AuthenticationErrorResponse authenticationErrorResponse) { + LOG.error( + "Bad authentication response, error={}", authenticationErrorResponse.getErrorObject()); + throw new TechnicalException("Bad authentication response"); + } + + LOG.debug("Authentication response successful"); + AuthenticationSuccessResponse successResponse = (AuthenticationSuccessResponse) response; + + OIDCProviderMetadata metadata = client.getConfiguration().getProviderMetadata(); + if (metadata.supportsAuthorizationResponseIssuerParam() + && !metadata.getIssuer().equals(successResponse.getIssuer())) { + throw new TechnicalException("Issuer mismatch, possible mix-up attack."); + } + + // Optional state validation + validateStateIfRequired(req, resp, successResponse); + + // Build Credentials + OidcCredentials credentials = buildCredentials(successResponse); + + // Validations + validateAndSendTokenRequest(req, credentials, computedCallbackUrl); + + // Log Error if the Refresh Token is null + if (credentials.getRefreshToken() == null) { + LOG.error("Refresh token is null for user session: {}", req.getSession().getId()); + } + + validateNonceIfRequired(req, credentials.getIdToken().getJWTClaimsSet()); + + // Put Credentials in Session + req.getSession().setAttribute(OIDC_CREDENTIAL_PROFILE, credentials); + + // Redirect + sendRedirectWithToken(resp, credentials, serverUrl, claimsOrder); + } catch (Exception e) { + getErrorMessage(resp, e); + } + } + + private OidcCredentials buildCredentials(AuthenticationSuccessResponse successResponse) { + OidcCredentials credentials = new OidcCredentials(); + // get authorization code + AuthorizationCode code = successResponse.getAuthorizationCode(); + if (code != null) { + credentials.setCode(code); + } + // get ID token + JWT idToken = successResponse.getIDToken(); + if (idToken != null) { + credentials.setIdToken(idToken); + } + // get access token + AccessToken accessToken = successResponse.getAccessToken(); + if (accessToken != null) { + credentials.setAccessToken(accessToken); + } + + return credentials; + } + + private void validateNonceIfRequired(HttpServletRequest req, JWTClaimsSet claimsSet) + throws BadJOSEException { + if (client.getConfiguration().isUseNonce()) { + String expectedNonce = + (String) req.getSession().getAttribute(client.getNonceSessionAttributeName()); + if (CommonHelper.isNotBlank(expectedNonce)) { + String tokenNonce; + try { + tokenNonce = claimsSet.getStringClaim("nonce"); + } catch (java.text.ParseException var10) { + throw new BadJWTException("Invalid JWT nonce (nonce) claim: " + var10.getMessage()); + } + + if (tokenNonce == null) { + throw BadJWTExceptions.MISSING_NONCE_CLAIM_EXCEPTION; + } + + if (!expectedNonce.equals(tokenNonce)) { + throw new BadJWTException("Unexpected JWT nonce (nonce) claim: " + tokenNonce); + } + } else { + throw new TechnicalException("Missing nonce parameter from Session."); + } + } + } + + private void validateStateIfRequired( + HttpServletRequest req, + HttpServletResponse resp, + AuthenticationSuccessResponse successResponse) { + if (client.getConfiguration().isWithState()) { + // Validate state for CSRF mitigation + State requestState = + (State) req.getSession().getAttribute(client.getStateSessionAttributeName()); + if (requestState == null || CommonHelper.isBlank(requestState.getValue())) { + getErrorMessage(resp, new TechnicalException("Missing state parameter")); + return; + } + + State responseState = successResponse.getState(); + if (responseState == null) { + throw new TechnicalException("Missing state parameter"); + } + + LOG.debug("Request state: {}/response state: {}", requestState, responseState); + if (!requestState.equals(responseState)) { + throw new TechnicalException( + "State parameter is different from the one sent in authentication request."); + } + } + } + + private void validateAndSendTokenRequest( + HttpServletRequest req, OidcCredentials oidcCredentials, String computedCallbackUrl) + throws IOException, ParseException, URISyntaxException { + if (oidcCredentials.getCode() != null) { + LOG.debug("Initiating Token Request for User Session: {} ", req.getSession().getId()); + CodeVerifier verifier = + (CodeVerifier) + req.getSession().getAttribute(client.getCodeVerifierSessionAttributeName()); + // Token request + TokenRequest request = + createTokenRequest( + new AuthorizationCodeGrant( + oidcCredentials.getCode(), new URI(computedCallbackUrl), verifier)); + executeTokenRequest(request, oidcCredentials); + } + } + + protected Map> retrieveParameters(HttpServletRequest request) { + Map requestParameters = request.getParameterMap(); + Map> map = new HashMap<>(); + for (var entry : requestParameters.entrySet()) { + map.put(entry.getKey(), Arrays.asList(entry.getValue())); + } + return map; + } + + protected TokenRequest createTokenRequest(final AuthorizationGrant grant) { + if (client.getConfiguration().getClientAuthenticationMethod() != null) { + return new TokenRequest( + client.getConfiguration().findProviderMetadata().getTokenEndpointURI(), + this.clientAuthentication, + grant); + } else { + return new TokenRequest( + client.getConfiguration().findProviderMetadata().getTokenEndpointURI(), + new ClientID(client.getConfiguration().getClientId()), + grant); + } + } + + private void executeTokenRequest(TokenRequest request, OidcCredentials credentials) + throws IOException, ParseException { + HTTPRequest tokenHttpRequest = request.toHTTPRequest(); + client.getConfiguration().configureHttpRequest(tokenHttpRequest); + + HTTPResponse httpResponse = tokenHttpRequest.send(); + LOG.debug( + "Token response: status={}, content={}", + httpResponse.getStatusCode(), + httpResponse.getContent()); + + TokenResponse response = OIDCTokenResponseParser.parse(httpResponse); + if (response instanceof TokenErrorResponse tokenErrorResponse) { + ErrorObject errorObject = tokenErrorResponse.getErrorObject(); + throw new TechnicalException( + "Bad token response, error=" + + errorObject.getCode() + + "," + + " description=" + + errorObject.getDescription()); + } + LOG.debug("Token response successful"); + OIDCTokenResponse tokenSuccessResponse = (OIDCTokenResponse) response; + + OIDCTokens oidcTokens = tokenSuccessResponse.getOIDCTokens(); + credentials.setAccessToken(oidcTokens.getAccessToken()); + credentials.setRefreshToken(oidcTokens.getRefreshToken()); + if (oidcTokens.getIDToken() != null) { + credentials.setIdToken(oidcTokens.getIDToken()); + } + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/security/AuthLoginServlet.java b/openmetadata-service/src/main/java/org/openmetadata/service/security/AuthLoginServlet.java new file mode 100644 index 000000000000..ff8d79581b16 --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/security/AuthLoginServlet.java @@ -0,0 +1,145 @@ +package org.openmetadata.service.security; + +import static org.openmetadata.service.security.SecurityUtil.getErrorMessage; +import static org.openmetadata.service.security.SecurityUtil.getUserCredentialsFromSession; +import static org.openmetadata.service.security.SecurityUtil.sendRedirectWithToken; + +import com.nimbusds.oauth2.sdk.id.State; +import com.nimbusds.oauth2.sdk.pkce.CodeChallenge; +import com.nimbusds.oauth2.sdk.pkce.CodeChallengeMethod; +import com.nimbusds.oauth2.sdk.pkce.CodeVerifier; +import com.nimbusds.openid.connect.sdk.AuthenticationRequest; +import com.nimbusds.openid.connect.sdk.Nonce; +import java.io.IOException; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.stream.Collectors; +import javax.servlet.annotation.WebServlet; +import javax.servlet.http.HttpServlet; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import lombok.extern.slf4j.Slf4j; +import org.pac4j.core.exception.TechnicalException; +import org.pac4j.core.util.CommonHelper; +import org.pac4j.oidc.client.GoogleOidcClient; +import org.pac4j.oidc.client.OidcClient; +import org.pac4j.oidc.config.OidcConfiguration; +import org.pac4j.oidc.credentials.OidcCredentials; + +@WebServlet("/api/v1/auth/login") +@Slf4j +public class AuthLoginServlet extends HttpServlet { + public static final String OIDC_CREDENTIAL_PROFILE = "oidcCredentialProfile"; + private final OidcClient client; + private final List claimsOrder; + private final String serverUrl; + + public AuthLoginServlet(OidcClient oidcClient, String serverUrl, List claimsOrder) { + this.client = oidcClient; + this.serverUrl = serverUrl; + this.claimsOrder = claimsOrder; + } + + @Override + protected void doGet(HttpServletRequest req, HttpServletResponse resp) { + try { + LOG.debug("Performing Auth Login For User Session: {} ", req.getSession().getId()); + Optional credentials = getUserCredentialsFromSession(req, client); + if (credentials.isPresent()) { + LOG.debug("Auth Tokens Located from Session: {} ", req.getSession().getId()); + sendRedirectWithToken(resp, credentials.get(), serverUrl, claimsOrder); + } else { + LOG.debug("Performing Auth Code Flow to Idp: {} ", req.getSession().getId()); + Map params = buildParams(); + + params.put(OidcConfiguration.REDIRECT_URI, client.getCallbackUrl()); + + addStateAndNonceParameters(req, params); + + // This is always used to prompt the user to login + if (client instanceof GoogleOidcClient) { + params.put(OidcConfiguration.PROMPT, "consent"); + } else { + params.put(OidcConfiguration.PROMPT, "login"); + } + params.put(OidcConfiguration.MAX_AGE, "0"); + + String location = buildAuthenticationRequestUrl(params); + LOG.debug("Authentication request url: {}", location); + + resp.sendRedirect(location); + } + } catch (Exception e) { + getErrorMessage(resp, new TechnicalException(e)); + } + } + + protected Map buildParams() { + Map authParams = new HashMap<>(); + authParams.put(OidcConfiguration.SCOPE, client.getConfiguration().getScope()); + authParams.put(OidcConfiguration.RESPONSE_TYPE, client.getConfiguration().getResponseType()); + authParams.put(OidcConfiguration.RESPONSE_MODE, "query"); + authParams.putAll(client.getConfiguration().getCustomParams()); + authParams.put(OidcConfiguration.CLIENT_ID, client.getConfiguration().getClientId()); + + return new HashMap<>(authParams); + } + + protected void addStateAndNonceParameters( + final HttpServletRequest request, final Map params) { + // Init state for CSRF mitigation + if (client.getConfiguration().isWithState()) { + State state = new State(CommonHelper.randomString(10)); + params.put(OidcConfiguration.STATE, state.getValue()); + request.getSession().setAttribute(client.getStateSessionAttributeName(), state); + } + + // Init nonce for replay attack mitigation + if (client.getConfiguration().isUseNonce()) { + Nonce nonce = new Nonce(); + params.put(OidcConfiguration.NONCE, nonce.getValue()); + request.getSession().setAttribute(client.getNonceSessionAttributeName(), nonce.getValue()); + } + + CodeChallengeMethod pkceMethod = client.getConfiguration().findPkceMethod(); + if (pkceMethod != null) { + CodeVerifier verfifier = new CodeVerifier(CommonHelper.randomString(10)); + request.getSession().setAttribute(client.getCodeVerifierSessionAttributeName(), verfifier); + params.put( + OidcConfiguration.CODE_CHALLENGE, + CodeChallenge.compute(pkceMethod, verfifier).getValue()); + params.put(OidcConfiguration.CODE_CHALLENGE_METHOD, pkceMethod.getValue()); + } + } + + protected String buildAuthenticationRequestUrl(final Map params) { + // Build authentication request query string + String queryString; + try { + queryString = + AuthenticationRequest.parse( + params.entrySet().stream() + .collect( + Collectors.toMap( + Map.Entry::getKey, e -> Collections.singletonList(e.getValue())))) + .toQueryString(); + } catch (Exception e) { + throw new TechnicalException(e); + } + return client.getConfiguration().getProviderMetadata().getAuthorizationEndpointURI().toString() + + '?' + + queryString; + } + + public static void writeJsonResponse(HttpServletResponse response, String message) + throws IOException { + response.setContentType("application/json"); + response.setCharacterEncoding("UTF-8"); + response.getOutputStream().print(message); + response.getOutputStream().flush(); + response.setStatus(HttpServletResponse.SC_OK); + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/security/AuthLogoutServlet.java b/openmetadata-service/src/main/java/org/openmetadata/service/security/AuthLogoutServlet.java new file mode 100644 index 000000000000..4703a7500334 --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/security/AuthLogoutServlet.java @@ -0,0 +1,36 @@ +package org.openmetadata.service.security; + +import javax.servlet.annotation.WebServlet; +import javax.servlet.http.HttpServlet; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import javax.servlet.http.HttpSession; +import lombok.extern.slf4j.Slf4j; + +@WebServlet("/api/v1/auth/logout") +@Slf4j +public class AuthLogoutServlet extends HttpServlet { + private final String url; + + public AuthLogoutServlet(String url) { + this.url = url; + } + + @Override + protected void doGet( + final HttpServletRequest httpServletRequest, final HttpServletResponse httpServletResponse) { + try { + LOG.debug("Performing application logout"); + HttpSession session = httpServletRequest.getSession(false); + if (session != null) { + LOG.debug("Invalidating the session for logout"); + session.invalidate(); + httpServletResponse.sendRedirect(url); + } else { + LOG.error("No session store available for this web context"); + } + } catch (Exception ex) { + LOG.error("[Auth Logout] Error while performing logout", ex); + } + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/security/AuthRefreshServlet.java b/openmetadata-service/src/main/java/org/openmetadata/service/security/AuthRefreshServlet.java new file mode 100644 index 000000000000..a40a7614b7ec --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/security/AuthRefreshServlet.java @@ -0,0 +1,58 @@ +package org.openmetadata.service.security; + +import static org.openmetadata.service.security.AuthLoginServlet.writeJsonResponse; +import static org.openmetadata.service.security.SecurityUtil.getErrorMessage; +import static org.openmetadata.service.security.SecurityUtil.getUserCredentialsFromSession; + +import java.util.Optional; +import javax.servlet.annotation.WebServlet; +import javax.servlet.http.HttpServlet; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import lombok.extern.slf4j.Slf4j; +import org.openmetadata.service.auth.JwtResponse; +import org.openmetadata.service.util.JsonUtils; +import org.pac4j.core.exception.TechnicalException; +import org.pac4j.oidc.client.OidcClient; +import org.pac4j.oidc.credentials.OidcCredentials; + +@WebServlet("/api/v1/auth/refresh") +@Slf4j +public class AuthRefreshServlet extends HttpServlet { + private final OidcClient client; + private final String baseUrl; + + public AuthRefreshServlet(OidcClient oidcClient, String url) { + this.client = oidcClient; + this.baseUrl = url; + } + + @Override + protected void doGet(HttpServletRequest req, HttpServletResponse resp) { + try { + LOG.debug("Performing Auth Refresh For User Session: {} ", req.getSession().getId()); + Optional credentials = getUserCredentialsFromSession(req, client); + if (credentials.isPresent()) { + LOG.debug("Credentials Found For User Session: {} ", req.getSession().getId()); + JwtResponse jwtResponse = new JwtResponse(); + jwtResponse.setAccessToken(credentials.get().getIdToken().getParsedString()); + jwtResponse.setExpiryDuration( + credentials + .get() + .getIdToken() + .getJWTClaimsSet() + .getExpirationTime() + .toInstant() + .getEpochSecond()); + writeJsonResponse(resp, JsonUtils.pojoToJson(jwtResponse)); + } else { + LOG.debug( + "Credentials Not Found For User Session: {}, Redirect to Logout ", + req.getSession().getId()); + resp.sendRedirect(String.format("%s/logout", baseUrl)); + } + } catch (Exception e) { + getErrorMessage(resp, new TechnicalException(e)); + } + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/security/JwtFilter.java b/openmetadata-service/src/main/java/org/openmetadata/service/security/JwtFilter.java index 62accf5d38dd..de1e214d5c41 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/security/JwtFilter.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/security/JwtFilter.java @@ -151,8 +151,7 @@ public void filter(ContainerRequestContext requestContext) { // validate access token if (claims.containsKey(TOKEN_TYPE) - && ServiceTokenType.PERSONAL_ACCESS.equals( - ServiceTokenType.fromValue(claims.get(TOKEN_TYPE).asString()))) { + && ServiceTokenType.PERSONAL_ACCESS.value().equals(claims.get(TOKEN_TYPE).asString())) { validatePersonalAccessToken(tokenFromHeader, userName); } @@ -218,8 +217,10 @@ public String validateAndReturnUsername(Map claims) { domain = StringUtils.EMPTY; } - // validate principal domain - if (enforcePrincipalDomain && !domain.equals(principalDomain)) { + // validate principal domain, for users + boolean isBot = + claims.containsKey(BOT_CLAIM) && Boolean.TRUE.equals(claims.get(BOT_CLAIM).asBoolean()); + if (!isBot && (enforcePrincipalDomain && !domain.equals(principalDomain))) { throw new AuthenticationException( String.format( "Not Authorized! Email does not match the principal domain %s", principalDomain)); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/security/SecurityUtil.java b/openmetadata-service/src/main/java/org/openmetadata/service/security/SecurityUtil.java index 0e72cf56c819..6af158f28c12 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/security/SecurityUtil.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/security/SecurityUtil.java @@ -13,19 +13,77 @@ package org.openmetadata.service.security; +import static org.openmetadata.service.security.AuthLoginServlet.OIDC_CREDENTIAL_PROFILE; +import static org.pac4j.core.util.CommonHelper.assertNotNull; +import static org.pac4j.core.util.CommonHelper.isNotEmpty; + +import com.fasterxml.jackson.core.type.TypeReference; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableMap.Builder; +import com.nimbusds.jose.JOSEException; +import com.nimbusds.jose.JWSAlgorithm; +import com.nimbusds.jwt.JWT; +import com.nimbusds.oauth2.sdk.auth.ClientAuthentication; +import com.nimbusds.oauth2.sdk.auth.ClientAuthenticationMethod; +import com.nimbusds.oauth2.sdk.auth.ClientSecretBasic; +import com.nimbusds.oauth2.sdk.auth.ClientSecretPost; +import com.nimbusds.oauth2.sdk.auth.PrivateKeyJWT; +import com.nimbusds.oauth2.sdk.auth.Secret; +import com.nimbusds.oauth2.sdk.id.ClientID; +import com.nimbusds.oauth2.sdk.token.BearerAccessToken; +import java.io.BufferedWriter; +import java.io.IOException; +import java.io.OutputStreamWriter; +import java.net.HttpURLConnection; +import java.net.URL; +import java.nio.charset.StandardCharsets; import java.security.Principal; +import java.security.PrivateKey; +import java.text.ParseException; +import java.time.Instant; +import java.util.Arrays; +import java.util.Collection; +import java.util.Date; +import java.util.HashMap; +import java.util.List; import java.util.Map; +import java.util.Optional; +import java.util.TreeMap; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; import javax.ws.rs.client.Invocation; import javax.ws.rs.client.WebTarget; import javax.ws.rs.core.SecurityContext; +import lombok.SneakyThrows; +import lombok.extern.slf4j.Slf4j; import org.openmetadata.common.utils.CommonUtil; +import org.openmetadata.schema.security.client.OidcClientConfig; import org.openmetadata.service.OpenMetadataApplicationConfig; +import org.openmetadata.service.util.JsonUtils; +import org.pac4j.core.context.HttpConstants; +import org.pac4j.core.exception.TechnicalException; +import org.pac4j.core.util.CommonHelper; +import org.pac4j.core.util.HttpUtils; +import org.pac4j.oidc.client.AzureAd2Client; +import org.pac4j.oidc.client.GoogleOidcClient; +import org.pac4j.oidc.client.OidcClient; +import org.pac4j.oidc.config.AzureAd2OidcConfiguration; +import org.pac4j.oidc.config.OidcConfiguration; +import org.pac4j.oidc.config.PrivateKeyJWTClientAuthnMethodConfig; +import org.pac4j.oidc.credentials.OidcCredentials; +import org.pac4j.oidc.credentials.authenticator.OidcAuthenticator; +@Slf4j public final class SecurityUtil { public static final String DEFAULT_PRINCIPAL_DOMAIN = "openmetadata.org"; + private static final Collection SUPPORTED_METHODS = + Arrays.asList( + ClientAuthenticationMethod.CLIENT_SECRET_POST, + ClientAuthenticationMethod.CLIENT_SECRET_BASIC, + ClientAuthenticationMethod.PRIVATE_KEY_JWT, + ClientAuthenticationMethod.NONE); + private SecurityUtil() {} public static String getUserName(SecurityContext securityContext) { @@ -66,4 +124,313 @@ public static Invocation.Builder addHeaders(WebTarget target, Map metadataMethods = + configuration.findProviderMetadata().getTokenEndpointAuthMethods(); + + ClientAuthenticationMethod preferredMethod = getPreferredAuthenticationMethod(configuration); + + final ClientAuthenticationMethod chosenMethod; + if (isNotEmpty(metadataMethods)) { + if (preferredMethod != null) { + if (metadataMethods.contains(preferredMethod)) { + chosenMethod = preferredMethod; + } else { + throw new TechnicalException( + "Preferred authentication method (" + + preferredMethod + + ") not supported " + + "by provider according to provider metadata (" + + metadataMethods + + ")."); + } + } else { + chosenMethod = firstSupportedMethod(metadataMethods); + } + } else { + chosenMethod = + preferredMethod != null ? preferredMethod : ClientAuthenticationMethod.getDefault(); + LOG.info( + "Provider metadata does not provide Token endpoint authentication methods. Using: {}", + chosenMethod); + } + + if (ClientAuthenticationMethod.CLIENT_SECRET_POST.equals(chosenMethod)) { + Secret clientSecret = new Secret(configuration.getSecret()); + clientAuthenticationMechanism = new ClientSecretPost(clientID, clientSecret); + } else if (ClientAuthenticationMethod.CLIENT_SECRET_BASIC.equals(chosenMethod)) { + Secret clientSecret = new Secret(configuration.getSecret()); + clientAuthenticationMechanism = new ClientSecretBasic(clientID, clientSecret); + } else if (ClientAuthenticationMethod.PRIVATE_KEY_JWT.equals(chosenMethod)) { + PrivateKeyJWTClientAuthnMethodConfig privateKetJwtConfig = + configuration.getPrivateKeyJWTClientAuthnMethodConfig(); + assertNotNull("privateKetJwtConfig", privateKetJwtConfig); + JWSAlgorithm jwsAlgo = privateKetJwtConfig.getJwsAlgorithm(); + assertNotNull("privateKetJwtConfig.getJwsAlgorithm()", jwsAlgo); + PrivateKey privateKey = privateKetJwtConfig.getPrivateKey(); + assertNotNull("privateKetJwtConfig.getPrivateKey()", privateKey); + String keyID = privateKetJwtConfig.getKeyID(); + try { + clientAuthenticationMechanism = + new PrivateKeyJWT( + clientID, + configuration.findProviderMetadata().getTokenEndpointURI(), + jwsAlgo, + privateKey, + keyID, + null); + } catch (final JOSEException e) { + throw new TechnicalException( + "Cannot instantiate private key JWT client authentication method", e); + } + } + } + + return clientAuthenticationMechanism; + } + + private static ClientAuthenticationMethod getPreferredAuthenticationMethod( + OidcConfiguration config) { + ClientAuthenticationMethod configurationMethod = config.getClientAuthenticationMethod(); + if (configurationMethod == null) { + return null; + } + + if (!SUPPORTED_METHODS.contains(configurationMethod)) { + throw new TechnicalException( + "Configured authentication method (" + configurationMethod + ") is not supported."); + } + + return configurationMethod; + } + + private static ClientAuthenticationMethod firstSupportedMethod( + final List metadataMethods) { + Optional firstSupported = + metadataMethods.stream().filter(SUPPORTED_METHODS::contains).findFirst(); + if (firstSupported.isPresent()) { + return firstSupported.get(); + } else { + throw new TechnicalException( + "None of the Token endpoint provider metadata authentication methods are supported: " + + metadataMethods); + } + } + + @SneakyThrows + public static void getErrorMessage(HttpServletResponse resp, Exception e) { + resp.setContentType("text/html; charset=UTF-8"); + LOG.error("[Auth Callback Servlet] Failed in Auth Login : {}", e.getMessage()); + resp.getOutputStream() + .println( + String.format( + "

[Auth Callback Servlet] Failed in Auth Login : %s

", e.getMessage())); + } + + public static void sendRedirectWithToken( + HttpServletResponse response, + OidcCredentials credentials, + String serverUrl, + List claimsOrder) + throws ParseException, IOException { + JWT jwt = credentials.getIdToken(); + Map claims = new TreeMap<>(String.CASE_INSENSITIVE_ORDER); + claims.putAll(jwt.getJWTClaimsSet().getClaims()); + String preferredJwtClaim = + claimsOrder.stream() + .filter(claims::containsKey) + .findFirst() + .map(claims::get) + .map(String.class::cast) + .orElseThrow( + () -> + new AuthenticationException( + "Invalid JWT token, none of the following claims are present " + + claimsOrder)); + + String email = (String) jwt.getJWTClaimsSet().getClaim("email"); + String userName; + if (preferredJwtClaim.contains("@")) { + userName = preferredJwtClaim.split("@")[0]; + } else { + userName = preferredJwtClaim; + } + + String url = + String.format( + "%s/auth/callback?id_token=%s&email=%s&name=%s", + serverUrl, credentials.getIdToken().getParsedString(), email, userName); + response.sendRedirect(url); + } + + public static boolean isCredentialsExpired(OidcCredentials credentials) throws ParseException { + Date expiration = credentials.getIdToken().getJWTClaimsSet().getExpirationTime(); + return expiration != null && expiration.toInstant().isBefore(Instant.now().plusSeconds(30)); + } + + public static Optional getUserCredentialsFromSession( + HttpServletRequest request, OidcClient client) throws ParseException { + OidcCredentials credentials = + (OidcCredentials) request.getSession().getAttribute(OIDC_CREDENTIAL_PROFILE); + if (credentials != null && credentials.getRefreshToken() != null) { + removeOrRenewOidcCredentials(request, client, credentials); + return Optional.of(credentials); + } else { + if (credentials == null) { + LOG.error("No credentials found against session. ID: {}", request.getSession().getId()); + } else { + LOG.error("No refresh token found against session. ID: {}", request.getSession().getId()); + } + } + return Optional.empty(); + } + + private static void removeOrRenewOidcCredentials( + HttpServletRequest request, OidcClient client, OidcCredentials credentials) + throws ParseException { + boolean profilesUpdated = false; + if (SecurityUtil.isCredentialsExpired(credentials)) { + LOG.debug("Expired credentials found, trying to renew."); + profilesUpdated = true; + if (client.getConfiguration() + instanceof AzureAd2OidcConfiguration azureAd2OidcConfiguration) { + refreshAccessTokenAzureAd2Token(azureAd2OidcConfiguration, credentials); + } else { + OidcAuthenticator authenticator = new OidcAuthenticator(client.getConfiguration(), client); + authenticator.refresh(credentials); + } + } + if (profilesUpdated) { + request.getSession().setAttribute(OIDC_CREDENTIAL_PROFILE, credentials); + } + } + + private static void refreshAccessTokenAzureAd2Token( + AzureAd2OidcConfiguration azureConfig, OidcCredentials azureAdProfile) { + HttpURLConnection connection = null; + try { + Map headers = new HashMap<>(); + headers.put( + HttpConstants.CONTENT_TYPE_HEADER, HttpConstants.APPLICATION_FORM_ENCODED_HEADER_VALUE); + headers.put(HttpConstants.ACCEPT_HEADER, HttpConstants.APPLICATION_JSON); + // get the token endpoint from discovery URI + URL tokenEndpointURL = azureConfig.findProviderMetadata().getTokenEndpointURI().toURL(); + connection = HttpUtils.openPostConnection(tokenEndpointURL, headers); + + BufferedWriter out = + new BufferedWriter( + new OutputStreamWriter(connection.getOutputStream(), StandardCharsets.UTF_8)); + out.write(azureConfig.makeOauth2TokenRequest(azureAdProfile.getRefreshToken().getValue())); + out.close(); + + int responseCode = connection.getResponseCode(); + if (responseCode != 200) { + throw new TechnicalException( + "request for access token failed: " + HttpUtils.buildHttpErrorMessage(connection)); + } + var body = HttpUtils.readBody(connection); + Map res = JsonUtils.readValue(body, new TypeReference<>() {}); + azureAdProfile.setAccessToken(new BearerAccessToken((String) res.get("access_token"))); + } catch (final IOException e) { + throw new TechnicalException(e); + } finally { + HttpUtils.closeConnection(connection); + } + } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/security/policyevaluator/CompiledRule.java b/openmetadata-service/src/main/java/org/openmetadata/service/security/policyevaluator/CompiledRule.java index 3ceecfbabec4..ba9c50a1989f 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/security/policyevaluator/CompiledRule.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/security/policyevaluator/CompiledRule.java @@ -18,6 +18,7 @@ import org.openmetadata.service.security.policyevaluator.SubjectContext.PolicyContext; import org.springframework.expression.Expression; import org.springframework.expression.spel.standard.SpelExpressionParser; +import org.springframework.expression.spel.support.SimpleEvaluationContext; /** This class is used in a single threaded model and hence does not have concurrency support */ @Slf4j @@ -54,8 +55,13 @@ public static void validateExpression(String condition, Class clz) { } Expression expression = parseExpression(condition); RuleEvaluator ruleEvaluator = new RuleEvaluator(); + SimpleEvaluationContext context = + SimpleEvaluationContext.forReadOnlyDataBinding() + .withInstanceMethods() + .withRootObject(ruleEvaluator) + .build(); try { - expression.getValue(ruleEvaluator, clz); + expression.getValue(context, clz); } catch (Exception exception) { // Remove unnecessary class details in the exception message String message = @@ -216,7 +222,12 @@ private boolean matchExpression( return true; } RuleEvaluator ruleEvaluator = new RuleEvaluator(policyContext, subjectContext, resourceContext); - return Boolean.TRUE.equals(expr.getValue(ruleEvaluator, Boolean.class)); + SimpleEvaluationContext context = + SimpleEvaluationContext.forReadOnlyDataBinding() + .withInstanceMethods() + .withRootObject(ruleEvaluator) + .build(); + return Boolean.TRUE.equals(expr.getValue(context, Boolean.class)); } public static boolean overrideAccess(Access newAccess, Access currentAccess) { diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/security/saml/OMMicrometerHttpFilter.java b/openmetadata-service/src/main/java/org/openmetadata/service/security/saml/OMMicrometerHttpFilter.java index f9ee4a9e02d4..c765fa98eaa6 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/security/saml/OMMicrometerHttpFilter.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/security/saml/OMMicrometerHttpFilter.java @@ -12,10 +12,8 @@ */ package org.openmetadata.service.security.saml; -import static org.openmetadata.service.util.MicrometerBundleSingleton.getWebAnalyticEvents; import static org.openmetadata.service.util.MicrometerBundleSingleton.prometheusMeterRegistry; -import io.github.maksymdolgykh.dropwizard.micrometer.MicrometerBundle; import io.micrometer.core.instrument.Timer; import java.io.IOException; import javax.servlet.Filter; @@ -25,9 +23,8 @@ import javax.servlet.ServletRequest; import javax.servlet.ServletResponse; import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; import lombok.extern.slf4j.Slf4j; -import org.openmetadata.common.utils.CommonUtil; +import org.openmetadata.service.util.MicrometerBundleSingleton; /** * This is OMMicrometerHttpFilter is similar to MicrometerHttpFilter with support to handle OM Servlets, and provide @@ -53,16 +50,9 @@ public void doFilter(ServletRequest request, ServletResponse response, FilterCha long startTime = System.nanoTime(); chain.doFilter(request, response); double elapsed = (System.nanoTime() - startTime) / 1.0E9; - String requestPath = ((HttpServletRequest) request).getPathInfo(); - if (CommonUtil.nullOrEmpty(requestPath)) { - requestPath = ((HttpServletRequest) request).getServletPath(); - } - String responseStatus = String.valueOf(((HttpServletResponse) response).getStatus()); String requestMethod = ((HttpServletRequest) request).getMethod(); - MicrometerBundle.httpRequests - .labels(requestMethod, responseStatus, requestPath) - .observe(elapsed); - timer.stop(getWebAnalyticEvents()); + MicrometerBundleSingleton.httpRequests.labels(requestMethod).observe(elapsed); + timer.stop(MicrometerBundleSingleton.getRequestsLatencyTimer()); } @Override diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/socket/SocketAddressFilter.java b/openmetadata-service/src/main/java/org/openmetadata/service/socket/SocketAddressFilter.java index 78ba04993601..9c3f867cefac 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/socket/SocketAddressFilter.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/socket/SocketAddressFilter.java @@ -65,13 +65,7 @@ public void doFilter(ServletRequest request, ServletResponse response, FilterCha if (enableSecureSocketConnection) { String tokenWithType = httpServletRequest.getHeader("Authorization"); requestWrapper.addHeader("Authorization", tokenWithType); - String token = JwtFilter.extractToken(tokenWithType); - // validate token - DecodedJWT jwt = jwtFilter.validateAndReturnDecodedJwtToken(token); - // validate Domain and Username - Map claims = new TreeMap<>(String.CASE_INSENSITIVE_ORDER); - claims.putAll(jwt.getClaims()); - jwtFilter.validateAndReturnUsername(claims); + validatePrefixedTokenRequest(jwtFilter, tokenWithType); } // Goes to default servlet. chain.doFilter(requestWrapper, response); @@ -85,4 +79,14 @@ public void doFilter(ServletRequest request, ServletResponse response, FilterCha @Override public void init(FilterConfig filterConfig) {} + + public static void validatePrefixedTokenRequest(JwtFilter jwtFilter, String prefixedToken) { + String token = JwtFilter.extractToken(prefixedToken); + // validate token + DecodedJWT jwt = jwtFilter.validateAndReturnDecodedJwtToken(token); + // validate Domain and Username + Map claims = new TreeMap<>(String.CASE_INSENSITIVE_ORDER); + claims.putAll(jwt.getClaims()); + jwtFilter.validateAndReturnUsername(claims); + } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/util/EntityUtil.java b/openmetadata-service/src/main/java/org/openmetadata/service/util/EntityUtil.java index 005155298980..78f33bdf98bd 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/util/EntityUtil.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/util/EntityUtil.java @@ -16,6 +16,7 @@ import static org.openmetadata.common.utils.CommonUtil.listOrEmpty; import static org.openmetadata.common.utils.CommonUtil.nullOrEmpty; import static org.openmetadata.schema.type.Include.ALL; +import static org.openmetadata.schema.type.Include.NON_DELETED; import java.io.IOException; import java.security.MessageDigest; @@ -538,7 +539,7 @@ public static List getEntityReferences(String entityType, List< } List references = new ArrayList<>(); for (String fqn : fqns) { - references.add(getEntityReference(entityType, fqn)); + references.add(Entity.getEntityReferenceByName(entityType, fqn, NON_DELETED)); } return references; } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/util/MicrometerBundleSingleton.java b/openmetadata-service/src/main/java/org/openmetadata/service/util/MicrometerBundleSingleton.java index d04d68ec0674..c8bf1e58169b 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/util/MicrometerBundleSingleton.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/util/MicrometerBundleSingleton.java @@ -18,30 +18,47 @@ import io.github.maksymdolgykh.dropwizard.micrometer.MicrometerBundle; import io.micrometer.core.instrument.Timer; import io.micrometer.prometheus.PrometheusMeterRegistry; +import io.prometheus.client.Histogram; +import lombok.Getter; import org.openmetadata.service.OpenMetadataApplicationConfig; public class MicrometerBundleSingleton { - private static final MicrometerBundle instance = new MicrometerBundle(); + @Getter private static final MicrometerBundle instance = new MicrometerBundle(); // We'll use this registry to add monitoring around Ingestion Pipelines public static final PrometheusMeterRegistry prometheusMeterRegistry = prometheusRegistry; - private static Timer webAnalyticEvents; + @Getter private static Timer requestsLatencyTimer; + @Getter private static Timer jdbiLatencyTimer; private MicrometerBundleSingleton() {} - public static MicrometerBundle getInstance() { - return instance; - } + private static final double[] latencyBuckets = new double[] {.01, .1, 1, 2, 5, 10, 20, 60}; + + public static final Histogram httpRequests = + Histogram.build() + .name("http_server_requests_sec") + .help("HTTP methods duration") + .labelNames("method") + .buckets(latencyBuckets) + .register(prometheusMeterRegistry.getPrometheusRegistry()); + + public static final Histogram jdbiRequests = + Histogram.build() + .name("jdbi_requests_seconds") + .help("jdbi requests duration distribution") + .buckets(latencyBuckets) + .register(MicrometerBundle.prometheusRegistry.getPrometheusRegistry()); - public static void setWebAnalyticsEvents(OpenMetadataApplicationConfig config) { - webAnalyticEvents = - Timer.builder("latency_requests") - .description("Request latency in seconds.") + public static void initLatencyEvents(OpenMetadataApplicationConfig config) { + requestsLatencyTimer = + Timer.builder("http_latency_requests") + .description("HTTP request latency in seconds.") .publishPercentiles(config.getEventMonitorConfiguration().getLatency()) - .publishPercentileHistogram() .register(prometheusMeterRegistry); - } - public static Timer getWebAnalyticEvents() { - return webAnalyticEvents; + jdbiLatencyTimer = + Timer.builder("jdbi_latency_requests") + .description("JDBI queries latency in seconds.") + .publishPercentiles(config.getEventMonitorConfiguration().getLatency()) + .register(prometheusMeterRegistry); } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/util/OpenMetadataOperations.java b/openmetadata-service/src/main/java/org/openmetadata/service/util/OpenMetadataOperations.java index 7a2fd47138eb..28640c5c883f 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/util/OpenMetadataOperations.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/util/OpenMetadataOperations.java @@ -38,6 +38,7 @@ import org.openmetadata.schema.ServiceEntityInterface; import org.openmetadata.schema.entity.app.App; import org.openmetadata.schema.entity.app.AppSchedule; +import org.openmetadata.schema.entity.app.ScheduleTimeline; import org.openmetadata.schema.entity.app.ScheduledExecutionContext; import org.openmetadata.schema.entity.services.ingestionPipelines.IngestionPipeline; import org.openmetadata.schema.services.connections.metadata.OpenMetadataConnection; @@ -46,6 +47,7 @@ import org.openmetadata.sdk.PipelineServiceClient; import org.openmetadata.service.Entity; import org.openmetadata.service.OpenMetadataApplicationConfig; +import org.openmetadata.service.apps.ApplicationHandler; import org.openmetadata.service.apps.scheduler.AppScheduler; import org.openmetadata.service.clients.pipeline.PipelineServiceClientFactory; import org.openmetadata.service.fernet.Fernet; @@ -60,6 +62,7 @@ import org.openmetadata.service.search.SearchRepository; import org.openmetadata.service.secrets.SecretsManager; import org.openmetadata.service.secrets.SecretsManagerFactory; +import org.openmetadata.service.secrets.SecretsManagerUpdateService; import org.openmetadata.service.util.jdbi.DatabaseAuthenticationProviderFactory; import org.slf4j.LoggerFactory; import picocli.CommandLine; @@ -183,6 +186,7 @@ public Integer dropCreate() { LOG.info("OpenMetadata Database Schema is Updated."); LOG.info("create indexes."); searchRepository.createIndexes(); + Entity.cleanup(); return 0; } catch (Exception e) { LOG.error("Failed to drop create due to ", e); @@ -207,6 +211,9 @@ public Integer migrate( LOG.info("Update Search Indexes."); searchRepository.updateIndexes(); printChangeLog(); + // update entities secrets if required + new SecretsManagerUpdateService(secretsManager, config.getClusterName()).updateEntities(); + Entity.cleanup(); return 0; } catch (Exception e) { LOG.error("Failed to db migration due to ", e); @@ -238,14 +245,14 @@ public Integer reIndex( boolean recreateIndexes) { try { parseConfig(); + ApplicationHandler.initialize(config); AppScheduler.initialize(config, collectionDAO, searchRepository); App searchIndexApp = new App() .withId(UUID.randomUUID()) .withName("SearchIndexApp") .withClassName("org.openmetadata.service.apps.bundles.searchIndex.SearchIndexApp") - .withAppSchedule( - new AppSchedule().withScheduleType(AppSchedule.ScheduleTimeline.DAILY)) + .withAppSchedule(new AppSchedule().withScheduleTimeline(ScheduleTimeline.DAILY)) .withAppConfiguration( new EventPublisherJob() .withEntities(new HashSet<>(List.of("all"))) @@ -293,6 +300,24 @@ public Integer deployPipelines() { } } + @Command( + name = "migrate-secrets", + description = + "Migrate secrets from DB to the configured Secrets Manager. " + + "Note that this does not support migrating between external Secrets Managers") + public Integer migrateSecrets() { + try { + LOG.info("Migrating Secrets from DB..."); + parseConfig(); + // update entities secrets if required + new SecretsManagerUpdateService(secretsManager, config.getClusterName()).updateEntities(); + return 0; + } catch (Exception e) { + LOG.error("Failed to deploy pipelines due to ", e); + return 1; + } + } + private void deployPipeline( IngestionPipeline pipeline, PipelineServiceClient pipelineServiceClient, @@ -434,7 +459,6 @@ private void validateAndRunSystemDataMigrations(boolean force) { jdbi, nativeSQLScriptRootPath, connType, extensionSQLScriptRootPath, force); workflow.loadMigrations(); workflow.runMigrationWorkflows(); - Entity.cleanup(); } private void printToAsciiTable(List columns, List> rows, String emptyText) { diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/util/SubscriptionUtil.java b/openmetadata-service/src/main/java/org/openmetadata/service/util/SubscriptionUtil.java index 3c77de459fd6..091cf26c8a5f 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/util/SubscriptionUtil.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/util/SubscriptionUtil.java @@ -20,8 +20,10 @@ import static org.openmetadata.service.events.subscription.AlertsRuleEvaluator.getEntity; import java.util.ArrayList; +import java.util.HashMap; import java.util.HashSet; import java.util.List; +import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.UUID; @@ -155,35 +157,43 @@ private static Set getTaskAssignees( Thread thread = AlertsRuleEvaluator.getThread(event); List assignees = thread.getTask().getAssignees(); Set receiversList = new HashSet<>(); - List teams = new ArrayList<>(); - List users = new ArrayList<>(); + Map teams = new HashMap<>(); + Map users = new HashMap<>(); + + Team tempTeamVar = null; + User tempUserVar = null; if (!nullOrEmpty(assignees)) { for (EntityReference reference : assignees) { if (Entity.USER.equals(reference.getType())) { - users.add(Entity.getEntity(USER, reference.getId(), "", Include.NON_DELETED)); + tempUserVar = Entity.getEntity(USER, reference.getId(), "profile", Include.NON_DELETED); + users.put(tempUserVar.getId(), tempUserVar); } else if (TEAM.equals(reference.getType())) { - teams.add(Entity.getEntity(TEAM, reference.getId(), "", Include.NON_DELETED)); + tempTeamVar = Entity.getEntity(TEAM, reference.getId(), "profile", Include.NON_DELETED); + teams.put(tempTeamVar.getId(), tempTeamVar); } } } for (Post post : thread.getPosts()) { - users.add(Entity.getEntityByName(USER, post.getFrom(), "", Include.NON_DELETED)); + tempUserVar = Entity.getEntityByName(USER, post.getFrom(), "profile", Include.NON_DELETED); + users.put(tempUserVar.getId(), tempUserVar); List mentions = MessageParser.getEntityLinks(post.getMessage()); for (MessageParser.EntityLink link : mentions) { if (USER.equals(link.getEntityType())) { - users.add(Entity.getEntity(link, "", Include.NON_DELETED)); + tempUserVar = Entity.getEntity(link, "profile", Include.NON_DELETED); + users.put(tempUserVar.getId(), tempUserVar); } else if (TEAM.equals(link.getEntityType())) { - teams.add(Entity.getEntity(link, "", Include.NON_DELETED)); + tempTeamVar = Entity.getEntity(link, "profile", Include.NON_DELETED); + teams.put(tempTeamVar.getId(), tempTeamVar); } } } // Users - receiversList.addAll(getEmailOrWebhookEndpointForUsers(users, type)); + receiversList.addAll(getEmailOrWebhookEndpointForUsers(users.values().stream().toList(), type)); // Teams - receiversList.addAll(getEmailOrWebhookEndpointForTeams(teams, type)); + receiversList.addAll(getEmailOrWebhookEndpointForTeams(teams.values().stream().toList(), type)); return receiversList; } @@ -192,36 +202,45 @@ public static Set handleConversationNotification( SubscriptionDestination.SubscriptionType type, ChangeEvent event) { Thread thread = AlertsRuleEvaluator.getThread(event); Set receiversList = new HashSet<>(); - List teams = new ArrayList<>(); - List users = new ArrayList<>(); - - users.add(Entity.getEntityByName(USER, thread.getCreatedBy(), "", Include.NON_DELETED)); + Map teams = new HashMap<>(); + Map users = new HashMap<>(); + + Team tempTeamVar = null; + User tempUserVar = null; + tempUserVar = + Entity.getEntityByName(USER, thread.getCreatedBy(), "profile", Include.NON_DELETED); + users.put(tempUserVar.getId(), tempUserVar); List mentions = MessageParser.getEntityLinks(thread.getMessage()); for (MessageParser.EntityLink link : mentions) { if (USER.equals(link.getEntityType())) { - users.add(Entity.getEntity(link, "", Include.NON_DELETED)); + tempUserVar = Entity.getEntity(link, "profile", Include.NON_DELETED); + users.put(tempUserVar.getId(), tempUserVar); } else if (TEAM.equals(link.getEntityType())) { - teams.add(Entity.getEntity(link, "", Include.NON_DELETED)); + tempTeamVar = Entity.getEntity(link, "", Include.NON_DELETED); + teams.put(tempTeamVar.getId(), tempTeamVar); } } for (Post post : thread.getPosts()) { - users.add(Entity.getEntityByName(USER, post.getFrom(), "", Include.NON_DELETED)); + tempUserVar = Entity.getEntityByName(USER, post.getFrom(), "profile", Include.NON_DELETED); + users.put(tempUserVar.getId(), tempUserVar); mentions = MessageParser.getEntityLinks(post.getMessage()); for (MessageParser.EntityLink link : mentions) { if (USER.equals(link.getEntityType())) { - users.add(Entity.getEntity(link, "", Include.NON_DELETED)); + tempUserVar = Entity.getEntity(link, "profile", Include.NON_DELETED); + users.put(tempUserVar.getId(), tempUserVar); } else if (TEAM.equals(link.getEntityType())) { - teams.add(Entity.getEntity(link, "", Include.NON_DELETED)); + tempTeamVar = Entity.getEntity(link, "profile", Include.NON_DELETED); + teams.put(tempTeamVar.getId(), tempTeamVar); } } } // Users - receiversList.addAll(getEmailOrWebhookEndpointForUsers(users, type)); + receiversList.addAll(getEmailOrWebhookEndpointForUsers(users.values().stream().toList(), type)); // Teams - receiversList.addAll(getEmailOrWebhookEndpointForTeams(teams, type)); + receiversList.addAll(getEmailOrWebhookEndpointForTeams(teams.values().stream().toList(), type)); return receiversList; } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/util/NotificationHandler.java b/openmetadata-service/src/main/java/org/openmetadata/service/util/WebsocketNotificationHandler.java similarity index 67% rename from openmetadata-service/src/main/java/org/openmetadata/service/util/NotificationHandler.java rename to openmetadata-service/src/main/java/org/openmetadata/service/util/WebsocketNotificationHandler.java index 1bd2397e69b6..84fc8caf81ab 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/util/NotificationHandler.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/util/WebsocketNotificationHandler.java @@ -15,10 +15,7 @@ import static org.openmetadata.service.Entity.TEAM; import static org.openmetadata.service.Entity.USER; -import static org.openmetadata.service.util.EmailUtil.getSmtpSettings; -import freemarker.template.TemplateException; -import java.io.IOException; import java.time.Instant; import java.util.HashSet; import java.util.List; @@ -38,61 +35,73 @@ import org.openmetadata.schema.type.Relationship; import org.openmetadata.service.Entity; import org.openmetadata.service.jdbi3.CollectionDAO; -import org.openmetadata.service.jdbi3.UserRepository; import org.openmetadata.service.resources.feeds.MessageParser; import org.openmetadata.service.socket.WebSocketManager; @Slf4j -public class NotificationHandler { +public class WebsocketNotificationHandler { private final ExecutorService threadScheduler; - public NotificationHandler() { + public WebsocketNotificationHandler() { this.threadScheduler = Executors.newFixedThreadPool(1); } + public void processNotifications(ContainerResponseContext responseContext) { + threadScheduler.submit( + () -> { + try { + handleNotifications(responseContext); + } catch (Exception ex) { + LOG.error("[NotificationHandler] Failed to use mapper in converting to Json", ex); + } + }); + } + private void handleNotifications(ContainerResponseContext responseContext) { int responseCode = responseContext.getStatus(); if (responseCode == Response.Status.CREATED.getStatusCode() && responseContext.getEntity() != null && responseContext.getEntity().getClass().equals(Thread.class)) { Thread thread = (Thread) responseContext.getEntity(); + switch (thread.getType()) { + case Task -> handleTaskNotification(thread); + case Conversation -> handleConversationNotification(thread); + case Announcement -> handleAnnouncementNotification(thread); + } } } public static void handleTaskNotification(Thread thread) { String jsonThread = JsonUtils.pojoToJson(thread); if (thread.getPostsCount() == 0) { - Set receiversList = getTaskAssignees(thread); + List assignees = thread.getTask().getAssignees(); + Set receiversList = new HashSet<>(); + // Update Assignee + assignees.forEach( + e -> { + if (Entity.USER.equals(e.getType())) { + receiversList.add(e.getId()); + } else if (Entity.TEAM.equals(e.getType())) { + // fetch all that are there in the team + List records = + Entity.getCollectionDAO() + .relationshipDAO() + .findTo(e.getId(), TEAM, Relationship.HAS.ordinal(), Entity.USER); + records.forEach(eRecord -> receiversList.add(eRecord.getId())); + } + }); + // Send WebSocket Notification WebSocketManager.getInstance() .sendToManyWithUUID(receiversList, WebSocketManager.TASK_BROADCAST_CHANNEL, jsonThread); - - // Send Email Notification If Enabled - // TODO: This needs to be handled from the Alerts - handleEmailNotifications(receiversList, thread); + } else { + List mentions; + Post latestPost = thread.getPosts().get(thread.getPostsCount() - 1); + mentions = MessageParser.getEntityLinks(latestPost.getMessage()); + notifyMentionedUsers(mentions, jsonThread); } } - public static Set getTaskAssignees(Thread thread) { - List assignees = thread.getTask().getAssignees(); - Set receiversList = new HashSet<>(); - assignees.forEach( - e -> { - if (Entity.USER.equals(e.getType())) { - receiversList.add(e.getId()); - } else if (Entity.TEAM.equals(e.getType())) { - // fetch all that are there in the team - List records = - Entity.getCollectionDAO() - .relationshipDAO() - .findTo(e.getId(), TEAM, Relationship.HAS.ordinal(), Entity.USER); - records.forEach(eRecord -> receiversList.add(eRecord.getId())); - } - }); - - return receiversList; - } - private void handleAnnouncementNotification(Thread thread) { String jsonThread = JsonUtils.pojoToJson(thread); AnnouncementDetails announcementDetails = thread.getAnnouncement(); @@ -104,7 +113,7 @@ private void handleAnnouncementNotification(Thread thread) { } } - public static void handleConversationNotification(Thread thread) { + private void handleConversationNotification(Thread thread) { String jsonThread = JsonUtils.pojoToJson(thread); WebSocketManager.getInstance() .broadCastMessageToAll(WebSocketManager.FEED_BROADCAST_CHANNEL, jsonThread); @@ -115,6 +124,11 @@ public static void handleConversationNotification(Thread thread) { Post latestPost = thread.getPosts().get(thread.getPostsCount() - 1); mentions = MessageParser.getEntityLinks(latestPost.getMessage()); } + notifyMentionedUsers(mentions, jsonThread); + } + + private static void notifyMentionedUsers( + List mentions, String jsonThread) { mentions.forEach( entityLink -> { String fqn = entityLink.getEntityFQN(); @@ -135,26 +149,4 @@ public static void handleConversationNotification(Thread thread) { } }); } - - public static void handleEmailNotifications(Set userList, Thread thread) { - UserRepository repository = (UserRepository) Entity.getEntityRepository(USER); - userList.forEach( - id -> { - try { - User user = repository.get(null, id, repository.getFields("name,email,href")); - EmailUtil.sendTaskAssignmentNotificationToUser( - user.getName(), - user.getEmail(), - String.format( - "%s/users/%s/tasks", getSmtpSettings().getOpenMetadataUrl(), user.getName()), - thread, - EmailUtil.getTaskAssignmentSubject(), - EmailUtil.TASK_NOTIFICATION_TEMPLATE); - } catch (IOException ex) { - LOG.error("Task Email Notification Failed :", ex); - } catch (TemplateException ex) { - LOG.error("Task Email Notification Template Parsing Exception :", ex); - } - }); - } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/util/jdbi/OMSqlLogger.java b/openmetadata-service/src/main/java/org/openmetadata/service/util/jdbi/OMSqlLogger.java new file mode 100644 index 000000000000..54aade1c858a --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/util/jdbi/OMSqlLogger.java @@ -0,0 +1,32 @@ +package org.openmetadata.service.util.jdbi; + +import java.time.temporal.ChronoUnit; +import java.util.concurrent.TimeUnit; +import lombok.extern.slf4j.Slf4j; +import org.jdbi.v3.core.statement.SqlLogger; +import org.jdbi.v3.core.statement.StatementContext; +import org.openmetadata.service.util.MicrometerBundleSingleton; + +@Slf4j +public class OMSqlLogger implements SqlLogger { + @Override + public void logBeforeExecution(StatementContext context) { + if (LOG.isDebugEnabled()) { + LOG.debug("sql {}, parameters {}", context.getRenderedSql(), context.getBinding()); + } + } + + @Override + public void logAfterExecution(StatementContext context) { + long elapsedTime = context.getElapsedTime(ChronoUnit.SECONDS); + MicrometerBundleSingleton.jdbiRequests.observe(elapsedTime); + MicrometerBundleSingleton.getJdbiLatencyTimer().record(elapsedTime, TimeUnit.SECONDS); + if (LOG.isDebugEnabled()) { + LOG.debug( + "sql {}, parameters {}, timeTaken {} ms", + context.getRenderedSql(), + context.getBinding(), + context.getElapsedTime(ChronoUnit.MILLIS)); + } + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/workflows/searchIndex/PaginatedEntitiesSource.java b/openmetadata-service/src/main/java/org/openmetadata/service/workflows/searchIndex/PaginatedEntitiesSource.java index c6900e0cfaa0..60e5ce027b12 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/workflows/searchIndex/PaginatedEntitiesSource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/workflows/searchIndex/PaginatedEntitiesSource.java @@ -72,7 +72,7 @@ public ResultList readNext(Map contex private ResultList read(String cursor) throws SearchIndexException { LOG.debug("[PaginatedEntitiesSource] Fetching a Batch of Size: {} ", batchSize); EntityRepository entityRepository = Entity.getEntityRepository(entityType); - ResultList result = null; + ResultList result; try { result = entityRepository.listAfterWithSkipFailure( @@ -83,39 +83,42 @@ private ResultList read(String cursor) throws SearchI cursor); if (!result.getErrors().isEmpty()) { lastFailedCursor = this.cursor; - throw new SearchIndexException( - new IndexingError() - .withErrorSource(READER) - .withLastFailedCursor(lastFailedCursor) - .withSubmittedCount(batchSize) - .withSuccessCount(result.getData().size()) - .withFailedCount(result.getErrors().size()) - .withMessage( - "Issues in Reading A Batch For Entities. Check Errors Corresponding to Entities.") - .withFailedEntities(result.getErrors())); + if (result.getPaging().getAfter() == null) { + isDone = true; + } else { + this.cursor = result.getPaging().getAfter(); + } + updateStats(result.getData().size(), result.getErrors().size()); + return result; } LOG.debug( "[PaginatedEntitiesSource] Batch Stats :- %n Submitted : {} Success: {} Failed: {}", batchSize, result.getData().size(), result.getErrors().size()); updateStats(result.getData().size(), result.getErrors().size()); - } catch (SearchIndexException ex) { + } catch (Exception e) { lastFailedCursor = this.cursor; - if (result.getPaging().getAfter() == null) { - isDone = true; + int remainingRecords = + stats.getTotalRecords() - stats.getFailedRecords() - stats.getSuccessRecords(); + int submittedRecords; + if (remainingRecords - batchSize <= 0) { + submittedRecords = remainingRecords; + updateStats(0, remainingRecords); + this.cursor = null; + this.isDone = true; } else { - this.cursor = result.getPaging().getAfter(); + submittedRecords = batchSize; + String decodedCursor = RestUtil.decodeCursor(cursor); + this.cursor = + RestUtil.encodeCursor(String.valueOf(Integer.parseInt(decodedCursor) + batchSize)); + updateStats(0, batchSize); } - updateStats(result.getData().size(), result.getErrors().size()); - throw ex; - } catch (Exception e) { - lastFailedCursor = this.cursor; IndexingError indexingError = new IndexingError() .withErrorSource(READER) - .withSubmittedCount(batchSize) + .withSubmittedCount(submittedRecords) .withSuccessCount(0) - .withFailedCount(batchSize) + .withFailedCount(submittedRecords) .withMessage( "Issues in Reading A Batch For Entities. No Relationship Issue , Json Processing or DB issue.") .withLastFailedCursor(lastFailedCursor) diff --git a/openmetadata-service/src/main/resources/elasticsearch/en/dashboard_index_mapping.json b/openmetadata-service/src/main/resources/elasticsearch/en/dashboard_index_mapping.json index 575bdfd43c11..a03d55e24941 100644 --- a/openmetadata-service/src/main/resources/elasticsearch/en/dashboard_index_mapping.json +++ b/openmetadata-service/src/main/resources/elasticsearch/en/dashboard_index_mapping.json @@ -196,6 +196,16 @@ } } }, + "project": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "normalizer": "lowercase_normalizer", + "ignore_above": 256 + } + } + }, "domain" : { "properties": { "id": { diff --git a/openmetadata-service/src/main/resources/elasticsearch/en/test_suite_index_mapping.json b/openmetadata-service/src/main/resources/elasticsearch/en/test_suite_index_mapping.json index 353405394c1b..6b58935c6a2d 100644 --- a/openmetadata-service/src/main/resources/elasticsearch/en/test_suite_index_mapping.json +++ b/openmetadata-service/src/main/resources/elasticsearch/en/test_suite_index_mapping.json @@ -84,6 +84,30 @@ "entityType": { "type": "keyword" }, + "testCaseResultSummary": { + "type": "nested", + "properties": { + "status": { + "type": "keyword" + }, + "testCaseName": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + }, + "ngram": { + "type": "text", + "analyzer": "om_ngram" + } + } + }, + "timestamp": { + "type": "long" + } + } + }, "suggest": { "type": "completion", "contexts": [ diff --git a/openmetadata-service/src/main/resources/elasticsearch/jp/dashboard_index_mapping.json b/openmetadata-service/src/main/resources/elasticsearch/jp/dashboard_index_mapping.json index 6f43943ecd0a..99c0a17618af 100644 --- a/openmetadata-service/src/main/resources/elasticsearch/jp/dashboard_index_mapping.json +++ b/openmetadata-service/src/main/resources/elasticsearch/jp/dashboard_index_mapping.json @@ -222,6 +222,16 @@ } } }, + "project": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "normalizer": "lowercase_normalizer", + "ignore_above": 256 + } + } + }, "dataModels": { "properties": { "id": { diff --git a/openmetadata-service/src/main/resources/elasticsearch/jp/test_suite_index_mapping.json b/openmetadata-service/src/main/resources/elasticsearch/jp/test_suite_index_mapping.json index 6c7ccb3130de..6062d4074c90 100644 --- a/openmetadata-service/src/main/resources/elasticsearch/jp/test_suite_index_mapping.json +++ b/openmetadata-service/src/main/resources/elasticsearch/jp/test_suite_index_mapping.json @@ -82,6 +82,27 @@ "entityType": { "type": "keyword" }, + "testCaseResultSummary": { + "type": "nested", + "properties": { + "status": { + "type": "keyword" + }, + "testCaseName": { + "type": "text", + "analyzer": "om_analyzer_jp", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "timestamp": { + "type": "long" + } + } + }, "suggest": { "type": "completion", "contexts": [ diff --git a/openmetadata-service/src/main/resources/elasticsearch/zh/dashboard_index_mapping.json b/openmetadata-service/src/main/resources/elasticsearch/zh/dashboard_index_mapping.json index 3e5a2abdb14c..bc0c865b51b4 100644 --- a/openmetadata-service/src/main/resources/elasticsearch/zh/dashboard_index_mapping.json +++ b/openmetadata-service/src/main/resources/elasticsearch/zh/dashboard_index_mapping.json @@ -123,6 +123,16 @@ } } }, + "project": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "normalizer": "lowercase_normalizer", + "ignore_above": 256 + } + } + }, "domain" : { "properties": { "id": { diff --git a/openmetadata-service/src/main/resources/elasticsearch/zh/test_suite_index_mapping.json b/openmetadata-service/src/main/resources/elasticsearch/zh/test_suite_index_mapping.json index 8860dd5c4eef..c1c45b7b82b9 100644 --- a/openmetadata-service/src/main/resources/elasticsearch/zh/test_suite_index_mapping.json +++ b/openmetadata-service/src/main/resources/elasticsearch/zh/test_suite_index_mapping.json @@ -67,6 +67,28 @@ "entityType": { "type": "keyword" }, + "testCaseResultSummary": { + "type": "nested", + "properties": { + "status": { + "type": "keyword" + }, + "testCaseName": { + "type": "text", + "analyzer": "ik_max_word", + "search_analyzer": "ik_smart", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "timestamp": { + "type": "long" + } + } + }, "fqnParts": { "type": "keyword" }, diff --git a/openmetadata-service/src/main/resources/emailTemplates/dataInsightReport.ftl b/openmetadata-service/src/main/resources/emailTemplates/dataInsightReport.ftl index 7d5398121cfd..489548a3d0be 100644 --- a/openmetadata-service/src/main/resources/emailTemplates/dataInsightReport.ftl +++ b/openmetadata-service/src/main/resources/emailTemplates/dataInsightReport.ftl @@ -84,7 +84,7 @@ - +
View DetailsView Report
@@ -104,46 +104,140 @@ - +
+ + + + + + + + + + +
+

Total Data Assets
${totalAssetObj.totalDataAssets}
${totalAssetObj.percentChangeMessage} [${totalAssetObj.numberOfDaysChange} Days Change]

+ 		+

Data Assets with Description
${descriptionObj.totalAssets}
${descriptionObj.percentChangeMessage} [${descriptionObj.numberOfDaysChange} Days Change]

+
+

Total Data Assets with Owner
${ownershipObj.totalAssets}
${ownershipObj.percentChangeMessage} [${ownershipObj.numberOfDaysChange} Days Change]

+ 		+

Data Assets with Tiers
${tierObj.totalAssets}
${tierObj.percentChangeMessage} [${tierObj.numberOfDaysChange} Days Change]

+
+ - @@ -507,7 +612,7 @@ <#list tierObj.dateMap?keys as key>
- + + + +
 
+ + + + + @@ -475,6 +576,10 @@ + + + +
+ - + + - + + + + + + + + + + + + + +
  + + + + + + +
Total Data Assets
+
+  
${totalAssetObj.completeMessage}
 
${totalAssetObj.percentChangeMessage} [${totalAssetObj.numberOfDaysChange} Days Change]
+ 		+ + + + + + + + + + @@ -337,12 +434,16 @@ - <#if ownershipObj.kpiAvailable> + <#if ownershipObj.kpiAvailable> + + + +
0
Total Data Assets
- - <#if descriptionObj.kpiAvailable> + <#if descriptionObj.kpiAvailable> - + + + + +
- + + + - - - - +
+ - + + + + + + @@ -151,12 +245,11 @@
- + + + + + +
+ - - - - - - - - - - - - - + + <#list totalAssetObj.dateMap?keys as key> + + +
 
 
Total Data Assets
${totalAssetObj.completeMessage}
${totalAssetObj.totalDataAssets} + + + + + + + + + +
 
 
+
 
+
+ + + + + + +
 
+
+ + + + + <#list totalAssetObj.dateMap?keys as key> + + + +
${key}
 
  
@@ -199,12 +292,16 @@
 
Target KPI: ${descriptionObj.targetKpi}% | Current KPI: ${descriptionObj.percentCompleted}%
${descriptionObj.percentChangeMessage} [${descriptionObj.numberOfDaysChange} Days Change]
 
Target KPI: ${ownershipObj.targetKpi}% | Current KPI: ${ownershipObj.percentCompleted}%
${ownershipObj.percentChangeMessage} [${ownershipObj.numberOfDaysChange} Days Change]
 
${tierObj.percentChangeMessage} [${tierObj.numberOfDaysChange} Days Change]
- + diff --git a/openmetadata-service/src/main/resources/json/data/app/DataInsightsApplication.json b/openmetadata-service/src/main/resources/json/data/app/DataInsightsApplication.json index 590d467790ae..4e3c99da6b76 100644 --- a/openmetadata-service/src/main/resources/json/data/app/DataInsightsApplication.json +++ b/openmetadata-service/src/main/resources/json/data/app/DataInsightsApplication.json @@ -3,7 +3,7 @@ "displayName": "Data Insights", "appConfiguration": {}, "appSchedule": { - "scheduleType": "Custom", + "scheduleTimeline": "Custom", "cronExpression": "0 0 1/1 * *" } } \ No newline at end of file diff --git a/openmetadata-service/src/main/resources/json/data/app/SearchIndexingApplication.json b/openmetadata-service/src/main/resources/json/data/app/SearchIndexingApplication.json index 660c7e1bc43b..a373325802e3 100644 --- a/openmetadata-service/src/main/resources/json/data/app/SearchIndexingApplication.json +++ b/openmetadata-service/src/main/resources/json/data/app/SearchIndexingApplication.json @@ -38,12 +38,12 @@ "storageService", "dataProduct" ], - "recreateIndex": false, + "recreateIndex": true, "batchSize": "100", "searchIndexMappingLanguage": "EN" }, "appSchedule": { - "scheduleType": "Custom", + "scheduleTimeline": "Custom", "cronExpression": "0 0 * * *" } } \ No newline at end of file diff --git a/openmetadata-service/src/main/resources/json/data/appMarketPlaceDefinition/SearchIndexingApplication.json b/openmetadata-service/src/main/resources/json/data/appMarketPlaceDefinition/SearchIndexingApplication.json index 357a8027eed2..a5371efbf407 100644 --- a/openmetadata-service/src/main/resources/json/data/appMarketPlaceDefinition/SearchIndexingApplication.json +++ b/openmetadata-service/src/main/resources/json/data/appMarketPlaceDefinition/SearchIndexingApplication.json @@ -52,7 +52,7 @@ "storageService", "dataProduct" ], - "recreateIndex": false, + "recreateIndex": true, "batchSize": "100", "searchIndexMappingLanguage": "EN" } diff --git a/openmetadata-service/src/main/resources/json/data/eventsubscription/ActivityFeedEvents.json b/openmetadata-service/src/main/resources/json/data/eventsubscription/ActivityFeedEvents.json index f2f6cb774e5b..1abf827e2668 100644 --- a/openmetadata-service/src/main/resources/json/data/eventsubscription/ActivityFeedEvents.json +++ b/openmetadata-service/src/main/resources/json/data/eventsubscription/ActivityFeedEvents.json @@ -20,7 +20,7 @@ { "name": "matchAnyFieldChange", "effect": "exclude", - "condition": "matchAnyFieldChange({'usageSummary'})" + "condition": "matchAnyFieldChange({'usageSummary', 'sourceHash', 'lifeCycle'})" } ] }, @@ -81,7 +81,9 @@ { "name": "fieldChangeList", "input": [ - "usageSummary" + "usageSummary", + "sourceHash", + "lifeCycle" ] } ] diff --git a/openmetadata-service/src/main/resources/json/data/testConnections/metadata/alation.json b/openmetadata-service/src/main/resources/json/data/testConnections/metadata/alation.json index 889832dd1af3..56b786902823 100644 --- a/openmetadata-service/src/main/resources/json/data/testConnections/metadata/alation.json +++ b/openmetadata-service/src/main/resources/json/data/testConnections/metadata/alation.json @@ -9,6 +9,13 @@ "errorMessage": "Failed to connect to Alation, please validate the credentials", "shortCircuit": true, "mandatory": true + }, + { + "name": "CheckDbAccess", + "description": "Check if the Alation backend database is reachable with the given credentials.", + "errorMessage": "Failed to connect to Alation Database, please validate the credentials", + "shortCircuit": false, + "mandatory": false } ] } \ No newline at end of file diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/OpenMetadataApplicationTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/OpenMetadataApplicationTest.java index 7a5d41c05a60..f402f39a7cef 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/OpenMetadataApplicationTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/OpenMetadataApplicationTest.java @@ -21,6 +21,7 @@ import io.dropwizard.testing.ConfigOverride; import io.dropwizard.testing.ResourceHelpers; import io.dropwizard.testing.junit5.DropwizardAppExtension; +import java.net.URI; import java.util.HashSet; import java.util.Set; import javax.ws.rs.client.Client; @@ -35,6 +36,7 @@ import org.jdbi.v3.core.Jdbi; import org.jdbi.v3.sqlobject.SqlObjectPlugin; import org.jdbi.v3.sqlobject.SqlObjects; +import org.jetbrains.annotations.NotNull; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.TestInstance; @@ -61,7 +63,7 @@ public abstract class OpenMetadataApplicationTest { public static final boolean RUN_ELASTIC_SEARCH_TESTCASES = false; - private static final Set configOverrides = new HashSet<>(); + protected static final Set configOverrides = new HashSet<>(); private static final String JDBC_CONTAINER_CLASS_NAME = "org.testcontainers.containers.MySQLContainer"; @@ -80,7 +82,7 @@ public abstract class OpenMetadataApplicationTest { } @BeforeAll - public static void createApplication() throws Exception { + public void createApplication() throws Exception { String jdbcContainerClassName = System.getProperty("jdbcContainerClassName"); String jdbcContainerImage = System.getProperty("jdbcContainerImage"); String elasticSearchContainerImage = System.getProperty("elasticSearchContainerClassName"); @@ -155,9 +157,7 @@ public static void createApplication() throws Exception { ConfigOverride.config("migrationConfiguration.nativePath", nativeMigrationScriptsLocation)); ConfigOverride[] configOverridesArray = configOverrides.toArray(new ConfigOverride[0]); - APP = - new DropwizardAppExtension<>( - OpenMetadataApplication.class, CONFIG_PATH, configOverridesArray); + APP = getApp(configOverridesArray); // Run System Migrations jdbi = Jdbi.create( @@ -176,6 +176,13 @@ public static void createApplication() throws Exception { createClient(); } + @NotNull + protected DropwizardAppExtension getApp( + ConfigOverride[] configOverridesArray) { + return new DropwizardAppExtension<>( + OpenMetadataApplication.class, CONFIG_PATH, configOverridesArray); + } + private static void createClient() { ClientConfig config = new ClientConfig(); config.connectorProvider(new JettyConnectorProvider()); @@ -187,7 +194,7 @@ private static void createClient() { } @AfterAll - public static void stopApplication() throws Exception { + public void stopApplication() throws Exception { // If BeforeAll causes and exception AfterAll still gets called before that exception is thrown. // If a NullPointerException is thrown during the cleanup of above it will eat the initial error if (APP != null) { @@ -210,6 +217,11 @@ public static WebTarget getResource(String collection) { return client.target(format("http://localhost:%s/api/v1/%s", APP.getLocalPort(), collection)); } + public static WebTarget getResourceAsURI(String collection) { + return client.target( + URI.create((format("http://localhost:%s/api/v1/%s", APP.getLocalPort(), collection)))); + } + public static WebTarget getConfigResource(String resource) { return client.target( format("http://localhost:%s/api/v1/system/config/%s", APP.getLocalPort(), resource)); diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/jdbi3/ListFilterTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/jdbi3/ListFilterTest.java index ddffdde3b6eb..cf6b389f17bc 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/jdbi3/ListFilterTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/jdbi3/ListFilterTest.java @@ -2,6 +2,8 @@ import static org.junit.jupiter.api.Assertions.*; +import java.util.ArrayList; +import java.util.List; import org.junit.jupiter.api.Test; class ListFilterTest { @@ -15,4 +17,34 @@ void test_escapeApostrophe() { assertEquals("a''b\\_c\\_d", ListFilter.escape("a'b_c_d")); assertEquals("a\\_b\\_c\\_d", ListFilter.escape("a_b_c_d")); } + + @Test + void addCondition() { + String condition; + ListFilter filter = new ListFilter(); + + condition = filter.addCondition(List.of("a", "b")); + assertEquals("a AND b", condition); + + condition = filter.addCondition(List.of("foo=`abcf`", "", "")); + assertEquals("foo=`abcf`", condition); + + condition = filter.addCondition(List.of("foo=`abcf`", "v in ('A', 'B')", "x > 6")); + assertEquals("foo=`abcf` AND v in ('A', 'B') AND x > 6", condition); + + condition = filter.addCondition(new ArrayList<>()); + assertEquals("", condition); + } + + @Test + void getCondition() { + ListFilter filter = new ListFilter(); + String condition = filter.getCondition("foo"); + assertEquals("WHERE foo.deleted = FALSE", condition); + + filter = new ListFilter(); + filter.addQueryParam("testCaseStatus", "Failed"); + condition = filter.getCondition("foo"); + assertEquals("WHERE foo.deleted = FALSE AND status = 'Failed'", condition); + } } diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/resources/EntityResourceTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/resources/EntityResourceTest.java index eeae3923a524..3b767f8fe753 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/resources/EntityResourceTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/resources/EntityResourceTest.java @@ -100,8 +100,10 @@ import org.apache.http.util.EntityUtils; import org.awaitility.Awaitility; import org.json.JSONArray; +import org.json.JSONException; import org.json.JSONObject; import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.Assumptions; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestInfo; @@ -230,6 +232,10 @@ public abstract class EntityResourceTest patchEntityAndCheck(entity, originalJson, ADMIN_AUTH_HEADERS, MINOR_UPDATE, change), + NOT_FOUND, + String.format("domain instance for %s not found", domainReference.getId())); + } + + @Test + @Execution(ExecutionMode.CONCURRENT) + void patchWrongDataProducts(TestInfo test) throws IOException { + Assumptions.assumeTrue(supportsDataProducts); + T entity = createEntity(createRequest(test, 0), ADMIN_AUTH_HEADERS); + + // Add random domain reference + EntityReference dataProductReference = new EntityReference().withId(UUID.randomUUID()); + String originalJson = JsonUtils.pojoToJson(entity); + ChangeDescription change = getChangeDescription(entity, MINOR_UPDATE); + entity.setDataProducts(List.of(dataProductReference)); + + assertResponse( + () -> patchEntityAndCheck(entity, originalJson, ADMIN_AUTH_HEADERS, MINOR_UPDATE, change), + NOT_FOUND, + String.format("dataProduct instance for %s not found", dataProductReference.getId())); + } + @Test @Execution(ExecutionMode.CONCURRENT) protected void get_entityListWithPagination_200(TestInfo test) throws IOException { @@ -1877,7 +1926,7 @@ void delete_systemEntity() throws IOException { } @Test - protected void checkIndexCreated() throws IOException { + protected void checkIndexCreated() throws IOException, JSONException { if (RUN_ELASTIC_SEARCH_TESTCASES) { RestClient client = getSearchClient(); Request request = new Request("GET", "/_cat/indices"); @@ -2610,6 +2659,9 @@ protected final void validateChangeEvents( ChangeDescription expectedChangeDescription, Map authHeaders) throws IOException { + if (!runWebhookTests) { + return; + } validateChangeEvents( entityInterface, timestamp, @@ -2751,6 +2803,9 @@ private void validateDeletedEvent( EventType expectedEventType, Double expectedVersion, Map authHeaders) { + if (!runWebhookTests) { + return; + } String updatedBy = SecurityUtil.getPrincipalName(authHeaders); EventHolder eventHolder = new EventHolder(); diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/resources/apps/AppsResourceTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/resources/apps/AppsResourceTest.java index 4bcb11ce0c05..3754f2579ea8 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/resources/apps/AppsResourceTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/resources/apps/AppsResourceTest.java @@ -1,24 +1,36 @@ package org.openmetadata.service.resources.apps; +import static javax.ws.rs.core.Response.Status.BAD_REQUEST; +import static javax.ws.rs.core.Response.Status.OK; import static org.openmetadata.service.util.TestUtils.ADMIN_AUTH_HEADERS; +import static org.openmetadata.service.util.TestUtils.assertResponseContains; +import static org.openmetadata.service.util.TestUtils.readResponse; +import java.io.IOException; import java.util.Map; +import java.util.concurrent.TimeUnit; +import javax.ws.rs.client.WebTarget; +import javax.ws.rs.core.Response; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; import org.apache.http.client.HttpResponseException; import org.junit.jupiter.api.Test; import org.openmetadata.schema.entity.app.App; import org.openmetadata.schema.entity.app.AppMarketPlaceDefinition; +import org.openmetadata.schema.entity.app.AppRunRecord; import org.openmetadata.schema.entity.app.AppSchedule; import org.openmetadata.schema.entity.app.CreateApp; import org.openmetadata.schema.entity.app.CreateAppMarketPlaceDefinitionReq; +import org.openmetadata.schema.entity.app.ScheduleTimeline; import org.openmetadata.service.Entity; import org.openmetadata.service.exception.EntityNotFoundException; import org.openmetadata.service.resources.EntityResourceTest; +import org.openmetadata.service.security.SecurityUtil; import org.openmetadata.service.util.TestUtils; @Slf4j public class AppsResourceTest extends EntityResourceTest { + private static final String SYSTEM_APP_NAME = "systemApp"; public AppsResourceTest() { super(Entity.APPLICATION, App.class, AppResource.AppList.class, "apps", AppResource.FIELDS); @@ -36,7 +48,8 @@ public CreateApp createRequest(String name) { appMarketPlaceDefinition = appMarketPlaceResourceTest.getEntityByName(name, ADMIN_AUTH_HEADERS); } catch (EntityNotFoundException | HttpResponseException ex) { - CreateAppMarketPlaceDefinitionReq req = appMarketPlaceResourceTest.createRequest(name); + CreateAppMarketPlaceDefinitionReq req = + appMarketPlaceResourceTest.createRequest(name).withSystem(name.equals(SYSTEM_APP_NAME)); appMarketPlaceDefinition = appMarketPlaceResourceTest.createAndCheckEntity(req, ADMIN_AUTH_HEADERS); } @@ -44,7 +57,7 @@ public CreateApp createRequest(String name) { return new CreateApp() .withName(appMarketPlaceDefinition.getName()) .withAppConfiguration(appMarketPlaceDefinition.getAppConfiguration()) - .withAppSchedule(new AppSchedule().withScheduleType(AppSchedule.ScheduleTimeline.HOURLY)); + .withAppSchedule(new AppSchedule().withScheduleTimeline(ScheduleTimeline.HOURLY)); } @Test @@ -54,6 +67,40 @@ protected void post_entityCreateWithInvalidName_400() { // Does not apply since the App is already validated in the AppMarketDefinition } + @Test + void delete_systemApp_400() throws IOException { + CreateApp systemAppRequest = createRequest(SYSTEM_APP_NAME); + App systemApp = createAndCheckEntity(systemAppRequest, ADMIN_AUTH_HEADERS); + assertResponseContains( + () -> deleteEntity(systemApp.getId(), ADMIN_AUTH_HEADERS), + BAD_REQUEST, + "of type SystemApp can not be deleted"); + } + + @Test + void post_trigger_app_200() throws HttpResponseException, InterruptedException { + postTriggerApp("SearchIndexingApplication", ADMIN_AUTH_HEADERS); + TimeUnit.MILLISECONDS.sleep(200); + AppRunRecord latestRun = getLatestAppRun("SearchIndexingApplication", ADMIN_AUTH_HEADERS); + assert latestRun.getStatus().equals(AppRunRecord.Status.RUNNING); + TimeUnit timeout = TimeUnit.SECONDS; + long timeoutValue = 30; + long startTime = System.currentTimeMillis(); + while (latestRun.getStatus().equals(AppRunRecord.Status.RUNNING)) { + // skip this loop in CI because it causes weird problems + if (TestUtils.isCI()) { + break; + } + assert !latestRun.getStatus().equals(AppRunRecord.Status.FAILED); + if (System.currentTimeMillis() - startTime > timeout.toMillis(timeoutValue)) { + throw new AssertionError( + String.format("Expected the app to succeed within %d %s", timeoutValue, timeout)); + } + TimeUnit.MILLISECONDS.sleep(500); + latestRun = getLatestAppRun("SearchIndexingApplication", ADMIN_AUTH_HEADERS); + } + } + @Override public void validateCreatedEntity( App createdEntity, CreateApp request, Map authHeaders) @@ -85,4 +132,17 @@ public App validateGetWithDifferentFields(App entity, boolean byName) public void assertFieldChange(String fieldName, Object expected, Object actual) { assertCommonFieldChange(fieldName, expected, actual); } + + private void postTriggerApp(String appName, Map authHeaders) + throws HttpResponseException { + WebTarget target = getResource("apps/trigger").path(appName); + Response response = SecurityUtil.addHeaders(target, authHeaders).post(null); + readResponse(response, OK.getStatusCode()); + } + + private AppRunRecord getLatestAppRun(String appName, Map authHeaders) + throws HttpResponseException { + WebTarget target = getResource(String.format("apps/name/%s/runs/latest", appName)); + return TestUtils.get(target, AppRunRecord.class, authHeaders); + } } diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/resources/databases/TableResourceTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/resources/databases/TableResourceTest.java index d49421312c01..ab9ce3ece827 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/resources/databases/TableResourceTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/resources/databases/TableResourceTest.java @@ -2299,9 +2299,10 @@ void testImportExport() throws IOException { Column c1 = new Column().withName("c1").withDataType(STRUCT); Column c11 = new Column().withName("c11").withDataType(INT); Column c2 = new Column().withName("c2").withDataType(INT); + Column c3 = new Column().withName("c3").withDataType(BIGINT); c1.withChildren(listOf(c11)); CreateTable createTable = - createRequest("s1").withColumns(listOf(c1, c2)).withTableConstraints(null); + createRequest("s1").withColumns(listOf(c1, c2, c3)).withTableConstraints(null); Table table = createEntity(createTable, ADMIN_AUTH_HEADERS); // Headers: name, displayName, description, owner, tags, retentionPeriod, sourceUrl, domain @@ -2313,7 +2314,8 @@ void testImportExport() throws IOException { + "dsp1-new,desc1,type,PII.Sensitive", user1, escapeCsv(DOMAIN.getFullyQualifiedName())), ",,,,,,,,c1.c11,dsp11-new,desc11,type1,PII.Sensitive", - ",,,,,,,,c2,,,,"); + ",,,,,,,,c2,,,,", + ",,,,,,,,c3,,,,"); // Update created entity with changes importCsvAndValidate(table.getFullyQualifiedName(), TableCsv.HEADERS, null, updateRecords); diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/resources/domains/DataProductResourceTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/resources/domains/DataProductResourceTest.java index 1f1e74d54f2d..7e69d726ef8d 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/resources/domains/DataProductResourceTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/resources/domains/DataProductResourceTest.java @@ -1,9 +1,11 @@ package org.openmetadata.service.resources.domains; +import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.openmetadata.common.utils.CommonUtil.listOf; import static org.openmetadata.service.Entity.FIELD_ASSETS; +import static org.openmetadata.service.Entity.TABLE; import static org.openmetadata.service.util.EntityUtil.fieldAdded; import static org.openmetadata.service.util.EntityUtil.fieldDeleted; import static org.openmetadata.service.util.TestUtils.*; @@ -14,6 +16,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.UUID; import javax.ws.rs.core.Response.Status; import org.apache.http.client.HttpResponseException; import org.junit.jupiter.api.Test; @@ -24,7 +27,10 @@ import org.openmetadata.schema.entity.domains.DataProduct; import org.openmetadata.schema.entity.type.Style; import org.openmetadata.schema.type.ChangeDescription; +import org.openmetadata.schema.type.EntityReference; import org.openmetadata.service.Entity; +import org.openmetadata.service.exception.EntityNotFoundException; +import org.openmetadata.service.jdbi3.TableRepository; import org.openmetadata.service.resources.EntityResourceTest; import org.openmetadata.service.resources.databases.TableResourceTest; import org.openmetadata.service.resources.domains.DataProductResource.DataProductList; @@ -160,6 +166,20 @@ void test_listWithDomainFilter(TestInfo test) throws HttpResponseException { assertTrue(list.stream().anyMatch(s -> s.getName().equals(p4.getName()))); } + @Test + void testValidateDataProducts() { + UUID rdnUUID = UUID.randomUUID(); + EntityReference entityReference = new EntityReference().withId(rdnUUID); + TableRepository entityRepository = (TableRepository) Entity.getEntityRepository(TABLE); + + assertThatThrownBy( + () -> { + entityRepository.validateDataProducts(List.of(entityReference)); + }) + .isInstanceOf(EntityNotFoundException.class) + .hasMessage(String.format("dataProduct instance for %s not found", rdnUUID)); + } + private void entityInDataProduct( EntityInterface entity, EntityInterface product, boolean inDataProduct) throws HttpResponseException { diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/resources/domains/DomainResourceTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/resources/domains/DomainResourceTest.java index f416213aa738..9f55c8451dbc 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/resources/domains/DomainResourceTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/resources/domains/DomainResourceTest.java @@ -1,7 +1,10 @@ package org.openmetadata.service.resources.domains; +import static javax.ws.rs.core.Response.Status.NOT_FOUND; +import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.openmetadata.common.utils.CommonUtil.listOf; +import static org.openmetadata.service.Entity.TABLE; import static org.openmetadata.service.security.SecurityUtil.authHeaders; import static org.openmetadata.service.util.EntityUtil.fieldAdded; import static org.openmetadata.service.util.EntityUtil.fieldDeleted; @@ -13,10 +16,12 @@ import static org.openmetadata.service.util.TestUtils.assertEntityReferenceNames; import static org.openmetadata.service.util.TestUtils.assertListNotNull; import static org.openmetadata.service.util.TestUtils.assertListNull; +import static org.openmetadata.service.util.TestUtils.assertResponse; import java.io.IOException; import java.util.List; import java.util.Map; +import java.util.UUID; import javax.ws.rs.core.Response.Status; import org.apache.http.client.HttpResponseException; import org.junit.jupiter.api.Test; @@ -26,7 +31,10 @@ import org.openmetadata.schema.entity.domains.Domain; import org.openmetadata.schema.entity.type.Style; import org.openmetadata.schema.type.ChangeDescription; +import org.openmetadata.schema.type.EntityReference; import org.openmetadata.service.Entity; +import org.openmetadata.service.exception.EntityNotFoundException; +import org.openmetadata.service.jdbi3.TableRepository; import org.openmetadata.service.resources.EntityResourceTest; import org.openmetadata.service.resources.domains.DomainResource.DomainList; import org.openmetadata.service.util.JsonUtils; @@ -113,6 +121,37 @@ void testInheritedPermissionFromParent(TestInfo test) throws IOException { createEntity(create, authHeaders(DATA_CONSUMER.getName())); } + @Test + void testValidateDomain() { + UUID rdnUUID = UUID.randomUUID(); + EntityReference entityReference = new EntityReference().withId(rdnUUID); + TableRepository entityRepository = (TableRepository) Entity.getEntityRepository(TABLE); + + assertThatThrownBy( + () -> { + entityRepository.validateDomain(entityReference); + }) + .isInstanceOf(EntityNotFoundException.class) + .hasMessage(String.format("domain instance for %s not found", rdnUUID)); + } + + @Test + void patchWrongExperts(TestInfo test) throws IOException { + Domain entity = createEntity(createRequest(test, 0), ADMIN_AUTH_HEADERS); + + // Add random domain reference + EntityReference expertReference = + new EntityReference().withId(UUID.randomUUID()).withType(Entity.USER); + String originalJson = JsonUtils.pojoToJson(entity); + ChangeDescription change = getChangeDescription(entity, MINOR_UPDATE); + entity.setExperts(List.of(expertReference)); + + assertResponse( + () -> patchEntityAndCheck(entity, originalJson, ADMIN_AUTH_HEADERS, MINOR_UPDATE, change), + NOT_FOUND, + String.format("user instance for %s not found", expertReference.getId())); + } + @Override public CreateDomain createRequest(String name) { return new CreateDomain() diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/resources/dqtests/TestCaseResourceTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/resources/dqtests/TestCaseResourceTest.java index f1e8d30e2597..cf9e80d5ff35 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/resources/dqtests/TestCaseResourceTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/resources/dqtests/TestCaseResourceTest.java @@ -79,6 +79,7 @@ import org.openmetadata.service.util.ResultList; import org.openmetadata.service.util.TestUtils; import org.openmetadata.service.util.incidentSeverityClassifier.IncidentSeverityClassifierInterface; +import org.testcontainers.shaded.com.google.common.collect.ImmutableMap; @TestMethodOrder(MethodOrderer.OrderAnnotation.class) @Slf4j @@ -167,7 +168,6 @@ void patch_entityComputePassedFailedRowCount(TestInfo test) throws IOException { TestCase entity = createEntity( createRequest(getEntityName(test), "description", null, null), ADMIN_AUTH_HEADERS); - ChangeDescription change = getChangeDescription(entity, MINOR_UPDATE); String json = JsonUtils.pojoToJson(entity); entity.setComputePassedFailedRowCount(true); @@ -381,17 +381,14 @@ void put_testCaseResults_200(TestInfo test) throws IOException, ParseException { ADMIN_AUTH_HEADERS); verifyTestCaseResults(testCaseResults, testCase1ResultList, 4); - TestSummary testSummary = getTestSummary(ADMIN_AUTH_HEADERS, null); - assertNotEquals(0, testSummary.getFailed()); - assertNotEquals(0, testSummary.getSuccess()); - assertNotEquals(0, testSummary.getTotal()); - assertEquals(0, testSummary.getAborted()); - - String randomUUID = UUID.randomUUID().toString(); - assertResponseContains( - () -> getTestSummary(ADMIN_AUTH_HEADERS, randomUUID), - NOT_FOUND, - "testSuite instance for " + randomUUID + " not found"); + TestSummary testSummary; + if (supportsSearchIndex && RUN_ELASTIC_SEARCH_TESTCASES) { + testSummary = getTestSummary(null); + assertNotEquals(0, testSummary.getFailed()); + assertNotEquals(0, testSummary.getSuccess()); + assertNotEquals(0, testSummary.getTotal()); + assertEquals(0, testSummary.getAborted()); + } // Test that we can get the test summary for a logical test suite and that // adding a logical test suite does not change the total number of tests @@ -403,25 +400,30 @@ void put_testCaseResults_200(TestInfo test) throws IOException, ParseException { testCaseIds.add(testCase1.getId()); testSuiteResourceTest.addTestCasesToLogicalTestSuite(logicalTestSuite, testCaseIds); - testSummary = getTestSummary(ADMIN_AUTH_HEADERS, logicalTestSuite.getId().toString()); - assertEquals(1, testSummary.getTotal()); - assertEquals(1, testSummary.getFailed()); + if (supportsSearchIndex && RUN_ELASTIC_SEARCH_TESTCASES) { + testSummary = getTestSummary(logicalTestSuite.getId().toString()); + assertEquals(1, testSummary.getTotal()); + assertEquals(1, testSummary.getFailed()); + } // add a new test case to the logical test suite to validate if the // summary is updated correctly testCaseIds.clear(); testCaseIds.add(testCase.getId()); testSuiteResourceTest.addTestCasesToLogicalTestSuite(logicalTestSuite, testCaseIds); - - testSummary = getTestSummary(ADMIN_AUTH_HEADERS, logicalTestSuite.getId().toString()); - assertEquals(2, testSummary.getTotal()); + if (supportsSearchIndex && RUN_ELASTIC_SEARCH_TESTCASES) { + testSummary = getTestSummary(logicalTestSuite.getId().toString()); + assertEquals(2, testSummary.getTotal()); + } // remove test case from logical test suite and validate // the summary is updated as expected deleteLogicalTestCase(logicalTestSuite, testCase.getId()); - testSummary = getTestSummary(ADMIN_AUTH_HEADERS, logicalTestSuite.getId().toString()); - assertEquals(1, testSummary.getTotal()); + if (supportsSearchIndex && RUN_ELASTIC_SEARCH_TESTCASES) { + testSummary = getTestSummary(logicalTestSuite.getId().toString()); + assertEquals(1, testSummary.getTotal()); + } } @Test @@ -458,37 +460,48 @@ void test_resultSummaryCascadeToAllSuites(TestInfo test) throws IOException, Par testCaseIds.add(testCase1.getId()); testSuiteResourceTest.addTestCasesToLogicalTestSuite(logicalTestSuite, testCaseIds); - // test we get the right summary for the executable test suite - TestSummary executableTestSummary = - getTestSummary(ADMIN_AUTH_HEADERS, testCase.getTestSuite().getId().toString()); TestSuite testSuite = testSuiteResourceTest.getEntity(testCase.getTestSuite().getId(), "*", ADMIN_AUTH_HEADERS); - assertEquals(testSuite.getTests().size(), executableTestSummary.getTotal()); + if (supportsSearchIndex && RUN_ELASTIC_SEARCH_TESTCASES) { + // test we get the right summary for the executable test suite + TestSummary executableTestSummary = + getTestSummary(testCase.getTestSuite().getId().toString()); + assertEquals(testSuite.getTests().size(), executableTestSummary.getTotal()); + } // test we get the right summary for the logical test suite - TestSummary logicalTestSummary = - getTestSummary(ADMIN_AUTH_HEADERS, logicalTestSuite.getId().toString()); - assertEquals(1, logicalTestSummary.getTotal()); + + if (supportsSearchIndex && RUN_ELASTIC_SEARCH_TESTCASES) { + TestSummary logicalTestSummary = getTestSummary(logicalTestSuite.getId().toString()); + assertEquals(1, logicalTestSummary.getTotal()); + } testCaseIds.clear(); testCaseIds.add(testCase.getId()); testSuiteResourceTest.addTestCasesToLogicalTestSuite(logicalTestSuite, testCaseIds); - logicalTestSummary = getTestSummary(ADMIN_AUTH_HEADERS, logicalTestSuite.getId().toString()); - assertEquals(2, logicalTestSummary.getTotal()); + if (supportsSearchIndex && RUN_ELASTIC_SEARCH_TESTCASES) { + TestSummary logicalTestSummary = getTestSummary(logicalTestSuite.getId().toString()); + assertEquals(2, logicalTestSummary.getTotal()); + } deleteEntity(testCase1.getId(), ADMIN_AUTH_HEADERS); - executableTestSummary = - getTestSummary(ADMIN_AUTH_HEADERS, testCase.getTestSuite().getId().toString()); testSuite = testSuiteResourceTest.getEntity(testCase.getTestSuite().getId(), "*", ADMIN_AUTH_HEADERS); - assertEquals(testSuite.getTests().size(), executableTestSummary.getTotal()); - logicalTestSummary = getTestSummary(ADMIN_AUTH_HEADERS, logicalTestSuite.getId().toString()); - assertEquals(2, logicalTestSummary.getTotal()); + if (supportsSearchIndex && RUN_ELASTIC_SEARCH_TESTCASES) { + TestSummary executableTestSummary = + getTestSummary(testCase.getTestSuite().getId().toString()); + assertEquals(testSuite.getTests().size(), executableTestSummary.getTotal()); + TestSummary logicalTestSummary = getTestSummary(logicalTestSuite.getId().toString()); + assertEquals(2, logicalTestSummary.getTotal()); + } // check the deletion of the test case from the executable test suite // cascaded to the logical test suite deleteLogicalTestCase(logicalTestSuite, testCase.getId()); - logicalTestSummary = getTestSummary(ADMIN_AUTH_HEADERS, logicalTestSuite.getId().toString()); - // check the deletion of the test case from the logical test suite is reflected in the summary - assertEquals(1, logicalTestSummary.getTotal()); + + if (supportsSearchIndex && RUN_ELASTIC_SEARCH_TESTCASES) { + TestSummary logicalTestSummary = getTestSummary(logicalTestSuite.getId().toString()); + // check the deletion of the test case from the logical test suite is reflected in the summary + assertEquals(1, logicalTestSummary.getTotal()); + } } @Test @@ -524,14 +537,33 @@ void test_sensitivePIITestCase(TestInfo test) throws IOException { createAndCheckEntity(create, ADMIN_AUTH_HEADERS); // Owner can see the results - ResultList testCases = - getTestCases(10, "*", sensitiveColumnLink, false, authHeaders(USER1_REF.getName())); + Map queryParamsOne = + ImmutableMap.of( + "limit", + 10, + "entityLink", + sensitiveColumnLink, + "orderByLastExecutionDate", + false, + "fields", + "*"); + ResultList testCases = getTestCases(queryParamsOne, authHeaders(USER1_REF.getName())); assertNotNull(testCases.getData().get(0).getDescription()); assertListNotEmpty(testCases.getData().get(0).getParameterValues()); // Owner can see the results + Map queryParamsTwo = + ImmutableMap.of( + "limit", + 10, + "entityLink", + sensitiveColumnLink, + "orderByLastExecutionDate", + false, + "fields", + "*"); ResultList maskedTestCases = - getTestCases(10, "*", sensitiveColumnLink, false, authHeaders(USER2_REF.getName())); + getTestCases(queryParamsTwo, authHeaders(USER2_REF.getName())); assertNull(maskedTestCases.getData().get(0).getDescription()); assertEquals(0, maskedTestCases.getData().get(0).getParameterValues().size()); } @@ -562,8 +594,12 @@ void put_testCase_list_200(TestInfo test) throws IOException { new TestCaseParameterValue().withValue("20").withName("missingCountValue"))); createAndCheckEntity(create1, ADMIN_AUTH_HEADERS); expectedTestCaseList.add(create1); - ResultList testCaseList = - getTestCases(10, "*", TABLE_LINK_2, false, ADMIN_AUTH_HEADERS); + Map queryParams = new HashMap<>(); + queryParams.put("limit", 10); + queryParams.put("entityLink", TABLE_LINK_2); + queryParams.put("fields", "*"); + queryParams.put("orderByLastExecutionDate", false); + ResultList testCaseList = getTestCases(queryParams, ADMIN_AUTH_HEADERS); verifyTestCases(testCaseList, expectedTestCaseList, 2); CreateTestCase create3 = @@ -577,10 +613,11 @@ void put_testCase_list_200(TestInfo test) throws IOException { createAndCheckEntity(create3, ADMIN_AUTH_HEADERS); expectedColTestCaseList.add(create3); - testCaseList = getTestCases(10, "*", TABLE_LINK_2, false, ADMIN_AUTH_HEADERS); + testCaseList = getTestCases(queryParams, ADMIN_AUTH_HEADERS); verifyTestCases(testCaseList, expectedTestCaseList, 2); - testCaseList = getTestCases(10, "*", TABLE_COLUMN_LINK_2, false, ADMIN_AUTH_HEADERS); + queryParams.put("entityLink", TABLE_COLUMN_LINK_2); + testCaseList = getTestCases(queryParams, ADMIN_AUTH_HEADERS); verifyTestCases(testCaseList, expectedColTestCaseList, 1); for (int i = 3; i < 12; i++) { @@ -595,14 +632,24 @@ void put_testCase_list_200(TestInfo test) throws IOException { createAndCheckEntity(create4, ADMIN_AUTH_HEADERS); expectedColTestCaseList.add(create4); } - testCaseList = getTestCases(10, "*", TABLE_COLUMN_LINK_2, false, ADMIN_AUTH_HEADERS); + + queryParams.put("entityLink", TABLE_COLUMN_LINK_2); + testCaseList = getTestCases(queryParams, ADMIN_AUTH_HEADERS); verifyTestCases(testCaseList, expectedColTestCaseList, 10); - testCaseList = getTestCases(12, "*", TABLE_LINK_2, true, ADMIN_AUTH_HEADERS); + queryParams.put("entityLink", TABLE_LINK_2); + queryParams.put("limit", 12); + queryParams.put("includeAllTests", true); + queryParams.put("include", "all"); + testCaseList = getTestCases(queryParams, ADMIN_AUTH_HEADERS); expectedTestCaseList.addAll(expectedColTestCaseList); verifyTestCases(testCaseList, expectedTestCaseList, 12); - testCaseList = getTestCases(12, "*", TEST_SUITE1, false, ADMIN_AUTH_HEADERS); + queryParams.remove("includeAllTests"); + queryParams.remove("include"); + queryParams.remove("entityLink"); + queryParams.put("testSuiteId", TEST_SUITE1.getId().toString()); + testCaseList = getTestCases(queryParams, ADMIN_AUTH_HEADERS); verifyTestCases(testCaseList, expectedTestCaseList, 12); } @@ -678,8 +725,7 @@ void patch_testCaseResults_noChange(TestInfo test) throws IOException, ParseExce testCaseResult.setTestCaseStatus(TestCaseStatus.Failed); JsonPatch patch = JsonUtils.getJsonPatch(original, JsonUtils.pojoToJson(testCaseResult)); - patchTestCaseResult( - testCase.getFullyQualifiedName(), dateToTimestamp("2021-09-09"), patch, ADMIN_AUTH_HEADERS); + patchTestCaseResult(testCase.getFullyQualifiedName(), dateToTimestamp("2021-09-09"), patch); ResultList testCaseResultResultListUpdated = getTestCaseResults( @@ -708,7 +754,7 @@ public void delete_entity_as_non_admin_401(TestInfo test) throws HttpResponseExc } @Test - public void add_EmptyTestCaseToLogicalTestSuite_200(TestInfo test) throws IOException { + void add_EmptyTestCaseToLogicalTestSuite_200(TestInfo test) throws IOException { TestSuiteResourceTest testSuiteResourceTest = new TestSuiteResourceTest(); // Create a logical Test Suite CreateTestSuite createLogicalTestSuite = testSuiteResourceTest.createRequest(test); @@ -719,34 +765,14 @@ public void add_EmptyTestCaseToLogicalTestSuite_200(TestInfo test) throws IOExce } @Test - public void delete_testCaseFromLogicalTestSuite(TestInfo test) throws IOException { + void delete_testCaseFromLogicalTestSuite(TestInfo test) throws IOException { TestSuiteResourceTest testSuiteResourceTest = new TestSuiteResourceTest(); // Create a logical Test Suite CreateTestSuite createLogicalTestSuite = testSuiteResourceTest.createRequest(test); TestSuite logicalTestSuite = testSuiteResourceTest.createEntity(createLogicalTestSuite, ADMIN_AUTH_HEADERS); // Create an executable test suite - TableResourceTest tableResourceTest = new TableResourceTest(); - CreateTable tableReq = - tableResourceTest - .createRequest(test) - .withName(test.getDisplayName()) - .withDatabaseSchema(DATABASE_SCHEMA.getFullyQualifiedName()) - .withOwner(USER1_REF) - .withColumns( - List.of( - new Column() - .withName(C1) - .withDisplayName("c1") - .withDataType(ColumnDataType.VARCHAR) - .withDataLength(10))) - .withOwner(USER1_REF); - Table table = tableResourceTest.createAndCheckEntity(tableReq, ADMIN_AUTH_HEADERS); - CreateTestSuite createExecutableTestSuite = - testSuiteResourceTest.createRequest(table.getFullyQualifiedName()); - TestSuite executableTestSuite = - testSuiteResourceTest.createExecutableTestSuite( - createExecutableTestSuite, ADMIN_AUTH_HEADERS); + TestSuite executableTestSuite = createExecutableTestSuite(test); List testCases = new ArrayList<>(); @@ -764,55 +790,70 @@ public void delete_testCaseFromLogicalTestSuite(TestInfo test) throws IOExceptio logicalTestSuite, testCases.stream().map(TestCase::getId).collect(Collectors.toList())); // Verify that the test cases are in the logical test suite - ResultList logicalTestSuiteTestCases = - getTestCases(100, "*", logicalTestSuite, false, ADMIN_AUTH_HEADERS); + Map queryParams = new HashMap<>(); + queryParams.put("limit", 100); + queryParams.put("fields", "*"); + queryParams.put("testSuiteId", logicalTestSuite.getId().toString()); + ResultList logicalTestSuiteTestCases = getTestCases(queryParams, ADMIN_AUTH_HEADERS); assertEquals(testCases.size(), logicalTestSuiteTestCases.getData().size()); // Delete a logical test case and check that it is deleted from the logical test suite but not // from the executable test suite UUID logicalTestCaseIdToDelete = testCases.get(0).getId(); deleteLogicalTestCase(logicalTestSuite, logicalTestCaseIdToDelete); - logicalTestSuiteTestCases = getTestCases(100, "*", logicalTestSuite, false, ADMIN_AUTH_HEADERS); + logicalTestSuiteTestCases = getTestCases(queryParams, ADMIN_AUTH_HEADERS); assertTrue(assertTestCaseIdNotInList(logicalTestSuiteTestCases, logicalTestCaseIdToDelete)); + + queryParams.put("testSuiteId", executableTestSuite.getId().toString()); ResultList executableTestSuiteTestCases = - getTestCases(100, "*", executableTestSuite, false, ADMIN_AUTH_HEADERS); + getTestCases(queryParams, ADMIN_AUTH_HEADERS); assertEquals(testCases.size(), executableTestSuiteTestCases.getData().size()); // Soft Delete a test case from the executable test suite and check that it is deleted from the // executable test suite and from the logical test suite UUID executableTestCaseIdToDelete = testCases.get(1).getId(); deleteEntity(executableTestCaseIdToDelete, false, false, ADMIN_AUTH_HEADERS); - logicalTestSuiteTestCases = getTestCases(100, "*", logicalTestSuite, false, ADMIN_AUTH_HEADERS); + queryParams.put("testSuiteId", logicalTestSuite.getId().toString()); + logicalTestSuiteTestCases = getTestCases(queryParams, ADMIN_AUTH_HEADERS); assertEquals(3, logicalTestSuiteTestCases.getData().size()); assertTrue(assertTestCaseIdNotInList(logicalTestSuiteTestCases, executableTestCaseIdToDelete)); - logicalTestSuiteTestCases = getTestCases(100, "*", logicalTestSuite, true, ADMIN_AUTH_HEADERS); + + queryParams.put("includeAllTests", true); + queryParams.put("include", "all"); + logicalTestSuiteTestCases = getTestCases(queryParams, ADMIN_AUTH_HEADERS); assertEquals(4, logicalTestSuiteTestCases.getData().size()); - executableTestSuiteTestCases = - getTestCases(100, "*", executableTestSuite, false, ADMIN_AUTH_HEADERS); + queryParams.put("testSuiteId", executableTestSuite.getId().toString()); + queryParams.remove("includeAllTests"); + queryParams.remove("include"); + executableTestSuiteTestCases = getTestCases(queryParams, ADMIN_AUTH_HEADERS); assertEquals(4, executableTestSuiteTestCases.getData().size()); assertTrue( assertTestCaseIdNotInList(executableTestSuiteTestCases, executableTestCaseIdToDelete)); - executableTestSuiteTestCases = - getTestCases(100, "*", executableTestSuite, true, ADMIN_AUTH_HEADERS); + + queryParams.put("includeAllTests", true); + queryParams.put("include", "all"); + executableTestSuiteTestCases = getTestCases(queryParams, ADMIN_AUTH_HEADERS); assertEquals(5, executableTestSuiteTestCases.getData().size()); // Hard Delete a test case from the executable test suite and check that it is deleted from the // executable test suite and from the logical test suite deleteEntity(executableTestCaseIdToDelete, false, true, ADMIN_AUTH_HEADERS); - logicalTestSuiteTestCases = getTestCases(100, "*", logicalTestSuite, true, ADMIN_AUTH_HEADERS); + + queryParams.put("testSuiteId", logicalTestSuite.getId().toString()); + logicalTestSuiteTestCases = getTestCases(queryParams, ADMIN_AUTH_HEADERS); assertEquals(3, logicalTestSuiteTestCases.getData().size()); assertTrue(assertTestCaseIdNotInList(logicalTestSuiteTestCases, executableTestCaseIdToDelete)); - executableTestSuiteTestCases = - getTestCases(100, "*", executableTestSuite, true, ADMIN_AUTH_HEADERS); + queryParams.put("testSuiteId", executableTestSuite.getId().toString()); + executableTestSuiteTestCases = getTestCases(queryParams, ADMIN_AUTH_HEADERS); assertEquals(4, executableTestSuiteTestCases.getData().size()); assertTrue( assertTestCaseIdNotInList(executableTestSuiteTestCases, executableTestCaseIdToDelete)); } @Test - public void list_allTestSuitesFromTestCase_200(TestInfo test) throws IOException { + void list_allTestSuitesFromTestCase_200(TestInfo test) throws IOException { TestSuiteResourceTest testSuiteResourceTest = new TestSuiteResourceTest(); // Create a logical Test Suite CreateTestSuite createLogicalTestSuite = testSuiteResourceTest.createRequest(test); @@ -852,38 +893,20 @@ public void list_allTestSuitesFromTestCase_200(TestInfo test) throws IOException } @Test - public void test_testCaseResultState(TestInfo test) throws IOException, ParseException { + void test_testCaseResultState(TestInfo test) throws IOException, ParseException { // Create table for our test TestSuiteResourceTest testSuiteResourceTest = new TestSuiteResourceTest(); - TableResourceTest tableResourceTest = new TableResourceTest(); - CreateTable tableReq = - tableResourceTest - .createRequest(test) - .withName(test.getDisplayName()) - .withDatabaseSchema(DATABASE_SCHEMA.getFullyQualifiedName()) - .withOwner(USER1_REF) - .withColumns( - List.of( - new Column() - .withName(C1) - .withDisplayName("c1") - .withDataType(ColumnDataType.VARCHAR) - .withDataLength(10))) - .withOwner(USER1_REF); - Table testTable = tableResourceTest.createAndCheckEntity(tableReq, ADMIN_AUTH_HEADERS); - // create testSuite - CreateTestSuite createExecutableTestSuite = - testSuiteResourceTest.createRequest(testTable.getFullyQualifiedName()); - TestSuite testSuite = - testSuiteResourceTest.createExecutableTestSuite( - createExecutableTestSuite, ADMIN_AUTH_HEADERS); + TestSuite testSuite = createExecutableTestSuite(test); // create testCase CreateTestCase createTestCase = new CreateTestCase() .withName(test.getDisplayName()) .withDescription(test.getDisplayName()) - .withEntityLink(String.format("<#E::table::%s>", testTable.getFullyQualifiedName())) + .withEntityLink( + String.format( + "<#E::table::%s>", + testSuite.getExecutableEntityReference().getFullyQualifiedName())) .withTestSuite(testSuite.getFullyQualifiedName()) .withTestDefinition(TEST_DEFINITION1.getFullyQualifiedName()); TestCase testCase = createAndCheckEntity(createTestCase, ADMIN_AUTH_HEADERS); @@ -953,8 +976,7 @@ public void test_testCaseResultState(TestInfo test) throws IOException, ParseExc TestCaseResult testCaseResult = storedTestCase.getTestCaseResult(); String original = JsonUtils.pojoToJson(testCaseResult); JsonPatch patch = JsonUtils.getJsonPatch(original, JsonUtils.pojoToJson(testCaseResult)); - patchTestCaseResult( - testCase.getFullyQualifiedName(), dateToTimestamp("2023-08-14"), patch, ADMIN_AUTH_HEADERS); + patchTestCaseResult(testCase.getFullyQualifiedName(), dateToTimestamp("2023-08-14"), patch); // add a new test case result for the 16th and check the state is correctly updated testCaseResult = @@ -1001,13 +1023,18 @@ public void test_testCaseResultState(TestInfo test) throws IOException, ParseExc } @Test - public void test_listTestCaseByExecutionTime(TestInfo test) throws IOException, ParseException { + void test_listTestCaseByExecutionTime(TestInfo test) throws IOException, ParseException { // if we have no test cases create some for (int i = 0; i < 10; i++) { createAndCheckEntity(createRequest(test, i), ADMIN_AUTH_HEADERS); } + + HashMap queryParams = new HashMap<>(); + queryParams.put("limit", 10); + queryParams.put("fields", "*"); + queryParams.put("orderByLastExecutionDate", false); ResultList nonExecutionSortedTestCases = - getTestCases(10, null, null, "*", false, ADMIN_AUTH_HEADERS); + getTestCases(queryParams, ADMIN_AUTH_HEADERS); TestCase lastTestCaseInList = nonExecutionSortedTestCases @@ -1038,8 +1065,8 @@ public void test_listTestCaseByExecutionTime(TestInfo test) throws IOException, .withTimestamp(TestUtils.dateToTimestamp(todayString)), ADMIN_AUTH_HEADERS); - ResultList executionSortedTestCases = - getTestCases(10, null, null, "*", true, ADMIN_AUTH_HEADERS); + queryParams.put("orderByLastExecutionDate", true); + ResultList executionSortedTestCases = getTestCases(queryParams, ADMIN_AUTH_HEADERS); assertEquals(lastTestCaseInList.getId(), executionSortedTestCases.getData().get(0).getId()); assertEquals( lastTestCaseInList.getId(), @@ -1082,8 +1109,11 @@ void test_listTestCaseByExecutionTimePagination_200(TestInfo test) ADMIN_AUTH_HEADERS); // List all entities and use it for checking pagination - ResultList allEntities = - getTestCases(1000000, null, null, "*", true, ADMIN_AUTH_HEADERS); + Map queryParams = new HashMap<>(); + queryParams.put("limit", 1000000); + queryParams.put("fields", "*"); + queryParams.put("orderByLastExecutionDate", true); + ResultList allEntities = getTestCases(queryParams, ADMIN_AUTH_HEADERS); paginate(maxEntities, allEntities, null); @@ -1095,9 +1125,9 @@ void test_listTestCaseByExecutionTimePagination_200(TestInfo test) List testCaseIds = createdTestCase.stream().map(TestCase::getId).collect(Collectors.toList()); testSuiteResourceTest.addTestCasesToLogicalTestSuite(logicalTestSuite, testCaseIds); - allEntities = - getTestCases( - 1000000, null, null, "*", null, logicalTestSuite, false, true, ADMIN_AUTH_HEADERS); + + queryParams.put("testSuiteId", logicalTestSuite.getId().toString()); + allEntities = getTestCases(queryParams, ADMIN_AUTH_HEADERS); paginate(maxEntities, allEntities, logicalTestSuite); } @@ -1286,7 +1316,7 @@ void test_listTestCaseFailureStatusPagination(TestInfo test) throws IOException, ResultList allEntities = getTestCaseFailureStatus(1000000, null, false, startTs, endTs, null); - paginateTestCaseFailureStatus(maxEntities, allEntities, null, startTs, endTs); + paginateTestCaseFailureStatus(maxEntities, allEntities, startTs, endTs); } @Test @@ -1309,7 +1339,7 @@ void patch_TestCaseResultFailure(TestInfo test) throws HttpResponseException { .withSeverity(Severity.Severity1)); JsonPatch patch = JsonUtils.getJsonPatch(original, updated); TestCaseResolutionStatus patched = - patchTestCaseResultFailureStatus(testCaseFailureStatus.getId(), patch, ADMIN_AUTH_HEADERS); + patchTestCaseResultFailureStatus(testCaseFailureStatus.getId(), patch); TestCaseResolutionStatus stored = getTestCaseFailureStatus(testCaseFailureStatus.getId()); // check our patch fields have been updated @@ -1338,15 +1368,13 @@ void patch_TestCaseResultFailureUnauthorizedFields(TestInfo test) throws HttpRes JsonPatch patch = JsonUtils.getJsonPatch(original, updated); assertResponse( - () -> - patchTestCaseResultFailureStatus( - testCaseFailureStatus.getId(), patch, ADMIN_AUTH_HEADERS), + () -> patchTestCaseResultFailureStatus(testCaseFailureStatus.getId(), patch), BAD_REQUEST, "Field testCaseResolutionStatusType is not allowed to be updated"); } @Test - public void test_testCaseResolutionTaskResolveWorkflowThruFeed(TestInfo test) + void test_testCaseResolutionTaskResolveWorkflowThruFeed(TestInfo test) throws HttpResponseException, ParseException { Long startTs = System.currentTimeMillis(); FeedResourceTest feedResourceTest = new FeedResourceTest(); @@ -1421,7 +1449,7 @@ public void test_testCaseResolutionTaskResolveWorkflowThruFeed(TestInfo test) } @Test - public void test_testCaseResolutionTaskCloseWorkflowThruFeed(TestInfo test) + void test_testCaseResolutionTaskCloseWorkflowThruFeed(TestInfo test) throws HttpResponseException, ParseException { Long startTs = System.currentTimeMillis(); FeedResourceTest feedResourceTest = new FeedResourceTest(); @@ -1489,7 +1517,7 @@ public void test_testCaseResolutionTaskCloseWorkflowThruFeed(TestInfo test) } @Test - public void test_testCaseResolutionTaskWorkflowThruAPI(TestInfo test) + void test_testCaseResolutionTaskWorkflowThruAPI(TestInfo test) throws HttpResponseException, ParseException { TestCase testCaseEntity = createEntity(createRequest(getEntityName(test)), ADMIN_AUTH_HEADERS); @@ -1538,7 +1566,7 @@ public void test_testCaseResolutionTaskWorkflowThruAPI(TestInfo test) } @Test - public void unauthorizedTestCaseResolutionFlow(TestInfo test) + void unauthorizedTestCaseResolutionFlow(TestInfo test) throws HttpResponseException, ParseException { TestCase testCaseEntity = createEntity(createRequest(getEntityName(test)), ADMIN_AUTH_HEADERS); // Add failed test case, which will create a NEW incident @@ -1568,7 +1596,7 @@ public void unauthorizedTestCaseResolutionFlow(TestInfo test) } @Test - public void testInferSeverity(TestInfo test) { + void testInferSeverity() { IncidentSeverityClassifierInterface severityClassifier = IncidentSeverityClassifierInterface.getInstance(); // TEST_TABLE1 has no tier information, hence severity should be null as the classifier won't be @@ -1585,12 +1613,117 @@ public void testInferSeverity(TestInfo test) { assertNotNull(severity); } + @Test + void get_listTestCaseWithStatusAndType(TestInfo test) + throws HttpResponseException, ParseException, IOException { + TestSuite testSuite = createExecutableTestSuite(test); + + int testCaseEntries = 15; + + List createdTestCase = new ArrayList<>(); + for (int i = 0; i < testCaseEntries; i++) { + if (i % 2 == 0) { + // Create column level test case + createdTestCase.add( + createEntity( + createRequest(test, i + 1) + .withEntityLink(TABLE_COLUMN_LINK) + .withTestSuite(testSuite.getFullyQualifiedName()), + ADMIN_AUTH_HEADERS)); + continue; + } + createdTestCase.add( + createEntity( + createRequest(test, i + 1).withTestSuite(testSuite.getFullyQualifiedName()), + ADMIN_AUTH_HEADERS)); + } + + for (int i = 0; i < testCaseEntries; i++) { + // Even number = Failed (8), Odd number = Success (7), 9 = Aborted (1) + TestCaseStatus result = null; + if (i % 2 == 0) { + result = TestCaseStatus.Failed; + } else if (i == 9) { + result = TestCaseStatus.Aborted; + } else { + result = TestCaseStatus.Success; + } + TestCaseResult testCaseResult = + new TestCaseResult() + .withResult("result") + .withTestCaseStatus(result) + .withTimestamp(TestUtils.dateToTimestamp("2024-01-01")); + putTestCaseResult( + createdTestCase.get(i).getFullyQualifiedName(), testCaseResult, ADMIN_AUTH_HEADERS); + } + + Map queryParams = new HashMap<>(); + queryParams.put("limit", 100); + queryParams.put("testSuiteId", testSuite.getId().toString()); + // Assert we get all 15 test cases + ResultList testCases = getTestCases(queryParams, ADMIN_AUTH_HEADERS); + assertEquals(testCaseEntries, testCases.getData().size()); + + // Assert we get 8 failed test cases + queryParams.put("testCaseStatus", TestCaseStatus.Failed); + testCases = getTestCases(queryParams, ADMIN_AUTH_HEADERS); + assertEquals(8, testCases.getData().size()); + + // Assert we get 7 success test cases + queryParams.put("testCaseStatus", TestCaseStatus.Success); + testCases = getTestCases(queryParams, ADMIN_AUTH_HEADERS); + assertEquals(6, testCases.getData().size()); + + // Assert we get 1 aborted test cases + queryParams.put("testCaseStatus", TestCaseStatus.Aborted); + testCases = getTestCases(queryParams, ADMIN_AUTH_HEADERS); + assertEquals(1, testCases.getData().size()); + + queryParams.remove("testCaseStatus"); + + // Assert we get 7 column level test cases + queryParams.put("testCaseType", "column"); + testCases = getTestCases(queryParams, ADMIN_AUTH_HEADERS); + assertEquals(8, testCases.getData().size()); + + // Assert we get 8 table level test cases + queryParams.put("testCaseType", "table"); + testCases = getTestCases(queryParams, ADMIN_AUTH_HEADERS); + assertEquals(7, testCases.getData().size()); + } + public void deleteTestCaseResult(String fqn, Long timestamp, Map authHeaders) throws HttpResponseException { WebTarget target = getCollection().path("/" + fqn + "/testCaseResult/" + timestamp); TestUtils.delete(target, authHeaders); } + private TestSuite createExecutableTestSuite(TestInfo test) throws IOException { + TestSuiteResourceTest testSuiteResourceTest = new TestSuiteResourceTest(); + TableResourceTest tableResourceTest = new TableResourceTest(); + CreateTable tableReq = + tableResourceTest + .createRequest(test) + .withName(test.getDisplayName()) + .withDatabaseSchema(DATABASE_SCHEMA.getFullyQualifiedName()) + .withOwner(USER1_REF) + .withColumns( + List.of( + new Column() + .withName(C1) + .withDisplayName("c1") + .withDataType(ColumnDataType.VARCHAR) + .withDataLength(10))) + .withOwner(USER1_REF); + Table table = tableResourceTest.createAndCheckEntity(tableReq, ADMIN_AUTH_HEADERS); + CreateTestSuite createExecutableTestSuite = + testSuiteResourceTest.createRequest(table.getFullyQualifiedName()); + TestSuite executableTestSuite = + testSuiteResourceTest.createExecutableTestSuite( + createExecutableTestSuite, ADMIN_AUTH_HEADERS); + return executableTestSuite; + } + private void deleteLogicalTestCase(TestSuite testSuite, UUID testCaseId) throws IOException { WebTarget target = getCollection() @@ -1621,76 +1754,28 @@ public TestCase getTestCase(String fqn, Map authHeaders) return TestUtils.get(target, TestCase.class, authHeaders); } - private TestSummary getTestSummary(Map authHeaders, String testSuiteId) - throws IOException { + private TestSummary getTestSummary(String testSuiteId) throws IOException { TestSuiteResourceTest testSuiteResourceTest = new TestSuiteResourceTest(); - return testSuiteResourceTest.getTestSummary(authHeaders, testSuiteId); + return testSuiteResourceTest.getTestSummary(ADMIN_AUTH_HEADERS, testSuiteId); } public ResultList getTestCases( - Integer limit, - String before, - String after, - String fields, - String link, - TestSuite testSuite, - Boolean includeAll, - Boolean orderByLastExecutionDate, - Map authHeaders) + Map queryParams, Map authHeaders) throws HttpResponseException { WebTarget target = getCollection(); - target = target.queryParam("fields", fields); - target = limit != null ? target.queryParam("limit", limit) : target; - target = before != null ? target.queryParam("before", before) : target; - target = after != null ? target.queryParam("after", after) : target; - target = link != null ? target.queryParam("entityLink", link) : target; - target = testSuite != null ? target.queryParam("testSuiteId", testSuite.getId()) : target; - target = - orderByLastExecutionDate ? target.queryParam("orderByLastExecutionDate", true) : target; - if (includeAll) { - target = target.queryParam("includeAllTests", true); - target = target.queryParam("include", "all"); + for (Map.Entry entry : queryParams.entrySet()) { + if (entry.getValue() == null || entry.getValue().toString().isEmpty()) { + continue; + } + target = target.queryParam(entry.getKey(), entry.getValue()); } return TestUtils.get(target, TestCaseResource.TestCaseList.class, authHeaders); } - public ResultList getTestCases( - Integer limit, - String fields, - String link, - Boolean includeAll, - Map authHeaders) - throws HttpResponseException { - return getTestCases(limit, null, null, fields, link, null, includeAll, false, authHeaders); - } - - public ResultList getTestCases( - Integer limit, - String fields, - TestSuite testSuite, - Boolean includeAll, - Map authHeaders) - throws HttpResponseException { - return getTestCases(limit, null, null, fields, null, testSuite, includeAll, false, authHeaders); - } - - public ResultList getTestCases( - Integer limit, - String before, - String after, - String fields, - Boolean orderByLastExecutionDate, - Map authHeaders) - throws HttpResponseException { - return getTestCases( - limit, before, after, fields, null, null, false, orderByLastExecutionDate, authHeaders); - } - - private TestCaseResult patchTestCaseResult( - String testCaseFqn, Long timestamp, JsonPatch patch, Map authHeaders) + private TestCaseResult patchTestCaseResult(String testCaseFqn, Long timestamp, JsonPatch patch) throws HttpResponseException { WebTarget target = getCollection().path("/" + testCaseFqn + "/testCaseResult/" + timestamp); - return TestUtils.patch(target, patch, TestCaseResult.class, authHeaders); + return TestUtils.patch(target, patch, TestCaseResult.class, ADMIN_AUTH_HEADERS); } private void verifyTestCaseResults( @@ -1743,8 +1828,13 @@ private void paginate(Integer maxEntities, ResultList allEntities, Tes ResultList forwardPage; ResultList backwardPage; do { // For each limit (or page size) - forward scroll till the end - forwardPage = - getTestCases(limit, null, after, "*", null, testSuite, false, true, ADMIN_AUTH_HEADERS); + Map queryParams = new HashMap<>(); + queryParams.put("limit", limit); + queryParams.put("after", after == null ? "" : after); + queryParams.put("fields", "*"); + queryParams.put("testSuiteId", testSuite == null ? "" : testSuite.getId().toString()); + queryParams.put("orderByLastExecutionDate", true); + forwardPage = getTestCases(queryParams, ADMIN_AUTH_HEADERS); after = forwardPage.getPaging().getAfter(); before = forwardPage.getPaging().getBefore(); assertEntityPagination(allEntities.getData(), forwardPage, limit, indexInAllTables); @@ -1753,10 +1843,10 @@ private void paginate(Integer maxEntities, ResultList allEntities, Tes assertNull(before); } else { // Make sure scrolling back based on before cursor returns the correct result - backwardPage = - getTestCases( - limit, before, null, "*", null, testSuite, false, true, ADMIN_AUTH_HEADERS); - getTestCases(limit, before, null, "*", true, ADMIN_AUTH_HEADERS); + queryParams.remove("after"); + queryParams.put("before", before); + backwardPage = getTestCases(queryParams, ADMIN_AUTH_HEADERS); + // getTestCases(queryParams, ADMIN_AUTH_HEADERS); assertEntityPagination( allEntities.getData(), backwardPage, limit, (indexInAllTables - limit)); } @@ -1769,9 +1859,21 @@ private void paginate(Integer maxEntities, ResultList allEntities, Tes pageCount = 0; indexInAllTables = totalRecords - limit - forwardPage.getData().size(); do { - forwardPage = - getTestCases( - limit, before, null, "*", null, testSuite, false, true, ADMIN_AUTH_HEADERS); + Map queryParams = + ImmutableMap.of( + "limit", + limit, + "before", + before == null ? "" : before, + "fields", + "*", + "testSuiteId", + testSuite == null ? "" : testSuite.getId().toString(), + "includeAllTests", + false, + "orderByLastExecutionDate", + true); + forwardPage = getTestCases(queryParams, ADMIN_AUTH_HEADERS); before = forwardPage.getPaging().getBefore(); assertEntityPagination(allEntities.getData(), forwardPage, limit, indexInAllTables); pageCount++; @@ -1918,22 +2020,10 @@ private TestCaseResolutionStatus createTestCaseFailureStatus( ADMIN_AUTH_HEADERS); } - private void createTestCaseResolutionStatus( - List createTestCaseFailureStatus) - throws HttpResponseException { - WebTarget target = getCollection().path("/testCaseIncidentStatus"); - - for (CreateTestCaseResolutionStatus testCaseFailureStatus : createTestCaseFailureStatus) { - TestUtils.post( - target, testCaseFailureStatus, TestCaseResolutionStatus.class, 200, ADMIN_AUTH_HEADERS); - } - } - private TestCaseResolutionStatus patchTestCaseResultFailureStatus( - UUID testCaseFailureStatusId, JsonPatch patch, Map authHeaders) - throws HttpResponseException { + UUID testCaseFailureStatusId, JsonPatch patch) throws HttpResponseException { WebTarget target = getCollection().path("/testCaseIncidentStatus/" + testCaseFailureStatusId); - return TestUtils.patch(target, patch, TestCaseResolutionStatus.class, authHeaders); + return TestUtils.patch(target, patch, TestCaseResolutionStatus.class, ADMIN_AUTH_HEADERS); } private TestCaseResolutionStatus getTestCaseFailureStatus(UUID testCaseFailureStatusId) @@ -1945,7 +2035,6 @@ private TestCaseResolutionStatus getTestCaseFailureStatus(UUID testCaseFailureSt private void paginateTestCaseFailureStatus( Integer maxEntities, ResultList allEntities, - Boolean latest, Long startTs, Long endTs) throws HttpResponseException { @@ -1960,7 +2049,7 @@ private void paginateTestCaseFailureStatus( ResultList forwardPage; ResultList backwardPage; do { // For each limit (or page size) - forward scroll till the end - forwardPage = getTestCaseFailureStatus(limit, after, latest, startTs, endTs, null); + forwardPage = getTestCaseFailureStatus(limit, after, null, startTs, endTs, null); after = forwardPage.getPaging().getAfter(); before = forwardPage.getPaging().getBefore(); assertEntityPagination(allEntities.getData(), forwardPage, limit, indexInAllTables); @@ -1969,7 +2058,7 @@ private void paginateTestCaseFailureStatus( assertNull(before); } else { // Make sure scrolling back based on before cursor returns the correct result - backwardPage = getTestCaseFailureStatus(limit, before, latest, startTs, endTs, null); + backwardPage = getTestCaseFailureStatus(limit, before, null, startTs, endTs, null); assertEntityPagination( allEntities.getData(), backwardPage, limit, (indexInAllTables - limit)); } @@ -1982,7 +2071,7 @@ private void paginateTestCaseFailureStatus( pageCount = 0; indexInAllTables = totalRecords - limit - forwardPage.getData().size(); do { - forwardPage = getTestCaseFailureStatus(limit, before, latest, startTs, endTs, null); + forwardPage = getTestCaseFailureStatus(limit, before, null, startTs, endTs, null); before = forwardPage.getPaging().getBefore(); assertEntityPagination(allEntities.getData(), forwardPage, limit, indexInAllTables); pageCount++; diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/resources/dqtests/TestSuiteResourceTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/resources/dqtests/TestSuiteResourceTest.java index 84f90fbbe90f..5ff8a0b71b63 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/resources/dqtests/TestSuiteResourceTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/resources/dqtests/TestSuiteResourceTest.java @@ -2,6 +2,7 @@ import static javax.ws.rs.core.Response.Status.BAD_REQUEST; import static javax.ws.rs.core.Response.Status.NOT_FOUND; +import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNull; import static org.openmetadata.service.util.TestUtils.ADMIN_AUTH_HEADERS; @@ -11,6 +12,8 @@ import static org.openmetadata.service.util.TestUtils.assertResponse; import static org.openmetadata.service.util.TestUtils.assertResponseContains; +import es.org.elasticsearch.search.aggregations.AggregationBuilder; +import es.org.elasticsearch.search.aggregations.AggregationBuilders; import java.io.IOException; import java.text.ParseException; import java.util.ArrayList; @@ -19,6 +22,7 @@ import java.util.Map; import java.util.UUID; import java.util.stream.Collectors; +import javax.json.JsonObject; import javax.ws.rs.client.WebTarget; import javax.ws.rs.core.Response; import org.apache.http.client.HttpResponseException; @@ -41,6 +45,7 @@ import org.openmetadata.service.Entity; import org.openmetadata.service.resources.EntityResourceTest; import org.openmetadata.service.resources.databases.TableResourceTest; +import org.openmetadata.service.search.elasticsearch.ElasticSearchClient; import org.openmetadata.service.util.JsonUtils; import org.openmetadata.service.util.ResultList; import org.openmetadata.service.util.TestUtils; @@ -58,6 +63,7 @@ public TestSuiteResourceTest() { TestSuiteResource.TestSuiteList.class, "dataQuality/testSuites", TestSuiteResource.FIELDS); + supportsSearchIndex = true; } public void setupTestSuites(TestInfo test) throws IOException { @@ -496,6 +502,121 @@ protected void post_entityCreateWithInvalidName_400() { TestUtils.getEntityNameLengthError(entityClass)); } + @Test + void buildElasticsearchAggregationFromJson(TestInfo test) { + JsonObject aggregationJson; + List actual; + List expected = new ArrayList<>(); + String aggregationQuery; + + // Test aggregation with nested aggregation + aggregationQuery = + """ + { + "aggregations": { + "test_case_results": { + "nested": { + "path": "testCaseResultSummary" + }, + "aggs": { + "status_counts": { + "terms": { + "field": "testCaseResultSummary.status" + } + } + } + } + } + } + """; + + expected.add( + AggregationBuilders.nested("testCaseResultSummary", "testCaseResultSummary") + .subAggregation( + AggregationBuilders.terms("status_counts").field("testCaseResultSummary.status"))); + + aggregationJson = JsonUtils.readJson(aggregationQuery).asJsonObject(); + actual = ElasticSearchClient.buildAggregation(aggregationJson.getJsonObject("aggregations")); + assertThat(actual).hasSameElementsAs(expected); + + // Test aggregation with multiple aggregations + aggregationQuery = + """ + { + "aggregations": { + "my-first-agg-name": { + "terms": { + "field": "my-field" + } + }, + "my-second-agg-name": { + "terms": { + "field": "my-other-field" + } + } + } + } + """; + aggregationJson = JsonUtils.readJson(aggregationQuery).asJsonObject(); + + expected.clear(); + expected.addAll( + List.of( + AggregationBuilders.terms("my-second-agg-name").field("my-other-field"), + AggregationBuilders.terms("my-first-agg-name").field("my-field"))); + + actual = ElasticSearchClient.buildAggregation(aggregationJson.getJsonObject("aggregations")); + assertThat(actual).hasSameElementsAs(expected); + + // Test aggregation with multiple aggregations including a nested one which has itself multiple + // aggregations + aggregationQuery = + """ + { + "aggregations": { + "my-first-agg-name": { + "terms": { + "field": "my-field" + } + }, + "test_case_results": { + "nested": { + "path": "testCaseResultSummary" + }, + "aggs": { + "status_counts": { + "terms": { + "field": "testCaseResultSummary.status" + } + }, + "other_status_counts": { + "terms": { + "field": "testCaseResultSummary.status" + } + } + } + } + } + } + """; + aggregationJson = JsonUtils.readJson(aggregationQuery).asJsonObject(); + + expected.clear(); + expected.addAll( + List.of( + AggregationBuilders.nested("testCaseResultSummary", "testCaseResultSummary") + .subAggregation( + AggregationBuilders.terms("status_counts") + .field("testCaseResultSummary.status")) + .subAggregation( + AggregationBuilders.terms("other_status_counts") + .field("testCaseResultSummary.status")), + AggregationBuilders.terms("my-first-agg-name").field("my-field"))); + + actual = ElasticSearchClient.buildAggregation(aggregationJson.getJsonObject("aggregations")); + assertThat(actual).hasSameElementsAs(expected); + } + @Test void delete_LogicalTestSuite_200(TestInfo test) throws IOException { TestCaseResourceTest testCaseResourceTest = new TestCaseResourceTest(); diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/resources/events/EventSubscriptionResourceTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/resources/events/EventSubscriptionResourceTest.java index 09ef641f2008..ed46384bf8a5 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/resources/events/EventSubscriptionResourceTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/resources/events/EventSubscriptionResourceTest.java @@ -643,7 +643,7 @@ public CreateEventSubscription createRequest(String name) { .withEnabled(true) .withBatchSize(10) .withRetries(0) - .withPollInterval(0) + .withPollInterval(1) .withAlertType(CreateEventSubscription.AlertType.NOTIFICATION); } diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/resources/feeds/FeedResourceTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/resources/feeds/FeedResourceTest.java index a2212ec285f9..a3e3eac3a130 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/resources/feeds/FeedResourceTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/resources/feeds/FeedResourceTest.java @@ -89,6 +89,7 @@ import org.openmetadata.schema.entity.teams.Team; import org.openmetadata.schema.entity.teams.User; import org.openmetadata.schema.type.AnnouncementDetails; +import org.openmetadata.schema.type.ChatbotDetails; import org.openmetadata.schema.type.Column; import org.openmetadata.schema.type.ColumnDataType; import org.openmetadata.schema.type.EntityReference; @@ -518,6 +519,17 @@ void post_validAnnouncementAndList_200() throws IOException { assertEquals(totalAnnouncementCount + 3, announcements.getData().size()); } + @Test + void post_validAI_200() throws IOException { + // Sample EntityLink + String about = String.format("<#E::%s::%s>", Entity.BOT, "ingestion-bot"); + createAI(USER.getName(), about, "First AI", "query", USER_AUTH_HEADERS); + createAI(USER.getName(), about, "Second AI", "query", USER_AUTH_HEADERS); + + // List all the AI and make sure the number of AI increased by 2 + assertEquals(2, listAI(null, null, ADMIN_AUTH_HEADERS).getPaging().getTotal()); + } + @Test void post_invalidAnnouncement_400() throws IOException { // create two announcements with same start time in the future @@ -922,6 +934,37 @@ void patch_thread_reactions_200() throws IOException { assertEquals(TEST_USER_NAME, patched.getUpdatedBy()); } + @Test + void patch_ai_200() throws IOException { + String about = String.format("<#E::%s::%s>", Entity.BOT, "ingestion-bot"); + + // Create thread without AI + CreateThread create = + new CreateThread() + .withFrom(USER.getName()) + .withMessage("message") + .withAbout(about) + .withType(ThreadType.Chatbot); + Thread thread = createAndCheck(create, ADMIN_AUTH_HEADERS); + String originalJson = JsonUtils.pojoToJson(thread); + + Thread updated = thread.withChatbot(new ChatbotDetails().withQuery("query")); + Thread patched = patchThreadAndCheck(updated, originalJson, TEST_AUTH_HEADERS); + + assertNotEquals(patched.getUpdatedAt(), thread.getUpdatedAt()); + assertEquals(TEST_USER_NAME, patched.getUpdatedBy()); + assertEquals("query", patched.getChatbot().getQuery()); + + // Patch again to update the query + String originalJson2 = JsonUtils.pojoToJson(patched); + Thread updated2 = patched.withChatbot(new ChatbotDetails().withQuery("query2")); + Thread patched2 = patchThreadAndCheck(updated2, originalJson2, TEST_AUTH_HEADERS); + + assertNotEquals(patched2.getUpdatedAt(), patched.getUpdatedAt()); + assertEquals(TEST_USER_NAME, patched2.getUpdatedBy()); + assertEquals("query2", patched2.getChatbot().getQuery()); + } + @Test void patch_announcement_200() throws IOException { LocalDateTime now = LocalDateTime.now(); @@ -1550,6 +1593,22 @@ public ThreadList listAnnouncements( null); } + public ThreadList listAI(String entityLink, Integer limitPosts, Map authHeaders) + throws HttpResponseException { + return listThreads( + entityLink, + limitPosts, + authHeaders, + null, + null, + null, + ThreadType.Chatbot.toString(), + null, + null, + null, + null); + } + public ThreadList listThreads( String entityLink, Integer limitPosts, Map authHeaders) throws HttpResponseException { @@ -1776,6 +1835,19 @@ public Thread createAnnouncement( return createAndCheck(create, authHeaders); } + public Thread createAI( + String fromUser, String about, String message, String query, Map authHeaders) + throws HttpResponseException { + CreateThread create = + new CreateThread() + .withFrom(fromUser) + .withMessage(message) + .withAbout(about) + .withType(ThreadType.Chatbot) + .withChatbotDetails(new ChatbotDetails().withQuery(query)); + return createAndCheck(create, authHeaders); + } + public void validateTaskList( UUID expectedAssignee, String expectedSuggestion, diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/resources/feeds/SuggestionsResourceTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/resources/feeds/SuggestionsResourceTest.java index ae90ddb55083..5fbd966ebf8a 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/resources/feeds/SuggestionsResourceTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/resources/feeds/SuggestionsResourceTest.java @@ -374,6 +374,160 @@ void put_rejectSuggestion_200(TestInfo test) throws IOException { assertEquals(SuggestionStatus.Rejected, suggestion2.getStatus()); } + @Test + @Order(3) + void put_acceptAllSuggestions_200() throws IOException { + CreateSuggestion create = create().withEntityLink(TABLE_LINK); + createAndCheck(create, USER_AUTH_HEADERS); + // Add another suggestion + createAndCheck(create, USER_AUTH_HEADERS); + // And now update tags + create = createTagSuggestion().withEntityLink(TABLE_LINK); + createAndCheck(create, USER_AUTH_HEADERS); + + SuggestionsResource.SuggestionList suggestionList = + listSuggestions(TABLE.getFullyQualifiedName(), null, null, null, USER_AUTH_HEADERS); + assertEquals(3, suggestionList.getData().size()); + + acceptAllSuggestions( + TABLE.getFullyQualifiedName(), + USER.getId(), + SuggestionType.SuggestDescription, + USER_AUTH_HEADERS); + + suggestionList = + listSuggestions( + TABLE.getFullyQualifiedName(), + null, + USER_AUTH_HEADERS, + null, + null, + SuggestionStatus.Open.toString(), + null, + null); + // We still have the tag suggestion open, since we only accepted the descriptions + assertEquals(1, suggestionList.getPaging().getTotal()); + + // Now we accept the pending one + acceptAllSuggestions( + TABLE.getFullyQualifiedName(), + USER.getId(), + SuggestionType.SuggestTagLabel, + USER_AUTH_HEADERS); + + suggestionList = + listSuggestions( + TABLE.getFullyQualifiedName(), + null, + USER_AUTH_HEADERS, + null, + null, + SuggestionStatus.Open.toString(), + null, + null); + assertEquals(0, suggestionList.getPaging().getTotal()); + } + + @Test + @Order(4) + void put_rejectAllSuggestions_200() throws IOException { + CreateSuggestion create = create().withEntityLink(TABLE_LINK); + createAndCheck(create, USER_AUTH_HEADERS); + // Add another suggestion + createAndCheck(create, USER_AUTH_HEADERS); + // And now update tags + create = createTagSuggestion().withEntityLink(TABLE_LINK); + createAndCheck(create, USER_AUTH_HEADERS); + + SuggestionsResource.SuggestionList suggestionList = + listSuggestions(TABLE.getFullyQualifiedName(), null, null, null, USER_AUTH_HEADERS); + assertEquals(3, suggestionList.getData().size()); + + rejectAllSuggestions( + TABLE.getFullyQualifiedName(), + USER.getId(), + SuggestionType.SuggestDescription, + USER_AUTH_HEADERS); + + suggestionList = + listSuggestions( + TABLE.getFullyQualifiedName(), + null, + USER_AUTH_HEADERS, + null, + null, + SuggestionStatus.Open.toString(), + null, + null); + assertEquals(1, suggestionList.getPaging().getTotal()); + + // Now we reject the pending one + rejectAllSuggestions( + TABLE.getFullyQualifiedName(), + USER.getId(), + SuggestionType.SuggestTagLabel, + USER_AUTH_HEADERS); + + suggestionList = + listSuggestions( + TABLE.getFullyQualifiedName(), + null, + USER_AUTH_HEADERS, + null, + null, + SuggestionStatus.Open.toString(), + null, + null); + assertEquals(0, suggestionList.getPaging().getTotal()); + } + + @Test + @Order(5) + void put_acceptAllColumnSuggestions_200() throws IOException { + CreateSuggestion create = create().withEntityLink(TABLE_LINK); + createAndCheck(create, USER_AUTH_HEADERS); + // Add another suggestion at one column level + create = + create().withEntityLink(TABLE_COLUMN1_LINK).withDescription("Update column1 description"); + createAndCheck(create, USER_AUTH_HEADERS); + // And now update another column description + create = + create().withEntityLink(TABLE_COLUMN2_LINK).withDescription("Update column2 description"); + createAndCheck(create, USER_AUTH_HEADERS); + + SuggestionsResource.SuggestionList suggestionList = + listSuggestions(TABLE.getFullyQualifiedName(), null, null, null, USER_AUTH_HEADERS); + assertEquals(3, suggestionList.getData().size()); + + acceptAllSuggestions( + TABLE.getFullyQualifiedName(), + USER.getId(), + SuggestionType.SuggestDescription, + USER_AUTH_HEADERS); + + suggestionList = + listSuggestions( + TABLE.getFullyQualifiedName(), + null, + USER_AUTH_HEADERS, + null, + null, + SuggestionStatus.Open.toString(), + null, + null); + assertEquals(0, suggestionList.getPaging().getTotal()); + + TableResourceTest tableResourceTest = new TableResourceTest(); + Table table = tableResourceTest.getEntity(TABLE.getId(), "columns", USER_AUTH_HEADERS); + for (Column column : table.getColumns()) { + if (column.getName().equals(C1)) { + assertEquals("Update column1 description", column.getDescription()); + } else if (column.getName().equals(C2)) { + assertEquals("Update column2 description", column.getDescription()); + } + } + } + public Suggestion createSuggestion(CreateSuggestion create, Map authHeaders) throws HttpResponseException { return TestUtils.post(getResource("suggestions"), create, Suggestion.class, authHeaders); @@ -451,6 +605,32 @@ public SuggestionsResource.SuggestionList listSuggestions( return listSuggestions(entityFQN, limit, authHeaders, null, null, null, before, after); } + public void acceptAllSuggestions( + String entityFQN, UUID userId, SuggestionType suggestionType, Map authHeaders) + throws HttpResponseException { + WebTarget target = getResource("suggestions/accept-all"); + target = entityFQN != null ? target.queryParam("entityFQN", entityFQN) : target; + target = userId != null ? target.queryParam("userId", userId) : target; + target = + suggestionType != null + ? target.queryParam("suggestionType", suggestionType.toString()) + : target; + TestUtils.put(target, null, Response.Status.OK, authHeaders); + } + + public void rejectAllSuggestions( + String entityFQN, UUID userId, SuggestionType suggestionType, Map authHeaders) + throws HttpResponseException { + WebTarget target = getResource("suggestions/reject-all"); + target = entityFQN != null ? target.queryParam("entityFQN", entityFQN) : target; + target = userId != null ? target.queryParam("userId", userId) : target; + target = + suggestionType != null + ? target.queryParam("suggestionType", suggestionType.toString()) + : target; + TestUtils.put(target, null, Response.Status.OK, authHeaders); + } + public Suggestion createAndCheck(CreateSuggestion create, Map authHeaders) throws HttpResponseException { // Validate returned thread from POST diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/resources/glossary/GlossaryTermResourceTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/resources/glossary/GlossaryTermResourceTest.java index 201d7d344e32..d9489c0ec394 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/resources/glossary/GlossaryTermResourceTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/resources/glossary/GlossaryTermResourceTest.java @@ -18,6 +18,7 @@ import static javax.ws.rs.core.Response.Status.BAD_REQUEST; import static javax.ws.rs.core.Response.Status.FORBIDDEN; +import static javax.ws.rs.core.Response.Status.NOT_FOUND; import static org.junit.jupiter.api.Assertions.*; import static org.openmetadata.common.utils.CommonUtil.listOf; import static org.openmetadata.common.utils.CommonUtil.listOrEmpty; @@ -53,6 +54,7 @@ import java.util.Map; import java.util.Objects; import java.util.Set; +import java.util.UUID; import javax.ws.rs.core.Response; import org.apache.http.client.HttpResponseException; import org.junit.jupiter.api.MethodOrderer; @@ -644,6 +646,23 @@ void delete_recursive(TestInfo test) throws IOException { assertTrue(table.getTags().isEmpty()); // tag t1 is removed } + @Test + void patchWrongReviewers(TestInfo test) throws IOException { + GlossaryTerm entity = createEntity(createRequest(test, 0), ADMIN_AUTH_HEADERS); + + // Add random domain reference + EntityReference reviewerReference = + new EntityReference().withId(UUID.randomUUID()).withType(Entity.USER); + String originalJson = JsonUtils.pojoToJson(entity); + ChangeDescription change = getChangeDescription(entity, MINOR_UPDATE); + entity.setReviewers(List.of(reviewerReference)); + + assertResponse( + () -> patchEntityAndCheck(entity, originalJson, ADMIN_AUTH_HEADERS, MINOR_UPDATE, change), + NOT_FOUND, + String.format("user instance for %s not found", reviewerReference.getId())); + } + public GlossaryTerm createTerm(Glossary glossary, GlossaryTerm parent, String termName) throws IOException { return createTerm(glossary, parent, termName, glossary.getReviewers()); diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/resources/lineage/LineageResourceTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/resources/lineage/LineageResourceTest.java index 2f7d8299eac9..be87e695b7ae 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/resources/lineage/LineageResourceTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/resources/lineage/LineageResourceTest.java @@ -26,6 +26,7 @@ import java.io.IOException; import java.net.URISyntaxException; +import java.net.URLEncoder; import java.util.ArrayList; import java.util.Arrays; import java.util.List; @@ -42,14 +43,23 @@ import org.junit.jupiter.api.TestInfo; import org.junit.jupiter.api.TestMethodOrder; import org.openmetadata.schema.EntityInterface; +import org.openmetadata.schema.api.data.CreateContainer; +import org.openmetadata.schema.api.data.CreateDashboard; import org.openmetadata.schema.api.data.CreateDashboardDataModel; +import org.openmetadata.schema.api.data.CreateMlModel; import org.openmetadata.schema.api.data.CreateTable; +import org.openmetadata.schema.api.data.CreateTopic; import org.openmetadata.schema.api.lineage.AddLineage; +import org.openmetadata.schema.entity.data.Container; +import org.openmetadata.schema.entity.data.Dashboard; import org.openmetadata.schema.entity.data.DashboardDataModel; +import org.openmetadata.schema.entity.data.MlModel; import org.openmetadata.schema.entity.data.Table; +import org.openmetadata.schema.entity.data.Topic; import org.openmetadata.schema.entity.teams.Role; import org.openmetadata.schema.entity.teams.User; import org.openmetadata.schema.type.ColumnLineage; +import org.openmetadata.schema.type.ContainerDataModel; import org.openmetadata.schema.type.Edge; import org.openmetadata.schema.type.EntitiesEdge; import org.openmetadata.schema.type.EntityLineage; @@ -58,11 +68,15 @@ import org.openmetadata.schema.type.MetadataOperation; import org.openmetadata.service.Entity; import org.openmetadata.service.OpenMetadataApplicationTest; +import org.openmetadata.service.resources.dashboards.DashboardResourceTest; import org.openmetadata.service.resources.databases.TableResourceTest; import org.openmetadata.service.resources.datamodels.DashboardDataModelResourceTest; +import org.openmetadata.service.resources.mlmodels.MlModelResourceTest; +import org.openmetadata.service.resources.storages.ContainerResourceTest; import org.openmetadata.service.resources.teams.RoleResource; import org.openmetadata.service.resources.teams.RoleResourceTest; import org.openmetadata.service.resources.teams.UserResourceTest; +import org.openmetadata.service.resources.topics.TopicResourceTest; import org.openmetadata.service.util.TestUtils; @Slf4j @@ -71,10 +85,13 @@ public class LineageResourceTest extends OpenMetadataApplicationTest { public static final List
 
TABLES = new ArrayList<>(); public static final int TABLE_COUNT = 10; private static final String DATA_STEWARD_ROLE_NAME = "DataSteward"; - private static DashboardDataModel DATA_MODEL; - private static Table TABLE_DATA_MODEL_LINEAGE; + private static Topic TOPIC; + private static Container CONTAINER; + private static MlModel ML_MODEL; + + private static Dashboard DASHBOARD; @BeforeAll public static void setup(TestInfo test) throws IOException, URISyntaxException { @@ -93,6 +110,25 @@ public static void setup(TestInfo test) throws IOException, URISyntaxException { CreateTable createTable = tableResourceTest.createRequest(test, TABLE_COUNT); createTable.setColumns(createDashboardDataModel.getColumns()); TABLE_DATA_MODEL_LINEAGE = tableResourceTest.createEntity(createTable, ADMIN_AUTH_HEADERS); + TopicResourceTest topicResourceTest = new TopicResourceTest(); + CreateTopic topicRequest = + topicResourceTest + .createRequest(test) + .withMessageSchema(TopicResourceTest.SCHEMA.withSchemaFields(TopicResourceTest.fields)); + TOPIC = topicResourceTest.createEntity(topicRequest, ADMIN_AUTH_HEADERS); + ContainerResourceTest containerResourceTest = new ContainerResourceTest(); + ContainerDataModel dataModel = + new ContainerDataModel().withColumns(ContainerResourceTest.dataModelColumns); + CreateContainer containerRequest = + containerResourceTest.createRequest(test).withDataModel(dataModel); + CONTAINER = containerResourceTest.createEntity(containerRequest, ADMIN_AUTH_HEADERS); + MlModelResourceTest mlModelResourceTest = new MlModelResourceTest(); + CreateMlModel createMlModel = + mlModelResourceTest.createRequest(test).withMlFeatures(MlModelResourceTest.ML_FEATURES); + ML_MODEL = mlModelResourceTest.createEntity(createMlModel, ADMIN_AUTH_HEADERS); + DashboardResourceTest dashboardResourceTest1 = new DashboardResourceTest(); + CreateDashboard createDashboard = dashboardResourceTest1.createRequest(test); + DASHBOARD = dashboardResourceTest1.createEntity(createDashboard, ADMIN_AUTH_HEADERS); } @Order(1) @@ -300,26 +336,22 @@ void put_lineageWithDetails() throws HttpResponseException { details.getColumnsLineage().clear(); details .getColumnsLineage() - .add(new ColumnLineage().withFromColumns(List.of(t1c1FQN, t3c1FQN)).withToColumn(t2c1FQN)); + .add(new ColumnLineage().withFromColumns(List.of(t1c1FQN, t1c3FQN)).withToColumn(t2c1FQN)); addEdge(TABLES.get(0), TABLES.get(1), details, ADMIN_AUTH_HEADERS); // Finally, add detailed column level lineage details.getColumnsLineage().clear(); List lineage = details.getColumnsLineage(); - lineage.add( - new ColumnLineage().withFromColumns(List.of(t1c1FQN, t3c1FQN)).withToColumn(t2c1FQN)); - lineage.add( - new ColumnLineage().withFromColumns(List.of(t1c2FQN, t3c2FQN)).withToColumn(t2c2FQN)); - lineage.add( - new ColumnLineage().withFromColumns(List.of(t1c3FQN, t3c3FQN)).withToColumn(t2c3FQN)); + lineage.add(new ColumnLineage().withFromColumns(List.of(t1c1FQN)).withToColumn(t2c1FQN)); + lineage.add(new ColumnLineage().withFromColumns(List.of(t1c2FQN)).withToColumn(t2c2FQN)); + lineage.add(new ColumnLineage().withFromColumns(List.of(t1c3FQN)).withToColumn(t2c3FQN)); addEdge(TABLES.get(0), TABLES.get(1), details, ADMIN_AUTH_HEADERS); } @Order(4) @Test - void putLineageFromDashboardDataModelToTable() throws HttpResponseException { - + void putLineageFromEntityToEntity() throws HttpResponseException { // Add column lineage dashboard.d1 -> table.c1 LineageDetails details = new LineageDetails(); String d1c1FQN = DATA_MODEL.getColumns().get(0).getFullyQualifiedName(); @@ -333,13 +365,83 @@ void putLineageFromDashboardDataModelToTable() throws HttpResponseException { lineage.add(new ColumnLineage().withFromColumns(List.of(c1c1FQN)).withToColumn(d1c1FQN)); lineage.add(new ColumnLineage().withFromColumns(List.of(c1c2FQN)).withToColumn(d1c2FQN)); lineage.add(new ColumnLineage().withFromColumns(List.of(c1c3FQN)).withToColumn(d1c3FQN)); - addEdge(TABLE_DATA_MODEL_LINEAGE, DATA_MODEL, details, ADMIN_AUTH_HEADERS); + LineageDetails topicToTable = new LineageDetails(); + String f1FQN = TOPIC.getMessageSchema().getSchemaFields().get(0).getFullyQualifiedName(); + String f2FQN = TOPIC.getMessageSchema().getSchemaFields().get(0).getFullyQualifiedName(); + String f1t1 = TABLE_DATA_MODEL_LINEAGE.getColumns().get(0).getFullyQualifiedName(); + String f2t2 = TABLE_DATA_MODEL_LINEAGE.getColumns().get(1).getFullyQualifiedName(); + List topicToTableLineage = topicToTable.getColumnsLineage(); + topicToTableLineage.add(new ColumnLineage().withFromColumns(List.of(f1FQN)).withToColumn(f1t1)); + topicToTableLineage.add(new ColumnLineage().withFromColumns(List.of(f2FQN)).withToColumn(f2t2)); + addEdge(TOPIC, TABLE_DATA_MODEL_LINEAGE, topicToTable, ADMIN_AUTH_HEADERS); + String f3FQN = "test_non_existent_filed"; + topicToTableLineage.add( + new ColumnLineage().withFromColumns(List.of(f3FQN)).withToColumn(d1c1FQN)); assertResponse( - () -> addEdge(DATA_MODEL, TABLE_DATA_MODEL_LINEAGE, details, ADMIN_AUTH_HEADERS), + () -> addEdge(TOPIC, TABLE_DATA_MODEL_LINEAGE, topicToTable, ADMIN_AUTH_HEADERS), BAD_REQUEST, - "Column level lineage is only allowed between two tables or from table to dashboard."); + String.format("Invalid field name %s", f3FQN)); + + LineageDetails topicToContainer = new LineageDetails(); + String f1c1 = CONTAINER.getDataModel().getColumns().get(0).getFullyQualifiedName(); + String f2c2 = CONTAINER.getDataModel().getColumns().get(1).getFullyQualifiedName(); + List topicToContainerLineage = topicToContainer.getColumnsLineage(); + topicToContainerLineage.add( + new ColumnLineage().withFromColumns(List.of(f1FQN)).withToColumn(f1c1)); + topicToContainerLineage.add( + new ColumnLineage().withFromColumns(List.of(f2FQN)).withToColumn(f2c2)); + addEdge(TOPIC, CONTAINER, topicToContainer, ADMIN_AUTH_HEADERS); + String f2c3FQN = "test_non_existent_container_column"; + topicToContainerLineage.add( + new ColumnLineage().withFromColumns(List.of(f2FQN)).withToColumn(f2c3FQN)); + assertResponse( + () -> addEdge(TOPIC, CONTAINER, topicToContainer, ADMIN_AUTH_HEADERS), + BAD_REQUEST, + String.format("Invalid fully qualified column name %s", f2c3FQN)); + + LineageDetails containerToTable = new LineageDetails(); + List containerToTableLineage = containerToTable.getColumnsLineage(); + containerToTableLineage.add( + new ColumnLineage().withFromColumns(List.of(f1c1)).withToColumn(f1t1)); + containerToTableLineage.add( + new ColumnLineage().withFromColumns(List.of(f2c2)).withToColumn(f2t2)); + addEdge(CONTAINER, TABLE_DATA_MODEL_LINEAGE, containerToTable, ADMIN_AUTH_HEADERS); + + LineageDetails tableToMlModel = new LineageDetails(); + String m1f1 = ML_MODEL.getMlFeatures().get(0).getFullyQualifiedName(); + String m2f2 = ML_MODEL.getMlFeatures().get(1).getFullyQualifiedName(); + List tableToMlModelLineage = tableToMlModel.getColumnsLineage(); + tableToMlModelLineage.add( + new ColumnLineage().withFromColumns(List.of(f1t1)).withToColumn(m1f1)); + tableToMlModelLineage.add( + new ColumnLineage().withFromColumns(List.of(f2t2)).withToColumn(m2f2)); + addEdge(TABLE_DATA_MODEL_LINEAGE, ML_MODEL, tableToMlModel, ADMIN_AUTH_HEADERS); + String m3f3 = "test_non_existent_feature"; + tableToMlModelLineage.add( + new ColumnLineage().withFromColumns(List.of(f2t2)).withToColumn(m3f3)); + assertResponse( + () -> addEdge(TABLE_DATA_MODEL_LINEAGE, ML_MODEL, tableToMlModel, ADMIN_AUTH_HEADERS), + BAD_REQUEST, + String.format("Invalid feature name %s", m3f3)); + + LineageDetails tableToDashboard = new LineageDetails(); + String c1d1 = DASHBOARD.getCharts().get(0).getFullyQualifiedName(); + String c2d1 = DASHBOARD.getCharts().get(1).getFullyQualifiedName(); + + List tableToDashboardLineage = tableToDashboard.getColumnsLineage(); + tableToDashboardLineage.add( + new ColumnLineage().withFromColumns(List.of(f1t1)).withToColumn(c1d1)); + tableToDashboardLineage.add( + new ColumnLineage().withFromColumns(List.of(f2t2)).withToColumn(c2d1)); + addEdge(TABLE_DATA_MODEL_LINEAGE, DASHBOARD, tableToDashboard, ADMIN_AUTH_HEADERS); + + deleteEdgeByName( + TOPIC.getEntityReference().getType(), + TOPIC.getFullyQualifiedName(), + CONTAINER.getEntityReference().getType(), + CONTAINER.getFullyQualifiedName()); } @Order(5) @@ -386,6 +488,11 @@ public void deleteEdge(Table from, Table to) throws HttpResponseException { deleteEdge(from, to, ADMIN_AUTH_HEADERS); } + public void deleteEdgeByName(String fromEntity, String fromFQN, String toEntity, String toFQN) + throws HttpResponseException { + deleteLineageByName(fromEntity, fromFQN, toEntity, toFQN, ADMIN_AUTH_HEADERS); + } + private void deleteEdge(Table from, Table to, Map authHeaders) throws HttpResponseException { EntitiesEdge edge = @@ -425,6 +532,21 @@ public void deleteLineage(EntitiesEdge edge, Map authHeaders) TestUtils.delete(target, authHeaders); } + public void deleteLineageByName( + String fromEntity, + String fromFQN, + String toEntity, + String toFQN, + Map authHeaders) + throws HttpResponseException { + WebTarget target = + getResourceAsURI( + String.format( + "lineage/%s/name/%s/%s/name/%s", + fromEntity, URLEncoder.encode(fromFQN), toEntity, URLEncoder.encode(toFQN))); + TestUtils.delete(target, authHeaders); + } + private void validateLineage(AddLineage addLineage, Map authHeaders) throws HttpResponseException { EntityReference from = addLineage.getEdge().getFromEntity(); diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/resources/metadata/TypeResourceTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/resources/metadata/TypeResourceTest.java index 635d73d71985..07f5a4f93e71 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/resources/metadata/TypeResourceTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/resources/metadata/TypeResourceTest.java @@ -42,7 +42,9 @@ import org.openmetadata.schema.entity.type.Category; import org.openmetadata.schema.entity.type.CustomProperty; import org.openmetadata.schema.type.ChangeDescription; +import org.openmetadata.schema.type.CustomPropertyConfig; import org.openmetadata.schema.type.EntityReference; +import org.openmetadata.schema.type.customproperties.EnumConfig; import org.openmetadata.service.Entity; import org.openmetadata.service.resources.EntityResourceTest; import org.openmetadata.service.resources.types.TypeResource; @@ -66,6 +68,7 @@ public TypeResourceTest() { public void setupTypes() throws HttpResponseException { INT_TYPE = getEntityByName("integer", "", ADMIN_AUTH_HEADERS); STRING_TYPE = getEntityByName("string", "", ADMIN_AUTH_HEADERS); + ENUM_TYPE = getEntityByName("enum", "", ADMIN_AUTH_HEADERS); } @Override @@ -86,8 +89,8 @@ public void post_entityCreateWithInvalidName_400() { @Test void put_patch_customProperty_200() throws IOException { - Type tableEntity = getEntityByName("table", "customProperties", ADMIN_AUTH_HEADERS); - assertTrue(listOrEmpty(tableEntity.getCustomProperties()).isEmpty()); + Type topicEntity = getEntityByName("topic", "customProperties", ADMIN_AUTH_HEADERS); + assertTrue(listOrEmpty(topicEntity.getCustomProperties()).isEmpty()); // Add a custom property with name intA with type integer with PUT CustomProperty fieldA = @@ -95,33 +98,169 @@ void put_patch_customProperty_200() throws IOException { .withName("intA") .withDescription("intA") .withPropertyType(INT_TYPE.getEntityReference()); - ChangeDescription change = getChangeDescription(tableEntity, MINOR_UPDATE); + ChangeDescription change = getChangeDescription(topicEntity, MINOR_UPDATE); fieldAdded(change, "customProperties", new ArrayList<>(List.of(fieldA))); - tableEntity = + topicEntity = addCustomPropertyAndCheck( - tableEntity.getId(), fieldA, ADMIN_AUTH_HEADERS, MINOR_UPDATE, change); - assertCustomProperties(new ArrayList<>(List.of(fieldA)), tableEntity.getCustomProperties()); + topicEntity.getId(), fieldA, ADMIN_AUTH_HEADERS, MINOR_UPDATE, change); + assertCustomProperties(new ArrayList<>(List.of(fieldA)), topicEntity.getCustomProperties()); // Changing custom property description with PUT fieldA.withDescription("updated"); - change = getChangeDescription(tableEntity, MINOR_UPDATE); + change = getChangeDescription(topicEntity, MINOR_UPDATE); fieldUpdated(change, EntityUtil.getCustomField(fieldA, "description"), "intA", "updated"); - tableEntity = + topicEntity = addCustomPropertyAndCheck( - tableEntity.getId(), fieldA, ADMIN_AUTH_HEADERS, MINOR_UPDATE, change); - assertCustomProperties(new ArrayList<>(List.of(fieldA)), tableEntity.getCustomProperties()); + topicEntity.getId(), fieldA, ADMIN_AUTH_HEADERS, MINOR_UPDATE, change); + assertCustomProperties(new ArrayList<>(List.of(fieldA)), topicEntity.getCustomProperties()); // Changing custom property description with PATCH // Changes from this PATCH is consolidated with the previous changes fieldA.withDescription("updated2"); + String json = JsonUtils.pojoToJson(topicEntity); + topicEntity.setCustomProperties(List.of(fieldA)); + change = getChangeDescription(topicEntity, CHANGE_CONSOLIDATED); + fieldUpdated(change, EntityUtil.getCustomField(fieldA, "description"), "intA", "updated2"); + topicEntity = + patchEntityAndCheck(topicEntity, json, ADMIN_AUTH_HEADERS, CHANGE_CONSOLIDATED, change); + + // Add a second property with name intB with type integer + // Note that since this is PUT operation, the previous changes are not consolidated + EntityReference typeRef = + new EntityReference() + .withType(INT_TYPE.getEntityReference().getType()) + .withId(INT_TYPE.getEntityReference().getId()); + CustomProperty fieldB = + new CustomProperty().withName("intB").withDescription("intB").withPropertyType(typeRef); + change = getChangeDescription(topicEntity, MINOR_UPDATE); + fieldAdded(change, "customProperties", new ArrayList<>(List.of(fieldB))); + topicEntity = + addCustomPropertyAndCheck( + topicEntity.getId(), fieldB, ADMIN_AUTH_HEADERS, MINOR_UPDATE, change); + fieldB.setPropertyType(INT_TYPE.getEntityReference()); + assertEquals(2, topicEntity.getCustomProperties().size()); + assertCustomProperties( + new ArrayList<>(List.of(fieldA, fieldB)), topicEntity.getCustomProperties()); + } + + @Test + void put_patch_customProperty_enum_200() throws IOException { + Type tableEntity = getEntityByName("table", "customProperties", ADMIN_AUTH_HEADERS); + assertTrue(listOrEmpty(tableEntity.getCustomProperties()).isEmpty()); + + // Add a custom property with name intA with type integer with PUT + CustomProperty enumFieldA = + new CustomProperty() + .withName("enumTest") + .withDescription("enumTest") + .withPropertyType(ENUM_TYPE.getEntityReference()); + ChangeDescription change = getChangeDescription(tableEntity, MINOR_UPDATE); + fieldAdded(change, "customProperties", new ArrayList<>(List.of(enumFieldA))); + Type finalTableEntity = tableEntity; + ChangeDescription finalChange = change; + assertResponseContains( + () -> + addCustomPropertyAndCheck( + finalTableEntity.getId(), + enumFieldA, + ADMIN_AUTH_HEADERS, + MINOR_UPDATE, + finalChange), + Status.BAD_REQUEST, + "Enum Custom Property Type must have EnumConfig."); + enumFieldA.setCustomPropertyConfig(new CustomPropertyConfig().withConfig(new EnumConfig())); + ChangeDescription change1 = getChangeDescription(tableEntity, MINOR_UPDATE); + Type tableEntity1 = tableEntity; + assertResponseContains( + () -> + addCustomPropertyAndCheck( + tableEntity1.getId(), enumFieldA, ADMIN_AUTH_HEADERS, MINOR_UPDATE, change1), + Status.BAD_REQUEST, + "Enum Custom Property Type must have EnumConfig populated with values."); + + enumFieldA.setCustomPropertyConfig( + new CustomPropertyConfig() + .withConfig(new EnumConfig().withValues(List.of("A", "B", "C", "C")))); + ChangeDescription change7 = getChangeDescription(tableEntity, MINOR_UPDATE); + Type tableEntity2 = tableEntity; + assertResponseContains( + () -> + addCustomPropertyAndCheck( + tableEntity2.getId(), enumFieldA, ADMIN_AUTH_HEADERS, MINOR_UPDATE, change7), + Status.BAD_REQUEST, + "Enum Custom Property values cannot have duplicates."); + + enumFieldA.setCustomPropertyConfig( + new CustomPropertyConfig().withConfig(new EnumConfig().withValues(List.of("A", "B", "C")))); + tableEntity = + addCustomPropertyAndCheck( + tableEntity.getId(), enumFieldA, ADMIN_AUTH_HEADERS, MINOR_UPDATE, change); + assertCustomProperties(new ArrayList<>(List.of(enumFieldA)), tableEntity.getCustomProperties()); + CustomPropertyConfig prevConfig = enumFieldA.getCustomPropertyConfig(); + // Changing custom property description with PUT + enumFieldA.withDescription("updatedEnumTest"); + ChangeDescription change2 = getChangeDescription(tableEntity, MINOR_UPDATE); + fieldUpdated( + change2, + EntityUtil.getCustomField(enumFieldA, "description"), + "enumTest", + "updatedEnumTest"); + tableEntity = + addCustomPropertyAndCheck( + tableEntity.getId(), enumFieldA, ADMIN_AUTH_HEADERS, MINOR_UPDATE, change2); + assertCustomProperties(new ArrayList<>(List.of(enumFieldA)), tableEntity.getCustomProperties()); + + enumFieldA.setCustomPropertyConfig( + new CustomPropertyConfig().withConfig(new EnumConfig().withValues(List.of("A", "B")))); + ChangeDescription change3 = getChangeDescription(tableEntity, MINOR_UPDATE); + assertResponseContains( + () -> + addCustomPropertyAndCheck( + tableEntity1.getId(), enumFieldA, ADMIN_AUTH_HEADERS, MINOR_UPDATE, change3), + Status.BAD_REQUEST, + "Existing Enum Custom Property values cannot be removed."); + + enumFieldA.setCustomPropertyConfig( + new CustomPropertyConfig() + .withConfig(new EnumConfig().withValues(List.of("A", "B", "C", "C")))); + ChangeDescription change4 = getChangeDescription(tableEntity, MINOR_UPDATE); + assertResponseContains( + () -> + addCustomPropertyAndCheck( + tableEntity1.getId(), enumFieldA, ADMIN_AUTH_HEADERS, MINOR_UPDATE, change4), + Status.BAD_REQUEST, + "Enum Custom Property values cannot have duplicates."); + + ChangeDescription change5 = getChangeDescription(tableEntity, MINOR_UPDATE); + enumFieldA.setCustomPropertyConfig( + new CustomPropertyConfig() + .withConfig(new EnumConfig().withValues(List.of("A", "B", "C", "D")))); + fieldUpdated( + change5, + EntityUtil.getCustomField(enumFieldA, "customPropertyConfig"), + prevConfig, + enumFieldA.getCustomPropertyConfig()); + tableEntity = + addCustomPropertyAndCheck( + tableEntity.getId(), enumFieldA, ADMIN_AUTH_HEADERS, MINOR_UPDATE, change5); + assertCustomProperties(new ArrayList<>(List.of(enumFieldA)), tableEntity.getCustomProperties()); + + // Changing custom property description with PATCH + // Changes from this PATCH is consolidated with the previous changes + enumFieldA.withDescription("updated2"); String json = JsonUtils.pojoToJson(tableEntity); - tableEntity.setCustomProperties(List.of(fieldA)); + tableEntity.setCustomProperties(List.of(enumFieldA)); change = getChangeDescription(tableEntity, CHANGE_CONSOLIDATED); - fieldUpdated(change, EntityUtil.getCustomField(fieldA, "description"), "intA", "updated2"); + fieldUpdated( + change5, + EntityUtil.getCustomField(enumFieldA, "description"), + "updatedEnumTest", + "updated2"); + tableEntity = - patchEntityAndCheck(tableEntity, json, ADMIN_AUTH_HEADERS, CHANGE_CONSOLIDATED, change); + patchEntityAndCheck(tableEntity, json, ADMIN_AUTH_HEADERS, CHANGE_CONSOLIDATED, change5); - // Add a second property with name intB with type integer + /* // Add a second property with name intB with type integer // Note that since this is PUT operation, the previous changes are not consolidated EntityReference typeRef = new EntityReference() @@ -137,7 +276,7 @@ void put_patch_customProperty_200() throws IOException { fieldB.setPropertyType(INT_TYPE.getEntityReference()); assertEquals(2, tableEntity.getCustomProperties().size()); assertCustomProperties( - new ArrayList<>(List.of(fieldA, fieldB)), tableEntity.getCustomProperties()); + new ArrayList<>(List.of(fieldA, fieldB)), tableEntity.getCustomProperties());*/ } @Test @@ -225,7 +364,6 @@ public void compareEntities(Type expected, Type patched, Map aut assertEquals(expected.getSchema(), patched.getSchema()); assertEquals(expected.getCategory(), patched.getCategory()); assertEquals(expected.getNameSpace(), patched.getNameSpace()); - assertEquals(expected.getCustomProperties(), patched.getCustomProperties()); } @Override @@ -239,6 +377,9 @@ public void assertFieldChange(String fieldName, Object expected, Object actual) List actualProperties = JsonUtils.readObjects(actual.toString(), CustomProperty.class); TestUtils.assertCustomProperties(expectedProperties, actualProperties); + } else if (fieldName.contains("customPropertyConfig")) { + String expectedStr = JsonUtils.pojoToJson(expected); + String actualStr = JsonUtils.pojoToJson(actual); } else { assertCommonFieldChange(fieldName, expected, actual); } diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/resources/mlmodels/MlModelResourceTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/resources/mlmodels/MlModelResourceTest.java index 2ffffe4a87a4..8fad246ee360 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/resources/mlmodels/MlModelResourceTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/resources/mlmodels/MlModelResourceTest.java @@ -465,12 +465,12 @@ public MlModel validateGetWithDifferentFields(MlModel model, boolean byName) model.getUsageSummary()); // .../models?fields=mlFeatures,mlHyperParameters - fields = "owner,dashboard,followers,tags,usageSummary"; + fields = "owner,followers,tags,usageSummary"; model = byName ? getEntityByName(model.getFullyQualifiedName(), fields, ADMIN_AUTH_HEADERS) : getEntity(model.getId(), fields, ADMIN_AUTH_HEADERS); - assertListNotNull(model.getDashboard(), model.getUsageSummary()); + assertListNotNull(model.getUsageSummary()); // Checks for other owner, tags, and followers is done in the base class return model; } @@ -482,7 +482,6 @@ public CreateMlModel createRequest(String name) { .withAlgorithm(ALGORITHM) .withMlFeatures(ML_FEATURES) .withMlHyperParameters(ML_HYPERPARAMS) - .withDashboard(DASHBOARD.getFullyQualifiedName()) .withService(MLFLOW_REFERENCE.getFullyQualifiedName()); } diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/resources/query/QueryResourceTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/resources/query/QueryResourceTest.java index 9e99c42a54c2..7bee5c9a0943 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/resources/query/QueryResourceTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/resources/query/QueryResourceTest.java @@ -48,6 +48,7 @@ public QueryResourceTest() { super( Entity.QUERY, Query.class, QueryResource.QueryList.class, "queries", QueryResource.FIELDS); supportsSearchIndex = true; + runWebhookTests = false; } @BeforeAll diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/resources/system/SystemResourceTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/resources/system/SystemResourceTest.java index 04ed3eee26e4..e6217dfc9203 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/resources/system/SystemResourceTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/resources/system/SystemResourceTest.java @@ -40,6 +40,7 @@ import org.openmetadata.schema.security.client.GoogleSSOClientConfig; import org.openmetadata.schema.settings.Settings; import org.openmetadata.schema.settings.SettingsType; +import org.openmetadata.schema.system.ValidationResponse; import org.openmetadata.schema.util.EntitiesCount; import org.openmetadata.schema.util.ServicesCount; import org.openmetadata.service.OpenMetadataApplicationConfig; @@ -301,6 +302,19 @@ void botUserCountCheck(TestInfo test) throws HttpResponseException { Assertions.assertEquals(beforeUserCount, afterUserCount); } + @Test + void validate_test() throws HttpResponseException { + ValidationResponse response = getValidation(); + + // Check migrations are OK + Assertions.assertEquals(Boolean.TRUE, response.getMigrations().getPassed()); + } + + private static ValidationResponse getValidation() throws HttpResponseException { + WebTarget target = getResource("system/status"); + return TestUtils.get(target, ValidationResponse.class, ADMIN_AUTH_HEADERS); + } + private static EntitiesCount getEntitiesCount() throws HttpResponseException { WebTarget target = getResource("system/entities/count"); return TestUtils.get(target, EntitiesCount.class, ADMIN_AUTH_HEADERS); diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/resources/topics/TopicResourceTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/resources/topics/TopicResourceTest.java index 5d625979675a..30ae6f49316f 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/resources/topics/TopicResourceTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/resources/topics/TopicResourceTest.java @@ -71,12 +71,22 @@ @Slf4j public class TopicResourceTest extends EntityResourceTest { - private static final String SCHEMA_TEXT = + public static final String SCHEMA_TEXT = "{\"namespace\":\"org.open-metadata.kafka\",\"name\":\"Customer\",\"type\":\"record\"," + "\"fields\":[{\"name\":\"id\",\"type\":\"string\"},{\"name\":\"first_name\",\"type\":\"string\"},{\"name\":\"last_name\",\"type\":\"string\"}," + "{\"name\":\"email\",\"type\":\"string\"},{\"name\":\"address_line_1\",\"type\":\"string\"},{\"name\":\"address_line_2\",\"type\":\"string\"}," + "{\"name\":\"post_code\",\"type\":\"string\"},{\"name\":\"country\",\"type\":\"string\"}]}"; - private static final MessageSchema schema = + public static final List fields = + Arrays.asList( + getField("id", FieldDataType.STRING, null), + getField("first_name", FieldDataType.STRING, null), + getField("last_name", FieldDataType.STRING, null), + getField("email", FieldDataType.STRING, null), + getField("address_line_1", FieldDataType.STRING, null), + getField("address_line_2", FieldDataType.STRING, null), + getField("post_code", FieldDataType.STRING, null), + getField("county", FieldDataType.STRING, PERSONAL_DATA_TAG_LABEL)); + public static final MessageSchema SCHEMA = new MessageSchema().withSchemaText(SCHEMA_TEXT).withSchemaType(SchemaType.Avro); public TopicResourceTest() { @@ -171,16 +181,6 @@ void put_topicAttributes_200_ok(TestInfo test) throws IOException { @Test void put_topicSchemaFields_200_ok(TestInfo test) throws IOException { - List fields = - Arrays.asList( - getField("id", FieldDataType.STRING, null), - getField("first_name", FieldDataType.STRING, null), - getField("last_name", FieldDataType.STRING, null), - getField("email", FieldDataType.STRING, null), - getField("address_line_1", FieldDataType.STRING, null), - getField("address_line_2", FieldDataType.STRING, null), - getField("post_code", FieldDataType.STRING, null), - getField("county", FieldDataType.STRING, PERSONAL_DATA_TAG_LABEL)); CreateTopic createTopic = createRequest(test) @@ -191,7 +191,7 @@ void put_topicSchemaFields_200_ok(TestInfo test) throws IOException { .withReplicationFactor(1) .withRetentionTime(1.0) .withRetentionSize(1.0) - .withMessageSchema(schema.withSchemaFields(fields)) + .withMessageSchema(SCHEMA.withSchemaFields(fields)) .withCleanupPolicies(List.of(CleanupPolicy.COMPACT)); // Patch and update the topic @@ -221,7 +221,7 @@ void patch_topicAttributes_200_ok(TestInfo test) throws IOException { .withReplicationFactor(1) .withRetentionTime(1.0) .withRetentionSize(1.0) - .withMessageSchema(schema.withSchemaFields(fields)) + .withMessageSchema(SCHEMA.withSchemaFields(fields)) .withCleanupPolicies(List.of(CleanupPolicy.COMPACT)); // Patch and update the topic @@ -236,7 +236,7 @@ void patch_topicAttributes_200_ok(TestInfo test) throws IOException { .withReplicationFactor(2) .withRetentionTime(2.0) .withRetentionSize(2.0) - .withMessageSchema(schema.withSchemaFields(fields)) + .withMessageSchema(SCHEMA.withSchemaFields(fields)) .withCleanupPolicies(List.of(CleanupPolicy.DELETE)); ChangeDescription change = getChangeDescription(topic, MINOR_UPDATE); @@ -284,7 +284,7 @@ void test_mutuallyExclusiveTags(TestInfo testInfo) { Field field = getField("first_name", FieldDataType.STRING, null) .withTags(listOf(TIER1_TAG_LABEL, TIER2_TAG_LABEL)); - create1.withMessageSchema(schema.withSchemaFields(List.of(field))); + create1.withMessageSchema(SCHEMA.withSchemaFields(List.of(field))); assertResponse( () -> createEntity(create1, ADMIN_AUTH_HEADERS), BAD_REQUEST, @@ -304,7 +304,7 @@ void test_mutuallyExclusiveTags(TestInfo testInfo) { getField("testNested", FieldDataType.STRING, null) .withTags(listOf(TIER1_TAG_LABEL, TIER2_TAG_LABEL)); Field field1 = getField("test", FieldDataType.RECORD, null).withChildren(List.of(nestedField)); - create2.setMessageSchema(schema.withSchemaFields(List.of(field1))); + create2.setMessageSchema(SCHEMA.withSchemaFields(List.of(field1))); assertResponse( () -> createEntity(create2, ADMIN_AUTH_HEADERS), BAD_REQUEST, diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/util/TestUtils.java b/openmetadata-service/src/test/java/org/openmetadata/service/util/TestUtils.java index 93251f5bf66e..c3670b383f7f 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/util/TestUtils.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/util/TestUtils.java @@ -202,6 +202,10 @@ public static void assertCustomProperties( } } + public static boolean isCI() { + return System.getenv("CI") != null; + } + public enum UpdateType { CREATED, // Not updated instead entity was created NO_CHANGE, // PUT/PATCH made no change to the entity and the version remains the same diff --git a/openmetadata-service/src/test/resources/openmetadata-secure-test.yaml b/openmetadata-service/src/test/resources/openmetadata-secure-test.yaml index 332d6c5510b6..1a6027d3c884 100644 --- a/openmetadata-service/src/test/resources/openmetadata-secure-test.yaml +++ b/openmetadata-service/src/test/resources/openmetadata-secure-test.yaml @@ -140,6 +140,7 @@ authorizerConfiguration: - "all" authenticationConfiguration: + clientType: "public" provider: "basic" providerName: "" publicKeyUrls: diff --git a/openmetadata-shaded-deps/elasticsearch-dep/pom.xml b/openmetadata-shaded-deps/elasticsearch-dep/pom.xml index 6beda6e056e3..5d6d929c3b98 100644 --- a/openmetadata-shaded-deps/elasticsearch-dep/pom.xml +++ b/openmetadata-shaded-deps/elasticsearch-dep/pom.xml @@ -5,7 +5,7 @@ openmetadata-shaded-deps org.open-metadata - 1.4.0-SNAPSHOT + 1.3.4 4.0.0 elasticsearch-deps diff --git a/openmetadata-shaded-deps/opensearch-dep/pom.xml b/openmetadata-shaded-deps/opensearch-dep/pom.xml index 0e6cc2ae231f..444307550cd3 100644 --- a/openmetadata-shaded-deps/opensearch-dep/pom.xml +++ b/openmetadata-shaded-deps/opensearch-dep/pom.xml @@ -5,7 +5,7 @@ openmetadata-shaded-deps org.open-metadata - 1.4.0-SNAPSHOT + 1.3.4 4.0.0 opensearch-deps diff --git a/openmetadata-shaded-deps/pom.xml b/openmetadata-shaded-deps/pom.xml index fc21ef779683..8d99351c1326 100644 --- a/openmetadata-shaded-deps/pom.xml +++ b/openmetadata-shaded-deps/pom.xml @@ -5,7 +5,7 @@ platform org.open-metadata - 1.4.0-SNAPSHOT + 1.3.4 4.0.0 openmetadata-shaded-deps diff --git a/openmetadata-spec/pom.xml b/openmetadata-spec/pom.xml index be3cf6c811b8..e4b29d08d902 100644 --- a/openmetadata-spec/pom.xml +++ b/openmetadata-spec/pom.xml @@ -5,7 +5,7 @@ platform org.open-metadata - 1.4.0-SNAPSHOT + 1.3.4 4.0.0 diff --git a/openmetadata-spec/src/main/java/org/openmetadata/sdk/PipelineServiceClient.java b/openmetadata-spec/src/main/java/org/openmetadata/sdk/PipelineServiceClient.java index 1ed740a388c8..89779ad3db0a 100644 --- a/openmetadata-spec/src/main/java/org/openmetadata/sdk/PipelineServiceClient.java +++ b/openmetadata-spec/src/main/java/org/openmetadata/sdk/PipelineServiceClient.java @@ -96,8 +96,8 @@ public abstract class PipelineServiceClient { "test_suite_task", PipelineType.DATA_INSIGHT.toString(), "data_insight_task", - PipelineType.ELASTIC_SEARCH_REINDEX.toString(), - "elasticsearch_reindex_task"); + PipelineType.APPLICATION.toString(), + "application_task"); public static final String SERVER_VERSION; diff --git a/openmetadata-spec/src/main/resources/json/schema/api/data/createCustomProperty.json b/openmetadata-spec/src/main/resources/json/schema/api/data/createCustomProperty.json index 1bd4ee78f0bc..7261753a98c6 100644 --- a/openmetadata-spec/src/main/resources/json/schema/api/data/createCustomProperty.json +++ b/openmetadata-spec/src/main/resources/json/schema/api/data/createCustomProperty.json @@ -4,23 +4,6 @@ "title": "CreateCustomPropertyRequest", "description": "Create Custom Property Model entity request", "type": "object", - "definitions": { - "propertyType": { - "description": "Property Type", - "type": "object", - "properties": { - "type": { - "description": "Property type", - "type": "string", - "default": "type" - }, - "id": { - "description": "Unique identifier of this instance.", - "$ref": "../../type/basic.json#/definitions/uuid" - } - } - } - }, "properties": { "name": { "description": "Name that identifies this Custom Property model.", @@ -31,10 +14,14 @@ "$ref": "../../type/basic.json#/definitions/markdown" }, "propertyType": { - "description": "Property Type", - "$ref": "#/definitions/propertyType" + "description": "Property Type.", + "$ref": "../../type/customProperty.json#/definitions/propertyType" + }, + "customPropertyConfig": { + "description": "Config to define constraints around CustomProperty.", + "$ref": "../../type/customProperty.json#/definitions/customPropertyConfig" } }, - "required": ["name"], + "required": ["name", "propertyType"], "additionalProperties": false } diff --git a/openmetadata-spec/src/main/resources/json/schema/api/feed/createThread.json b/openmetadata-spec/src/main/resources/json/schema/api/feed/createThread.json index 55a312765e8a..d466729e16d3 100644 --- a/openmetadata-spec/src/main/resources/json/schema/api/feed/createThread.json +++ b/openmetadata-spec/src/main/resources/json/schema/api/feed/createThread.json @@ -55,6 +55,10 @@ }, "announcementDetails": { "$ref": "../../entity/feed/thread.json#/definitions/announcementDetails" + }, + "chatbotDetails": { + "description": "Details about the Chatbot conversation. This is only applicable if thread is of type Chatbot.", + "$ref": "../../entity/feed/thread.json#/definitions/chatbotDetails" } }, "required": ["message", "from", "about"], diff --git a/openmetadata-spec/src/main/resources/json/schema/configuration/appsPrivateConfiguration.json b/openmetadata-spec/src/main/resources/json/schema/configuration/appsPrivateConfiguration.json index f0e1672ea96a..0ac3ba5b7c35 100644 --- a/openmetadata-spec/src/main/resources/json/schema/configuration/appsPrivateConfiguration.json +++ b/openmetadata-spec/src/main/resources/json/schema/configuration/appsPrivateConfiguration.json @@ -16,6 +16,11 @@ "type": "string", "description": "Application Name" }, + "preview": { + "type": "boolean", + "description": "Flag to enable/disable preview for the application. If the app is in preview mode, it can't be installed.", + "default": false + }, "parameters": { "javaType": "org.openmetadata.schema.api.configuration.apps.Parameters", "description": "Parameters to initialize the Applications.", diff --git a/openmetadata-spec/src/main/resources/json/schema/configuration/authenticationConfiguration.json b/openmetadata-spec/src/main/resources/json/schema/configuration/authenticationConfiguration.json index c75f06b989a1..70412355bf43 100644 --- a/openmetadata-spec/src/main/resources/json/schema/configuration/authenticationConfiguration.json +++ b/openmetadata-spec/src/main/resources/json/schema/configuration/authenticationConfiguration.json @@ -18,6 +18,16 @@ } }, "properties": { + "clientType": { + "javaType": "org.openmetadata.schema.api.security.ClientType", + "description": "Client Type", + "type": "string", + "enum": [ + "public", + "confidential" + ], + "default": "public" + }, "provider": { "$ref": "../entity/services/connections/metadata/openMetadataConnection.json#/definitions/authProvider" }, @@ -67,6 +77,10 @@ "samlConfiguration": { "description": "Saml Configuration that is applicable only when the provider is Saml", "$ref": "../security/client/samlSSOClientConfig.json" + }, + "oidcConfiguration": { + "description": "Oidc Configuration for Confidential Client Type", + "$ref": "../security/client/oidcClientConfig.json" } }, "required": ["provider", "providerName", "publicKeyUrls", "authority", "callbackUrl", "clientId", "jwtPrincipalClaims"], diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/applications/app.json b/openmetadata-spec/src/main/resources/json/schema/entity/applications/app.json index 2546da2a2bab..98a6cf6c41c3 100644 --- a/openmetadata-spec/src/main/resources/json/schema/entity/applications/app.json +++ b/openmetadata-spec/src/main/resources/json/schema/entity/applications/app.json @@ -25,15 +25,17 @@ ] }, "scheduleTimeline": { + "javaType": "org.openmetadata.schema.entity.app.ScheduleTimeline", + "description": "This schema defines the Application ScheduleTimeline Options", "type": "string", - "enum": ["Hourly"," Daily", "Weekly", "Monthly", "Custom"], + "enum": ["Hourly"," Daily", "Weekly", "Monthly", "Custom", "None"], "default": "Weekly" }, "appSchedule": { "javaType": "org.openmetadata.schema.entity.app.AppSchedule", "description": "This schema defines the type of application.", "properties": { - "scheduleType": { + "scheduleTimeline": { "$ref": "#/definitions/scheduleTimeline" }, "cronExpression": { @@ -41,7 +43,7 @@ "type": "string" } }, - "required": ["scheduleType"], + "required": ["scheduleTimeline"], "additionalProperties": false }, "appType": { @@ -195,6 +197,11 @@ "type": "boolean", "default": true }, + "system": { + "description": "A system app cannot be uninstalled or modified.", + "type": "boolean", + "default": false + }, "appConfiguration": { "description": "Application Configuration object.", "$ref": "./configuration/applicationConfig.json#/definitions/appConfig" @@ -203,6 +210,11 @@ "description": "Application Private configuration loaded at runtime.", "$ref": "./configuration/applicationConfig.json#/definitions/privateConfig" }, + "preview": { + "type": "boolean", + "description": "Flag to enable/disable preview for the application. If the app is in preview mode, it can't be installed.", + "default": false + }, "pipelines": { "description": "References to pipelines deployed for this database service to extract metadata, usage, lineage etc..", "$ref": "../../type/entityReferenceList.json" diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/applications/appRunRecord.json b/openmetadata-spec/src/main/resources/json/schema/entity/applications/appRunRecord.json index 5f9378d7dc5a..9ef46dcb9b30 100644 --- a/openmetadata-spec/src/main/resources/json/schema/entity/applications/appRunRecord.json +++ b/openmetadata-spec/src/main/resources/json/schema/entity/applications/appRunRecord.json @@ -25,21 +25,8 @@ ] }, "runType": { - "javaType": "org.openmetadata.schema.entity.app.AppRunType", "description": "This schema defines the type of application Run.", - "type": "string", - "enum": [ - "Scheduled", - "OnDemand" - ], - "javaEnums": [ - { - "name": "Scheduled" - }, - { - "name": "OnDemand" - } - ] + "type": "string" }, "startTime": { "description": "Start of the job status.", diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/applications/configuration/external/metaPilotAppConfig.json b/openmetadata-spec/src/main/resources/json/schema/entity/applications/configuration/external/metaPilotAppConfig.json index 4ac75978557f..84c4f6cac4f8 100644 --- a/openmetadata-spec/src/main/resources/json/schema/entity/applications/configuration/external/metaPilotAppConfig.json +++ b/openmetadata-spec/src/main/resources/json/schema/entity/applications/configuration/external/metaPilotAppConfig.json @@ -11,34 +11,6 @@ "type": "string", "enum": ["MetaPilot"], "default": "MetaPilot" - }, - "serviceDatabases": { - "title": "Service Databases", - "description": "Choose the service and its databases you want to generate descriptions from.", - "type": "object", - "properties": { - "service": { - "title": "Service Name", - "placeholder": "Search Service", - "description": "Service Name to get descriptions from.", - "$ref": "../../../../type/entityReference.json", - "format": "autoComplete", - "autoCompleteType": "database_service_search_index" - }, - "databases": { - "title": "Databases", - "description": "List of database names from the Service to get descriptions from.", - "type": "array", - "items": { - "placeholder": "Search Databases", - "$ref": "../../../../type/entityReference.json", - "format": "autoComplete", - "autoCompleteType": "database_search_index" - } - } - }, - "additionalProperties": false, - "required": ["service", "databases"] } }, "properties": { @@ -48,13 +20,24 @@ "$ref": "#/definitions/metaPilotAppType", "default": "MetaPilot" }, - "serviceDatabases": { - "title": "Service Databases", + "descriptionDatabases": { + "title": "Databases for Automated Description Generation", "description": "Services and Databases configured to get the descriptions from.", "type": "array", "items": { - "$ref": "#/definitions/serviceDatabases" + "placeholder": "Search Databases", + "$ref": "../../../../type/entityReference.json", + "format": "autoComplete", + "autoCompleteType": "database_search_index" } + }, + "defaultScope": { + "title": "Default Chatbot Database Scope", + "description": "Default database scope for the chatbot.", + "$ref": "../../../../type/entityReference.json", + "format": "autoComplete", + "autoCompleteType": "database_search_index", + "placeholder": "Search Databases" } }, "additionalProperties": false diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/applications/createAppRequest.json b/openmetadata-spec/src/main/resources/json/schema/entity/applications/createAppRequest.json index 07a82aca6db6..035a63052d51 100644 --- a/openmetadata-spec/src/main/resources/json/schema/entity/applications/createAppRequest.json +++ b/openmetadata-spec/src/main/resources/json/schema/entity/applications/createAppRequest.json @@ -1,7 +1,7 @@ { "$id": "https://open-metadata.org/schema/entity/applications/createAppRequest.json", "$schema": "http://json-schema.org/draft-07/schema#", - "title": "CreateApp", + "title": "CreateAppRequest", "javaType": "org.openmetadata.schema.entity.app.CreateApp", "javaInterfaces": ["org.openmetadata.schema.CreateEntity"], "description": "This schema defines the create applications request for Open-Metadata.", diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/applications/marketplace/appMarketPlaceDefinition.json b/openmetadata-spec/src/main/resources/json/schema/entity/applications/marketplace/appMarketPlaceDefinition.json index 649e72b2a2b6..c6bb27f464d1 100644 --- a/openmetadata-spec/src/main/resources/json/schema/entity/applications/marketplace/appMarketPlaceDefinition.json +++ b/openmetadata-spec/src/main/resources/json/schema/entity/applications/marketplace/appMarketPlaceDefinition.json @@ -130,6 +130,16 @@ "type": "string" }, "uniqueItems": true + }, + "system": { + "description": "A system app cannot be uninstalled or modified.", + "type": "boolean", + "default": false + }, + "preview": { + "type": "boolean", + "description": "Flag to enable/disable preview for the application. If the app is in preview mode, it can't be installed.", + "default": false } }, "additionalProperties": false, diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/applications/marketplace/createAppMarketPlaceDefinitionReq.json b/openmetadata-spec/src/main/resources/json/schema/entity/applications/marketplace/createAppMarketPlaceDefinitionReq.json index f94d1ca89df6..d6a238895908 100644 --- a/openmetadata-spec/src/main/resources/json/schema/entity/applications/marketplace/createAppMarketPlaceDefinitionReq.json +++ b/openmetadata-spec/src/main/resources/json/schema/entity/applications/marketplace/createAppMarketPlaceDefinitionReq.json @@ -1,7 +1,7 @@ { "$id": "https://open-metadata.org/schema/entity/applications/marketplace/createAppMarketPlaceDefinitionReq.json", "$schema": "http://json-schema.org/draft-07/schema#", - "title": "CreateAppMarketPlaceDefinitionReq", + "title": "CreateAppMarketPlaceDefinitionRequest", "javaType": "org.openmetadata.schema.entity.app.CreateAppMarketPlaceDefinitionReq", "javaInterfaces": ["org.openmetadata.schema.CreateEntity"], "description": "This schema defines the applications for Open-Metadata.", @@ -97,6 +97,11 @@ "type": "string" }, "uniqueItems": true + }, + "system": { + "description": "A system app cannot be uninstalled or modified.", + "type": "boolean", + "default": false } }, "additionalProperties": false, diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/feed/thread.json b/openmetadata-spec/src/main/resources/json/schema/entity/feed/thread.json index 4bef6569d3d6..f78f6e5038cf 100644 --- a/openmetadata-spec/src/main/resources/json/schema/entity/feed/thread.json +++ b/openmetadata-spec/src/main/resources/json/schema/entity/feed/thread.json @@ -109,7 +109,7 @@ "javaType": "org.openmetadata.schema.type.ThreadType", "type": "string", "description": "Type of thread.", - "enum": ["Conversation", "Task", "Announcement"], + "enum": ["Conversation", "Task", "Announcement", "Chatbot"], "javaEnums": [ { "name": "Conversation" @@ -119,6 +119,9 @@ }, { "name": "Announcement" + }, + { + "name": "Chatbot" } ], "default": "Conversation" @@ -144,6 +147,17 @@ "required": ["startTime", "endTime"], "additionalProperties": false }, + "chatbotDetails": { + "javaType": "org.openmetadata.schema.type.ChatbotDetails", + "description": "Details about the Chatbot conversation. This is only applicable if thread is of type Chatbot.", + "type": "object", + "properties": { + "query": { + "description": "The query being discussed with the Chatbot", + "type": "string" + } + } + }, "post": { "javaType": "org.openmetadata.schema.type.Post", "type": "object", @@ -245,6 +259,10 @@ "announcement": { "description": "Details about the announcement. This is only applicable if thread is of type announcement.", "$ref": "#/definitions/announcementDetails" + }, + "chatbot": { + "description": "Details about the Chatbot conversation. This is only applicable if thread is of type Chatbot.", + "$ref": "#/definitions/chatbotDetails" } }, "required": ["id", "about", "message"], diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/connectionBasicType.json b/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/connectionBasicType.json index 1c822a39767b..3f3fa95cab2d 100644 --- a/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/connectionBasicType.json +++ b/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/connectionBasicType.json @@ -81,6 +81,12 @@ "type": "string", "default": "" }, + "filePathPattern": { + "title": "File Path Pattern", + "description": "Provide the pattern of the path where the generated sample data file needs to be stored.", + "type": "string", + "default": "{service_name}/{database_name}/{database_schema_name}/{table_name}/sample_data.parquet" + }, "overwriteData": { "title": "Overwrite Sample Data", "description": "When this field enabled a single parquet file will be created to store sample data, otherwise we will create a new file per day", diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/database/azureSQLConnection.json b/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/database/azureSQLConnection.json index b2b0a2cdc880..1cb390f00242 100644 --- a/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/database/azureSQLConnection.json +++ b/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/database/azureSQLConnection.json @@ -9,13 +9,17 @@ "azureSQLType": { "description": "Service type.", "type": "string", - "enum": ["AzureSQL"], + "enum": [ + "AzureSQL" + ], "default": "AzureSQL" }, "azureSQLScheme": { "description": "SQLAlchemy driver scheme options.", "type": "string", - "enum": ["mssql+pyodbc"], + "enum": [ + "mssql+pyodbc" + ], "default": "mssql+pyodbc" } }, @@ -59,6 +63,37 @@ "type": "string", "default": "ODBC Driver 18 for SQL Server" }, + "authenticationMode": { + "title": "Authentication Mode", + "description": "This parameter determines the mode of authentication for connecting to AzureSQL using ODBC. If 'Active Directory Password' is selected, you need to provide the password. If 'Active Directory Integrated' is selected, password is not required as it uses the logged-in user's credentials. This mode is useful for establishing secure and seamless connections with AzureSQL.", + "properties": { + "authentication": { + "title": "Authentication", + "description": "Authentication from Connection String for AzureSQL.", + "type": "string", + "enum": [ + "ActiveDirectoryIntegrated", + "ActiveDirectoryPassword" + ] + }, + "encrypt": { + "title": "Encrypt", + "description": "Encrypt from Connection String for AzureSQL.", + "type": "boolean" + }, + "trustServerCertificate": { + "title": "Trust Server Certificate", + "description": "Trust Server Certificate from Connection String for AzureSQL.", + "type": "boolean" + }, + "connectionTimeout": { + "title": "Connection Timeout", + "description": "Connection Timeout from Connection String for AzureSQL.", + "type": "integer", + "default": 30 + } + } + }, "ingestAllDatabases": { "title": "Ingest All Databases", "description": "Ingest data from all databases in Azuresql. You can use databaseFilterPattern on top of this.", @@ -102,5 +137,9 @@ } }, "additionalProperties": false, - "required": ["hostPort", "username", "database"] -} + "required": [ + "hostPort", + "database", + "username" + ] +} \ No newline at end of file diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/database/common/azureConfig.json b/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/database/common/azureConfig.json new file mode 100644 index 000000000000..364f69347c88 --- /dev/null +++ b/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/database/common/azureConfig.json @@ -0,0 +1,15 @@ +{ + "$id": "https://open-metadata.org/schema/entity/services/connections/database/common/azureConfig.json", + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Azure Configuration Source", + "description": "Azure Database Connection Config", + "type": "object", + "javaType": "org.openmetadata.schema.services.connections.database.common.AzureConfig", + "properties": { + "azureConfig": { + "title": "Azure Credentials Configuration", + "$ref": "../../../../../security/credentials/azureCredentials.json" + } + }, + "additionalProperties": false +} \ No newline at end of file diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/database/mongoDBConnection.json b/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/database/mongoDBConnection.json index 7b453a16653a..8fc6a817e2ef 100644 --- a/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/database/mongoDBConnection.json +++ b/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/database/mongoDBConnection.json @@ -66,10 +66,6 @@ "supportsMetadataExtraction": { "title": "Supports Metadata Extraction", "$ref": "../connectionBasicType.json#/definitions/supportsMetadataExtraction" - }, - "supportsProfiler": { - "title": "Supports Profiler", - "$ref": "../connectionBasicType.json#/definitions/supportsProfiler" } }, "required": ["hostPort"], diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/database/mysqlConnection.json b/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/database/mysqlConnection.json index c11c496c4424..2a96d10ed68b 100644 --- a/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/database/mysqlConnection.json +++ b/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/database/mysqlConnection.json @@ -9,13 +9,17 @@ "mySQLType": { "description": "Service type.", "type": "string", - "enum": ["Mysql"], + "enum": [ + "Mysql" + ], "default": "Mysql" }, "mySQLScheme": { "description": "SQLAlchemy driver scheme options.", "type": "string", - "enum": ["mysql+pymysql"], + "enum": [ + "mysql+pymysql" + ], "default": "mysql+pymysql" } }, @@ -46,6 +50,9 @@ }, { "$ref": "./common/iamAuthConfig.json" + }, + { + "$ref": "./common/azureConfig.json" } ] }, @@ -108,5 +115,8 @@ } }, "additionalProperties": false, - "required": ["hostPort", "username"] -} + "required": [ + "hostPort", + "username" + ] +} \ No newline at end of file diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/database/postgresConnection.json b/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/database/postgresConnection.json index ac0445d63de6..b4e32b29c9e2 100644 --- a/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/database/postgresConnection.json +++ b/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/database/postgresConnection.json @@ -50,6 +50,9 @@ }, { "$ref": "./common/iamAuthConfig.json" + }, + { + "$ref": "./common/azureConfig.json" } ] }, diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/database/sapHana/sapHanaHDBConnection.json b/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/database/sapHana/sapHanaHDBConnection.json new file mode 100644 index 000000000000..bbd240879a68 --- /dev/null +++ b/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/database/sapHana/sapHanaHDBConnection.json @@ -0,0 +1,16 @@ +{ + "$id": "https://open-metadata.org/schema/entity/services/connections/database/sapHana/sapHanaHDBConnection.json", + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "SapHanaHDBConnection", + "description": "Sap Hana Database HDB User Store Connection Config", + "type": "object", + "javaType": "org.openmetadata.schema.services.connections.database.sapHana.SapHanaHDBConnection", + "properties": { + "userKey": { + "title": "User Key", + "description": "HDB Store User Key generated from the command `hdbuserstore SET `", + "type": "string" + } + }, + "additionalProperties": false +} diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/database/sapHana/sapHanaSQLConnection.json b/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/database/sapHana/sapHanaSQLConnection.json new file mode 100644 index 000000000000..8b79e256ce8b --- /dev/null +++ b/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/database/sapHana/sapHanaSQLConnection.json @@ -0,0 +1,38 @@ +{ + "$id": "https://open-metadata.org/schema/entity/services/connections/database/sapHana/sapHanaConnection.json", + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "SapHanaSQLConnection", + "description": "Sap Hana Database SQL Connection Config", + "type": "object", + "javaType": "org.openmetadata.schema.services.connections.database.sapHana.SapHanaSQLConnection", + "properties": { + "hostPort": { + "title": "Host and Port", + "description": "Host and port of the Hana service.", + "type": "string" + }, + "username": { + "title": "Username", + "description": "Username to connect to Hana. This user should have privileges to read all the metadata.", + "type": "string" + }, + "password": { + "title": "Password", + "description": "Password to connect to Hana.", + "type": "string", + "format": "password" + }, + "databaseSchema": { + "title": "Database Schema", + "description": "Database Schema of the data source. This is an optional parameter, if you would like to restrict the metadata reading to a single schema. When left blank, OpenMetadata Ingestion attempts to scan all the schemas.", + "type": "string" + }, + "database": { + "title": "Database", + "description": "Database of the data source.", + "type": "string" + } + }, + "additionalProperties": false, + "required": ["username", "password", "hostPort"] +} diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/database/sapHanaConnection.json b/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/database/sapHanaConnection.json index e65fdd53e7ac..1458159de932 100644 --- a/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/database/sapHanaConnection.json +++ b/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/database/sapHanaConnection.json @@ -17,56 +17,8 @@ "type": "string", "enum": ["hana"], "default": "hana" - }, - "sqlConnection": { - "title": "SQL Connection", - "description": "Options to connect to SAP Hana by passing the database information", - "type": "object", - "properties": { - "hostPort": { - "title": "Host and Port", - "description": "Host and port of the Hana service.", - "type": "string" - }, - "username": { - "title": "Username", - "description": "Username to connect to Hana. This user should have privileges to read all the metadata.", - "type": "string" - }, - "password": { - "title": "Password", - "description": "Password to connect to Hana.", - "type": "string", - "format": "password" - }, - "databaseSchema": { - "title": "Database Schema", - "description": "Database Schema of the data source. This is an optional parameter, if you would like to restrict the metadata reading to a single schema. When left blank, OpenMetadata Ingestion attempts to scan all the schemas.", - "type": "string" - }, - "database": { - "title": "Database", - "description": "Database of the data source.", - "type": "string" - } - }, - "additionalProperties": false, - "required": ["username", "password", "hostPort"] - }, - "hdbUserStoreConnection": { - "title": "HDB User Store Connection", - "description": "Use HDB User Store to avoid entering connection-related information manually. This store needs to be present on the client running the ingestion.", - "type": "object", - "properties": { - "userKey": { - "title": "User Key", - "description": "HDB Store User Key generated from the command `hdbuserstore SET `", - "type": "string" - } - }, - "additionalProperties": false } - }, + }, "properties": { "type": { "title": "Service Type", @@ -86,10 +38,10 @@ "description": "Choose between Database connection or HDB User Store connection.", "oneOf": [ { - "$ref": "#/definitions/sqlConnection" + "$ref": "sapHana/sapHanaSQLConnection.json" }, { - "$ref": "#/definitions/hdbUserStoreConnection" + "$ref": "sapHana/sapHanaHDBConnection.json" } ] }, diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/metadata/alationConnection.json b/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/metadata/alationConnection.json index 10ec93c85594..693d48149ece 100644 --- a/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/metadata/alationConnection.json +++ b/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/metadata/alationConnection.json @@ -1,85 +1,126 @@ { - "$id": "https://open-metadata.org/schema/entity/services/connections/metadata/AlationConnection.json", - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "AlationConnection", - "description": "Alation Connection Config", - "type": "object", - "javaType": "org.openmetadata.schema.services.connections.metadata.AlationConnection", - "definitions": { - "alationType": { - "description": "Service type.", - "type": "string", - "enum": ["Alation"], - "default": "Alation" - } - }, - "properties": { - "type": { - "description": "Service Type", - "$ref": "#/definitions/alationType", - "default": "Alation" - }, - "hostPort": { - "description": "Host and port of the Alation service.", - "title": "Host and Port", - "type": "string", - "format": "uri", - "expose": true - }, - "authType": { - "mask": true, - "title": "Authentication type for Alation", - "description": "Types of methods used to authenticate to the alation instance", - "oneOf": [ - { - "$ref": "../../../../security/credentials/basicAuth.json" - }, - { - "$ref": "../../../../security/credentials/apiAccessTokenAuth.json" - } - ] - }, - "projectName": { - "title": "Project Name", - "description": "Project name to create the refreshToken. Can be anything", - "type": "string", - "default": "AlationAPI" - }, - "paginationLimit": { - "title": "Pagination Limit", - "description": "Pagination limit used for Alation APIs pagination", - "type": "integer", - "default": 10 - }, - "includeUndeployedDatasources": { - "title": "Include Undeployed Datasources", - "description": "Specifies if undeployed datasources should be included while ingesting.", - "type": "boolean", - "default": false - }, - "includeHiddenDatasources": { - "title": "Include Hidden Datasources", - "description": "Specifies if hidden datasources should be included while ingesting.", - "type": "boolean", - "default": false - }, - "alationTagClassificationName": { - "title": "Alation Tags Classification Name", - "description": "Custom OpenMetadata Classification name for alation tags.", - "type": "string", - "default": "alationTags" - }, - "connectionOptions": { - "$ref": "../connectionBasicType.json#/definitions/connectionOptions" - }, - "connectionArguments": { - "$ref": "../connectionBasicType.json#/definitions/connectionArguments" - }, - "supportsMetadataExtraction": { - "$ref": "../connectionBasicType.json#/definitions/supportsMetadataExtraction" - } - }, - "required": ["hostPort", "authType"], - "additionalProperties": false - } - \ No newline at end of file + "$id": "https://open-metadata.org/schema/entity/services/connections/metadata/AlationConnection.json", + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "AlationConnection", + "description": "Alation Connection Config", + "type": "object", + "javaType": "org.openmetadata.schema.services.connections.metadata.AlationConnection", + "definitions": { + "alationType": { + "description": "Service type.", + "type": "string", + "enum": ["Alation"], + "default": "Alation" + } + }, + "properties": { + "type": { + "description": "Service Type", + "$ref": "#/definitions/alationType", + "default": "Alation" + }, + "hostPort": { + "description": "Host and port of the Alation service.", + "title": "Host and Port", + "type": "string", + "format": "uri", + "expose": true + }, + "authType": { + "mask": true, + "title": "Authentication type for Alation", + "description": "Types of methods used to authenticate to the alation instance", + "oneOf": [ + { + "$ref": "../../../../security/credentials/basicAuth.json" + }, + { + "$ref": "../../../../security/credentials/apiAccessTokenAuth.json" + } + ] + }, + "connection":{ + "title": "Alation Database Connection", + "description": "Choose between mysql and postgres connection for alation database", + "oneOf": [ + { + "$ref": "../database/postgresConnection.json" + }, + { + "$ref": "../database/mysqlConnection.json" + } + ] + }, + "projectName": { + "title": "Project Name", + "description": "Project name to create the refreshToken. Can be anything", + "type": "string", + "default": "AlationAPI" + }, + "paginationLimit": { + "title": "Pagination Limit", + "description": "Pagination limit used for Alation APIs pagination", + "type": "integer", + "default": 10 + }, + "includeUndeployedDatasources": { + "title": "Include Undeployed Datasources", + "description": "Specifies if undeployed datasources should be included while ingesting.", + "type": "boolean", + "default": false + }, + "includeHiddenDatasources": { + "title": "Include Hidden Datasources", + "description": "Specifies if hidden datasources should be included while ingesting.", + "type": "boolean", + "default": false + }, + "ingestDatasources": { + "title": "Ingest Datasources", + "description": "Specifies if Datasources are to be ingested while running the ingestion job.", + "type": "boolean", + "default": true + }, + "ingestUsersAndGroups": { + "title": "Ingest Users and Groups", + "description": "Specifies if Users and Groups are to be ingested while running the ingestion job.", + "type": "boolean", + "default": true + }, + "ingestDomains": { + "title": "Ingest Domains", + "description": "Specifies if Domains are to be ingested while running the ingestion job.", + "type": "boolean", + "default": true + }, + "ingestKnowledgeArticles": { + "title": "Ingest Knowledge Articles", + "description": "Specifies if Knowledge Articles are to be ingested while running the ingestion job.", + "type": "boolean", + "default": true + }, + "ingestDashboards": { + "title": "Ingest Dashboards", + "description": "Specifies if Dashboards are to be ingested while running the ingestion job.", + "type": "boolean", + "default": true + }, + "alationTagClassificationName": { + "title": "Alation Tags Classification Name", + "description": "Custom OpenMetadata Classification name for alation tags.", + "type": "string", + "default": "alationTags" + }, + "connectionOptions": { + "$ref": "../connectionBasicType.json#/definitions/connectionOptions" + }, + "connectionArguments": { + "$ref": "../connectionBasicType.json#/definitions/connectionArguments" + }, + "supportsMetadataExtraction": { + "$ref": "../connectionBasicType.json#/definitions/supportsMetadataExtraction" + } + }, + "required": ["hostPort", "authType"], + "additionalProperties": false +} diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/type.json b/openmetadata-spec/src/main/resources/json/schema/entity/type.json index f0e85aa8b3d1..adb0ce9f460c 100644 --- a/openmetadata-spec/src/main/resources/json/schema/entity/type.json +++ b/openmetadata-spec/src/main/resources/json/schema/entity/type.json @@ -25,26 +25,6 @@ "name": "Entity" } ] - }, - "customProperty": { - "description": "Type used for adding custom property to an entity to extend it.", - "type": "object", - "javaType": "org.openmetadata.schema.entity.type.CustomProperty", - "properties": { - "name": { - "description": "Name of the entity property. Note a property name must be unique for an entity. Property name must follow camelCase naming adopted by openMetadata - must start with lower case with no space, underscore, or dots.", - "$ref": "#/definitions/entityName" - }, - "description": { - "$ref": "../type/basic.json#/definitions/markdown" - }, - "propertyType": { - "description": "Reference to a property type. Only property types are allowed and entity types are not allowed as custom properties to extend an existing entity", - "$ref": "../type/entityReference.json" - } - }, - "required": ["name", "description", "propertyType"], - "additionalProperties": false } }, "properties": { @@ -84,7 +64,7 @@ "description": "Custom properties added to extend the entity. Only available for entity type", "type": "array", "items": { - "$ref": "#/definitions/customProperty" + "$ref": "../type/customProperty.json" } }, "version": { diff --git a/openmetadata-spec/src/main/resources/json/schema/events/eventSubscription.json b/openmetadata-spec/src/main/resources/json/schema/events/eventSubscription.json index 6e5dce12d148..dc33000d5482 100644 --- a/openmetadata-spec/src/main/resources/json/schema/events/eventSubscription.json +++ b/openmetadata-spec/src/main/resources/json/schema/events/eventSubscription.json @@ -86,6 +86,7 @@ "Admins", "Assignees", "Owners", + "Mentions", "Followers", "External" ] @@ -333,7 +334,7 @@ "pollInterval": { "description": "Poll Interval in seconds.", "type": "integer", - "default": 10 + "default": 60 }, "input": { "description": "Input for the Filters.", diff --git a/openmetadata-spec/src/main/resources/json/schema/metadataIngestion/dbtPipeline.json b/openmetadata-spec/src/main/resources/json/schema/metadataIngestion/dbtPipeline.json index eebfcb8fc8f8..985d16aab572 100644 --- a/openmetadata-spec/src/main/resources/json/schema/metadataIngestion/dbtPipeline.json +++ b/openmetadata-spec/src/main/resources/json/schema/metadataIngestion/dbtPipeline.json @@ -82,5 +82,6 @@ "title": "Database Filter Pattern" } }, - "additionalProperties": false + "additionalProperties": false, + "required": ["dbtConfigSource"] } diff --git a/openmetadata-spec/src/main/resources/json/schema/metadataIngestion/dbtconfig/dbtAzureConfig.json b/openmetadata-spec/src/main/resources/json/schema/metadataIngestion/dbtconfig/dbtAzureConfig.json index 60e9fcaf6da2..dfdbb0b37c20 100644 --- a/openmetadata-spec/src/main/resources/json/schema/metadataIngestion/dbtconfig/dbtAzureConfig.json +++ b/openmetadata-spec/src/main/resources/json/schema/metadataIngestion/dbtconfig/dbtAzureConfig.json @@ -6,6 +6,12 @@ "description": "DBT Catalog, Manifest and Run Results files in Azure bucket. We will search for catalog.json, manifest.json and run_results.json.", "javaType": "org.openmetadata.schema.metadataIngestion.dbtconfig.DbtAzureConfig", "properties": { + "dbtConfigType": { + "description": "dbt Configuration type", + "type": "string", + "enum": ["azure"], + "default": "azure" + }, "dbtSecurityConfig": { "title": "DBT Azure Security Config", "$ref": "../../security/credentials/azureCredentials.json" @@ -30,5 +36,5 @@ } }, "additionalProperties": false, - "required": ["dbtSecurityConfig"] + "required": ["dbtSecurityConfig", "dbtConfigType"] } diff --git a/openmetadata-spec/src/main/resources/json/schema/metadataIngestion/dbtconfig/dbtCloudConfig.json b/openmetadata-spec/src/main/resources/json/schema/metadataIngestion/dbtconfig/dbtCloudConfig.json index d335abcaf66e..2017dd68e68e 100644 --- a/openmetadata-spec/src/main/resources/json/schema/metadataIngestion/dbtconfig/dbtCloudConfig.json +++ b/openmetadata-spec/src/main/resources/json/schema/metadataIngestion/dbtconfig/dbtCloudConfig.json @@ -6,6 +6,12 @@ "javaType": "org.openmetadata.schema.metadataIngestion.dbtconfig.DbtCloudConfig", "type": "object", "properties": { + "dbtConfigType": { + "description": "dbt Configuration type", + "type": "string", + "enum": ["cloud"], + "default": "cloud" + }, "dbtCloudAuthToken": { "title": "dbt Cloud Authentication Token", "description": "dbt cloud account authentication token", @@ -36,5 +42,5 @@ } }, "additionalProperties": false, - "required": ["dbtCloudAuthToken", "dbtCloudAccountId", "dbtCloudUrl"] + "required": ["dbtCloudAuthToken", "dbtCloudAccountId", "dbtCloudUrl", "dbtConfigType"] } \ No newline at end of file diff --git a/openmetadata-spec/src/main/resources/json/schema/metadataIngestion/dbtconfig/dbtGCSConfig.json b/openmetadata-spec/src/main/resources/json/schema/metadataIngestion/dbtconfig/dbtGCSConfig.json index 417a977e6cb8..f9881afaa042 100644 --- a/openmetadata-spec/src/main/resources/json/schema/metadataIngestion/dbtconfig/dbtGCSConfig.json +++ b/openmetadata-spec/src/main/resources/json/schema/metadataIngestion/dbtconfig/dbtGCSConfig.json @@ -6,6 +6,12 @@ "description": "DBT Catalog, Manifest and Run Results files in GCS storage. We will search for catalog.json, manifest.json and run_results.json.", "javaType": "org.openmetadata.schema.metadataIngestion.dbtconfig.DbtGCSConfig", "properties": { + "dbtConfigType": { + "description": "dbt Configuration type", + "type": "string", + "enum": ["gcs"], + "default": "gcs" + }, "dbtSecurityConfig": { "title": "DBT GCS Security Config", "$ref": "../../security/credentials/gcpCredentials.json" @@ -30,5 +36,5 @@ } }, "additionalProperties": false, - "required": ["dbtSecurityConfig"] + "required": ["dbtSecurityConfig", "dbtConfigType"] } diff --git a/openmetadata-spec/src/main/resources/json/schema/metadataIngestion/dbtconfig/dbtHttpConfig.json b/openmetadata-spec/src/main/resources/json/schema/metadataIngestion/dbtconfig/dbtHttpConfig.json index 11cf830c5016..179573b67ecc 100644 --- a/openmetadata-spec/src/main/resources/json/schema/metadataIngestion/dbtconfig/dbtHttpConfig.json +++ b/openmetadata-spec/src/main/resources/json/schema/metadataIngestion/dbtconfig/dbtHttpConfig.json @@ -6,6 +6,12 @@ "javaType": "org.openmetadata.schema.metadataIngestion.dbtconfig.DbtHttpConfig", "type": "object", "properties": { + "dbtConfigType": { + "description": "dbt Configuration type", + "type": "string", + "enum": ["http"], + "default": "http" + }, "dbtCatalogHttpPath": { "title": "DBT Catalog HTTP File Path", "description": "DBT catalog http file path to extract dbt models with their column schemas.", @@ -23,5 +29,5 @@ } }, "additionalProperties": false, - "required": ["dbtManifestHttpPath"] + "required": ["dbtManifestHttpPath", "dbtConfigType"] } \ No newline at end of file diff --git a/openmetadata-spec/src/main/resources/json/schema/metadataIngestion/dbtconfig/dbtLocalConfig.json b/openmetadata-spec/src/main/resources/json/schema/metadataIngestion/dbtconfig/dbtLocalConfig.json index 80d92eb199e3..171b2a675f8e 100644 --- a/openmetadata-spec/src/main/resources/json/schema/metadataIngestion/dbtconfig/dbtLocalConfig.json +++ b/openmetadata-spec/src/main/resources/json/schema/metadataIngestion/dbtconfig/dbtLocalConfig.json @@ -6,6 +6,12 @@ "javaType": "org.openmetadata.schema.metadataIngestion.dbtconfig.DbtLocalConfig", "type": "object", "properties": { + "dbtConfigType": { + "description": "dbt Configuration type", + "type": "string", + "enum": ["local"], + "default": "local" + }, "dbtCatalogFilePath": { "title": "DBT Catalog File Path", "description": "DBT catalog file path to extract dbt models with their column schemas.", @@ -23,5 +29,5 @@ } }, "additionalProperties": false, - "required": ["dbtManifestFilePath"] + "required": ["dbtManifestFilePath", "dbtConfigType"] } \ No newline at end of file diff --git a/openmetadata-spec/src/main/resources/json/schema/metadataIngestion/dbtconfig/dbtS3Config.json b/openmetadata-spec/src/main/resources/json/schema/metadataIngestion/dbtconfig/dbtS3Config.json index 8844e8eade8a..47fa0be22381 100644 --- a/openmetadata-spec/src/main/resources/json/schema/metadataIngestion/dbtconfig/dbtS3Config.json +++ b/openmetadata-spec/src/main/resources/json/schema/metadataIngestion/dbtconfig/dbtS3Config.json @@ -6,6 +6,12 @@ "javaType": "org.openmetadata.schema.metadataIngestion.dbtconfig.DbtS3Config", "type": "object", "properties": { + "dbtConfigType": { + "description": "dbt Configuration type", + "type": "string", + "enum": ["s3"], + "default": "s3" + }, "dbtSecurityConfig": { "title": "DBT S3 Security Config", "$ref": "../../security/credentials/awsCredentials.json" @@ -30,5 +36,5 @@ } }, "additionalProperties": false, - "required": ["dbtSecurityConfig"] + "required": ["dbtSecurityConfig", "dbtConfigType"] } diff --git a/openmetadata-spec/src/main/resources/json/schema/security/client/oidcClientConfig.json b/openmetadata-spec/src/main/resources/json/schema/security/client/oidcClientConfig.json new file mode 100644 index 000000000000..2e45e5ea3a09 --- /dev/null +++ b/openmetadata-spec/src/main/resources/json/schema/security/client/oidcClientConfig.json @@ -0,0 +1,78 @@ +{ + "$id": "https://open-metadata.org/schema/security/client/oidcClientConfig.json", + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "OidcClientConfig", + "description": "Oidc client security configs.", + "type": "object", + "javaType": "org.openmetadata.schema.security.client.OidcClientConfig", + "properties": { + "type": { + "description": "IDP type (Example Google,Azure).", + "type": "string" + }, + "id": { + "description": "Client ID.", + "type": "string" + }, + "secret": { + "description": "Client Secret.", + "type": "string" + }, + "scope": { + "description": "Oidc Request Scopes.", + "type": "string", + "default": "openid email profile" + }, + "discoveryUri": { + "description": "Discovery Uri for the Client.", + "type": "string" + }, + "useNonce": { + "description": "Use Nonce.", + "type": "string", + "default": true + }, + "preferredJwsAlgorithm": { + "description": "Preferred Jws Algorithm.", + "type": "string", + "default": "RS256" + }, + "responseType": { + "description": "Auth0 Client Secret Key.", + "type": "string", + "default": "code" + }, + "disablePkce": { + "description": "Disable PKCE.", + "type": "boolean", + "default": true + }, + "maxClockSkew": { + "description": "Max Clock Skew", + "type": "string" + }, + "clientAuthenticationMethod": { + "description": "Client Authentication Method.", + "type": "string", + "enum": ["client_secret_basic", "client_secret_post", "client_secret_jwt", "private_key_jwt"] + }, + "customParams": { + "description": "Custom Params.", + "existingJavaType" : "java.util.Map", + "type" : "object" + }, + "tenant": { + "description": "Tenant in case of Azure.", + "type": "string" + }, + "serverUrl": { + "description": "Server Url.", + "type": "string" + }, + "callbackUrl": { + "description": "Callback Url.", + "type": "string" + } + }, + "additionalProperties": false +} diff --git a/openmetadata-spec/src/main/resources/json/schema/security/credentials/azureCredentials.json b/openmetadata-spec/src/main/resources/json/schema/security/credentials/azureCredentials.json index 330f178d565f..7db1e585769e 100644 --- a/openmetadata-spec/src/main/resources/json/schema/security/credentials/azureCredentials.json +++ b/openmetadata-spec/src/main/resources/json/schema/security/credentials/azureCredentials.json @@ -31,7 +31,12 @@ "title": "Key Vault Name", "description": "Key Vault Name", "type": "string" + }, + "scopes": { + "title": "Scopes", + "description": "Scopes to get access token, for e.g. api://6dfX33ab-XXXX-49df-XXXX-3459eX817d3e/.default", + "type": "string" } }, "additionalProperties": false -} +} \ No newline at end of file diff --git a/openmetadata-spec/src/main/resources/json/schema/security/credentials/gcpCredentials.json b/openmetadata-spec/src/main/resources/json/schema/security/credentials/gcpCredentials.json index e2a619484e8d..abf0ebbdcc94 100644 --- a/openmetadata-spec/src/main/resources/json/schema/security/credentials/gcpCredentials.json +++ b/openmetadata-spec/src/main/resources/json/schema/security/credentials/gcpCredentials.json @@ -40,6 +40,9 @@ }, { "$ref": "#/definitions/gcpCredentialsPath" + }, + { + "$ref": "gcpExternalAccount.json" } ] }, @@ -50,5 +53,7 @@ } }, "additionalProperties": false, - "required": ["gcpConfig"] -} + "required": [ + "gcpConfig" + ] +} \ No newline at end of file diff --git a/openmetadata-spec/src/main/resources/json/schema/security/credentials/gcpExternalAccount.json b/openmetadata-spec/src/main/resources/json/schema/security/credentials/gcpExternalAccount.json new file mode 100644 index 000000000000..37ac0397a1f2 --- /dev/null +++ b/openmetadata-spec/src/main/resources/json/schema/security/credentials/gcpExternalAccount.json @@ -0,0 +1,40 @@ +{ + "$id": "https:./open-metadata.org/schema/security/credentials/gcpExternalAccount.json", + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "javaType": "org.openmetadata.schema.security.credentials.GCPExternalAccount", + "title": "GCP External Account", + "description": "Pass the raw credential values provided by GCP", + "properties": { + "externalType": { + "title": "Credentials Type", + "description": "Google Cloud Platform account type.", + "type": "string", + "default": "external_account" + }, + "audience": { + "title": "Audience", + "description": "Google Security Token Service audience which contains the resource name for the workload identity pool and the provider identifier in that pool.", + "type": "string" + }, + "subjectTokenType": { + "title": "Subject Token Type", + "description": "Google Security Token Service subject token type based on the OAuth 2.0 token exchange spec.", + "type": "string" + }, + "tokenURL": { + "title": "Token URL", + "description": "Google Security Token Service token exchange endpoint.", + "type": "string" + }, + "credentialSource": { + "title": "Credential Source", + "description": "This object defines the mechanism used to retrieve the external credential from the local environment so that it can be exchanged for a GCP access token via the STS endpoint", + "type": "object", + "javaType": "org.openmetadata.schema.security.credentials.credentialSource", + "additionalProperties": { + "type": "string" + } + } + } +} \ No newline at end of file diff --git a/openmetadata-spec/src/main/resources/json/schema/security/credentials/gcpValues.json b/openmetadata-spec/src/main/resources/json/schema/security/credentials/gcpValues.json index fb5a5811fc0f..87f393d12a9c 100644 --- a/openmetadata-spec/src/main/resources/json/schema/security/credentials/gcpValues.json +++ b/openmetadata-spec/src/main/resources/json/schema/security/credentials/gcpValues.json @@ -22,7 +22,8 @@ "type": { "title": "Credentials Type", "description": "Google Cloud Platform account type.", - "type": "string" + "type": "string", + "default": "service_account" }, "projectId": { "title": "Project ID", @@ -86,4 +87,4 @@ } }, "additionalProperties": false -} +} \ No newline at end of file diff --git a/openmetadata-spec/src/main/resources/json/schema/system/validationResponse.json b/openmetadata-spec/src/main/resources/json/schema/system/validationResponse.json new file mode 100644 index 000000000000..d3749bfc65ca --- /dev/null +++ b/openmetadata-spec/src/main/resources/json/schema/system/validationResponse.json @@ -0,0 +1,53 @@ +{ + "$id": "https://open-metadata.org/schema/system/validationResponse.json", + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "SystemValidationResponse", + "description": "Define the system validation response", + "type": "object", + "javaType": "org.openmetadata.schema.system.ValidationResponse", + "definitions": { + "stepValidation": { + "javaType": "org.openmetadata.schema.system.StepValidation", + "type": "object", + "properties": { + "description": { + "description": "Validation description. What is being tested?", + "type": "string" + }, + "passed": { + "description": "Did the step validation successfully?", + "type": "boolean" + }, + "message": { + "description": "Results or exceptions to be shared after running the test.", + "type": "string", + "default": null + } + }, + "additionalProperties": false + } + }, + "properties": { + "database": { + "description": "Database connectivity check", + "$ref": "#/definitions/stepValidation" + }, + "searchInstance": { + "description": "Search instance connectivity check", + "$ref": "#/definitions/stepValidation" + }, + "pipelineServiceClient": { + "description": "Pipeline Service Client connectivity check", + "$ref": "#/definitions/stepValidation" + }, + "jwks": { + "description": "JWKs validation", + "$ref": "#/definitions/stepValidation" + }, + "migrations": { + "description": "List migration results", + "$ref": "#/definitions/stepValidation" + } + }, + "additionalProperties": false +} \ No newline at end of file diff --git a/openmetadata-spec/src/main/resources/json/schema/type/basic.json b/openmetadata-spec/src/main/resources/json/schema/type/basic.json index 22eddb637518..45a4ee7a894e 100644 --- a/openmetadata-spec/src/main/resources/json/schema/type/basic.json +++ b/openmetadata-spec/src/main/resources/json/schema/type/basic.json @@ -85,6 +85,14 @@ "type": "string", "format": "time" }, + "enum": { + "$comment" : "@om-field-type", + "description": "List of values in Enum.", + "type": "array", + "items": { + "type": "string" + } + }, "timezone": { "description": "Timezone of the user in the format `America/Los_Angeles`, `Brazil/East`, etc.", "type": "string", diff --git a/openmetadata-spec/src/main/resources/json/schema/type/customProperties/enumConfig.json b/openmetadata-spec/src/main/resources/json/schema/type/customProperties/enumConfig.json new file mode 100644 index 000000000000..459dbac8aab6 --- /dev/null +++ b/openmetadata-spec/src/main/resources/json/schema/type/customProperties/enumConfig.json @@ -0,0 +1,21 @@ +{ + "$id": "https://open-metadata.org/schema/type/customPropertyEnumConfig.json", + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "EnumConfig", + "type": "object", + "javaType": "org.openmetadata.schema.type.customproperties.EnumConfig", + "description": "Applies to Enum type, this config is used to define list of enum values", + "properties": { + "multiSelect": { + "type": "boolean", + "default": false + }, + "values": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "additionalProperties": false + } \ No newline at end of file diff --git a/openmetadata-spec/src/main/resources/json/schema/type/customProperty.json b/openmetadata-spec/src/main/resources/json/schema/type/customProperty.json new file mode 100644 index 000000000000..f88f5cd512b6 --- /dev/null +++ b/openmetadata-spec/src/main/resources/json/schema/type/customProperty.json @@ -0,0 +1,61 @@ +{ + "$id": "https://open-metadata.org/schema/type/customProperty.json", + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "CustomProperty", + "description": "This schema defines the custom property to an entity to extend it.", + "type": "object", + "javaType": "org.openmetadata.schema.entity.type.CustomProperty", + "definitions": { + "format": { + "description": "Applies to date interval, date, time format.", + "type": "string" + }, + "entityTypes": { + "description": "Applies to Entity References. Entity Types can be used to restrict what type of entities can be configured for a entity reference.", + "type": "string" + }, + "customPropertyConfig": { + "type": "object", + "javaType": "org.openmetadata.schema.type.CustomPropertyConfig", + "title": "CustomPropertyConfig", + "description": "Config to define constraints around CustomProperty", + "properties": { + "config": { + "oneOf": [ + { + "$ref": "../type/customProperties/enumConfig.json" + }, + { + "$ref": "#/definitions/format" + }, + { + "$ref": "#/definitions/entityTypes" + } + ] + } + }, + "additionalProperties": false + }, + "propertyType": { + "description": "Reference to a property type. Only property types are allowed and entity types are not allowed as custom properties to extend an existing entity", + "$ref": "../type/entityReference.json" + } + }, + "properties": { + "name": { + "description": "Name of the entity property. Note a property name must be unique for an entity. Property name must follow camelCase naming adopted by openMetadata - must start with lower case with no space, underscore, or dots.", + "$ref": "../type/basic.json#/definitions/entityName" + }, + "description": { + "$ref": "../type/basic.json#/definitions/markdown" + }, + "propertyType": { + "$ref": "#/definitions/propertyType" + }, + "customPropertyConfig": { + "$ref": "#/definitions/customPropertyConfig" + } + }, + "required": ["name", "description", "propertyType"], + "additionalProperties": false +} \ No newline at end of file diff --git a/openmetadata-ui/pom.xml b/openmetadata-ui/pom.xml index 1f298e55b7fb..efa68db3a622 100644 --- a/openmetadata-ui/pom.xml +++ b/openmetadata-ui/pom.xml @@ -5,7 +5,7 @@ platform org.open-metadata - 1.4.0-SNAPSHOT + 1.3.4 4.0.0 diff --git a/openmetadata-ui/src/main/resources/ui/cypress/common/AlertUtils.js b/openmetadata-ui/src/main/resources/ui/cypress/common/AlertUtils.ts similarity index 92% rename from openmetadata-ui/src/main/resources/ui/cypress/common/AlertUtils.js rename to openmetadata-ui/src/main/resources/ui/cypress/common/AlertUtils.ts index ef42d8c9dd5c..03153333ff2d 100644 --- a/openmetadata-ui/src/main/resources/ui/cypress/common/AlertUtils.js +++ b/openmetadata-ui/src/main/resources/ui/cypress/common/AlertUtils.ts @@ -19,10 +19,7 @@ import { verifyResponseStatusCode, } from './common'; -// eslint-disable-next-line spaced-comment -/// - -export const deleteAlertSteps = (name) => { +export const deleteAlertSteps = (name: string) => { cy.get('table').should('contain', name).click(); cy.get(`[data-testid="alert-delete-${name}"]`).click(); cy.get('.ant-modal-header').should( @@ -38,8 +35,8 @@ export const deleteAlertSteps = (name) => { }; export const addOwnerFilter = ( - filterNumber, - ownerName, + filterNumber: number, + ownerName: string, exclude = false, selectId = 'Owner' ) => { @@ -67,8 +64,8 @@ export const addOwnerFilter = ( }; export const addEntityFQNFilter = ( - filterNumber, - entityFQN, + filterNumber: number, + entityFQN: string, exclude = false, selectId = 'Entity FQN' ) => { @@ -96,8 +93,8 @@ export const addEntityFQNFilter = ( }; export const addEventTypeFilter = ( - filterNumber, - eventType, + filterNumber: number, + eventType: string, exclude = false ) => { // Select event type filter @@ -127,18 +124,17 @@ export const addEventTypeFilter = ( } }; -export const addUpdaterNameFilter = ( - filterNumber, - updaterName, +export const addFilterWithUsersListInput = ( + filterTestId: string, + filterNumber: number, + updaterName: string, exclude = false ) => { // Select updater name filter cy.get(`[data-testid="filter-select-${filterNumber}"]`).click({ waitForAnimations: true, }); - cy.get('[data-testid="Updater Name-filter-option"]') - .filter(':visible') - .click(); + cy.get(`[data-testid="${filterTestId}"]`).filter(':visible').click(); // Search and select user interceptURL('GET', `/api/v1/search/query?q=*`, 'getSearchResult'); @@ -158,7 +154,11 @@ export const addUpdaterNameFilter = ( } }; -export const addDomainFilter = (filterNumber, domainName, exclude = false) => { +export const addDomainFilter = ( + filterNumber: number, + domainName: string, + exclude = false +) => { // Select domain filter cy.get(`[data-testid="filter-select-${filterNumber}"]`).click({ waitForAnimations: true, @@ -180,7 +180,7 @@ export const addDomainFilter = (filterNumber, domainName, exclude = false) => { } }; -export const addGMEFilter = (filterNumber, exclude = false) => { +export const addGMEFilter = (filterNumber: number, exclude = false) => { // Select general metadata events filter cy.get(`[data-testid="filter-select-${filterNumber}"]`).click({ waitForAnimations: true, @@ -198,11 +198,11 @@ export const addGMEFilter = (filterNumber, exclude = false) => { }; export const addInternalDestination = ( - destinationNumber, - category, - type, - typeId, - searchText + destinationNumber: number, + category: string, + type: string, + typeId?: string, + searchText?: string ) => { // Select destination category cy.get(`[data-testid="destination-category-select-${destinationNumber}"]`) @@ -222,11 +222,12 @@ export const addInternalDestination = ( .scrollIntoView() .click(); interceptURL('GET', `/api/v1/search/query?q=*`, 'getSearchResult'); + cy.get( + `[data-testid="team-user-select-dropdown-${destinationNumber}"]` + ).should('be.visible'); cy.get( `[data-testid="team-user-select-dropdown-${destinationNumber}"] [data-testid="search-input"]` - ) - .click() - .type(searchText); + ).type(searchText); // Added wait for debounce functionality cy.wait(600); verifyResponseStatusCode('@getSearchResult', 200); @@ -259,7 +260,11 @@ export const addInternalDestination = ( ); }; -export const addExternalDestination = (destinationNumber, category, input) => { +export const addExternalDestination = ( + destinationNumber: number, + category: string, + input: string +) => { // Select destination category cy.get(`[data-testid="destination-category-select-${destinationNumber}"]`) .scrollIntoView() diff --git a/openmetadata-ui/src/main/resources/ui/cypress/common/CustomizeLandingPageUtils.js b/openmetadata-ui/src/main/resources/ui/cypress/common/CustomizeLandingPageUtils.ts similarity index 98% rename from openmetadata-ui/src/main/resources/ui/cypress/common/CustomizeLandingPageUtils.js rename to openmetadata-ui/src/main/resources/ui/cypress/common/CustomizeLandingPageUtils.ts index 5225d8ec6afb..c5bf3b33d5da 100644 --- a/openmetadata-ui/src/main/resources/ui/cypress/common/CustomizeLandingPageUtils.js +++ b/openmetadata-ui/src/main/resources/ui/cypress/common/CustomizeLandingPageUtils.ts @@ -10,8 +10,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -// eslint-disable-next-line spaced-comment -/// import { GlobalSettingOptions } from '../constants/settings.constant'; import { diff --git a/openmetadata-ui/src/main/resources/ui/cypress/common/DataInsightUtils.js b/openmetadata-ui/src/main/resources/ui/cypress/common/DataInsightUtils.ts similarity index 100% rename from openmetadata-ui/src/main/resources/ui/cypress/common/DataInsightUtils.js rename to openmetadata-ui/src/main/resources/ui/cypress/common/DataInsightUtils.ts diff --git a/openmetadata-ui/src/main/resources/ui/cypress/common/DomainUtils.js b/openmetadata-ui/src/main/resources/ui/cypress/common/DomainUtils.ts similarity index 100% rename from openmetadata-ui/src/main/resources/ui/cypress/common/DomainUtils.js rename to openmetadata-ui/src/main/resources/ui/cypress/common/DomainUtils.ts diff --git a/openmetadata-ui/src/main/resources/ui/cypress/common/Entities/ContainerClass.ts b/openmetadata-ui/src/main/resources/ui/cypress/common/Entities/ContainerClass.ts index 5030e19723d4..6805fd5427ff 100644 --- a/openmetadata-ui/src/main/resources/ui/cypress/common/Entities/ContainerClass.ts +++ b/openmetadata-ui/src/main/resources/ui/cypress/common/Entities/ContainerClass.ts @@ -14,6 +14,7 @@ import { EntityType } from '../../constants/Entity.interface'; import { STORAGE_SERVICE } from '../../constants/EntityConstant'; import { createSingleLevelEntity } from '../EntityUtils'; import { visitEntityDetailsPage } from '../Utils/Entity'; +import { getToken } from '../Utils/LocalStorage'; import EntityClass from './EntityClass'; class ContainerClass extends EntityClass { @@ -45,7 +46,7 @@ class ContainerClass extends EntityClass { // Handle creation here cy.getAllLocalStorage().then((data) => { - const token = Object.values(data)[0].oidcIdToken as string; + const token = getToken(data); createSingleLevelEntity({ token, diff --git a/openmetadata-ui/src/main/resources/ui/cypress/common/Entities/DashboardClass.ts b/openmetadata-ui/src/main/resources/ui/cypress/common/Entities/DashboardClass.ts index d8122fafcbb8..19492b3257be 100644 --- a/openmetadata-ui/src/main/resources/ui/cypress/common/Entities/DashboardClass.ts +++ b/openmetadata-ui/src/main/resources/ui/cypress/common/Entities/DashboardClass.ts @@ -17,6 +17,7 @@ import { } from '../../constants/EntityConstant'; import { createSingleLevelEntity } from '../EntityUtils'; import { deleteEntityViaREST, visitEntityDetailsPage } from '../Utils/Entity'; +import { getToken } from '../Utils/LocalStorage'; import EntityClass from './EntityClass'; class DashboardClass extends EntityClass { @@ -44,7 +45,7 @@ class DashboardClass extends EntityClass { // Handle creation here cy.getAllLocalStorage().then((data) => { - const token = Object.values(data)[0].oidcIdToken as string; + const token = getToken(data); createSingleLevelEntity({ token, @@ -64,7 +65,7 @@ class DashboardClass extends EntityClass { override cleanup() { super.cleanup(); cy.getAllLocalStorage().then((data) => { - const token = Object.values(data)[0].oidcIdToken as string; + const token = getToken(data); deleteEntityViaREST({ token, endPoint: EntityType.DashboardService, diff --git a/openmetadata-ui/src/main/resources/ui/cypress/common/Entities/DashboardServiceClass.ts b/openmetadata-ui/src/main/resources/ui/cypress/common/Entities/DashboardServiceClass.ts index 56240699cafa..00f81d685201 100644 --- a/openmetadata-ui/src/main/resources/ui/cypress/common/Entities/DashboardServiceClass.ts +++ b/openmetadata-ui/src/main/resources/ui/cypress/common/Entities/DashboardServiceClass.ts @@ -16,6 +16,7 @@ import { visitServiceDetailsPage } from '../../common/serviceUtils'; import { SERVICE_TYPE } from '../../constants/constants'; import { EntityType } from '../../constants/Entity.interface'; import { DASHBOARD_SERVICE } from '../../constants/EntityConstant'; +import { getToken } from '../Utils/LocalStorage'; import EntityClass from './EntityClass'; class DashboardServiceClass extends EntityClass { @@ -45,7 +46,7 @@ class DashboardServiceClass extends EntityClass { // Handle creation here cy.getAllLocalStorage().then((data) => { - const token = Object.values(data)[0].oidcIdToken as string; + const token = getToken(data); createSingleLevelEntity({ ...DASHBOARD_SERVICE, diff --git a/openmetadata-ui/src/main/resources/ui/cypress/common/Entities/DataModelClass.ts b/openmetadata-ui/src/main/resources/ui/cypress/common/Entities/DataModelClass.ts index 9097e4d3c1c6..3c37e90d6a9f 100644 --- a/openmetadata-ui/src/main/resources/ui/cypress/common/Entities/DataModelClass.ts +++ b/openmetadata-ui/src/main/resources/ui/cypress/common/Entities/DataModelClass.ts @@ -20,6 +20,7 @@ import { DASHBOARD_DATA_MODEL_DETAILS, DASHBOARD_SERVICE_DETAILS, } from '../../constants/EntityConstant'; +import { getToken } from '../Utils/LocalStorage'; import EntityClass from './EntityClass'; class DashboardDataModelClass extends EntityClass { @@ -51,7 +52,7 @@ class DashboardDataModelClass extends EntityClass { // Handle creation here cy.getAllLocalStorage().then((data) => { - const token = Object.values(data)[0].oidcIdToken as string; + const token = getToken(data); createEntityViaREST({ token, @@ -75,7 +76,7 @@ class DashboardDataModelClass extends EntityClass { override cleanup() { super.cleanup(); cy.getAllLocalStorage().then((data) => { - const token = Object.values(data)[0].oidcIdToken as string; + const token = getToken(data); deleteEntityViaREST({ token, endPoint: EntityType.DashboardService, diff --git a/openmetadata-ui/src/main/resources/ui/cypress/common/Entities/DatabaseClass.ts b/openmetadata-ui/src/main/resources/ui/cypress/common/Entities/DatabaseClass.ts index 88b788fbb697..30989a32927b 100644 --- a/openmetadata-ui/src/main/resources/ui/cypress/common/Entities/DatabaseClass.ts +++ b/openmetadata-ui/src/main/resources/ui/cypress/common/Entities/DatabaseClass.ts @@ -19,6 +19,7 @@ import { import { EntityType } from '../../constants/Entity.interface'; import { DATABASE_SERVICE } from '../../constants/EntityConstant'; import { GlobalSettingOptions } from '../../constants/settings.constant'; +import { getToken } from '../Utils/LocalStorage'; import EntityClass from './EntityClass'; class DatabaseClass extends EntityClass { @@ -60,7 +61,7 @@ class DatabaseClass extends EntityClass { // Handle creation here cy.getAllLocalStorage().then((data) => { - const token = Object.values(data)[0].oidcIdToken as string; + const token = getToken(data); createEntityTableViaREST({ token, @@ -76,7 +77,7 @@ class DatabaseClass extends EntityClass { override cleanup() { super.cleanup(); cy.getAllLocalStorage().then((data) => { - const token = Object.values(data)[0].oidcIdToken as string; + const token = getToken(data); deleteEntityViaREST({ token, endPoint: EntityType.DatabaseService, diff --git a/openmetadata-ui/src/main/resources/ui/cypress/common/Entities/DatabaseSchemaClass.ts b/openmetadata-ui/src/main/resources/ui/cypress/common/Entities/DatabaseSchemaClass.ts index a28c99a09b8b..62a2a0a4a9cc 100644 --- a/openmetadata-ui/src/main/resources/ui/cypress/common/Entities/DatabaseSchemaClass.ts +++ b/openmetadata-ui/src/main/resources/ui/cypress/common/Entities/DatabaseSchemaClass.ts @@ -19,6 +19,7 @@ import { import { EntityType } from '../../constants/Entity.interface'; import { DATABASE_SERVICE } from '../../constants/EntityConstant'; import { GlobalSettingOptions } from '../../constants/settings.constant'; +import { getToken } from '../Utils/LocalStorage'; import EntityClass from './EntityClass'; class DatabaseSchemaClass extends EntityClass { @@ -72,7 +73,7 @@ class DatabaseSchemaClass extends EntityClass { // Handle creation here cy.getAllLocalStorage().then((data) => { - const token = Object.values(data)[0].oidcIdToken as string; + const token = getToken(data); createEntityTableViaREST({ token, @@ -90,7 +91,7 @@ class DatabaseSchemaClass extends EntityClass { override cleanup() { super.cleanup(); cy.getAllLocalStorage().then((data) => { - const token = Object.values(data)[0].oidcIdToken as string; + const token = getToken(data); deleteEntityViaREST({ token, endPoint: EntityType.DatabaseService, diff --git a/openmetadata-ui/src/main/resources/ui/cypress/common/Entities/DatabaseServiceClass.ts b/openmetadata-ui/src/main/resources/ui/cypress/common/Entities/DatabaseServiceClass.ts index dac31a483e08..06fc8deb209b 100644 --- a/openmetadata-ui/src/main/resources/ui/cypress/common/Entities/DatabaseServiceClass.ts +++ b/openmetadata-ui/src/main/resources/ui/cypress/common/Entities/DatabaseServiceClass.ts @@ -16,6 +16,7 @@ import { createEntityTableViaREST } from '../../common/Utils/Entity'; import { SERVICE_TYPE } from '../../constants/constants'; import { EntityType } from '../../constants/Entity.interface'; import { DATABASE_SERVICE } from '../../constants/EntityConstant'; +import { getToken } from '../Utils/LocalStorage'; import EntityClass from './EntityClass'; class DatabaseServiceClass extends EntityClass { @@ -45,7 +46,7 @@ class DatabaseServiceClass extends EntityClass { // Handle creation here cy.getAllLocalStorage().then((data) => { - const token = Object.values(data)[0].oidcIdToken as string; + const token = getToken(data); createEntityTableViaREST({ ...DATABASE_SERVICE, diff --git a/openmetadata-ui/src/main/resources/ui/cypress/common/Entities/EntityClass.ts b/openmetadata-ui/src/main/resources/ui/cypress/common/Entities/EntityClass.ts index 241d71380fcf..b1d64e560db8 100644 --- a/openmetadata-ui/src/main/resources/ui/cypress/common/Entities/EntityClass.ts +++ b/openmetadata-ui/src/main/resources/ui/cypress/common/Entities/EntityClass.ts @@ -46,6 +46,7 @@ import { removeGlossaryTerm, udpateGlossaryTerm, } from '../Utils/Glossary'; +import { getToken } from '../Utils/LocalStorage'; import { addOwner, addTeamAsOwner, @@ -103,7 +104,7 @@ const glossaryDetails2 = { }; const glossaryTermDetails1 = { - name: 'CypressBankNumber', + name: `CypressBankNumber-${uuid()}`, displayName: 'Cypress BankNumber', description: 'A bank account number.', reviewers: [], @@ -189,7 +190,7 @@ class EntityClass { async setToken() { await new Promise((res) => cy.getAllLocalStorage().then((data) => { - const token = Object.values(data)[0].oidcIdToken; + const token = getToken(data); this.token = token; res(); @@ -222,7 +223,7 @@ class EntityClass { static preRequisitesForTests() { cy.getAllLocalStorage().then((data) => { - const token = Object.values(data)[0].oidcIdToken; + const token = getToken(data); // assign DevOps team to user @@ -296,7 +297,7 @@ class EntityClass { static postRequisitesForTests() { cy.getAllLocalStorage().then((data) => { - const token = Object.values(data)[0].oidcIdToken; + const token = getToken(data); // Remove devops as team // cy.get('[data-testid="dropdown-profile"]').click(); @@ -368,6 +369,10 @@ class EntityClass { addDomainToEntity(domainDetails1.displayName); } + validateDomainVersionForEntity() { + // override for entity + } + updateDomain() { addDomainToEntity(domainDetails2.displayName); } @@ -426,12 +431,14 @@ class EntityClass { assignGlossary() { assignGlossaryTerm( `${glossaryDetails1.name}.${glossaryTermDetails1.name}`, + glossaryTermDetails1.name, this.endPoint ); } updateGlossary() { udpateGlossaryTerm( `${glossaryDetails2.name}.${glossaryTermDetails2.name}`, + glossaryTermDetails2.name, this.endPoint ); } diff --git a/openmetadata-ui/src/main/resources/ui/cypress/common/Entities/MessagingServiceClass.ts b/openmetadata-ui/src/main/resources/ui/cypress/common/Entities/MessagingServiceClass.ts index deb615914cb7..68adf046d1b3 100644 --- a/openmetadata-ui/src/main/resources/ui/cypress/common/Entities/MessagingServiceClass.ts +++ b/openmetadata-ui/src/main/resources/ui/cypress/common/Entities/MessagingServiceClass.ts @@ -16,6 +16,7 @@ import { EntityType } from '../../constants/Entity.interface'; import { MESSAGING_SERVICE } from '../../constants/EntityConstant'; import { createSingleLevelEntity } from '../EntityUtils'; import { visitServiceDetailsPage } from '../serviceUtils'; +import { getToken } from '../Utils/LocalStorage'; import EntityClass from './EntityClass'; class MessagingServiceClass extends EntityClass { @@ -49,7 +50,7 @@ class MessagingServiceClass extends EntityClass { // Handle creation here cy.getAllLocalStorage().then((data) => { - const token = Object.values(data)[0].oidcIdToken as string; + const token = getToken(data); createSingleLevelEntity({ ...MESSAGING_SERVICE, diff --git a/openmetadata-ui/src/main/resources/ui/cypress/common/Entities/MlModelClass.ts b/openmetadata-ui/src/main/resources/ui/cypress/common/Entities/MlModelClass.ts index 11d83722e4ef..e2a00900006f 100644 --- a/openmetadata-ui/src/main/resources/ui/cypress/common/Entities/MlModelClass.ts +++ b/openmetadata-ui/src/main/resources/ui/cypress/common/Entities/MlModelClass.ts @@ -14,6 +14,7 @@ import { EntityType } from '../../constants/Entity.interface'; import { ML_MODEL_SERVICE } from '../../constants/EntityConstant'; import { createSingleLevelEntity } from '../EntityUtils'; import { visitEntityDetailsPage } from '../Utils/Entity'; +import { getToken } from '../Utils/LocalStorage'; import EntityClass from './EntityClass'; class MlModelClass extends EntityClass { @@ -41,7 +42,7 @@ class MlModelClass extends EntityClass { // Handle creation here cy.getAllLocalStorage().then((data) => { - const token = Object.values(data)[0].oidcIdToken as string; + const token = getToken(data); createSingleLevelEntity({ token, diff --git a/openmetadata-ui/src/main/resources/ui/cypress/common/Entities/MlModelServiceClass.ts b/openmetadata-ui/src/main/resources/ui/cypress/common/Entities/MlModelServiceClass.ts index 48e6c96af179..a2fa2654f91f 100644 --- a/openmetadata-ui/src/main/resources/ui/cypress/common/Entities/MlModelServiceClass.ts +++ b/openmetadata-ui/src/main/resources/ui/cypress/common/Entities/MlModelServiceClass.ts @@ -16,6 +16,7 @@ import { visitServiceDetailsPage } from '../../common/serviceUtils'; import { SERVICE_TYPE } from '../../constants/constants'; import { EntityType } from '../../constants/Entity.interface'; import { ML_MODEL_SERVICE } from '../../constants/EntityConstant'; +import { getToken } from '../Utils/LocalStorage'; import EntityClass from './EntityClass'; class MlModelServiceClass extends EntityClass { @@ -49,7 +50,7 @@ class MlModelServiceClass extends EntityClass { // Handle creation here cy.getAllLocalStorage().then((data) => { - const token = Object.values(data)[0].oidcIdToken as string; + const token = getToken(data); createSingleLevelEntity({ ...ML_MODEL_SERVICE, diff --git a/openmetadata-ui/src/main/resources/ui/cypress/common/Entities/PipelineClass.ts b/openmetadata-ui/src/main/resources/ui/cypress/common/Entities/PipelineClass.ts index cf9bff417e70..6ef0fa6bbd44 100644 --- a/openmetadata-ui/src/main/resources/ui/cypress/common/Entities/PipelineClass.ts +++ b/openmetadata-ui/src/main/resources/ui/cypress/common/Entities/PipelineClass.ts @@ -14,6 +14,7 @@ import { EntityType } from '../../constants/Entity.interface'; import { PIPELINE_SERVICE } from '../../constants/EntityConstant'; import { createSingleLevelEntity } from '../EntityUtils'; import { visitEntityDetailsPage } from '../Utils/Entity'; +import { getToken } from '../Utils/LocalStorage'; import EntityClass from './EntityClass'; class PipelineClass extends EntityClass { @@ -41,7 +42,7 @@ class PipelineClass extends EntityClass { // Handle creation here cy.getAllLocalStorage().then((data) => { - const token = Object.values(data)[0].oidcIdToken as string; + const token = getToken(data); createSingleLevelEntity({ token, diff --git a/openmetadata-ui/src/main/resources/ui/cypress/common/Entities/PipelineServiceClass.ts b/openmetadata-ui/src/main/resources/ui/cypress/common/Entities/PipelineServiceClass.ts index d86c28eed038..699b0c89bdd2 100644 --- a/openmetadata-ui/src/main/resources/ui/cypress/common/Entities/PipelineServiceClass.ts +++ b/openmetadata-ui/src/main/resources/ui/cypress/common/Entities/PipelineServiceClass.ts @@ -16,6 +16,7 @@ import { visitServiceDetailsPage } from '../../common/serviceUtils'; import { SERVICE_TYPE } from '../../constants/constants'; import { EntityType } from '../../constants/Entity.interface'; import { PIPELINE_SERVICE } from '../../constants/EntityConstant'; +import { getToken } from '../Utils/LocalStorage'; import EntityClass from './EntityClass'; class PipelineServiceClass extends EntityClass { @@ -49,7 +50,7 @@ class PipelineServiceClass extends EntityClass { // Handle creation here cy.getAllLocalStorage().then((data) => { - const token = Object.values(data)[0].oidcIdToken as string; + const token = getToken(data); createSingleLevelEntity({ ...PIPELINE_SERVICE, diff --git a/openmetadata-ui/src/main/resources/ui/cypress/common/Entities/SearchIndexClass.ts b/openmetadata-ui/src/main/resources/ui/cypress/common/Entities/SearchIndexClass.ts index 3115ce4313b5..7be90b6b1578 100644 --- a/openmetadata-ui/src/main/resources/ui/cypress/common/Entities/SearchIndexClass.ts +++ b/openmetadata-ui/src/main/resources/ui/cypress/common/Entities/SearchIndexClass.ts @@ -17,12 +17,13 @@ import { } from '../../constants/EntityConstant'; import { createSingleLevelEntity } from '../EntityUtils'; import { visitEntityDetailsPage } from '../Utils/Entity'; +import { getToken } from '../Utils/LocalStorage'; import EntityClass from './EntityClass'; class SearchIndexClass extends EntityClass { constructor() { const topicName = `cypress-search-index-${Date.now()}`; - super(topicName, SEARCH_INDEX_DETAILS, EntityType.SeachIndex); + super(topicName, SEARCH_INDEX_DETAILS, EntityType.SearchIndex); this.name = 'SearchIndex'; } @@ -45,7 +46,7 @@ class SearchIndexClass extends EntityClass { // Handle creation here cy.getAllLocalStorage().then((data) => { - const token = Object.values(data)[0].oidcIdToken; + const token = getToken(data); createSingleLevelEntity({ ...SEARCH_SERVICE, diff --git a/openmetadata-ui/src/main/resources/ui/cypress/common/Entities/SearchServiceClass.ts b/openmetadata-ui/src/main/resources/ui/cypress/common/Entities/SearchServiceClass.ts index 481ae00a8a1d..65e1d7a9da58 100644 --- a/openmetadata-ui/src/main/resources/ui/cypress/common/Entities/SearchServiceClass.ts +++ b/openmetadata-ui/src/main/resources/ui/cypress/common/Entities/SearchServiceClass.ts @@ -16,6 +16,7 @@ import { visitServiceDetailsPage } from '../../common/serviceUtils'; import { SERVICE_TYPE } from '../../constants/constants'; import { EntityType } from '../../constants/Entity.interface'; import { SEARCH_SERVICE } from '../../constants/EntityConstant'; +import { getToken } from '../Utils/LocalStorage'; import EntityClass from './EntityClass'; class SearchServiceClass extends EntityClass { @@ -45,7 +46,7 @@ class SearchServiceClass extends EntityClass { // Handle creation here cy.getAllLocalStorage().then((data) => { - const token = Object.values(data)[0].oidcIdToken as string; + const token = getToken(data); createSingleLevelEntity({ ...SEARCH_SERVICE, diff --git a/openmetadata-ui/src/main/resources/ui/cypress/common/Entities/StorageServiceClass.ts b/openmetadata-ui/src/main/resources/ui/cypress/common/Entities/StorageServiceClass.ts index 5002d9595fdd..e3baa8c20e04 100644 --- a/openmetadata-ui/src/main/resources/ui/cypress/common/Entities/StorageServiceClass.ts +++ b/openmetadata-ui/src/main/resources/ui/cypress/common/Entities/StorageServiceClass.ts @@ -16,6 +16,7 @@ import { visitServiceDetailsPage } from '../../common/serviceUtils'; import { SERVICE_TYPE } from '../../constants/constants'; import { EntityType } from '../../constants/Entity.interface'; import { STORAGE_SERVICE } from '../../constants/EntityConstant'; +import { getToken } from '../Utils/LocalStorage'; import EntityClass from './EntityClass'; class StorageServiceClass extends EntityClass { @@ -49,7 +50,7 @@ class StorageServiceClass extends EntityClass { // Handle creation here cy.getAllLocalStorage().then((data) => { - const token = Object.values(data)[0].oidcIdToken as string; + const token = getToken(data); createSingleLevelEntity({ ...STORAGE_SERVICE, diff --git a/openmetadata-ui/src/main/resources/ui/cypress/common/Entities/StoredProcedureClass.ts b/openmetadata-ui/src/main/resources/ui/cypress/common/Entities/StoredProcedureClass.ts index 72d6287fab06..ad96a34c2aa7 100644 --- a/openmetadata-ui/src/main/resources/ui/cypress/common/Entities/StoredProcedureClass.ts +++ b/openmetadata-ui/src/main/resources/ui/cypress/common/Entities/StoredProcedureClass.ts @@ -20,6 +20,7 @@ import { deleteEntityViaREST, visitEntityDetailsPage, } from '../Utils/Entity'; +import { getToken } from '../Utils/LocalStorage'; import EntityClass from './EntityClass'; class StoreProcedureClass extends EntityClass { @@ -51,7 +52,7 @@ class StoreProcedureClass extends EntityClass { // Handle creation here cy.getAllLocalStorage().then((data) => { - const token = Object.values(data)[0].oidcIdToken as string; + const token = getToken(data); createEntityTableViaREST({ token, @@ -73,7 +74,7 @@ class StoreProcedureClass extends EntityClass { override cleanup() { super.cleanup(); cy.getAllLocalStorage().then((data) => { - const token = Object.values(data)[0].oidcIdToken as string; + const token = getToken(data); deleteEntityViaREST({ token, endPoint: EntityType.DatabaseService, diff --git a/openmetadata-ui/src/main/resources/ui/cypress/common/Entities/TableClass.ts b/openmetadata-ui/src/main/resources/ui/cypress/common/Entities/TableClass.ts index 7ae59c299cae..b04622e009df 100644 --- a/openmetadata-ui/src/main/resources/ui/cypress/common/Entities/TableClass.ts +++ b/openmetadata-ui/src/main/resources/ui/cypress/common/Entities/TableClass.ts @@ -17,6 +17,7 @@ import { deleteEntityViaREST, visitEntityDetailsPage, } from '../Utils/Entity'; +import { getToken } from '../Utils/LocalStorage'; import EntityClass from './EntityClass'; class TableClass extends EntityClass { @@ -44,7 +45,7 @@ class TableClass extends EntityClass { // Handle creation here cy.getAllLocalStorage().then((data) => { - const token = Object.values(data)[0].oidcIdToken as string; + const token = getToken(data); createEntityTableViaREST({ token, @@ -58,7 +59,7 @@ class TableClass extends EntityClass { override cleanup() { super.cleanup(); cy.getAllLocalStorage().then((data) => { - const token = Object.values(data)[0].oidcIdToken as string; + const token = getToken(data); deleteEntityViaREST({ token, endPoint: EntityType.DatabaseService, diff --git a/openmetadata-ui/src/main/resources/ui/cypress/common/Entities/TopicClass.ts b/openmetadata-ui/src/main/resources/ui/cypress/common/Entities/TopicClass.ts index 60eb4e15f2ca..52160eef647e 100644 --- a/openmetadata-ui/src/main/resources/ui/cypress/common/Entities/TopicClass.ts +++ b/openmetadata-ui/src/main/resources/ui/cypress/common/Entities/TopicClass.ts @@ -17,6 +17,7 @@ import { } from '../../constants/EntityConstant'; import { createSingleLevelEntity } from '../EntityUtils'; import { visitEntityDetailsPage } from '../Utils/Entity'; +import { getToken } from '../Utils/LocalStorage'; import EntityClass from './EntityClass'; class TopicClass extends EntityClass { @@ -41,7 +42,7 @@ class TopicClass extends EntityClass { // Handle creation here cy.getAllLocalStorage().then((data) => { - const token = Object.values(data)[0].oidcIdToken; + const token = getToken(data); createSingleLevelEntity({ ...MESSAGING_SERVICE, diff --git a/openmetadata-ui/src/main/resources/ui/cypress/common/EntityUtils.js b/openmetadata-ui/src/main/resources/ui/cypress/common/EntityUtils.ts similarity index 74% rename from openmetadata-ui/src/main/resources/ui/cypress/common/EntityUtils.js rename to openmetadata-ui/src/main/resources/ui/cypress/common/EntityUtils.ts index 176229546305..1f69a9390c6e 100644 --- a/openmetadata-ui/src/main/resources/ui/cypress/common/EntityUtils.js +++ b/openmetadata-ui/src/main/resources/ui/cypress/common/EntityUtils.ts @@ -335,3 +335,126 @@ export const deleteEntityById = ({ entityType, token, entityFqn }) => { }); }); }; + +export const getTableDetails = (tableName: string, schemaFQN: string) => ({ + name: tableName, + description: 'description', + columns: [ + { + name: `cy-user_id-${uuid()}`, + dataType: 'NUMERIC', + dataTypeDisplay: 'numeric', + description: + 'Unique identifier for the user of your Shopify POS or your Shopify admin.', + }, + { + name: `cy-shop_id-${uuid()}`, + dataType: 'NUMERIC', + dataTypeDisplay: 'numeric', + description: + 'The ID of the store. This column is a foreign key reference to the shop_id column in the dim.shop table.', + }, + { + name: `cy-name-${uuid()}`, + dataType: 'VARCHAR', + dataLength: 100, + dataTypeDisplay: 'varchar', + description: 'Name of the staff member.', + children: [ + { + name: `cy-first_name-${uuid()}`, + dataType: 'VARCHAR', + dataLength: 100, + dataTypeDisplay: 'varchar', + description: 'First name of the staff member.', + }, + { + name: `cy-last_name-${uuid()}`, + dataType: 'VARCHAR', + dataLength: 100, + dataTypeDisplay: 'varchar', + }, + ], + }, + { + name: `cy-email-${uuid()}`, + dataType: 'VARCHAR', + dataLength: 100, + dataTypeDisplay: 'varchar', + description: 'Email address of the staff member.', + }, + ], + databaseSchema: schemaFQN, +}); + +export const getTableCreationDetails = () => { + const DATABASE_SERVICE_NAME = `cy-database-service-${uuid()}`; + const DATABASE_NAME = `cy-database-${uuid()}`; + const SCHEMA_NAME = `cy-database-schema-${uuid()}`; + const TABLE_1_NAME = `cy-table-1-${uuid()}`; + const TABLE_2_NAME = `cy-table-2-${uuid()}`; + const TABLE_3_NAME = `cy-table-3-${uuid()}`; + + const service = { + name: DATABASE_SERVICE_NAME, + serviceType: 'Mysql', + connection: { + config: { + type: 'Mysql', + scheme: 'mysql+pymysql', + username: 'username', + authType: { + password: 'password', + }, + hostPort: 'mysql:3306', + supportsMetadataExtraction: true, + supportsDBTExtraction: true, + supportsProfiler: true, + supportsQueryComment: true, + }, + }, + }; + const database = { + name: DATABASE_NAME, + service: DATABASE_SERVICE_NAME, + }; + + const schema = { + name: SCHEMA_NAME, + database: `${DATABASE_SERVICE_NAME}.${DATABASE_NAME}`, + }; + + const table1 = getTableDetails( + TABLE_1_NAME, + `${DATABASE_SERVICE_NAME}.${DATABASE_NAME}.${SCHEMA_NAME}` + ); + const table2 = getTableDetails( + TABLE_2_NAME, + `${DATABASE_SERVICE_NAME}.${DATABASE_NAME}.${SCHEMA_NAME}` + ); + const table3 = getTableDetails( + TABLE_3_NAME, + `${DATABASE_SERVICE_NAME}.${DATABASE_NAME}.${SCHEMA_NAME}` + ); + + return { + service, + database, + schema, + tables: [table1, table2, table3], + }; +}; + +export const getUserCreationDetails = () => { + const userName = `user${uuid()}`; + + return { + userName, + user: { + firstName: `first-name-${uuid()}`, + lastName: `last-name-${uuid()}`, + email: `${userName}@example.com`, + password: 'User@OMD123', + }, + }; +}; diff --git a/openmetadata-ui/src/main/resources/ui/cypress/common/GlossaryUtils.js b/openmetadata-ui/src/main/resources/ui/cypress/common/GlossaryUtils.ts similarity index 97% rename from openmetadata-ui/src/main/resources/ui/cypress/common/GlossaryUtils.js rename to openmetadata-ui/src/main/resources/ui/cypress/common/GlossaryUtils.ts index 8f209deb78ba..ddc624c4e795 100644 --- a/openmetadata-ui/src/main/resources/ui/cypress/common/GlossaryUtils.js +++ b/openmetadata-ui/src/main/resources/ui/cypress/common/GlossaryUtils.ts @@ -10,8 +10,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -// eslint-disable-next-line spaced-comment -/// import { DELETE_TERM } from '../constants/constants'; import { SidebarItem } from '../constants/Entity.interface'; diff --git a/openmetadata-ui/src/main/resources/ui/cypress/common/Services/BigQueryIngestionClass.ts b/openmetadata-ui/src/main/resources/ui/cypress/common/Services/BigQueryIngestionClass.ts index 9a28f419e5ea..fdf79e771678 100644 --- a/openmetadata-ui/src/main/resources/ui/cypress/common/Services/BigQueryIngestionClass.ts +++ b/openmetadata-ui/src/main/resources/ui/cypress/common/Services/BigQueryIngestionClass.ts @@ -35,10 +35,9 @@ class BigQueryIngestionClass extends ServiceBaseClass { fillConnectionDetails() { const clientEmail = Cypress.env('bigqueryClientEmail'); - cy.get('.form-group > #root\\/credentials\\/gcpConfig\\/type') + cy.get('#root\\/credentials\\/gcpConfig__oneof_select') .scrollIntoView() - .type('service_account'); - checkServiceFieldSectionHighlighting('type'); + .select('GCP Credentials Values'); cy.get('#root\\/credentials\\/gcpConfig\\/projectId') .scrollIntoView() .type(Cypress.env('bigqueryProjectId')); diff --git a/openmetadata-ui/src/main/resources/ui/cypress/common/Services/PostgresIngestionClass.ts b/openmetadata-ui/src/main/resources/ui/cypress/common/Services/PostgresIngestionClass.ts index c9bf3b3d1295..3e48c4bf465b 100644 --- a/openmetadata-ui/src/main/resources/ui/cypress/common/Services/PostgresIngestionClass.ts +++ b/openmetadata-ui/src/main/resources/ui/cypress/common/Services/PostgresIngestionClass.ts @@ -14,14 +14,13 @@ import { SERVICE_TYPE } from '../../constants/constants'; import { EntityType } from '../../constants/Entity.interface'; import { checkServiceFieldSectionHighlighting, - handleIngestionRetry, interceptURL, - scheduleIngestion, verifyResponseStatusCode, } from '../common'; import ServiceBaseClass from '../Entities/ServiceBaseClass'; import { visitServiceDetailsPage } from '../serviceUtils'; import { visitEntityDetailsPage } from '../Utils/Entity'; +import { handleIngestionRetry, scheduleIngestion } from '../Utils/Ingestion'; import { Services } from '../Utils/Services'; class PostgresIngestionClass extends ServiceBaseClass { @@ -137,7 +136,7 @@ class PostgresIngestionClass extends ServiceBaseClass { verifyResponseStatusCode('@serviceDetails', 200); verifyResponseStatusCode('@ingestionPipelines', 200); - handleIngestionRetry('database', true, 0, 'usage'); + handleIngestionRetry(0, 'usage'); }); }); diff --git a/openmetadata-ui/src/main/resources/ui/cypress/common/Services/RedshiftWithDBTIngestionClass.ts b/openmetadata-ui/src/main/resources/ui/cypress/common/Services/RedshiftWithDBTIngestionClass.ts index 8e674a32fc6b..4d10eebf9830 100644 --- a/openmetadata-ui/src/main/resources/ui/cypress/common/Services/RedshiftWithDBTIngestionClass.ts +++ b/openmetadata-ui/src/main/resources/ui/cypress/common/Services/RedshiftWithDBTIngestionClass.ts @@ -16,14 +16,13 @@ import { REDSHIFT } from '../../constants/service.constants'; import { GlobalSettingOptions } from '../../constants/settings.constant'; import { checkServiceFieldSectionHighlighting, - handleIngestionRetry, interceptURL, - scheduleIngestion, verifyResponseStatusCode, } from '../common'; import ServiceBaseClass from '../Entities/ServiceBaseClass'; import { searchServiceFromSettingPage } from '../serviceUtils'; import { visitEntityDetailsPage } from '../Utils/Entity'; +import { handleIngestionRetry, scheduleIngestion } from '../Utils/Ingestion'; import { Services } from '../Utils/Services'; class RedshiftWithDBTIngestionClass extends ServiceBaseClass { @@ -188,7 +187,7 @@ class RedshiftWithDBTIngestionClass extends ServiceBaseClass { verifyResponseStatusCode('@getIngestionPipelineStatus', 200); verifyResponseStatusCode('@serviceDetails', 200); verifyResponseStatusCode('@ingestionPipelines', 200); - handleIngestionRetry('database', true, 0, 'dbt'); + handleIngestionRetry(0, 'dbt'); }); }); diff --git a/openmetadata-ui/src/main/resources/ui/cypress/common/Services/S3IngestionClass.ts b/openmetadata-ui/src/main/resources/ui/cypress/common/Services/S3IngestionClass.ts index 0e7b2255a0ea..7ea1dbb40cda 100644 --- a/openmetadata-ui/src/main/resources/ui/cypress/common/Services/S3IngestionClass.ts +++ b/openmetadata-ui/src/main/resources/ui/cypress/common/Services/S3IngestionClass.ts @@ -17,7 +17,7 @@ import { Services } from '../Utils/Services'; class S3IngestionClass extends ServiceBaseClass { name: string; constructor() { - super(Services.Storage, 'cypress-s3-storage', 'S3', 'cypress-bucket'); + super(Services.Storage, 'cypress-s3-storage', 'S3', 'om-cypress-bucket'); } createService() { @@ -37,12 +37,14 @@ class S3IngestionClass extends ServiceBaseClass { Cypress.env('s3StorageSecretAccessKey') ); checkServiceFieldSectionHighlighting('awsSecretAccessKey'); - cy.get('#root\\/awsConfig\\/awsRegion').type('us'); + cy.get('#root\\/awsConfig\\/awsRegion').type('us-east-2'); checkServiceFieldSectionHighlighting('awsRegion'); - cy.get('#root\\/awsConfig\\/endPointURL').type( - Cypress.env('s3StorageEndPointUrl') - ); - checkServiceFieldSectionHighlighting('endPointURL'); + } + + fillIngestionDetails() { + cy.get('#root\\/containerFilterPattern\\/includes') + .scrollIntoView() + .type(`${this.entityName}{enter}`); } } diff --git a/openmetadata-ui/src/main/resources/ui/cypress/common/TagUtils.js b/openmetadata-ui/src/main/resources/ui/cypress/common/TagUtils.ts similarity index 100% rename from openmetadata-ui/src/main/resources/ui/cypress/common/TagUtils.js rename to openmetadata-ui/src/main/resources/ui/cypress/common/TagUtils.ts diff --git a/openmetadata-ui/src/main/resources/ui/cypress/common/TaskUtils.js b/openmetadata-ui/src/main/resources/ui/cypress/common/TaskUtils.ts similarity index 86% rename from openmetadata-ui/src/main/resources/ui/cypress/common/TaskUtils.js rename to openmetadata-ui/src/main/resources/ui/cypress/common/TaskUtils.ts index cff4f86f0c14..5955e093f0a9 100644 --- a/openmetadata-ui/src/main/resources/ui/cypress/common/TaskUtils.js +++ b/openmetadata-ui/src/main/resources/ui/cypress/common/TaskUtils.ts @@ -21,7 +21,20 @@ const owner = 'admin'; const assignee = 'adam.matthews2'; const secondAssignee = 'aaron_johnson0'; -export const verifyTaskDetails = (regexPattern) => { +export type TaskDetails = { + assignee: string; + term: string; + displayName?: string; + entity?: string; + serviceName?: string; + entityType?: string; + schemaName?: string; +}; + +export const verifyTaskDetails = ( + regexPattern: RegExp, + taskAssignee?: string +) => { cy.get('#task-panel').should('be.visible'); cy.get('[data-testid="task-title"]') .invoke('text') @@ -33,7 +46,7 @@ export const verifyTaskDetails = (regexPattern) => { cy.get('[data-testid="owner-link"]').should('contain', owner); - cy.get(`[data-testid="${assignee}"]`).should('be.visible'); + cy.get(`[data-testid="${taskAssignee ?? assignee}"]`).should('be.visible'); }; export const editAssignee = () => { @@ -61,7 +74,10 @@ export const editAssignee = () => { cy.get(`[data-testid="${assignee}"]`).should('be.visible'); }; -export const createDescriptionTask = (value, assigneeDisabled) => { +export const createDescriptionTask = ( + value: TaskDetails, + assigneeDisabled?: boolean +) => { interceptURL('POST', 'api/v1/feed', 'createTask'); cy.get('#title').should( @@ -98,7 +114,7 @@ export const createDescriptionTask = (value, assigneeDisabled) => { toastNotification('Task created successfully.'); }; -export const createAndUpdateDescriptionTask = (value) => { +export const createAndUpdateDescriptionTask = (value: TaskDetails) => { createDescriptionTask(value); // verify the task details diff --git a/openmetadata-ui/src/main/resources/ui/cypress/common/advancedSearch.js b/openmetadata-ui/src/main/resources/ui/cypress/common/Utils/AdvancedSearch.ts similarity index 59% rename from openmetadata-ui/src/main/resources/ui/cypress/common/advancedSearch.js rename to openmetadata-ui/src/main/resources/ui/cypress/common/Utils/AdvancedSearch.ts index 21849cd2d6ef..26d3ade88baf 100644 --- a/openmetadata-ui/src/main/resources/ui/cypress/common/advancedSearch.js +++ b/openmetadata-ui/src/main/resources/ui/cypress/common/Utils/AdvancedSearch.ts @@ -11,77 +11,26 @@ * limitations under the License. */ -import { SEARCH_ENTITY_TABLE } from '../constants/constants'; -import { SidebarItem } from '../constants/Entity.interface'; +import { EntityType, SidebarItem } from '../../constants/Entity.interface'; +import { SERVICE_CATEGORIES } from '../../constants/service.constants'; +import { interceptURL, verifyResponseStatusCode } from '../common'; import { - DATABASE_DETAILS, - DATABASE_SERVICE_DETAILS, - SCHEMA_DETAILS, - TABLE_DETAILS, -} from '../constants/EntityConstant'; -import { USER_CREDENTIALS } from '../constants/SearchIndexDetails.constants'; -import { interceptURL, uuid, verifyResponseStatusCode } from './common'; -import { createEntityTable } from './EntityUtils'; -import { visitEntityDetailsPage } from './Utils/Entity'; - -export const ADVANCE_SEARCH_TABLES = { - table1: TABLE_DETAILS, - table2: { - name: `cy-table2-${uuid()}`, - description: 'description', - columns: [ - { - name: 'cypress_first_name', - dataType: 'VARCHAR', - dataLength: 100, - dataTypeDisplay: 'varchar', - description: 'First name of the staff member.', - }, - { - name: 'cypress_last_name', - dataType: 'VARCHAR', - dataLength: 100, - dataTypeDisplay: 'varchar', - }, - { - name: 'cypress_email', - dataType: 'VARCHAR', - dataLength: 100, - dataTypeDisplay: 'varchar', - description: 'Email address of the staff member.', - }, - ], - databaseSchema: `${DATABASE_SERVICE_DETAILS.name}.${DATABASE_DETAILS.name}.${SCHEMA_DETAILS.name}`, - }, - table3: { - name: `cy-table3-${uuid()}`, - description: 'description', - columns: [ - { - name: 'cypress_user_id', - dataType: 'NUMERIC', - dataTypeDisplay: 'numeric', - description: - 'Unique identifier for the user of your Shopify POS or your Shopify admin.', - }, - { - name: 'cypress_shop_id', - dataType: 'NUMERIC', - dataTypeDisplay: 'numeric', - description: - 'The ID of the store. This column is a foreign key reference to the shop_id column in the dim.shop table.', - }, - ], - databaseSchema: `${DATABASE_SERVICE_DETAILS.name}.${DATABASE_DETAILS.name}.${SCHEMA_DETAILS.name}`, - }, -}; + createEntityTable, + getTableCreationDetails, + getUserCreationDetails, + hardDeleteService, +} from '../EntityUtils'; +import { deleteEntityViaREST, visitEntityDetailsPage } from './Entity'; -export const ADVANCE_SEARCH_DATABASE_SERVICE = { - service: DATABASE_SERVICE_DETAILS, - database: DATABASE_DETAILS, - schema: SCHEMA_DETAILS, - tables: Object.values(ADVANCE_SEARCH_TABLES), -}; +export const ADVANCE_SEARCH_DATABASE_SERVICE = getTableCreationDetails(); + +export const ADVANCE_SEARCH_DATABASE_SERVICE_2 = getTableCreationDetails(); + +export const USER_1 = getUserCreationDetails(); +export const USER_1_FULL_NAME = `${USER_1.user.firstName}${USER_1.user.lastName}`; + +export const USER_2 = getUserCreationDetails(); +export const USER_2_FULL_NAME = `${USER_2.user.firstName}${USER_2.user.lastName}`; export const CONDITIONS_MUST = { equalTo: { @@ -112,22 +61,34 @@ export const CONDITIONS_MUST_NOT = { filter: 'must_not', }, }; -const ownerFullName = `${USER_CREDENTIALS.firstName}${USER_CREDENTIALS.lastName}`; -export const FIELDS = { +export type AdvancedSearchFieldDetails = { + name: string; + testId: string; + searchTerm1?: string; + searchCriteriaFirstGroup: string; + responseValueFirstGroup: string; + searchCriteriaSecondGroup: string; + responseValueSecondGroup: string; + owner?: boolean; + createTagName?: string; + isLocalSearch?: boolean; +}; + +export const FIELDS: Record = { Owner: { name: 'Owner', - testid: '[title="Owner"]', - searchTerm1: ownerFullName, - searchCriteriaFirstGroup: ownerFullName, - responseValueFirstGroup: `"displayName":"${ownerFullName}"`, - searchCriteriaSecondGroup: 'Aaron Singh', + testId: '[title="Owner"]', + searchTerm1: USER_1_FULL_NAME, + searchCriteriaFirstGroup: USER_1_FULL_NAME, + responseValueFirstGroup: `"displayName":"${USER_1_FULL_NAME}"`, + searchCriteriaSecondGroup: USER_2_FULL_NAME, owner: true, - responseValueSecondGroup: 'Aaron Singh', + responseValueSecondGroup: USER_2_FULL_NAME, }, Tags: { name: 'Tags', - testid: '[title="Tags"]', + testId: '[title="Tags"]', createTagName: 'Personal', searchCriteriaFirstGroup: 'PersonalData.Personal', responseValueFirstGroup: '"tagFQN":"PersonalData.Personal"', @@ -136,7 +97,7 @@ export const FIELDS = { }, Tiers: { name: 'Tier', - testid: '[title="Tier"]', + testId: '[title="Tier"]', searchCriteriaFirstGroup: 'Tier.Tier1', responseValueFirstGroup: '"tagFQN":"Tier.Tier1"', searchCriteriaSecondGroup: 'Tier.Tier2', @@ -145,35 +106,37 @@ export const FIELDS = { }, Service: { name: 'Service', - testid: '[title="Service"]', - searchCriteriaFirstGroup: 'sample_data', - responseValueFirstGroup: `"name":"sample_data"`, - searchCriteriaSecondGroup: DATABASE_SERVICE_DETAILS.name, - responseValueSecondGroup: `"name":"${DATABASE_SERVICE_DETAILS.name}"`, + testId: '[title="Service"]', + searchCriteriaFirstGroup: ADVANCE_SEARCH_DATABASE_SERVICE_2.service.name, + responseValueFirstGroup: `"name":"${ADVANCE_SEARCH_DATABASE_SERVICE_2.service.name}"`, + searchCriteriaSecondGroup: ADVANCE_SEARCH_DATABASE_SERVICE.service.name, + responseValueSecondGroup: `"name":"${ADVANCE_SEARCH_DATABASE_SERVICE.service.name}"`, }, Database: { name: 'Database', - testid: '[title="Database"]', - searchCriteriaFirstGroup: 'ecommerce_db', - responseValueFirstGroup: `"name":"ecommerce_db"`, - searchCriteriaSecondGroup: DATABASE_DETAILS.name, - responseValueSecondGroup: `"name":"${DATABASE_DETAILS.name}"`, + testId: '[title="Database"]', + searchCriteriaFirstGroup: ADVANCE_SEARCH_DATABASE_SERVICE_2.database.name, + responseValueFirstGroup: `"name":"${ADVANCE_SEARCH_DATABASE_SERVICE_2.database.name}"`, + searchCriteriaSecondGroup: ADVANCE_SEARCH_DATABASE_SERVICE.database.name, + responseValueSecondGroup: `"name":"${ADVANCE_SEARCH_DATABASE_SERVICE.database.name}"`, }, Database_Schema: { name: 'Database Schema', - testid: '[title="Database Schema"]', - searchCriteriaFirstGroup: 'shopify', - responseValueFirstGroup: `"name":"shopify"`, - searchCriteriaSecondGroup: SCHEMA_DETAILS.name, - responseValueSecondGroup: `"name":"${SCHEMA_DETAILS.name}"`, + testId: '[title="Database Schema"]', + searchCriteriaFirstGroup: ADVANCE_SEARCH_DATABASE_SERVICE_2.schema.name, + responseValueFirstGroup: `"name":"${ADVANCE_SEARCH_DATABASE_SERVICE_2.schema.name}"`, + searchCriteriaSecondGroup: ADVANCE_SEARCH_DATABASE_SERVICE.schema.name, + responseValueSecondGroup: `"name":"${ADVANCE_SEARCH_DATABASE_SERVICE.schema.name}"`, }, Column: { name: 'Column', - testid: '[title="Column"]', - searchCriteriaFirstGroup: 'cypress_first_name', - responseValueFirstGroup: '"name":"cypress_first_name"', - searchCriteriaSecondGroup: 'cypress_user_id', - responseValueSecondGroup: '"name":"cypress_user_id"', + testId: '[title="Column"]', + searchCriteriaFirstGroup: + ADVANCE_SEARCH_DATABASE_SERVICE_2.tables[0].columns[0].name, + responseValueFirstGroup: `"name":"${ADVANCE_SEARCH_DATABASE_SERVICE_2.tables[0].columns[0].name}"`, + searchCriteriaSecondGroup: + ADVANCE_SEARCH_DATABASE_SERVICE.tables[0].columns[0].name, + responseValueSecondGroup: `"name":"${ADVANCE_SEARCH_DATABASE_SERVICE.tables[0].columns[0].name}"`, }, }; @@ -189,10 +152,10 @@ export const OPERATOR = { }; export const searchForField = ( - condition, - fieldId, - searchCriteria, - index, + condition: string, + fieldId: string, + searchCriteria: string, + index: number, isLocalSearch = false ) => { if (!isLocalSearch) { @@ -225,9 +188,9 @@ export const searchForField = ( .type(searchCriteria); // checking filter is working - cy.get( - `.ant-select-item-option-active[title="${searchCriteria}"]` - ).should('be.visible'); + cy.get(`.ant-select-dropdown [title="${searchCriteria}"]`).should( + 'be.visible' + ); // select value from dropdown if (!isLocalSearch) { @@ -247,13 +210,13 @@ export const goToAdvanceSearch = () => { cy.get('[data-testid="reset-btn"]').click(); }; -export const checkmustPaths = ( - condition, - field, - searchCriteria, - index, - responseSearch, - isLocalSearch +export const checkMustPaths = ( + condition: string, + field: string, + searchCriteria: string, + index: number, + responseSearch: string, + isLocalSearch: boolean ) => { goToAdvanceSearch(); @@ -279,7 +242,7 @@ export const checkmustPaths = ( }); }; -export const checkmust_notPaths = ( +export const checkMust_notPaths = ( condition, field, searchCriteria, @@ -309,95 +272,6 @@ export const checkmust_notPaths = ( }); }; -export const removeOwner = () => { - visitEntityDetailsPage({ - term: SEARCH_ENTITY_TABLE.table_1.term, - serviceName: SEARCH_ENTITY_TABLE.table_1.serviceName, - entity: SEARCH_ENTITY_TABLE.table_1.entity, - }); - interceptURL( - 'PATCH', - `/api/v1/${SEARCH_ENTITY_TABLE.table_1.entity}/*`, - 'patchOwner' - ); - cy.get('[data-testid="edit-owner"]').click(); - cy.get('[data-testid="remove-owner"]').click(); - verifyResponseStatusCode('@patchOwner', 200); - cy.get('[data-testid="owner-link"]').should('contain', 'No Owner'); -}; - -export const addOwner = ({ ownerName, term, serviceName, entity }) => { - visitEntityDetailsPage({ - term, - serviceName, - entity, - }); - - interceptURL( - 'GET', - '/api/v1/search/query?q=**%20AND%20teamType:Group&from=0&size=25&index=team_search_index&sort_field=displayName.keyword&sort_order=asc', - 'waitForTeams' - ); - - cy.get('[data-testid="edit-owner"]').click(); - - verifyResponseStatusCode('@waitForTeams', 200); - interceptURL('GET', '/api/v1/users?limit=25&isBot=false', 'getUsers'); - - cy.get('.ant-tabs [id*=tab-users]').click(); - verifyResponseStatusCode('@getUsers', 200); - - interceptURL( - 'GET', - `api/v1/search/query?q=*${encodeURI(ownerName)}*`, - 'searchOwner' - ); - - cy.get('[data-testid="owner-select-users-search-bar"]').type(ownerName); - - verifyResponseStatusCode('@searchOwner', 200); - - interceptURL('PATCH', '/api/v1/tables/*', 'tablePatch'); - - // Selecting the user - cy.get(`[title="${ownerName}"]`) - .should('exist') - .scrollIntoView() - .and('be.visible') - .click(); - - verifyResponseStatusCode('@tablePatch', 200); - - cy.get('[data-testid="owner-link"]') - .scrollIntoView() - .invoke('text') - .then((text) => { - expect(text).equal(ownerName); - }); -}; - -export const addTier = ({ term, serviceName, entity }) => { - visitEntityDetailsPage({ - term, - serviceName, - entity, - }); - - cy.get('[data-testid="edit-tier"]') - .scrollIntoView() - .should('exist') - .should('be.visible') - .click(); - - cy.get('[data-testid="select-tier-button"]') - .first() - .should('exist') - .should('be.visible') - .click(); - - cy.get('[data-testid="tier-dropdown"]').should('contain', 'Tier1'); -}; - export const addTag = ({ tag, term, serviceName, entity }) => { visitEntityDetailsPage({ term, @@ -428,7 +302,19 @@ export const addTag = ({ tag, term, serviceName, entity }) => { .contains(tag); }; -export const checkAddGroupWithOperator = ( +type CheckAddGroupWithOperatorArgs = { + condition_1: string; + condition_2: string; + fieldId: string; + searchCriteria_1: string; + searchCriteria_2: string; + index_1: number; + index_2: number; + operatorIndex: number; + isLocalSearch?: boolean; +}; + +export const checkAddGroupWithOperator = ({ condition_1, condition_2, fieldId, @@ -437,12 +323,8 @@ export const checkAddGroupWithOperator = ( index_1, index_2, operatorIndex, - filter_1, - filter_2, - response_1, - response_2, - isLocalSearch = false -) => { + isLocalSearch = false, +}: CheckAddGroupWithOperatorArgs) => { goToAdvanceSearch(); // Click on field dropdown cy.get('.rule--field > .ant-select > .ant-select-selector') @@ -564,11 +446,25 @@ export const checkAddGroupWithOperator = ( const resBody = JSON.stringify(response.body); expect(request.url).to.contain(encodeURI(searchCriteria_1)); - expect(resBody).to.not.include(response_2); + expect(resBody).to.not.include(response); }); }; -export const checkAddRuleWithOperator = ( +type CheckAddRuleWithOperatorArgs = { + condition_1: string; + condition_2: string; + fieldId: string; + searchCriteria_1: string; + searchCriteria_2: string; + index_1: number; + index_2: number; + operatorIndex: number; + filter_1: string; + filter_2: string; + response: string; +}; + +export const checkAddRuleWithOperator = ({ condition_1, condition_2, fieldId, @@ -579,9 +475,8 @@ export const checkAddRuleWithOperator = ( operatorIndex, filter_1, filter_2, - response_1, - response_2 -) => { + response, +}: CheckAddRuleWithOperatorArgs) => { goToAdvanceSearch(); // Click on field dropdown cy.get('.rule--field').eq(index_1).should('be.visible').click(); @@ -674,7 +569,7 @@ export const checkAddRuleWithOperator = ( 'GET', `/api/v1/search/query?q=&index=*&from=0&size=10&deleted=false&query_filter=*${filter_1}*${encodeURI( searchCriteria_1 - )}*${filter_2}*${encodeURI(response_2)}*`, + )}*${filter_2}*${encodeURI(response.replaceAll(' ', '+'))}*`, `search${searchCriteria_1}` ); @@ -685,130 +580,171 @@ export const checkAddRuleWithOperator = ( const resBody = JSON.stringify(response.body); expect(request.url).to.contain(encodeURI(searchCriteria_1)); - expect(resBody).to.not.include(response_2); + expect(resBody).to.not.include(response); }); }; -export const advanceSearchPreRequests = (token) => { +export const advanceSearchPreRequests = (testData, token: string) => { // Create Table hierarchy - createEntityTable({ token, ...ADVANCE_SEARCH_DATABASE_SERVICE, }); - // Create a new user + createEntityTable({ + token, + ...ADVANCE_SEARCH_DATABASE_SERVICE_2, + }); + cy.request({ method: 'POST', url: `/api/v1/users/signup`, headers: { Authorization: `Bearer ${token}` }, - body: USER_CREDENTIALS, + body: USER_1.user, }).then((response) => { - USER_CREDENTIALS.id = response.body.id; - }); + testData.user_1 = response.body; - // Add owner to table 1 - cy.request({ - method: 'GET', - url: `/api/v1/tables/name/${ADVANCE_SEARCH_TABLES.table1.databaseSchema}.${ADVANCE_SEARCH_TABLES.table1.name}`, - headers: { Authorization: `Bearer ${token}` }, - }).then((response) => { + // Add owner to table 1 cy.request({ - method: 'PATCH', - url: `/api/v1/tables/${response.body.id}`, - headers: { - Authorization: `Bearer ${token}`, - 'Content-Type': 'application/json-patch+json', - }, - body: [ - { - op: 'add', - path: '/owner', - value: { - id: USER_CREDENTIALS.id, - type: 'user', - }, + method: 'GET', + url: `/api/v1/tables/name/${ADVANCE_SEARCH_DATABASE_SERVICE.tables[0].databaseSchema}.${ADVANCE_SEARCH_DATABASE_SERVICE.tables[0].name}`, + headers: { Authorization: `Bearer ${token}` }, + }).then((response) => { + cy.request({ + method: 'PATCH', + url: `/api/v1/tables/${response.body.id}`, + headers: { + Authorization: `Bearer ${token}`, + 'Content-Type': 'application/json-patch+json', }, - ], + body: [ + { + op: 'add', + path: '/owner', + value: { + id: testData.user_1.id, + type: 'user', + }, + }, + ], + }); }); - }); - // Add Tier to table 2 - cy.request({ - method: 'GET', - url: `/api/v1/tables/name/${ADVANCE_SEARCH_TABLES.table2.databaseSchema}.${ADVANCE_SEARCH_TABLES.table2.name}`, - headers: { Authorization: `Bearer ${token}` }, - }).then((response) => { + // Create a new users cy.request({ - method: 'PATCH', - url: `/api/v1/tables/${response.body.id}`, - headers: { - Authorization: `Bearer ${token}`, - 'Content-Type': 'application/json-patch+json', - }, - body: [ - { - op: 'add', - path: '/tags/0', - value: { - name: 'Tier1', - tagFQN: 'Tier.Tier1', - labelType: 'Manual', - state: 'Confirmed', - }, + method: 'POST', + url: `/api/v1/users/signup`, + headers: { Authorization: `Bearer ${token}` }, + body: USER_2.user, + }).then((response) => { + testData.user_2 = response.body; + }); + + // Add Tier to table 2 + cy.request({ + method: 'GET', + url: `/api/v1/tables/name/${ADVANCE_SEARCH_DATABASE_SERVICE.tables[1].databaseSchema}.${ADVANCE_SEARCH_DATABASE_SERVICE.tables[1].name}`, + headers: { Authorization: `Bearer ${token}` }, + }).then((response) => { + cy.request({ + method: 'PATCH', + url: `/api/v1/tables/${response.body.id}`, + headers: { + Authorization: `Bearer ${token}`, + 'Content-Type': 'application/json-patch+json', }, - { - op: 'add', - path: '/tags/1', - value: { - name: 'SpecialCategory', - tagFQN: 'PersonalData.SpecialCategory', - labelType: 'Manual', - state: 'Confirmed', + body: [ + { + op: 'add', + path: '/tags/0', + value: { + name: 'Tier1', + tagFQN: 'Tier.Tier1', + labelType: 'Manual', + state: 'Confirmed', + }, }, - }, - ], + { + op: 'add', + path: '/tags/1', + value: { + name: 'SpecialCategory', + tagFQN: 'PersonalData.SpecialCategory', + labelType: 'Manual', + state: 'Confirmed', + }, + }, + ], + }); }); - }); - // Add Tag to table 3 - cy.request({ - method: 'GET', - url: `/api/v1/tables/name/${ADVANCE_SEARCH_TABLES.table3.databaseSchema}.${ADVANCE_SEARCH_TABLES.table3.name}`, - headers: { Authorization: `Bearer ${token}` }, - }).then((response) => { + // Add Tag to table 3 cy.request({ - method: 'PATCH', - url: `/api/v1/tables/${response.body.id}`, - headers: { - Authorization: `Bearer ${token}`, - 'Content-Type': 'application/json-patch+json', - }, - body: [ - { - op: 'add', - path: '/tags/0', - value: { - tagFQN: 'PersonalData.Personal', - source: 'Classification', - name: 'Personal', - description: - 'Data that can be used to directly or indirectly identify a person.', - labelType: 'Manual', - state: 'Confirmed', - }, + method: 'GET', + url: `/api/v1/tables/name/${ADVANCE_SEARCH_DATABASE_SERVICE.tables[2].databaseSchema}.${ADVANCE_SEARCH_DATABASE_SERVICE.tables[2].name}`, + headers: { Authorization: `Bearer ${token}` }, + }).then((response) => { + cy.request({ + method: 'PATCH', + url: `/api/v1/tables/${response.body.id}`, + headers: { + Authorization: `Bearer ${token}`, + 'Content-Type': 'application/json-patch+json', }, - { - op: 'add', - path: '/tags/1', - value: { - name: 'Tier2', - tagFQN: 'Tier.Tier2', - labelType: 'Manual', - state: 'Confirmed', + body: [ + { + op: 'add', + path: '/tags/0', + value: { + tagFQN: 'PersonalData.Personal', + source: 'Classification', + name: 'Personal', + description: + 'Data that can be used to directly or indirectly identify a person.', + labelType: 'Manual', + state: 'Confirmed', + }, }, - }, - ], + { + op: 'add', + path: '/tags/1', + value: { + name: 'Tier2', + tagFQN: 'Tier.Tier2', + labelType: 'Manual', + state: 'Confirmed', + }, + }, + ], + }); }); }); }; + +export const advancedSearchFlowCleanup = (token: string) => { + // Delete created services + hardDeleteService({ + token, + serviceFqn: ADVANCE_SEARCH_DATABASE_SERVICE.service.name, + serviceType: SERVICE_CATEGORIES.DATABASE_SERVICES, + }); + + hardDeleteService({ + token, + serviceFqn: ADVANCE_SEARCH_DATABASE_SERVICE_2.service.name, + serviceType: SERVICE_CATEGORIES.DATABASE_SERVICES, + }); + + // Delete created users + deleteEntityViaREST({ + token, + endPoint: EntityType.User, + entityName: USER_1.userName, + }); + + deleteEntityViaREST({ + token, + endPoint: EntityType.User, + entityName: USER_2.userName, + }); +}; diff --git a/openmetadata-ui/src/main/resources/ui/cypress/common/Utils/Apps.ts b/openmetadata-ui/src/main/resources/ui/cypress/common/Utils/Apps.ts new file mode 100644 index 000000000000..b452053eb228 --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/cypress/common/Utils/Apps.ts @@ -0,0 +1,41 @@ +/* + * Copyright 2024 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +export const checkAndDeleteApp = ({ + token, + applicationName, +}: { + token: string; + applicationName: string; +}) => { + cy.request({ + method: 'GET', + url: '/api/v1/apps?limit=20&include=non-deleted', + headers: { + Authorization: `Bearer ${token}`, + }, + }).then((response) => { + expect(response.status).to.eq(200); + + const app = response.body.data.find((app) => app.name === applicationName); + + if (app.name === applicationName) { + cy.request({ + method: 'DELETE', + url: `/api/v1/apps/name/${applicationName}?hardDelete=true`, + headers: { + Authorization: `Bearer ${token}`, + }, + }); + } + }); +}; diff --git a/openmetadata-ui/src/main/resources/ui/cypress/common/Utils/CustomProperty.ts b/openmetadata-ui/src/main/resources/ui/cypress/common/Utils/CustomProperty.ts index 434cbb58c165..91d7e11174d5 100644 --- a/openmetadata-ui/src/main/resources/ui/cypress/common/Utils/CustomProperty.ts +++ b/openmetadata-ui/src/main/resources/ui/cypress/common/Utils/CustomProperty.ts @@ -11,8 +11,17 @@ * limitations under the License. */ +import { + CUSTOM_PROPERTY_INVALID_NAMES, + CUSTOM_PROPERTY_NAME_VALIDATION_ERROR, +} from '../../constants/constants'; import { EntityType } from '../../constants/Entity.interface'; -import { interceptURL, uuid, verifyResponseStatusCode } from '../common'; +import { + descriptionBox, + interceptURL, + uuid, + verifyResponseStatusCode, +} from '../common'; export enum CustomPropertyType { STRING = 'String', @@ -188,3 +197,148 @@ export const deleteCustomProperties = ( export const customPropertiesArray = Array(10) .fill(null) .map(() => generateCustomProperties()); + +export const addCustomPropertiesForEntity = ( + propertyName: string, + customPropertyData: { description: string }, + customType: string, + value: { values: string[]; multiSelect: boolean } +) => { + // Add Custom property for selected entity + cy.get('[data-testid="add-field-button"]').click(); + + // validation should work + cy.get('[data-testid="create-button"]').scrollIntoView().click(); + + cy.get('#name_help').should('contain', 'Name is required'); + cy.get('#propertyType_help').should('contain', 'Property Type is required'); + + cy.get('#description_help').should('contain', 'Description is required'); + + // capital case validation + cy.get('[data-testid="name"]') + .scrollIntoView() + .type(CUSTOM_PROPERTY_INVALID_NAMES.CAPITAL_CASE); + cy.get('[role="alert"]').should( + 'contain', + CUSTOM_PROPERTY_NAME_VALIDATION_ERROR + ); + + // with underscore validation + cy.get('[data-testid="name"]') + .clear() + .type(CUSTOM_PROPERTY_INVALID_NAMES.WITH_UNDERSCORE); + cy.get('[role="alert"]').should( + 'contain', + CUSTOM_PROPERTY_NAME_VALIDATION_ERROR + ); + + // with space validation + cy.get('[data-testid="name"]') + .clear() + .type(CUSTOM_PROPERTY_INVALID_NAMES.WITH_SPACE); + cy.get('[role="alert"]').should( + 'contain', + CUSTOM_PROPERTY_NAME_VALIDATION_ERROR + ); + + // with dots validation + cy.get('[data-testid="name"]') + .clear() + .type(CUSTOM_PROPERTY_INVALID_NAMES.WITH_DOTS); + cy.get('[role="alert"]').should( + 'contain', + CUSTOM_PROPERTY_NAME_VALIDATION_ERROR + ); + + // should allow name in another languages + cy.get('[data-testid="name"]').clear().type('汝らヴェディア'); + // should not throw the validation error + cy.get('#name_help').should('not.exist'); + + cy.get('[data-testid="name"]').clear().type(propertyName); + + cy.get('[data-testid="propertyType"]').click(); + cy.get(`[title="${customType}"]`).click(); + + if (customType === 'Enum') { + value.values.forEach((val) => { + cy.get('#root\\/customPropertyConfig').type(`${val}{enter}`); + }); + + cy.clickOutside(); + + if (value.multiSelect) { + cy.get('#root\\/multiSelect').scrollIntoView().click(); + } + } + + cy.get(descriptionBox).clear().type(customPropertyData.description); + + // Check if the property got added + cy.intercept('/api/v1/metadata/types/name/*?fields=customProperties').as( + 'customProperties' + ); + cy.get('[data-testid="create-button"]').scrollIntoView().click(); + + cy.wait('@customProperties'); + cy.get('.ant-table-row').should('contain', propertyName); + + // Navigating to home page + cy.clickOnLogo(); +}; + +export const editCreatedProperty = (propertyName: string, type: string) => { + // Fetching for edit button + cy.get(`[data-row-key="${propertyName}"]`) + .find('[data-testid="edit-button"]') + .as('editButton'); + + if (type === 'Enum') { + cy.get(`[data-row-key="${propertyName}"]`) + .find('[data-testid="enum-config"]') + .should('contain', '["enum1","enum2","enum3"]'); + } + + cy.get('@editButton').click(); + + cy.get(descriptionBox).clear().type('This is new description'); + + if (type === 'Enum') { + cy.get('#root\\/customPropertyConfig').type(`updatedValue{enter}`); + + cy.clickOutside(); + } + + interceptURL('PATCH', '/api/v1/metadata/types/*', 'checkPatchForDescription'); + + cy.get('button[type="submit"]').scrollIntoView().click(); + + cy.wait('@checkPatchForDescription', { timeout: 15000 }); + + cy.get('.ant-modal-wrap').should('not.exist'); + + // Fetching for updated descriptions for the created custom property + cy.get(`[data-row-key="${propertyName}"]`) + .find('[data-testid="viewer-container"]') + .should('contain', 'This is new description'); + + if (type === 'Enum') { + cy.get(`[data-row-key="${propertyName}"]`) + .find('[data-testid="enum-config"]') + .should('contain', '["enum1","enum2","enum3","updatedValue"]'); + } +}; + +export const deleteCreatedProperty = (propertyName: string) => { + // Fetching for delete button + cy.get(`[data-row-key="${propertyName}"]`) + .scrollIntoView() + .find('[data-testid="delete-button"]') + .click(); + + // Checking property name is present on the delete pop-up + cy.get('[data-testid="body-text"]').should('contain', propertyName); + + cy.get('[data-testid="save-button"]').should('be.visible').click(); +}; diff --git a/openmetadata-ui/src/main/resources/ui/cypress/common/Utils/Entity.ts b/openmetadata-ui/src/main/resources/ui/cypress/common/Utils/Entity.ts index 275ebe18a86d..1215778ddf39 100644 --- a/openmetadata-ui/src/main/resources/ui/cypress/common/Utils/Entity.ts +++ b/openmetadata-ui/src/main/resources/ui/cypress/common/Utils/Entity.ts @@ -138,10 +138,10 @@ export const deleteEntityViaREST = ({ token, }: { entityName: string; - endPoint: EntityType; + endPoint: EntityType | string; token: Cypress.Storable; }) => { - // Create entity + // Delete entity cy.request({ method: 'DELETE', url: `/api/v1/${endPoint}/name/${entityName}?recursive=true&hardDelete=true`, @@ -163,7 +163,7 @@ export const visitEntityDetailsPage = ({ serviceName: string; entity: EntityType; dataTestId?: string; - entityType?: EntityType; + entityType?: string; entityFqn?: string; }) => { if (entity === EntityType.DataModel) { @@ -412,7 +412,7 @@ export const deleteEntity = ( displayName: string ) => { deletedEntityCommonChecks({ entityType: endPoint, deleted: false }); - + cy.clickOutside(); cy.get('[data-testid="manage-button"]').click(); cy.get('[data-testid="delete-button"]').scrollIntoView().click(); cy.get('[data-testid="delete-modal"]').then(() => { @@ -446,6 +446,7 @@ export const deleteEntity = ( ); deletedEntityCommonChecks({ entityType: endPoint, deleted: true }); + cy.clickOutside(); if (endPoint === EntityType.Table) { interceptURL( diff --git a/openmetadata-ui/src/main/resources/ui/cypress/common/Utils/Glossary.ts b/openmetadata-ui/src/main/resources/ui/cypress/common/Utils/Glossary.ts index 255ec9799b8b..a9775eecce25 100644 --- a/openmetadata-ui/src/main/resources/ui/cypress/common/Utils/Glossary.ts +++ b/openmetadata-ui/src/main/resources/ui/cypress/common/Utils/Glossary.ts @@ -15,7 +15,8 @@ import { EntityType } from '../../constants/Entity.interface'; import { interceptURL, verifyResponseStatusCode } from '../common'; export const assignGlossaryTerm = ( - glossaryTerm: string, + glossaryTermFQN: string, + glossaryTermName: string, endPoint: EntityType ) => { interceptURL('PATCH', `/api/v1/${endPoint}/*`, 'addGlossaryTerm'); @@ -25,26 +26,27 @@ export const assignGlossaryTerm = ( cy.get('[data-testid="tag-selector"] input') .should('be.visible') - .type(glossaryTerm); + .type(glossaryTermName); - cy.get(`[data-testid="tag-${glossaryTerm}"]`).click(); + cy.get(`[data-testid="tag-${glossaryTermFQN}"]`).click(); // to close popup cy.clickOutside(); cy.get( - `[data-testid="tag-selector"] [data-testid="selected-tag-${glossaryTerm}"]` + `[data-testid="tag-selector"] [data-testid="selected-tag-${glossaryTermFQN}"]` ).should('be.visible'); cy.get('[data-testid="saveAssociatedTag"]').scrollIntoView().click(); verifyResponseStatusCode('@addGlossaryTerm', 200); cy.get( - `[data-testid="entity-right-panel"] [data-testid="glossary-container"] [data-testid="tag-${glossaryTerm}"]` + `[data-testid="entity-right-panel"] [data-testid="glossary-container"] [data-testid="tag-${glossaryTermFQN}"]` ).should('be.visible'); }; export const udpateGlossaryTerm = ( - glossaryTerm: string, + glossaryTermFQN: string, + glossaryTermName: string, endPoint: EntityType ) => { interceptURL('PATCH', `/api/v1/${endPoint}/*`, 'addGlossaryTerm'); @@ -54,20 +56,20 @@ export const udpateGlossaryTerm = ( cy.get('[data-testid="tag-selector"] input') .should('be.visible') - .type(glossaryTerm); + .type(glossaryTermName); - cy.get(`[data-testid="tag-${glossaryTerm}"]`).click(); + cy.get(`[data-testid="tag-${glossaryTermFQN}"]`).click(); // to close popup cy.clickOutside(); cy.get( - `[data-testid="tag-selector"] [data-testid="selected-tag-${glossaryTerm}"]` + `[data-testid="tag-selector"] [data-testid="selected-tag-${glossaryTermFQN}"]` ).should('be.visible'); cy.get('[data-testid="saveAssociatedTag"]').scrollIntoView().click(); verifyResponseStatusCode('@addGlossaryTerm', 200); cy.get( - `[data-testid="entity-right-panel"] [data-testid="glossary-container"] [data-testid="tag-${glossaryTerm}"]` + `[data-testid="entity-right-panel"] [data-testid="glossary-container"] [data-testid="tag-${glossaryTermFQN}"]` ).should('be.visible'); }; diff --git a/openmetadata-ui/src/main/resources/ui/cypress/common/Utils/Ingestion.ts b/openmetadata-ui/src/main/resources/ui/cypress/common/Utils/Ingestion.ts new file mode 100644 index 000000000000..9706dd810da9 --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/cypress/common/Utils/Ingestion.ts @@ -0,0 +1,140 @@ +/* + * Copyright 2024 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { + BASE_WAIT_TIME, + interceptURL, + RETRY_TIMES, + verifyResponseStatusCode, +} from '../common'; + +const RETRIES_COUNT = 4; + +export const handleIngestionRetry = (count = 0, ingestionType = 'metadata') => { + let timer = BASE_WAIT_TIME; + const rowIndex = ingestionType === 'metadata' ? 1 : 2; + + interceptURL( + 'GET', + '/api/v1/services/ingestionPipelines?*', + 'ingestionPipelines' + ); + interceptURL( + 'GET', + '/api/v1/services/ingestionPipelines/*/pipelineStatus?startTs=*&endTs=*', + 'pipelineStatuses' + ); + interceptURL('GET', '/api/v1/services/*/name/*', 'serviceDetails'); + interceptURL('GET', '/api/v1/permissions?limit=100', 'allPermissions'); + + // ingestions page + let retryCount = count; + const testIngestionsTab = () => { + // click on the tab only for the first time + if (retryCount === 0) { + cy.get('[data-testid="ingestions"]').should('exist').and('be.visible'); + cy.get('[data-testid="ingestions"] >> [data-testid="count"]').should( + 'have.text', + rowIndex + ); + cy.get('[data-testid="ingestions"]').click(); + + if (ingestionType === 'metadata') { + verifyResponseStatusCode('@pipelineStatuses', 200, { + responseTimeout: 50000, + }); + } + } + }; + const checkSuccessState = () => { + testIngestionsTab(); + + if (retryCount !== 0) { + cy.wait('@allPermissions').then(() => { + cy.wait('@serviceDetails').then(() => { + verifyResponseStatusCode('@ingestionPipelines', 200); + verifyResponseStatusCode('@pipelineStatuses', 200, { + responseTimeout: 50000, + }); + }); + }); + } + + retryCount++; + + cy.get(`[data-row-key*="${ingestionType}"]`) + .find('[data-testid="pipeline-status"]') + .as('checkRun'); + // the latest run should be success + cy.get('@checkRun').then(($ingestionStatus) => { + const text = $ingestionStatus.text(); + if ( + text !== 'Success' && + text !== 'Failed' && + retryCount <= RETRY_TIMES + ) { + // retry after waiting with log1 method [20s,40s,80s,160s,320s] + cy.wait(timer); + timer *= 2; + cy.reload(); + checkSuccessState(); + } else { + cy.get('@checkRun').should('contain', 'Success'); + } + }); + }; + + checkSuccessState(); +}; + +export const scheduleIngestion = (hasRetryCount = true) => { + interceptURL( + 'POST', + '/api/v1/services/ingestionPipelines', + 'createIngestionPipelines' + ); + interceptURL( + 'POST', + '/api/v1/services/ingestionPipelines/deploy/*', + 'deployPipeline' + ); + interceptURL( + 'GET', + '/api/v1/services/ingestionPipelines/status', + 'getIngestionPipelineStatus' + ); + // Schedule & Deploy + cy.get('[data-testid="cron-type"]').should('be.visible').click(); + cy.get('.ant-select-item-option-content').contains('Hour').click(); + + if (hasRetryCount) { + cy.get('#retries') + .scrollIntoView() + .clear() + .type(RETRIES_COUNT + ''); + } + + cy.get('[data-testid="deploy-button"]').should('be.visible').click(); + + verifyResponseStatusCode('@createIngestionPipelines', 201); + verifyResponseStatusCode('@deployPipeline', 200, { + responseTimeout: 50000, + }); + verifyResponseStatusCode('@getIngestionPipelineStatus', 200); + // check success + cy.get('[data-testid="success-line"]', { timeout: 15000 }).should( + 'be.visible' + ); + cy.contains('has been created and deployed successfully').should( + 'be.visible' + ); +}; diff --git a/openmetadata-ui/src/main/resources/ui/cypress/common/Utils/LocalStorage.ts b/openmetadata-ui/src/main/resources/ui/cypress/common/Utils/LocalStorage.ts new file mode 100644 index 000000000000..593437c03c9f --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/cypress/common/Utils/LocalStorage.ts @@ -0,0 +1,22 @@ +/* + * Copyright 2024 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +export const getToken = (data: Cypress.StorageByOrigin) => { + try { + return ( + JSON.parse((Object.values(data)[0]['om-session'] as string) ?? '{}') + ?.state?.oidcIdToken ?? '' + ); + } catch (error) { + return ''; + } +}; diff --git a/openmetadata-ui/src/main/resources/ui/cypress/common/Utils/Login.ts b/openmetadata-ui/src/main/resources/ui/cypress/common/Utils/Login.ts new file mode 100644 index 000000000000..44112e5ef536 --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/cypress/common/Utils/Login.ts @@ -0,0 +1,48 @@ +/* + * Copyright 2024 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { interceptURL } from '../common'; + +/** + * Try Performing login with the given username and password. + * Particularly used for testing login. + * + * @param {string} username - The username for login + * @param {string} password - The password for login + * @return {void} + */ +export const performLogin = (username, password) => { + cy.visit('/'); + interceptURL('POST', '/api/v1/users/login', 'loginUser'); + cy.get('[id="email"]').should('be.visible').clear().type(username); + cy.get('[id="password"]').should('be.visible').clear().type(password); + + // Don't want to show any popup in the tests + cy.setCookie(`STAR_OMD_USER_${username.split('@')[0]}`, 'true'); + + // Get version and set cookie to hide version banner + cy.request({ + method: 'GET', + url: `api/v1/system/version`, + }).then((res) => { + const version = res.body.version; + const versionCookie = `VERSION_${version + .split('-')[0] + .replaceAll('.', '_')}`; + + cy.setCookie(versionCookie, 'true'); + window.localStorage.setItem('loggedInUsers', username.split('@')[0]); + }); + + cy.get('.ant-btn').contains('Login').should('be.visible').click(); + cy.wait('@loginUser'); +}; diff --git a/openmetadata-ui/src/main/resources/ui/cypress/common/Utils/Owner.ts b/openmetadata-ui/src/main/resources/ui/cypress/common/Utils/Owner.ts index cd72ba49dc75..ccd2b914fba1 100644 --- a/openmetadata-ui/src/main/resources/ui/cypress/common/Utils/Owner.ts +++ b/openmetadata-ui/src/main/resources/ui/cypress/common/Utils/Owner.ts @@ -11,6 +11,7 @@ * limitations under the License. */ import { interceptURL, uuid, verifyResponseStatusCode } from '../common'; +import { getToken } from './LocalStorage'; const userURL = '/api/v1/search/query?q=**%20AND%20isBot:false&from=0&size=0&index=user_search_index'; @@ -28,7 +29,7 @@ export const generateRandomUser = () => { export const validateOwnerAndTeamCounts = () => { cy.getAllLocalStorage().then((data) => { - const token = Object.values(data)[0].oidcIdToken; + const token = getToken(data); cy.request({ method: 'GET', @@ -130,6 +131,10 @@ export const removeOwner = (ownerName: string, dataTestId?: string) => { cy.get('[data-testid="select-owner-tabs"]').should('be.visible'); + cy.get( + '[data-testid="select-owner-tabs"] [data-testid="remove-owner"]' + ).scrollIntoView({ offset: { top: -100, left: 0 } }); + cy.get( '[data-testid="select-owner-tabs"] [data-testid="remove-owner"]' ).click(); diff --git a/openmetadata-ui/src/main/resources/ui/cypress/common/Utils/Services.ts b/openmetadata-ui/src/main/resources/ui/cypress/common/Utils/Services.ts index e76a72d3617d..b5adf59afb39 100644 --- a/openmetadata-ui/src/main/resources/ui/cypress/common/Utils/Services.ts +++ b/openmetadata-ui/src/main/resources/ui/cypress/common/Utils/Services.ts @@ -36,7 +36,7 @@ export const ServicesEntityMap = { [Services.Pipeline]: EntityType.Pipeline, [Services.MLModels]: EntityType.MlModel, [Services.Storage]: EntityType.Container, - [Services.Search]: EntityType.SeachIndex, + [Services.Search]: EntityType.SearchIndex, }; export const RETRY_TIMES = 4; diff --git a/openmetadata-ui/src/main/resources/ui/cypress/common/Utils/Teams.ts b/openmetadata-ui/src/main/resources/ui/cypress/common/Utils/Teams.ts index 17442ebed10e..302a94ca978c 100644 --- a/openmetadata-ui/src/main/resources/ui/cypress/common/Utils/Teams.ts +++ b/openmetadata-ui/src/main/resources/ui/cypress/common/Utils/Teams.ts @@ -12,11 +12,14 @@ */ import { + descriptionBox, interceptURL, toastNotification, verifyResponseStatusCode, } from '../common'; +const TEAM_TYPES = ['Department', 'Division', 'Group']; + export const commonTeamDetails = { username: 'Aaron Johnson', userId: 'aaron_johnson0', @@ -85,3 +88,119 @@ export const deleteTeamPermanently = (teamName: string) => { cy.get('table').should('not.contain', teamName); }; + +const getTeamType = ( + currentTeam: string +): { + childTeamType: string; + teamTypeOptions: typeof TEAM_TYPES; +} => { + switch (currentTeam) { + case 'BusinessUnit': + return { + childTeamType: 'Division', + teamTypeOptions: TEAM_TYPES, + }; + + case 'Division': + return { + childTeamType: 'Department', + teamTypeOptions: TEAM_TYPES, + }; + + case 'Department': + return { + childTeamType: 'Group', + teamTypeOptions: ['Department', 'Group'], + }; + } + + return { + childTeamType: '', + teamTypeOptions: [], + }; +}; + +const checkTeamTypeOptions = (type: string) => { + for (const teamType of getTeamType(type)?.teamTypeOptions) { + cy.get(`.ant-select-dropdown [title="${teamType}"]`) + .should('exist') + .should('be.visible'); + } +}; + +export const selectTeamHierarchy = (index: number) => { + if (index > 0) { + cy.get('[data-testid="team-type"]') + .invoke('text') + .then((text) => { + cy.log(text); + checkTeamTypeOptions(text); + cy.log('check type', text); + cy.get( + `.ant-select-dropdown [title="${getTeamType(text).childTeamType}"]` + ).click(); + }); + } else { + checkTeamTypeOptions('BusinessUnit'); + + cy.get(`.ant-select-dropdown [title='BusinessUnit']`) + .should('exist') + .should('be.visible') + .click(); + } +}; + +export const addTeam = ( + teamDetails: { + name: string; + displayName?: string; + teamType: string; + description: string; + ownername?: string; + email: string; + updatedName?: string; + username?: string; + userId?: string; + assetname?: string; + updatedEmail?: string; + }, + index?: number, + isHierarchy = false +) => { + interceptURL('GET', '/api/v1/teams*', 'addTeam'); + // Fetching the add button and clicking on it + if (index > 0) { + cy.get('[data-testid="add-placeholder-button"]').click(); + } else { + cy.get('[data-testid="add-team"]').click(); + } + + verifyResponseStatusCode('@addTeam', 200); + + // Entering team details + cy.get('[data-testid="name"]').type(teamDetails.name); + + cy.get('[data-testid="display-name"]').type(teamDetails.name); + + cy.get('[data-testid="email"]').type(teamDetails.email); + + cy.get('[data-testid="team-selector"]').click(); + + if (isHierarchy) { + selectTeamHierarchy(index); + } else { + cy.get(`.ant-select-dropdown [title="${teamDetails.teamType}"]`).click(); + } + + cy.get(descriptionBox).type(teamDetails.description); + + interceptURL('POST', '/api/v1/teams', 'saveTeam'); + interceptURL('GET', '/api/v1/team*', 'createTeam'); + + // Saving the created team + cy.get('[form="add-team-form"]').scrollIntoView().click(); + + verifyResponseStatusCode('@saveTeam', 201); + verifyResponseStatusCode('@createTeam', 200); +}; diff --git a/openmetadata-ui/src/main/resources/ui/cypress/common/Utils/Versions.ts b/openmetadata-ui/src/main/resources/ui/cypress/common/Utils/Versions.ts index e3c53097665d..0f648335b199 100644 --- a/openmetadata-ui/src/main/resources/ui/cypress/common/Utils/Versions.ts +++ b/openmetadata-ui/src/main/resources/ui/cypress/common/Utils/Versions.ts @@ -11,10 +11,184 @@ * limitations under the License. */ import { EntityType } from '../../constants/Entity.interface'; +import { DOMAIN_CREATION_DETAILS } from '../../constants/EntityConstant'; +import { SERVICE_CATEGORIES } from '../../constants/service.constants'; +import { + COMMON_PATCH_PAYLOAD, + DATABASE_DETAILS_FOR_VERSION_TEST, + DATABASE_SCHEMA_DETAILS_FOR_VERSION_TEST, + OWNER_DETAILS, + SERVICE_DETAILS_FOR_VERSION_TEST, +} from '../../constants/Version.constants'; import { interceptURL, verifyResponseStatusCode } from '../common'; +import { deleteEntityViaREST } from './Entity'; + +interface EntityReference { + id?: string; + name?: string; + displayName?: string; + fullyQualifiedName?: string; +} + +const serviceDetails = SERVICE_DETAILS_FOR_VERSION_TEST.Database; + +interface EntityCreationData { + user?: EntityReference; + domain?: EntityReference; + database?: EntityReference; + schema?: EntityReference; +} export const validateDomain = (domain: string, entityType: EntityType) => { interceptURL('GET', `/api/v1/${entityType}/*/versions/0.2`, 'getVersion'); cy.get('[data-testid="version-button"]').should('contain', '0.2').click(); verifyResponseStatusCode('@getVersion', 200); }; + +export const commonPrerequisites = ( + token: string, + data: EntityCreationData +) => { + // Create user + cy.request({ + method: 'POST', + url: `/api/v1/users/signup`, + headers: { Authorization: `Bearer ${token}` }, + body: OWNER_DETAILS, + }).then((response) => { + data.user = response.body; + }); + + cy.request({ + method: 'PUT', + url: `/api/v1/domains`, + headers: { Authorization: `Bearer ${token}` }, + body: DOMAIN_CREATION_DETAILS, + }).then((response) => { + data.domain = response.body; + }); +}; + +export const commonTestCleanup = (token: string, data: EntityCreationData) => { + deleteEntityViaREST({ + token, + endPoint: EntityType.Domain, + entityName: DOMAIN_CREATION_DETAILS.name, + }); + + deleteEntityViaREST({ + token, + endPoint: EntityType.User, + entityName: data.user.name, + }); + + deleteEntityViaREST({ + token, + endPoint: `services/${SERVICE_CATEGORIES.DATABASE_SERVICES}`, + entityName: serviceDetails.serviceName, + }); +}; + +export const databaseSchemaVersionPrerequisites = ( + token: string, + data: EntityCreationData +) => { + commonPrerequisites(token, data); + + // Create service + cy.request({ + method: 'POST', + url: `/api/v1/services/${serviceDetails.serviceCategory}`, + headers: { Authorization: `Bearer ${token}` }, + body: serviceDetails.entityCreationDetails, + }).then(() => { + // Create Database + cy.request({ + method: 'POST', + url: `/api/v1/databases`, + headers: { Authorization: `Bearer ${token}` }, + body: DATABASE_DETAILS_FOR_VERSION_TEST, + }).then((response) => { + data.database = response.body; + + // Create Database Schema + cy.request({ + method: 'PUT', + url: `/api/v1/databaseSchemas`, + headers: { Authorization: `Bearer ${token}` }, + body: DATABASE_SCHEMA_DETAILS_FOR_VERSION_TEST, + }).then((response) => { + data.schema = response.body; + + cy.request({ + method: 'PATCH', + url: `/api/v1/databaseSchemas/${data.schema.id}`, + headers: { + Authorization: `Bearer ${token}`, + 'Content-Type': 'application/json-patch+json', + }, + body: [ + ...COMMON_PATCH_PAYLOAD, + { + op: 'add', + path: '/domain', + value: { + id: data.domain.id, + type: 'domain', + name: DOMAIN_CREATION_DETAILS.name, + description: DOMAIN_CREATION_DETAILS.description, + }, + }, + ], + }); + }); + }); + }); +}; + +export const databaseVersionPrerequisites = ( + token: string, + data: EntityCreationData +) => { + commonPrerequisites(token, data); + + // Create service + cy.request({ + method: 'POST', + url: `/api/v1/services/${serviceDetails.serviceCategory}`, + headers: { Authorization: `Bearer ${token}` }, + body: serviceDetails.entityCreationDetails, + }).then(() => { + // Create Database + cy.request({ + method: 'POST', + url: `/api/v1/databases`, + headers: { Authorization: `Bearer ${token}` }, + body: DATABASE_DETAILS_FOR_VERSION_TEST, + }).then((response) => { + data.database = response.body; + + cy.request({ + method: 'PATCH', + url: `/api/v1/databases/${data.database.id}`, + headers: { + Authorization: `Bearer ${token}`, + 'Content-Type': 'application/json-patch+json', + }, + body: [ + ...COMMON_PATCH_PAYLOAD, + { + op: 'add', + path: '/domain', + value: { + id: data.domain.id, + type: 'domain', + name: DOMAIN_CREATION_DETAILS.name, + description: DOMAIN_CREATION_DETAILS.description, + }, + }, + ], + }); + }); + }); +}; diff --git a/openmetadata-ui/src/main/resources/ui/cypress/common/VersionUtils.js b/openmetadata-ui/src/main/resources/ui/cypress/common/VersionUtils.ts similarity index 95% rename from openmetadata-ui/src/main/resources/ui/cypress/common/VersionUtils.js rename to openmetadata-ui/src/main/resources/ui/cypress/common/VersionUtils.ts index 69c6ee704202..7ffa216a48cf 100644 --- a/openmetadata-ui/src/main/resources/ui/cypress/common/VersionUtils.js +++ b/openmetadata-ui/src/main/resources/ui/cypress/common/VersionUtils.ts @@ -10,8 +10,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -// eslint-disable-next-line spaced-comment -/// import { isUndefined } from 'lodash'; import { @@ -67,9 +65,10 @@ export const visitDataModelVersionPage = ( dataModelFQN, dataModelId, dataModelName, + serviceName, version ) => { - visitDataModelPage(dataModelFQN, dataModelName); + visitDataModelPage(dataModelFQN, dataModelName, serviceName); interceptURL( 'GET', diff --git a/openmetadata-ui/src/main/resources/ui/cypress/common/advancedSearchQuickFilters.js b/openmetadata-ui/src/main/resources/ui/cypress/common/advancedSearchQuickFilters.ts similarity index 100% rename from openmetadata-ui/src/main/resources/ui/cypress/common/advancedSearchQuickFilters.js rename to openmetadata-ui/src/main/resources/ui/cypress/common/advancedSearchQuickFilters.ts diff --git a/openmetadata-ui/src/main/resources/ui/cypress/common/common.js b/openmetadata-ui/src/main/resources/ui/cypress/common/common.js deleted file mode 100644 index 400490ec6ef9..000000000000 --- a/openmetadata-ui/src/main/resources/ui/cypress/common/common.js +++ /dev/null @@ -1,1125 +0,0 @@ -/* - * Copyright 2022 Collate. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * http://www.apache.org/licenses/LICENSE-2.0 - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// eslint-disable-next-line spaced-comment -/// - -import { - BASE_URL, - CUSTOM_PROPERTY_INVALID_NAMES, - CUSTOM_PROPERTY_NAME_VALIDATION_ERROR, - DELETE_TERM, - INVALID_NAMES, - NAME_VALIDATION_ERROR, -} from '../constants/constants'; -import { GlobalSettingOptions } from '../constants/settings.constant'; - -export const descriptionBox = - '.toastui-editor-md-container > .toastui-editor > .ProseMirror'; -export const uuid = () => Cypress._.random(0, 1e6); -export const RETRY_TIMES = 4; -export const BASE_WAIT_TIME = 20000; - -const RETRIES_COUNT = 4; - -const TEAM_TYPES = ['Department', 'Division', 'Group']; - -export const replaceAllSpacialCharWith_ = (text) => { - return text.replaceAll(/[&/\\#, +()$~%.'":*?<>{}]/g, '_'); -}; - -const isDatabaseService = (type) => type === 'database'; - -export const checkServiceFieldSectionHighlighting = (field) => { - cy.get(`[data-id="${field}"]`).should( - 'have.attr', - 'data-highlighted', - 'true' - ); -}; - -const getTeamType = (currentTeam) => { - switch (currentTeam) { - case 'BusinessUnit': - return { - childTeamType: 'Division', - teamTypeOptions: TEAM_TYPES, - }; - - case 'Division': - return { - childTeamType: 'Department', - teamTypeOptions: TEAM_TYPES, - }; - - case 'Department': - return { - childTeamType: 'Group', - teamTypeOptions: ['Department', 'Group'], - }; - } -}; - -const checkTeamTypeOptions = (type) => { - for (const teamType of getTeamType(type).teamTypeOptions) { - cy.get(`.ant-select-dropdown [title="${teamType}"]`) - .should('exist') - .should('be.visible'); - } -}; - -// intercepting URL with cy.intercept -export const interceptURL = (method, url, alias, callback) => { - cy.intercept({ method: method, url: url }, callback).as(alias); -}; - -// waiting for response and validating the response status code -export const verifyResponseStatusCode = ( - alias, - responseCode, - option, - hasMultipleResponseCode = false -) => { - if (hasMultipleResponseCode) { - return cy - .wait(alias, option) - .its('response.statusCode') - .should('be.oneOf', responseCode); - } else { - return cy - .wait(alias, option) - .its('response.statusCode') - .should('eq', responseCode); - } -}; - -// waiting for multiple response and validating the response status code -export const verifyMultipleResponseStatusCode = ( - alias = [], - responseCode = 200, - option -) => { - cy.wait(alias, option).then((data) => { - data.map((value) => expect(value.response.statusCode).eq(responseCode)); - }); -}; - -export const handleIngestionRetry = ( - type, - testIngestionButton, - count = 0, - ingestionType = 'metadata' -) => { - let timer = BASE_WAIT_TIME; - const rowIndex = ingestionType === 'metadata' ? 1 : 2; - - interceptURL( - 'GET', - '/api/v1/services/ingestionPipelines?*', - 'ingestionPipelines' - ); - interceptURL( - 'GET', - '/api/v1/services/ingestionPipelines/*/pipelineStatus?startTs=*&endTs=*', - 'pipelineStatuses' - ); - interceptURL('GET', '/api/v1/services/*/name/*', 'serviceDetails'); - interceptURL('GET', '/api/v1/permissions?limit=100', 'allPermissions'); - - // ingestions page - let retryCount = count; - const testIngestionsTab = () => { - // click on the tab only for the first time - if (retryCount === 0) { - cy.get('[data-testid="ingestions"]').should('exist').and('be.visible'); - cy.get('[data-testid="ingestions"] >> [data-testid="count"]').should( - 'have.text', - rowIndex - ); - cy.get('[data-testid="ingestions"]').click(); - - if (ingestionType === 'metadata') { - verifyResponseStatusCode('@pipelineStatuses', 200, { - responseTimeout: 50000, - }); - } - } - }; - const checkSuccessState = () => { - testIngestionsTab(); - - if (retryCount !== 0) { - cy.wait('@allPermissions').then(() => { - cy.wait('@serviceDetails').then(() => { - verifyResponseStatusCode('@ingestionPipelines', 200); - verifyResponseStatusCode('@pipelineStatuses', 200, { - responseTimeout: 50000, - }); - }); - }); - } - - retryCount++; - - cy.get(`[data-row-key*="${ingestionType}"]`) - .find('[data-testid="pipeline-status"]') - .as('checkRun'); - // the latest run should be success - cy.get('@checkRun').then(($ingestionStatus) => { - const text = $ingestionStatus.text(); - if ( - text !== 'Success' && - text !== 'Failed' && - retryCount <= RETRY_TIMES - ) { - // retry after waiting with log1 method [20s,40s,80s,160s,320s] - cy.wait(timer); - timer *= 2; - cy.reload(); - checkSuccessState(); - } else { - cy.get('@checkRun').should('contain', 'Success'); - } - }); - }; - - checkSuccessState(); -}; - -export const scheduleIngestion = (hasRetryCount = true) => { - interceptURL( - 'POST', - '/api/v1/services/ingestionPipelines', - 'createIngestionPipelines' - ); - interceptURL( - 'POST', - '/api/v1/services/ingestionPipelines/deploy/*', - 'deployPipeline' - ); - interceptURL( - 'GET', - '/api/v1/services/ingestionPipelines/status', - 'getIngestionPipelineStatus' - ); - // Schedule & Deploy - cy.get('[data-testid="cron-type"]').should('be.visible').click(); - cy.get('.ant-select-item-option-content').contains('Hour').click(); - - if (hasRetryCount) { - cy.get('#retries').scrollIntoView().clear().type(RETRIES_COUNT); - } - - cy.get('[data-testid="deploy-button"]').should('be.visible').click(); - - verifyResponseStatusCode('@createIngestionPipelines', 201); - verifyResponseStatusCode('@deployPipeline', 200, { - responseTimeout: 50000, - }); - verifyResponseStatusCode('@getIngestionPipelineStatus', 200); - // check success - cy.get('[data-testid="success-line"]', { timeout: 15000 }).should( - 'be.visible' - ); - cy.contains('has been created and deployed successfully').should( - 'be.visible' - ); -}; - -// Storing the created service name and the type of service for later use - -export const testServiceCreationAndIngestion = ({ - serviceType, - connectionInput, - addIngestionInput, - viewIngestionInput, - serviceName, - type = 'database', - testIngestionButton = true, - serviceCategory, - shouldAddIngestion = true, - allowTestConnection = true, -}) => { - // Storing the created service name and the type of service - // Select Service in step 1 - cy.get(`[data-testid="${serviceType}"]`).should('exist').click(); - cy.get('[data-testid="next-button"]').should('exist').click(); - - // Enter service name in step 2 - - // validation should work - cy.get('[data-testid="next-button"]').should('exist').click(); - - cy.get('#name_help').should('be.visible').contains('Name is required'); - - // invalid name validation should work - cy.get('[data-testid="service-name"]') - .should('exist') - .type(INVALID_NAMES.WITH_SPECIAL_CHARS); - cy.get('#name_help').should('be.visible').contains(NAME_VALIDATION_ERROR); - - cy.get('[data-testid="service-name"]') - .should('exist') - .clear() - .type(serviceName); - interceptURL('GET', '/api/v1/services/ingestionPipelines/ip', 'ipApi'); - interceptURL( - 'GET', - 'api/v1/services/ingestionPipelines/*', - 'ingestionPipelineStatus' - ); - // intercept the service requirement md file fetch request - interceptURL( - 'GET', - `en-US/${serviceCategory}/${serviceType}.md`, - 'getServiceRequirements' - ); - cy.get('[data-testid="next-button"]').should('exist').click(); - verifyResponseStatusCode('@ingestionPipelineStatus', 200); - verifyResponseStatusCode('@ipApi', 204); - - // Connection Details in step 3 - cy.get('[data-testid="add-new-service-container"]') - .parent() - .parent() - .scrollTo('top', { - ensureScrollable: false, - }); - cy.contains('Connection Details').scrollIntoView().should('be.visible'); - - // Requirement panel should be visible and fetch the requirements md file - cy.get('[data-testid="service-requirements"]').should('be.visible'); - verifyResponseStatusCode('@getServiceRequirements', [200, 304], {}, true); - - connectionInput(); - - // Test the connection - interceptURL( - 'GET', - '/api/v1/services/testConnectionDefinitions/name/*', - 'testConnectionStepDefinition' - ); - - interceptURL('POST', '/api/v1/automations/workflows', 'createWorkflow'); - - interceptURL( - 'POST', - '/api/v1/automations/workflows/trigger/*', - 'triggerWorkflow' - ); - - interceptURL('GET', '/api/v1/automations/workflows/*', 'getWorkflow'); - - if (allowTestConnection) { - cy.get('[data-testid="test-connection-btn"]').should('exist').click(); - - verifyResponseStatusCode('@testConnectionStepDefinition', 200); - - verifyResponseStatusCode('@createWorkflow', 201); - // added extra buffer time as triggerWorkflow API can take up to 2minute to provide result - verifyResponseStatusCode('@triggerWorkflow', 200, { - responseTimeout: 120000, - }); - cy.get('[data-testid="test-connection-modal"]').should('exist'); - cy.get('.ant-modal-footer > .ant-btn-primary') - .should('exist') - .contains('OK') - .click(); - verifyResponseStatusCode('@getWorkflow', 200); - cy.get('[data-testid="messag-text"]').then(($message) => { - if ($message.text().includes('partially successful')) { - cy.contains('Test connection partially successful').should('exist'); - } else { - cy.contains('Connection test was successful').should('exist'); - } - }); - } - interceptURL( - 'GET', - '/api/v1/services/ingestionPipelines/status', - 'getIngestionPipelineStatus' - ); - cy.get('[data-testid="submit-btn"]').should('exist').click(); - verifyResponseStatusCode('@getIngestionPipelineStatus', 200); - // check success - cy.get('[data-testid="success-line"]').should('be.visible'); - cy.contains(`"${serviceName}"`).should('be.visible'); - cy.contains('has been created successfully').should('be.visible'); - - if (shouldAddIngestion) { - cy.get('[data-testid="add-ingestion-button"]').should('be.visible').click(); - - // Add ingestion page - cy.get('[data-testid="add-ingestion-container"]').should('be.visible'); - - if (isDatabaseService(type)) { - // Set mark-deleted slider to off to disable it. - cy.get('#root\\/markDeletedTables').click(); - } - - addIngestionInput && addIngestionInput(); - - cy.get('[data-testid="submit-btn"]').scrollIntoView().click(); - - if (viewIngestionInput) { - // Go back and data should persist - cy.get('[data-testid="back-button"]').scrollIntoView().click(); - - viewIngestionInput(); - - // Go Next - cy.get('[data-testid="submit-btn"]').scrollIntoView().click(); - } - - scheduleIngestion(); - - cy.contains(`${replaceAllSpacialCharWith_(serviceName)}_metadata`).should( - 'be.visible' - ); - - // wait for ingestion to run - cy.clock(); - cy.wait(1000); - - interceptURL( - 'GET', - '/api/v1/services/ingestionPipelines?*', - 'ingestionPipelines' - ); - interceptURL('GET', '/api/v1/services/*/name/*', 'serviceDetails'); - - cy.get('[data-testid="view-service-button"]').click(); - verifyResponseStatusCode('@serviceDetails', 200); - verifyResponseStatusCode('@ingestionPipelines', 200); - handleIngestionRetry(type, testIngestionButton); - } -}; - -export const deleteCreatedService = ( - typeOfService, - serviceName, - apiService, - serviceCategory -) => { - // Click on settings page - // Services page - interceptURL('GET', '/api/v1/services/*', 'getServices'); - - cy.settingClick(typeOfService); - - verifyResponseStatusCode('@getServices', 200); - - interceptURL( - 'GET', - 'api/v1/search/query?q=*&from=0&size=15&index=*', - 'searchService' - ); - cy.get('[data-testid="searchbar"]').type(serviceName); - - verifyResponseStatusCode('@searchService', 200); - - // click on created service - cy.get(`[data-testid="service-name-${serviceName}"]`) - .should('exist') - .should('be.visible') - .click(); - - cy.get(`[data-testid="entity-header-display-name"]`) - .should('exist') - .should('be.visible') - .invoke('text') - .then((text) => { - expect(text).to.equal(serviceName); - }); - - verifyResponseStatusCode('@getServices', 200); - - // Clicking on permanent delete radio button and checking the service name - cy.get('[data-testid="manage-button"]') - .should('exist') - .should('be.visible') - .click(); - - cy.get('[data-menu-id*="delete-button"]').should('be.visible'); - cy.get('[data-testid="delete-button-title"]').click(); - - // Clicking on permanent delete radio button and checking the service name - cy.get('[data-testid="hard-delete-option"]').contains(serviceName).click(); - - cy.get('[data-testid="confirmation-text-input"]') - .should('be.visible') - .type(DELETE_TERM); - interceptURL('DELETE', `/api/v1/services/${apiService}/*`, 'deleteService'); - interceptURL( - 'GET', - '/api/v1/services/*/name/*?fields=owner', - 'serviceDetails' - ); - - cy.get('[data-testid="confirm-button"]').should('be.visible').click(); - verifyResponseStatusCode('@deleteService', 200); - - // Closing the toast notification - toastNotification(`"${serviceName}" deleted successfully!`); - - cy.get(`[data-testid="service-name-${serviceName}"]`).should('not.exist'); -}; - -export const goToAddNewServicePage = (service_type) => { - // Services page - interceptURL('GET', '/api/v1/services/*', 'getServiceList'); - cy.settingClick(service_type); - verifyResponseStatusCode('@getServiceList', 200); - - cy.get('[data-testid="add-service-button"]').should('be.visible').click(); - - // Add new service page - cy.url().should('include', '/add-service'); - cy.get('[data-testid="header"]').should('be.visible'); - cy.contains('Add New Service').should('be.visible'); - cy.get('[data-testid="service-category"]').should('be.visible'); -}; - -// add new tag to entity and its table -export const addNewTagToEntity = (entityObj, term) => { - const { name, fqn } = term; - - cy.get( - '[data-testid="classification-tags-0"] [data-testid="entity-tags"] [data-testid="add-tag"]' - ) - .eq(0) - .should('be.visible') - .scrollIntoView() - .click(); - - cy.get('[data-testid="tag-selector"] input').should('be.visible').type(name); - - cy.get(`[data-testid="tag-${fqn}"]`).should('be.visible').click(); - // to close popup - cy.clickOutside(); - - cy.get('[data-testid="tag-selector"] > .ant-select-selector').contains(name); - cy.get('[data-testid="saveAssociatedTag"]') - .scrollIntoView() - .should('be.visible') - .click(); - cy.get('[data-testid="classification-tags-0"] [data-testid="tags-container"]') - .scrollIntoView() - .contains(name) - .should('exist'); - if (term.color) { - cy.get( - '[data-testid="classification-tags-0"] [data-testid="tags-container"] [data-testid="icon"]' - ).should('be.visible'); - } -}; - -export const toastNotification = (msg, closeToast = true) => { - cy.get('.Toastify__toast-body').should('contain.text', msg); - cy.wait(200); - if (closeToast) { - cy.get('.Toastify__close-button').click(); - } -}; - -export const addCustomPropertiesForEntity = ( - propertyName, - customPropertyData, - customType, - value, - entityObj -) => { - // Add Custom property for selected entity - cy.get('[data-testid="add-field-button"]').click(); - - // validation should work - cy.get('[data-testid="create-button"]').scrollIntoView().click(); - - cy.get('#name_help').should('contain', 'Name is required'); - cy.get('#propertyType_help').should('contain', 'Property Type is required'); - - cy.get('#description_help').should('contain', 'Description is required'); - - // capital case validation - cy.get('[data-testid="name"]') - .scrollIntoView() - .type(CUSTOM_PROPERTY_INVALID_NAMES.CAPITAL_CASE); - cy.get('[role="alert"]').should( - 'contain', - CUSTOM_PROPERTY_NAME_VALIDATION_ERROR - ); - - // with underscore validation - cy.get('[data-testid="name"]') - .clear() - .type(CUSTOM_PROPERTY_INVALID_NAMES.WITH_UNDERSCORE); - cy.get('[role="alert"]').should( - 'contain', - CUSTOM_PROPERTY_NAME_VALIDATION_ERROR - ); - - // with space validation - cy.get('[data-testid="name"]') - .clear() - .type(CUSTOM_PROPERTY_INVALID_NAMES.WITH_SPACE); - cy.get('[role="alert"]').should( - 'contain', - CUSTOM_PROPERTY_NAME_VALIDATION_ERROR - ); - - // with dots validation - cy.get('[data-testid="name"]') - .clear() - .type(CUSTOM_PROPERTY_INVALID_NAMES.WITH_DOTS); - cy.get('[role="alert"]').should( - 'contain', - CUSTOM_PROPERTY_NAME_VALIDATION_ERROR - ); - - // should allow name in another languages - cy.get('[data-testid="name"]').clear().type('汝らヴェディア'); - // should not throw the validation error - cy.get('#name_help').should('not.exist'); - - cy.get('[data-testid="name"]').clear().type(propertyName); - - cy.get('[data-testid="propertyType"]').click(); - cy.get(`[title="${customType}"]`).click(); - - cy.get(descriptionBox).clear().type(customPropertyData.description); - - // Check if the property got added - cy.intercept('/api/v1/metadata/types/name/*?fields=customProperties').as( - 'customProperties' - ); - cy.get('[data-testid="create-button"]').scrollIntoView().click(); - - cy.wait('@customProperties'); - cy.get('.ant-table-row').should('contain', propertyName); - - // Navigating to home page - cy.clickOnLogo(); -}; - -export const editCreatedProperty = (propertyName) => { - // Fetching for edit button - cy.get(`[data-row-key="${propertyName}"]`) - .find('[data-testid="edit-button"]') - .as('editButton'); - - cy.get('@editButton').click(); - - cy.get(descriptionBox).clear().type('This is new description'); - - interceptURL('PATCH', '/api/v1/metadata/types/*', 'checkPatchForDescription'); - - cy.get('[data-testid="save"]').click(); - - cy.wait('@checkPatchForDescription', { timeout: 15000 }); - - cy.get('.ant-modal-wrap').should('not.exist'); - - // Fetching for updated descriptions for the created custom property - cy.get(`[data-row-key="${propertyName}"]`) - .find('[data-testid="viewer-container"]') - .should('contain', 'This is new description'); -}; - -export const deleteCreatedProperty = (propertyName) => { - // Fetching for delete button - cy.get(`[data-row-key="${propertyName}"]`) - .scrollIntoView() - .find('[data-testid="delete-button"]') - .click(); - - // Checking property name is present on the delete pop-up - cy.get('[data-testid="body-text"]').should('contain', propertyName); - - cy.get('[data-testid="save-button"]').should('be.visible').click(); -}; - -export const updateOwner = () => { - cy.get('[data-testid="avatar"]').click(); - cy.get('[data-testid="user-name"]') - .should('exist') - .invoke('text') - .then((text) => { - interceptURL('GET', '/api/v1/users?limit=15', 'getUsers'); - // Clicking on edit owner button - cy.get('[data-testid="edit-owner"]').click(); - - cy.get('.user-team-select-popover').contains('Users').click(); - cy.get('[data-testid="owner-select-users-search-bar"]').type(text); - cy.get('[data-testid="selectable-list"]') - .eq(1) - .find(`[title="${text.trim()}"]`) - .click(); - - // Asserting the added name - cy.get('[data-testid="owner-link"]').should('contain', text.trim()); - }); -}; - -export const mySqlConnectionInput = () => { - cy.get('#root\\/username').type(Cypress.env('mysqlUsername')); - checkServiceFieldSectionHighlighting('username'); - cy.get('#root\\/authType\\/password').type(Cypress.env('mysqlPassword')); - checkServiceFieldSectionHighlighting('password'); - cy.get('#root\\/hostPort').type(Cypress.env('mysqlHostPort')); - checkServiceFieldSectionHighlighting('hostPort'); - cy.get('#root\\/databaseSchema').type(Cypress.env('mysqlDatabaseSchema')); - checkServiceFieldSectionHighlighting('databaseSchema'); -}; - -export const login = (username, password) => { - cy.visit('/'); - interceptURL('POST', '/api/v1/users/login', 'loginUser'); - cy.get('[id="email"]').should('be.visible').clear().type(username); - cy.get('[id="password"]').should('be.visible').clear().type(password); - - // Don't want to show any popup in the tests - cy.setCookie(`STAR_OMD_USER_${username.split('@')[0]}`, 'true'); - - // Get version and set cookie to hide version banner - cy.request({ - method: 'GET', - url: `api/v1/system/version`, - }).then((res) => { - const version = res.body.version; - const versionCookie = `VERSION_${version - .split('-')[0] - .replaceAll('.', '_')}`; - - cy.setCookie(versionCookie, 'true'); - window.localStorage.setItem('loggedInUsers', username.split('@')[0]); - }); - - cy.get('.ant-btn').contains('Login').should('be.visible').click(); - cy.wait('@loginUser'); -}; - -export const selectTeamHierarchy = (index) => { - if (index > 0) { - cy.get('[data-testid="team-type"]') - .invoke('text') - .then((text) => { - cy.log(text); - checkTeamTypeOptions(text); - cy.log('check type', text); - cy.get( - `.ant-select-dropdown [title="${getTeamType(text).childTeamType}"]` - ).click(); - }); - } else { - checkTeamTypeOptions('BusinessUnit'); - - cy.get(`.ant-select-dropdown [title='BusinessUnit']`) - .should('exist') - .should('be.visible') - .click(); - } -}; - -export const addTeam = (teamDetails, index, isHierarchy) => { - interceptURL('GET', '/api/v1/teams*', 'addTeam'); - // Fetching the add button and clicking on it - if (index > 0) { - cy.get('[data-testid="add-placeholder-button"]').click(); - } else { - cy.get('[data-testid="add-team"]').click(); - } - - verifyResponseStatusCode('@addTeam', 200); - - // Entering team details - cy.get('[data-testid="name"]').type(teamDetails.name); - - cy.get('[data-testid="display-name"]').type(teamDetails.name); - - cy.get('[data-testid="email"]').type(teamDetails.email); - - cy.get('[data-testid="team-selector"]').click(); - - if (isHierarchy) { - selectTeamHierarchy(index); - } else { - cy.get(`.ant-select-dropdown [title="${teamDetails.teamType}"]`).click(); - } - - cy.get(descriptionBox).type(teamDetails.description); - - interceptURL('POST', '/api/v1/teams', 'saveTeam'); - interceptURL('GET', '/api/v1/team*', 'createTeam'); - - // Saving the created team - cy.get('[form="add-team-form"]').scrollIntoView().click(); - - verifyResponseStatusCode('@saveTeam', 201); - verifyResponseStatusCode('@createTeam', 200); -}; - -export const deleteEntity = ( - entityName, - serviceName, - entity, - successMessageEntityName, - deletionType = 'hard' -) => { - cy.get('[data-testid="manage-button"]').click(); - - cy.get('[data-testid="delete-button-title"]').click(); - - cy.get('.ant-modal-header').should('contain', entityName); - - cy.get(`[data-testid="${deletionType}-delete-option"]`).click(); - - cy.get('[data-testid="confirm-button"]').should('be.disabled'); - cy.get('[data-testid="confirmation-text-input"]').type(DELETE_TERM); - - interceptURL( - 'DELETE', - `api/v1/${entity}/*?hardDelete=${deletionType === 'hard'}&recursive=true`, - `${deletionType}DeleteTable` - ); - cy.get('[data-testid="confirm-button"]').should('not.be.disabled'); - cy.get('[data-testid="confirm-button"]').click(); - verifyResponseStatusCode(`@${deletionType}DeleteTable`, 200); - - toastNotification(`deleted successfully!`); -}; - -export const visitServiceDetailsPage = ( - settingsMenuId, - serviceCategory, - serviceName, - isServiceDeleted = false -) => { - interceptURL( - 'GET', - `/api/v1/search/query?q=*${isServiceDeleted ? 'deleted=true' : ''}`, - 'searchService' - ); - interceptURL('GET', `/api/v1/services/${serviceCategory}*`, 'getServices'); - cy.settingClick(settingsMenuId); - verifyResponseStatusCode('@getServices', 200); - - if (isServiceDeleted) { - cy.get('[data-testid="show-deleted-switch"]').click(); - } - - interceptURL( - 'GET', - `api/v1/services/${serviceCategory}/name/${serviceName}*`, - 'getServiceDetails' - ); - - cy.get('[data-testid="searchbar"]').type(serviceName); - - verifyResponseStatusCode('@searchService', 200); - - cy.get(`[data-testid="service-name-${serviceName}"]`).click(); - - verifyResponseStatusCode('@getServiceDetails', 200); -}; - -export const visitDataModelPage = (dataModelFQN, dataModelName) => { - interceptURL('GET', '/api/v1/services/dashboardServices*', 'getServices'); - cy.settingClick(GlobalSettingOptions.DASHBOARDS); - verifyResponseStatusCode('@getServices', 200); - - interceptURL( - 'GET', - 'api/v1/services/dashboardServices/name/sample_looker*', - 'getDashboardDetails' - ); - interceptURL( - 'GET', - '/api/v1/dashboard/datamodels?service=sample_looker*', - 'getDataModels' - ); - - cy.get('[data-testid="service-name-sample_looker"]').scrollIntoView().click(); - - verifyResponseStatusCode('@getDashboardDetails', 200); - verifyResponseStatusCode('@getDataModels', 200); - - cy.get('[data-testid="data-model"]').scrollIntoView().click(); - - verifyResponseStatusCode('@getDataModels', 200); - - interceptURL( - 'GET', - `/api/v1/dashboard/datamodels/name/${dataModelFQN}*`, - 'getDataModelDetails' - ); - - cy.get(`[data-testid="data-model-${dataModelName}"]`) - .scrollIntoView() - .click(); - - verifyResponseStatusCode('@getDataModelDetails', 200); -}; - -export const signupAndLogin = (email, password, firstName, lastName) => { - return new Cypress.Promise((resolve) => { - let createdUserId = ''; - interceptURL('GET', 'api/v1/system/config/auth', 'getLoginPage'); - cy.visit('/'); - verifyResponseStatusCode('@getLoginPage', 200); - - // Click on create account button - cy.get('[data-testid="signup"]').scrollIntoView().click(); - - // Enter first name - cy.get('[id="firstName"]').type(firstName); - - // Enter last name - cy.get('[id="lastName"]').type(lastName); - - // Enter email - cy.get('[id="email"]').type(email); - - // Enter password - cy.get('[id="password"]').type(password); - cy.get('[id="password"]') - .should('have.attr', 'type') - .should('eq', 'password'); - - // Confirm password - cy.get('[id="confirmPassword"]').type(password); - - // Click on create account button - cy.get('.ant-btn').contains('Create Account').click(); - - cy.url().should('eq', `${BASE_URL}/signin`).and('contain', 'signin'); - - // Login with the created user - login(email, password); - // cy.goToHomePage(true); - cy.url().should('eq', `${BASE_URL}/my-data`); - - // Verify user profile - cy.get('[data-testid="avatar"]').first().trigger('mouseover').click(); - cy.get('[data-testid="user-name"]') - .should('be.visible') - .invoke('text') - .should('contain', `${firstName}${lastName}`); - - interceptURL('GET', 'api/v1/users/name/*', 'getUserPage'); - - cy.get('[data-testid="user-name"]').click({ force: true }); - cy.wait('@getUserPage').then((response) => { - createdUserId = response.response.body.id; - resolve(createdUserId); // Resolve the promise with the createdUserId - }); - cy.get( - '[data-testid="user-profile"] [data-testid="user-profile-details"]' - ).should('contain', `${firstName}${lastName}`); - }); -}; - -export const addTags = (classificationName, tagName, entity) => { - cy.get( - '[data-testid="entity-right-panel"] [data-testid="tags-container"] [data-testid="add-tag"]' - ).click(); - - cy.get('[data-testid="tag-selector"] #tagsForm_tags').type(tagName); - - cy.get(`[data-testid="tag-${classificationName}.${tagName}"]`).click(); - - interceptURL('PATCH', `/api/v1/${entity}/*`, 'patchTag'); - - cy.get('[data-testid="saveAssociatedTag"]').click(); - - verifyResponseStatusCode('@patchTag', 200); - - cy.get( - `[data-testid="entity-right-panel"] [data-testid="tags-container"] [data-testid="tag-${classificationName}.${tagName}"]` - ) - .scrollIntoView() - .should('be.visible'); -}; - -export const removeTags = (classificationName, tagName, entity) => { - cy.get( - '[data-testid="entity-right-panel"] [data-testid="tags-container"] [data-testid="edit-button"]' - ).click(); - - cy.get( - `[data-testid="selected-tag-${classificationName}.${tagName}"] [data-testid="remove-tags"]` - ).click(); - - interceptURL('PATCH', `/api/v1/${entity}/*`, `patchTag`); - - cy.get('[data-testid="saveAssociatedTag"]').click(); - - verifyResponseStatusCode(`@patchTag`, 200); - - cy.get( - '[data-testid="entity-right-panel"] [data-testid="tags-container"]' - ).then(($body) => { - const manageButton = $body.find( - `[data-testid="tag-${classificationName}.${tagName}"]` - ); - - expect(manageButton.length).to.equal(0); - }); -}; - -export const addTableFieldTags = ( - dataRowKey, - classificationName, - tagName, - entity -) => { - cy.get( - `[data-row-key="${dataRowKey}"] [data-testid="tags-container"] [data-testid="add-tag"]` - ).click(); - - cy.get('[data-testid="tag-selector"] #tagsForm_tags') - .scrollIntoView() - .type(tagName); - - cy.get(`[data-testid="tag-${classificationName}.${tagName}"]`).click(); - - interceptURL('PATCH', `/api/v1/${entity}/*`, 'patchTag'); - - cy.get('[data-testid="saveAssociatedTag"]').click(); - - verifyResponseStatusCode('@patchTag', 200); - - cy.get( - `[data-row-key="${dataRowKey}"] [data-testid="tag-${classificationName}.${tagName}"]` - ) - .scrollIntoView() - .should('be.visible'); -}; - -export const removeTableFieldTags = ( - dataRowKey, - classificationName, - tagName, - entity -) => { - cy.get( - `[data-row-key="${dataRowKey}"] [data-testid="tags-container"] [data-testid="edit-button"]` - ).click(); - - cy.get( - `[data-testid="selected-tag-${classificationName}.${tagName}"] [data-testid="remove-tags"]` - ).click(); - - interceptURL('PATCH', `/api/v1/${entity}/*`, `patchTag`); - - cy.get('[data-testid="saveAssociatedTag"]').click(); - - verifyResponseStatusCode(`@patchTag`, 200); - - cy.get(`[data-row-key="${dataRowKey}"]`).then(($body) => { - const manageButton = $body.find( - `[data-testid="tag-${classificationName}.${tagName}"]` - ); - - expect(manageButton.length).to.equal(0); - }); -}; - -export const updateDescription = (description, entity) => { - cy.get( - '[data-testid="asset-description-container"] [data-testid="edit-description"]' - ).click(); - - cy.get(descriptionBox).should('be.visible').click().clear().type(description); - - interceptURL('PATCH', `/api/v1/${entity}/*`, 'updateDescription'); - - cy.get('[data-testid="save"]').click(); - - verifyResponseStatusCode('@updateDescription', 200); -}; - -export const updateTableFieldDescription = ( - dataRowKey, - description, - entity -) => { - cy.get( - `[data-row-key="${dataRowKey}"] [data-testid="description"] [data-testid="edit-button"]` - ).click(); - - cy.get(descriptionBox).should('be.visible').click().clear().type(description); - - interceptURL('PATCH', `/api/v1/${entity}/*`, 'updateDescription'); - - cy.get('[data-testid="save"]').click(); - - verifyResponseStatusCode('@updateDescription', 200); -}; - -export const visitDatabaseDetailsPage = ({ - settingsMenuId, - serviceCategory, - serviceName, - databaseRowKey, - databaseName, - isDeleted = false, -}) => { - visitServiceDetailsPage( - settingsMenuId, - serviceCategory, - serviceName, - isDeleted - ); - - if (isDeleted) { - interceptURL('GET', `/api/v1/databases*include=deleted*`, 'getDatabases'); - cy.get('[data-testid="show-deleted"]').click(); - verifyResponseStatusCode('@getDatabases', 200); - } - - cy.get(`[data-row-key="${databaseRowKey}"]`).contains(databaseName).click(); -}; - -export const visitDatabaseSchemaDetailsPage = ({ - settingsMenuId, - serviceCategory, - serviceName, - databaseRowKey, - databaseName, - databaseSchemaRowKey, - databaseSchemaName, - isDeleted = false, -}) => { - visitDatabaseDetailsPage({ - settingsMenuId, - serviceCategory, - serviceName, - databaseRowKey, - databaseName, - isDeleted, - }); - - if (isDeleted) { - interceptURL( - 'GET', - `/api/v1/databaseSchemas*include=deleted*`, - 'getDatabaseSchemas' - ); - cy.get('[data-testid="show-deleted"]').click(); - verifyResponseStatusCode('@getDatabaseSchemas', 200); - } - - cy.get(`[data-row-key="${databaseSchemaRowKey}"]`) - .contains(databaseSchemaName) - .click(); -}; diff --git a/openmetadata-ui/src/main/resources/ui/cypress/common/common.ts b/openmetadata-ui/src/main/resources/ui/cypress/common/common.ts new file mode 100644 index 000000000000..cb71c7e794c1 --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/cypress/common/common.ts @@ -0,0 +1,440 @@ +/* + * Copyright 2022 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { RouteHandler, WaitOptions } from 'cypress/types/net-stubbing'; +import { BASE_URL, DELETE_TERM, NEW_TAG } from '../constants/constants'; +import { GlobalSettingOptions } from '../constants/settings.constant'; + +export const descriptionBox = + '.toastui-editor-md-container > .toastui-editor > .ProseMirror'; +export const uuid = () => Cypress._.random(0, 1e6); +export const RETRY_TIMES = 4; +export const BASE_WAIT_TIME = 20000; + +export const replaceAllSpacialCharWith_ = (text: string) => { + return text.replaceAll(/[&/\\#, +()$~%.'":*?<>{}]/g, '_'); +}; + +export const checkServiceFieldSectionHighlighting = (field: string) => { + cy.get(`[data-id="${field}"]`).should( + 'have.attr', + 'data-highlighted', + 'true' + ); +}; + +// intercepting URL with cy.intercept +export const interceptURL = ( + method: string, + url: string, + alias: string, + callback?: RouteHandler +) => { + cy.intercept({ method: method, url: url }, callback).as(alias); +}; + +// waiting for response and validating the response status code +export const verifyResponseStatusCode = ( + alias: string, + responseCode: number, + option?: Partial, + hasMultipleResponseCode = false +) => { + if (hasMultipleResponseCode) { + return cy + .wait(alias, option) + .its('response.statusCode') + .should('be.oneOf', responseCode); + } else { + return cy + .wait(alias, option) + .its('response.statusCode') + .should('eq', responseCode); + } +}; + +// waiting for multiple response and validating the response status code +export const verifyMultipleResponseStatusCode = ( + alias: string[] = [], + responseCode = 200, + option?: Partial +) => { + cy.wait(alias, option).then((data) => { + data.map((value) => expect(value.response?.statusCode).eq(responseCode)); + }); +}; + +// Storing the created service name and the type of service for later use + +// add new tag to entity and its table +export const addNewTagToEntity = (term: typeof NEW_TAG) => { + const { name, fqn } = term; + + cy.get( + '[data-testid="classification-tags-0"] [data-testid="entity-tags"] [data-testid="add-tag"]' + ) + .eq(0) + .should('be.visible') + .scrollIntoView() + .click(); + + cy.get('[data-testid="tag-selector"] input').should('be.visible').type(name); + + cy.get(`[data-testid="tag-${fqn}"]`).should('be.visible').click(); + // to close popup + cy.clickOutside(); + + cy.get('[data-testid="tag-selector"] > .ant-select-selector').contains(name); + cy.get('[data-testid="saveAssociatedTag"]') + .scrollIntoView() + .should('be.visible') + .click(); + cy.get('[data-testid="classification-tags-0"] [data-testid="tags-container"]') + .scrollIntoView() + .contains(name) + .should('exist'); + if (term.color) { + cy.get( + '[data-testid="classification-tags-0"] [data-testid="tags-container"] [data-testid="icon"]' + ).should('be.visible'); + } +}; + +export const toastNotification = (msg: string, closeToast = true) => { + cy.get('.Toastify__toast-body').should('contain.text', msg); + cy.wait(200); + if (closeToast) { + cy.get('.Toastify__close-button').click(); + } +}; + +export const deleteEntity = ( + entityName: string, + serviceName: string, + entity: string, + successMessageEntityName: string, + deletionType = 'hard' +) => { + cy.get('[data-testid="manage-button"]').click(); + + cy.get('[data-testid="delete-button-title"]').click(); + + cy.get('.ant-modal-header').should('contain', entityName); + + cy.get(`[data-testid="${deletionType}-delete-option"]`).click(); + + cy.get('[data-testid="confirm-button"]').should('be.disabled'); + cy.get('[data-testid="confirmation-text-input"]').type(DELETE_TERM); + + interceptURL( + 'DELETE', + `api/v1/${entity}/*?hardDelete=${deletionType === 'hard'}&recursive=true`, + `${deletionType}DeleteTable` + ); + cy.get('[data-testid="confirm-button"]').should('not.be.disabled'); + cy.get('[data-testid="confirm-button"]').click(); + verifyResponseStatusCode(`@${deletionType}DeleteTable`, 200); + + toastNotification(`deleted successfully!`); +}; + +export const visitServiceDetailsPage = ( + settingsMenuId: string, + serviceCategory: string, + serviceName: string, + isServiceDeleted = false +) => { + interceptURL( + 'GET', + `/api/v1/search/query?q=*${isServiceDeleted ? 'deleted=true' : ''}`, + 'searchService' + ); + interceptURL('GET', `/api/v1/services/${serviceCategory}*`, 'getServices'); + cy.settingClick(settingsMenuId); + verifyResponseStatusCode('@getServices', 200); + + if (isServiceDeleted) { + cy.get('[data-testid="show-deleted-switch"]').click(); + } + + interceptURL( + 'GET', + `api/v1/services/${serviceCategory}/name/${serviceName}*`, + 'getServiceDetails' + ); + + cy.get('[data-testid="searchbar"]').type(serviceName); + + verifyResponseStatusCode('@searchService', 200); + + cy.get(`[data-testid="service-name-${serviceName}"]`).click(); + + verifyResponseStatusCode('@getServiceDetails', 200); +}; + +export const visitDataModelPage = ( + dataModelFQN: string, + dataModelName: string, + serviceName: string +) => { + interceptURL('GET', '/api/v1/services/dashboardServices*', 'getServices'); + cy.settingClick(GlobalSettingOptions.DASHBOARDS); + verifyResponseStatusCode('@getServices', 200); + + interceptURL( + 'GET', + `api/v1/services/dashboardServices/name/${serviceName}*`, + 'getDashboardDetails' + ); + interceptURL( + 'GET', + `/api/v1/dashboard/datamodels?service=${serviceName}*`, + 'getDataModels' + ); + + cy.get(`[data-testid="service-name-${serviceName}"]`) + .scrollIntoView() + .click(); + + verifyResponseStatusCode('@getDashboardDetails', 200); + verifyResponseStatusCode('@getDataModels', 200); + + cy.get('[data-testid="data-model"]').scrollIntoView().click(); + + verifyResponseStatusCode('@getDataModels', 200); + + interceptURL( + 'GET', + `/api/v1/dashboard/datamodels/name/${dataModelFQN}*`, + 'getDataModelDetails' + ); + + cy.get(`[data-testid="data-model-${dataModelName}"]`) + .scrollIntoView() + .click(); + + verifyResponseStatusCode('@getDataModelDetails', 200); +}; + +export const signupAndLogin = ( + email: string, + password: string, + firstName: string, + lastName: string +) => { + return new Cypress.Promise((resolve) => { + let createdUserId = ''; + interceptURL('GET', 'api/v1/system/config/auth', 'getLoginPage'); + cy.visit('/'); + verifyResponseStatusCode('@getLoginPage', 200); + + // Click on create account button + cy.get('[data-testid="signup"]').scrollIntoView().click(); + + // Enter first name + cy.get('[id="firstName"]').type(firstName); + + // Enter last name + cy.get('[id="lastName"]').type(lastName); + + // Enter email + cy.get('[id="email"]').type(email); + + // Enter password + cy.get('[id="password"]').type(password); + cy.get('[id="password"]') + .should('have.attr', 'type') + .should('eq', 'password'); + + // Confirm password + cy.get('[id="confirmPassword"]').type(password); + + // Click on create account button + cy.get('.ant-btn').contains('Create Account').click(); + + cy.url().should('eq', `${BASE_URL}/signin`).and('contain', 'signin'); + + // Login with the created user + cy.login(email, password); + + cy.url().should('eq', `${BASE_URL}/my-data`); + + // Verify user profile + cy.get('[data-testid="avatar"]').first().trigger('mouseover').click(); + cy.get('[data-testid="user-name"]') + .should('be.visible') + .invoke('text') + .should('contain', `${firstName}${lastName}`); + + interceptURL('GET', 'api/v1/users/name/*', 'getUserPage'); + + cy.get('[data-testid="user-name"]').click({ force: true }); + cy.wait('@getUserPage').then((response) => { + createdUserId = response.response?.body.id; + resolve(createdUserId); // Resolve the promise with the createdUserId + }); + cy.get( + '[data-testid="user-profile"] [data-testid="user-profile-details"]' + ).should('contain', `${firstName}${lastName}`); + }); +}; + +export const addTableFieldTags = ( + dataRowKey: string, + classificationName: string, + tagName: string, + entity: string +) => { + cy.get( + `[data-row-key="${dataRowKey}"] [data-testid="tags-container"] [data-testid="add-tag"]` + ).click(); + + cy.get('[data-testid="tag-selector"] #tagsForm_tags') + .scrollIntoView() + .type(tagName); + + cy.get(`[data-testid="tag-${classificationName}.${tagName}"]`).click(); + + interceptURL('PATCH', `/api/v1/${entity}/*`, 'patchTag'); + + cy.get('[data-testid="saveAssociatedTag"]').click(); + + verifyResponseStatusCode('@patchTag', 200); + + cy.get( + `[data-row-key="${dataRowKey}"] [data-testid="tag-${classificationName}.${tagName}"]` + ) + .scrollIntoView() + .should('be.visible'); +}; + +export const removeTableFieldTags = ( + dataRowKey: string, + classificationName: string, + tagName: string, + entity: string +) => { + cy.get( + `[data-row-key="${dataRowKey}"] [data-testid="tags-container"] [data-testid="edit-button"]` + ).click(); + + cy.get( + `[data-testid="selected-tag-${classificationName}.${tagName}"] [data-testid="remove-tags"]` + ).click(); + + interceptURL('PATCH', `/api/v1/${entity}/*`, `patchTag`); + + cy.get('[data-testid="saveAssociatedTag"]').click(); + + verifyResponseStatusCode(`@patchTag`, 200); + + cy.get(`[data-row-key="${dataRowKey}"]`).then(($body) => { + const manageButton = $body.find( + `[data-testid="tag-${classificationName}.${tagName}"]` + ); + + expect(manageButton.length).to.equal(0); + }); +}; + +export const updateTableFieldDescription = ( + dataRowKey: string, + description: string, + entity: string +) => { + cy.get( + `[data-row-key="${dataRowKey}"] [data-testid="description"] [data-testid="edit-button"]` + ).click(); + + cy.get(descriptionBox).should('be.visible').click().clear().type(description); + + interceptURL('PATCH', `/api/v1/${entity}/*`, 'updateDescription'); + + cy.get('[data-testid="save"]').click(); + + verifyResponseStatusCode('@updateDescription', 200); +}; + +export const visitDatabaseDetailsPage = ({ + settingsMenuId, + serviceCategory, + serviceName, + databaseRowKey, + databaseName, + isDeleted = false, +}: { + settingsMenuId: string; + serviceCategory: string; + serviceName: string; + databaseRowKey: string; + databaseName: string; + isDeleted?: boolean; +}) => { + visitServiceDetailsPage( + settingsMenuId, + serviceCategory, + serviceName, + isDeleted + ); + + if (isDeleted) { + interceptURL('GET', `/api/v1/databases*include=deleted*`, 'getDatabases'); + cy.get('[data-testid="show-deleted"]').click(); + verifyResponseStatusCode('@getDatabases', 200); + } + + cy.get(`[data-row-key="${databaseRowKey}"]`).contains(databaseName).click(); +}; + +export const visitDatabaseSchemaDetailsPage = ({ + settingsMenuId, + serviceCategory, + serviceName, + databaseRowKey, + databaseName, + databaseSchemaRowKey, + databaseSchemaName, + isDeleted = false, +}: { + settingsMenuId: string; + serviceCategory: string; + serviceName: string; + databaseRowKey: string; + databaseName: string; + databaseSchemaRowKey: string; + databaseSchemaName: string; + isDeleted?: boolean; +}) => { + visitDatabaseDetailsPage({ + settingsMenuId, + serviceCategory, + serviceName, + databaseRowKey, + databaseName, + isDeleted, + }); + + if (isDeleted) { + interceptURL( + 'GET', + `/api/v1/databaseSchemas*include=deleted*`, + 'getDatabaseSchemas' + ); + cy.get('[data-testid="show-deleted"]').click(); + verifyResponseStatusCode('@getDatabaseSchemas', 200); + } + + cy.get(`[data-row-key="${databaseSchemaRowKey}"]`) + .contains(databaseSchemaName) + .click(); +}; diff --git a/openmetadata-ui/src/main/resources/ui/cypress/common/serviceUtils.js b/openmetadata-ui/src/main/resources/ui/cypress/common/serviceUtils.ts similarity index 100% rename from openmetadata-ui/src/main/resources/ui/cypress/common/serviceUtils.js rename to openmetadata-ui/src/main/resources/ui/cypress/common/serviceUtils.ts diff --git a/openmetadata-ui/src/main/resources/ui/cypress/constants/Alert.constant.js b/openmetadata-ui/src/main/resources/ui/cypress/constants/Alert.constant.ts similarity index 96% rename from openmetadata-ui/src/main/resources/ui/cypress/constants/Alert.constant.js rename to openmetadata-ui/src/main/resources/ui/cypress/constants/Alert.constant.ts index 7b7d431e3df9..0a246e733eaf 100644 --- a/openmetadata-ui/src/main/resources/ui/cypress/constants/Alert.constant.js +++ b/openmetadata-ui/src/main/resources/ui/cypress/constants/Alert.constant.ts @@ -11,9 +11,6 @@ * limitations under the License. */ -// eslint-disable-next-line spaced-comment -/// - import { uuid } from '../common/common'; import { DATABASE_SERVICE, @@ -152,10 +149,6 @@ export const OBSERVABILITY_CREATION_DETAILS = { }, ], actions: [ - { - name: 'Get Schema Changes', - exclude: true, - }, { name: 'Get Test Case Status Updates', inputs: [ @@ -210,10 +203,6 @@ export const OBSERVABILITY_CREATION_DETAILS = { }, ], actions: [ - { - name: 'Get Schema Changes', - exclude: true, - }, { name: 'Get Test Case Status Updates belonging to a Test Suite', inputs: [ diff --git a/openmetadata-ui/src/main/resources/ui/cypress/constants/CustomProperty.constant.ts b/openmetadata-ui/src/main/resources/ui/cypress/constants/CustomProperty.constant.ts index 91c0fac6749d..0de91637271b 100644 --- a/openmetadata-ui/src/main/resources/ui/cypress/constants/CustomProperty.constant.ts +++ b/openmetadata-ui/src/main/resources/ui/cypress/constants/CustomProperty.constant.ts @@ -23,5 +23,5 @@ export const CustomPropertySupportedEntityList = [ EntityType.Container, EntityType.MlModel, EntityType.GlossaryTerm, - EntityType.SeachIndex, + EntityType.SearchIndex, ]; diff --git a/openmetadata-ui/src/main/resources/ui/cypress/constants/Entity.interface.ts b/openmetadata-ui/src/main/resources/ui/cypress/constants/Entity.interface.ts index fd48d3cf5b96..d447cb7abc43 100644 --- a/openmetadata-ui/src/main/resources/ui/cypress/constants/Entity.interface.ts +++ b/openmetadata-ui/src/main/resources/ui/cypress/constants/Entity.interface.ts @@ -21,7 +21,7 @@ export enum EntityType { Domain = 'domains', Glossary = 'glossaries', GlossaryTerm = 'glossaryTerms', - SeachIndex = 'searchIndexes', + SearchIndex = 'searchIndexes', DatabaseService = 'services/databaseServices', DashboardService = 'services/dashboardServices', StorageService = 'services/storageServices', @@ -33,6 +33,7 @@ export enum EntityType { Database = 'databases', DatabaseSchema = 'databaseSchemas', DataModel = 'dashboard/datamodels', + User = 'users', } export const EXPLORE_PAGE_TABS: Record< @@ -50,6 +51,7 @@ export const EXPLORE_PAGE_TABS: Record< | EntityType.GlossaryTerm | EntityType.Domain | EntityType.MetadataService + | EntityType.User >, string > = { @@ -59,7 +61,7 @@ export const EXPLORE_PAGE_TABS: Record< [EntityType.Topic]: 'topics', [EntityType.MlModel]: 'ml models', [EntityType.Container]: 'containers', - [EntityType.SeachIndex]: 'search indexes', + [EntityType.SearchIndex]: 'search indexes', [EntityType.Table]: 'tables', [EntityType.StoreProcedure]: 'stored procedures', [EntityType.Glossary]: 'glossaries', @@ -79,6 +81,7 @@ export const SEARCH_INDEX: Record< | EntityType.DatabaseSchema | EntityType.GlossaryTerm | EntityType.MetadataService + | EntityType.User >, string > = { @@ -88,7 +91,7 @@ export const SEARCH_INDEX: Record< [EntityType.Topic]: 'topic_search_index', [EntityType.MlModel]: 'mlmodel_search_index', [EntityType.Container]: 'container_search_index', - [EntityType.SeachIndex]: 'search_entity_search_index', + [EntityType.SearchIndex]: 'search_entity_search_index', [EntityType.Table]: 'table_search_index', [EntityType.StoreProcedure]: 'store_procedure_search_index', [EntityType.Glossary]: 'glossary_search_index', diff --git a/openmetadata-ui/src/main/resources/ui/cypress/constants/EntityConstant.js b/openmetadata-ui/src/main/resources/ui/cypress/constants/EntityConstant.ts similarity index 97% rename from openmetadata-ui/src/main/resources/ui/cypress/constants/EntityConstant.js rename to openmetadata-ui/src/main/resources/ui/cypress/constants/EntityConstant.ts index 8a7fdab3b62c..89d477c80d4b 100644 --- a/openmetadata-ui/src/main/resources/ui/cypress/constants/EntityConstant.js +++ b/openmetadata-ui/src/main/resources/ui/cypress/constants/EntityConstant.ts @@ -11,10 +11,8 @@ * limitations under the License. */ -// eslint-disable-next-line spaced-comment -/// - import { DATA_ASSETS, uuid } from './constants'; +import { EntityType } from './Entity.interface'; import { SERVICE_CATEGORIES } from './service.constants'; const DATABASE_SERVICE_NAME = `cy-database-service-${uuid()}`; @@ -384,7 +382,7 @@ export const VISIT_ENTITIES_DATA = { table: { term: DATABASE_SERVICE.entity.name, displayName: DATABASE_SERVICE.entity.name, - entity: DATA_ASSETS.tables, + entity: EntityType.Table, serviceName: DATABASE_SERVICE.service.name, schemaName: DATABASE_SERVICE.schema.name, entityType: 'Table', @@ -392,41 +390,41 @@ export const VISIT_ENTITIES_DATA = { topic: { term: MESSAGING_SERVICE.entity.name, displayName: MESSAGING_SERVICE.entity.name, - entity: DATA_ASSETS.topics, + entity: EntityType.Topic, serviceName: MESSAGING_SERVICE.service.name, entityType: 'Topic', }, dashboard: { term: DASHBOARD_SERVICE.entity.name, displayName: DASHBOARD_SERVICE.entity.name, - entity: DATA_ASSETS.dashboards, + entity: EntityType.Dashboard, serviceName: DASHBOARD_SERVICE.service.name, entityType: 'Dashboard', }, pipeline: { term: PIPELINE_SERVICE.entity.name, displayName: PIPELINE_SERVICE.entity.name, - entity: DATA_ASSETS.pipelines, + entity: EntityType.Pipeline, serviceName: PIPELINE_SERVICE.service.name, entityType: 'Pipeline', }, mlmodel: { term: ML_MODEL_SERVICE.entity.name, displayName: ML_MODEL_SERVICE.entity.name, - entity: DATA_ASSETS.mlmodels, + entity: EntityType.MlModel, serviceName: ML_MODEL_SERVICE.service.name, entityType: 'ML Model', }, storedProcedure: { term: STORED_PROCEDURE_DETAILS.name, displayName: STORED_PROCEDURE_DETAILS.name, - entity: DATA_ASSETS.storedProcedures, + entity: EntityType.StoreProcedure, serviceName: DATABASE_SERVICE_DETAILS.name, entityType: 'Stored Procedure', }, dataModel: { term: DASHBOARD_DATA_MODEL_DETAILS.name, - entity: DATA_ASSETS.dataModel, + entity: EntityType.DataModel, serviceName: DASHBOARD_DATA_MODEL_DETAILS.service, displayName: DASHBOARD_DATA_MODEL_DETAILS.name, entityType: 'Data Model', @@ -434,7 +432,7 @@ export const VISIT_ENTITIES_DATA = { container: { term: STORAGE_SERVICE.entity.name, displayName: STORAGE_SERVICE.entity.name, - entity: 'containers', + entity: EntityType.Container, serviceName: STORAGE_SERVICE.service.name, }, }; diff --git a/openmetadata-ui/src/main/resources/ui/cypress/constants/SearchIndexDetails.constants.js b/openmetadata-ui/src/main/resources/ui/cypress/constants/SearchIndexDetails.constants.ts similarity index 96% rename from openmetadata-ui/src/main/resources/ui/cypress/constants/SearchIndexDetails.constants.js rename to openmetadata-ui/src/main/resources/ui/cypress/constants/SearchIndexDetails.constants.ts index e162fb01735e..0648b92766a4 100644 --- a/openmetadata-ui/src/main/resources/ui/cypress/constants/SearchIndexDetails.constants.js +++ b/openmetadata-ui/src/main/resources/ui/cypress/constants/SearchIndexDetails.constants.ts @@ -11,11 +11,8 @@ * limitations under the License. */ -// eslint-disable-next-line spaced-comment -/// - import { uuid } from '../common/common'; -import { SEARCH_SERVICE } from '../constants/EntityConstant'; +import { SEARCH_SERVICE } from './EntityConstant'; export const TIER = 'Tier1'; export const TAG_1 = { diff --git a/openmetadata-ui/src/main/resources/ui/cypress/constants/SoftDeleteFlow.constants.js b/openmetadata-ui/src/main/resources/ui/cypress/constants/SoftDeleteFlow.constants.ts similarity index 99% rename from openmetadata-ui/src/main/resources/ui/cypress/constants/SoftDeleteFlow.constants.js rename to openmetadata-ui/src/main/resources/ui/cypress/constants/SoftDeleteFlow.constants.ts index f4daf9e55833..a3f27426c9b9 100644 --- a/openmetadata-ui/src/main/resources/ui/cypress/constants/SoftDeleteFlow.constants.js +++ b/openmetadata-ui/src/main/resources/ui/cypress/constants/SoftDeleteFlow.constants.ts @@ -11,9 +11,6 @@ * limitations under the License. */ -// eslint-disable-next-line spaced-comment -/// - import { uuid } from '../common/common'; import { DATA_ASSETS } from './constants'; import { diff --git a/openmetadata-ui/src/main/resources/ui/cypress/constants/Version.constants.js b/openmetadata-ui/src/main/resources/ui/cypress/constants/Version.constants.ts similarity index 93% rename from openmetadata-ui/src/main/resources/ui/cypress/constants/Version.constants.js rename to openmetadata-ui/src/main/resources/ui/cypress/constants/Version.constants.ts index 84fdd3e4102c..bf1e1250b208 100644 --- a/openmetadata-ui/src/main/resources/ui/cypress/constants/Version.constants.js +++ b/openmetadata-ui/src/main/resources/ui/cypress/constants/Version.constants.ts @@ -12,11 +12,28 @@ */ import { uuid } from '../common/common'; +import { EntityType } from './Entity.interface'; +import { DASHBOARD_DETAILS } from './EntityConstant'; import { SERVICE_CATEGORIES } from './service.constants'; import { GlobalSettingOptions } from './settings.constant'; -export const OWNER = 'Amber Green'; -export const REVIEWER = 'Amanda York'; +export const OWNER_NAME = `user${uuid()}`; + +export const OWNER_DETAILS = { + firstName: `first-name-${uuid()}`, + lastName: `last-name-${uuid()}`, + email: `${OWNER_NAME}@example.com`, + password: 'User@OMD123', +}; + +export const REVIEWER_NAME = `user${uuid()}`; + +export const REVIEWER_DETAILS = { + firstName: `first-name-${uuid()}`, + lastName: `last-name-${uuid()}`, + email: `${REVIEWER_NAME}@example.com`, + password: 'User@OMD123', +}; export const TIER = 'Tier1'; const TABLE_NAME = `cypress_version_table-${uuid()}`; @@ -26,10 +43,20 @@ const PIPELINE_NAME = `cypress_version_pipeline-${uuid()}`; const ML_MODEL_NAME = `cypress_version_ml_model-${uuid()}`; const CONTAINER_NAME = `cypress_version_container-${uuid()}`; const SEARCH_INDEX_NAME = `cypress_version_search_index-${uuid()}`; -const STORED_PROCEDURE_NAME = `cypress_version_stored_procedure-${uuid()}`; const DATA_MODEL_NAME = `cypress_version_data_model_${uuid()}`; -const TABLE_DETAILS_FOR_VERSION_TEST = { +type TableColumn = Record; +type TableColumnWithChildren = + | Record + | { + children?: TableColumn[]; + }; + +const TABLE_DETAILS_FOR_VERSION_TEST: { + name: string; + columns: TableColumnWithChildren[]; + databaseSchema: string; +} = { name: TABLE_NAME, columns: [ { @@ -615,37 +642,9 @@ const SEARCH_INDEX_PATCH_PAYLOAD = [ }, ]; -const STORED_PROCEDURE_DETAILS_FOR_VERSION_TEST = { - name: STORED_PROCEDURE_NAME, - databaseSchema: 'sample_data.ecommerce_db.shopify', - storedProcedureCode: { - langauge: 'SQL', - code: 'CREATE OR REPLACE PROCEDURE output_message(message VARCHAR)\nRETURNS VARCHAR NOT NULL\nLANGUAGE SQL\nAS\n$$\nBEGIN\n RETURN message;\nEND;\n$$\n;', - }, - tags: [], -}; - -const STORED_PROCEDURE_PATCH_PAYLOAD = [ - { - op: 'add', - path: '/tags/0', - value: { - labelType: 'Manual', - state: 'Confirmed', - source: 'Classification', - tagFQN: 'PersonalData.SpecialCategory', - }, - }, - { - op: 'add', - path: '/description', - value: `Description for ${STORED_PROCEDURE_NAME}`, - }, -]; - export const DATA_MODEL_DETAILS_FOR_VERSION_TEST = { name: DATA_MODEL_NAME, - service: 'sample_looker', + service: DASHBOARD_DETAILS.service, dataModelType: 'LookMlExplore', tags: [], columns: [ @@ -654,8 +653,7 @@ export const DATA_MODEL_DETAILS_FOR_VERSION_TEST = { dataType: 'VARCHAR', dataLength: 256, dataTypeDisplay: 'varchar', - fullyQualifiedName: - 'sample_looker.model.cypress_version_test_data_model.column_1', + fullyQualifiedName: `${DASHBOARD_DETAILS.service}.model.${DATA_MODEL_NAME}.column_1`, tags: [], ordinalPosition: 1, }, @@ -664,8 +662,7 @@ export const DATA_MODEL_DETAILS_FOR_VERSION_TEST = { dataType: 'NUMERIC', dataTypeDisplay: 'numeric', description: 'Description for column column_2', - fullyQualifiedName: - 'sample_looker.model.cypress_version_test_data_model.column_2', + fullyQualifiedName: `${DASHBOARD_DETAILS.service}.model.${DATA_MODEL_NAME}.column_2`, tags: [], ordinalPosition: 2, }, @@ -673,8 +670,7 @@ export const DATA_MODEL_DETAILS_FOR_VERSION_TEST = { name: 'column_3', dataType: 'NUMERIC', dataTypeDisplay: 'numeric', - fullyQualifiedName: - 'sample_looker.model.cypress_version_test_data_model.column_3', + fullyQualifiedName: `${DASHBOARD_DETAILS.service}.model.${DATA_MODEL_NAME}.column_3`, tags: [], ordinalPosition: 3, }, @@ -728,11 +724,31 @@ export const DATA_MODEL_PATCH_PAYLOAD = [ }, ]; -export const ENTITY_DETAILS_FOR_VERSION_TEST = { +export const ENTITY_DETAILS_FOR_VERSION_TEST: Record< + string, + { + name: string; + serviceName: string; + entity: EntityType; + entityCreationDetails: + | typeof TABLE_DETAILS_FOR_VERSION_TEST + | typeof TOPIC_DETAILS_FOR_VERSION_TEST + | typeof DASHBOARD_DETAILS_FOR_VERSION_TEST; + entityPatchPayload: typeof TABLE_PATCH_PAYLOAD; + isChildrenExist: boolean; + childFieldNameToCheck?: string; + columnDisplayNameToUpdate?: string; + childSelector?: string; + entityAddedDescription: string; + updatedTagEntityChildName?: string; + entityChildRemovedDescription?: string; + entityChildAddedDescription?: string; + } +> = { Table: { name: TABLE_NAME, serviceName: 'sample_data', - entity: 'tables', + entity: EntityType.Table, entityCreationDetails: TABLE_DETAILS_FOR_VERSION_TEST, entityPatchPayload: TABLE_PATCH_PAYLOAD, isChildrenExist: true, @@ -747,7 +763,7 @@ export const ENTITY_DETAILS_FOR_VERSION_TEST = { Topic: { name: TOPIC_NAME, serviceName: 'sample_kafka', - entity: 'topics', + entity: EntityType.Topic, entityCreationDetails: TOPIC_DETAILS_FOR_VERSION_TEST, entityPatchPayload: TOPIC_PATCH_PAYLOAD, isChildrenExist: true, @@ -761,7 +777,7 @@ export const ENTITY_DETAILS_FOR_VERSION_TEST = { Dashboard: { name: DASHBOARD_NAME, serviceName: 'sample_superset', - entity: 'dashboards', + entity: EntityType.Dashboard, entityCreationDetails: DASHBOARD_DETAILS_FOR_VERSION_TEST, entityPatchPayload: DASHBOARD_PATCH_PAYLOAD, isChildrenExist: false, @@ -770,7 +786,7 @@ export const ENTITY_DETAILS_FOR_VERSION_TEST = { Pipeline: { name: PIPELINE_NAME, serviceName: 'sample_airflow', - entity: 'pipelines', + entity: EntityType.Pipeline, entityCreationDetails: PIPELINE_DETAILS_FOR_VERSION_TEST, entityPatchPayload: PIPELINE_PATCH_PAYLOAD, isChildrenExist: true, @@ -784,7 +800,7 @@ export const ENTITY_DETAILS_FOR_VERSION_TEST = { 'ML Model': { name: ML_MODEL_NAME, serviceName: 'mlflow_svc', - entity: 'mlmodels', + entity: EntityType.MlModel, entityCreationDetails: ML_MODEL_DETAILS_FOR_VERSION_TEST, entityPatchPayload: ML_MODEL_PATCH_PAYLOAD, isChildrenExist: true, @@ -797,7 +813,7 @@ export const ENTITY_DETAILS_FOR_VERSION_TEST = { Container: { name: CONTAINER_NAME, serviceName: 's3_storage_sample', - entity: 'containers', + entity: EntityType.Container, entityCreationDetails: CONTAINER_DETAILS_FOR_VERSION_TEST, entityPatchPayload: CONTAINER_PATCH_PAYLOAD, isChildrenExist: true, @@ -811,7 +827,7 @@ export const ENTITY_DETAILS_FOR_VERSION_TEST = { 'Search Index': { name: SEARCH_INDEX_NAME, serviceName: 'elasticsearch_sample', - entity: 'searchIndexes', + entity: EntityType.SearchIndex, entityCreationDetails: SEARCH_INDEX_DETAILS_FOR_VERSION_TEST, entityPatchPayload: SEARCH_INDEX_PATCH_PAYLOAD, isChildrenExist: true, @@ -826,7 +842,7 @@ export const ENTITY_DETAILS_FOR_VERSION_TEST = { // 'Stored Procedure': { // name: STORED_PROCEDURE_NAME, // serviceName: 'sample_data', - // entity: 'storedProcedures', + // entity: EntityType.StoreProcedure, // entityCreationDetails: STORED_PROCEDURE_DETAILS_FOR_VERSION_TEST, // entityPatchPayload: STORED_PROCEDURE_PATCH_PAYLOAD, // isChildrenExist: false, @@ -840,7 +856,7 @@ export const ENTITY_DETAILS_FOR_VERSION_TEST = { export const DATA_MODEL_DETAILS = { name: DATA_MODEL_NAME, - entity: 'containers', + entity: EntityType.Container, entityAddedDescription: `Description for ${DATA_MODEL_NAME}`, updatedTagEntityChildName: 'column_1', entityChildRemovedDescription: 'Description for column column_2', diff --git a/openmetadata-ui/src/main/resources/ui/cypress/constants/advancedSearchQuickFilters.constants.js b/openmetadata-ui/src/main/resources/ui/cypress/constants/advancedSearchQuickFilters.constants.ts similarity index 99% rename from openmetadata-ui/src/main/resources/ui/cypress/constants/advancedSearchQuickFilters.constants.js rename to openmetadata-ui/src/main/resources/ui/cypress/constants/advancedSearchQuickFilters.constants.ts index 9ab90171d7d2..36b5b5d44ea7 100644 --- a/openmetadata-ui/src/main/resources/ui/cypress/constants/advancedSearchQuickFilters.constants.js +++ b/openmetadata-ui/src/main/resources/ui/cypress/constants/advancedSearchQuickFilters.constants.ts @@ -185,7 +185,7 @@ export const QUICK_FILTERS_BY_ASSETS = [ label: 'Glossaries', searchIndex: 'glossary_search_index', filters: GLOSSARY_DROPDOWN_ITEMS, - tab: 'glossaries-tab', + tab: 'glossary terms-tab', entity: DATA_ASSETS.glossaryTerms, }, { diff --git a/openmetadata-ui/src/main/resources/ui/cypress/constants/constants.js b/openmetadata-ui/src/main/resources/ui/cypress/constants/constants.ts similarity index 95% rename from openmetadata-ui/src/main/resources/ui/cypress/constants/constants.js rename to openmetadata-ui/src/main/resources/ui/cypress/constants/constants.ts index f359cc5e8ba4..ce346e9acb4b 100644 --- a/openmetadata-ui/src/main/resources/ui/cypress/constants/constants.js +++ b/openmetadata-ui/src/main/resources/ui/cypress/constants/constants.ts @@ -11,6 +11,7 @@ * limitations under the License. */ +import { EntityType } from './Entity.interface'; import { GlobalSettingOptions } from './settings.constant'; export const uuid = () => Cypress._.random(0, 1e6); @@ -58,7 +59,7 @@ export const SEARCH_INDEX = { export const DATA_QUALITY_SAMPLE_DATA_TABLE = { term: 'dim_address', - entity: DATA_ASSETS.tables, + entity: EntityType.Table, serviceName: 'sample_data', testCaseName: 'column_value_max_to_be_between', sqlTestCaseName: 'my_sql_test_case_cypress', @@ -72,14 +73,14 @@ export const SEARCH_ENTITY_TABLE = { table_1: { term: 'raw_customer', displayName: 'raw_customer', - entity: DATA_ASSETS.tables, + entity: EntityType.Table, serviceName: 'sample_data', entityType: 'Table', }, table_2: { term: 'fact_session', displayName: 'fact_session', - entity: DATA_ASSETS.tables, + entity: EntityType.Table, serviceName: 'sample_data', schemaName: 'shopify', entityType: 'Table', @@ -87,7 +88,7 @@ export const SEARCH_ENTITY_TABLE = { table_3: { term: 'raw_product_catalog', displayName: 'raw_product_catalog', - entity: DATA_ASSETS.tables, + entity: EntityType.Table, serviceName: 'sample_data', schemaName: 'shopify', entityType: 'Table', @@ -95,14 +96,14 @@ export const SEARCH_ENTITY_TABLE = { table_4: { term: 'dim_address', displayName: 'dim_address', - entity: DATA_ASSETS.tables, + entity: EntityType.Table, serviceName: 'sample_data', entityType: 'Table', }, table_5: { term: 'dim.api/client', displayName: 'dim.api/client', - entity: DATA_ASSETS.tables, + entity: EntityType.Table, serviceName: 'sample_data', entityType: 'Table', }, @@ -112,13 +113,13 @@ export const SEARCH_ENTITY_TOPIC = { topic_1: { term: 'shop_products', displayName: 'shop_products', - entity: DATA_ASSETS.topics, + entity: EntityType.Topic, serviceName: 'sample_kafka', entityType: 'Topic', }, topic_2: { term: 'orders', - entity: DATA_ASSETS.topics, + entity: EntityType.Topic, serviceName: 'sample_kafka', entityType: 'Topic', }, @@ -128,13 +129,13 @@ export const SEARCH_ENTITY_DASHBOARD = { dashboard_1: { term: 'Slack Dashboard', displayName: 'Slack Dashboard', - entity: DATA_ASSETS.dashboards, + entity: EntityType.Dashboard, serviceName: 'sample_superset', entityType: 'Dashboard', }, dashboard_2: { term: 'Unicode Test', - entity: DATA_ASSETS.dashboards, + entity: EntityType.Dashboard, serviceName: 'sample_superset', entityType: 'Dashboard', }, @@ -144,14 +145,14 @@ export const SEARCH_ENTITY_PIPELINE = { pipeline_1: { term: 'dim_product_etl', displayName: 'dim_product etl', - entity: DATA_ASSETS.pipelines, + entity: EntityType.Pipeline, serviceName: 'sample_airflow', entityType: 'Pipeline', }, pipeline_2: { term: 'dim_user_etl', displayName: 'dim_user etl', - entity: DATA_ASSETS.pipelines, + entity: EntityType.Pipeline, serviceName: 'sample_airflow', entityType: 'Pipeline', }, @@ -159,13 +160,13 @@ export const SEARCH_ENTITY_PIPELINE = { export const SEARCH_ENTITY_MLMODEL = { mlmodel_1: { term: 'forecast_sales', - entity: DATA_ASSETS.mlmodels, + entity: EntityType.MlModel, serviceName: 'mlflow_svc', entityType: 'ML Model', }, mlmodel_2: { term: 'eta_predictions', - entity: DATA_ASSETS.mlmodels, + entity: EntityType.MlModel, serviceName: 'mlflow_svc', displayName: 'ETA Predictions', entityType: 'ML Model', @@ -175,13 +176,13 @@ export const SEARCH_ENTITY_MLMODEL = { export const SEARCH_ENTITY_STORED_PROCEDURE = { stored_procedure_1: { term: 'update_dim_address_table', - entity: DATA_ASSETS.storedProcedures, + entity: EntityType.StoreProcedure, serviceName: 'sample_data', entityType: 'Stored Procedure', }, stored_procedure_2: { term: 'update_dim_address_table', - entity: DATA_ASSETS.storedProcedures, + entity: EntityType.StoreProcedure, serviceName: 'sample_data', displayName: 'update_dim_address_table', entityType: 'Stored Procedure', @@ -191,13 +192,13 @@ export const SEARCH_ENTITY_STORED_PROCEDURE = { export const SEARCH_ENTITY_DATA_MODEL = { data_model_1: { term: 'operations_view', - entity: DATA_ASSETS.dataModel, + entity: EntityType.DataModel, serviceName: 'sample_looker', entityType: 'Data Model', }, data_model_2: { term: 'orders_view', - entity: DATA_ASSETS.dataModel, + entity: EntityType.DataModel, serviceName: 'sample_looker', displayName: 'Orders View', entityType: 'Data Model', @@ -207,15 +208,15 @@ export const SEARCH_ENTITY_DATA_MODEL = { export const DELETE_ENTITY = { table: { term: 'dim.shop', - entity: DATA_ASSETS.tables, + entity: EntityType.Table, serviceName: 'sample_data', entityType: 'Table', }, topic: { term: 'shop_updates', - entity: DATA_ASSETS.topics, + entity: EntityType.Topic, serviceName: 'sample_kafka', - entityType: 'Table', + entityType: 'Topic', }, }; @@ -223,7 +224,7 @@ export const RECENT_SEARCH_TITLE = 'Recent Search Terms'; export const RECENT_VIEW_TITLE = 'Recent Views'; export const MY_DATA_TITLE = 'My Data'; export const FOLLOWING_TITLE = 'Following'; -export const TEAM_ENTITY = 'team_entity'; +export const TEAM_ENTITY = 'alert_entity'; export const NO_SEARCHED_TERMS = 'No searched terms'; export const DELETE_TERM = 'DELETE'; @@ -253,8 +254,8 @@ export const NEW_COLUMN_TEST_CASE = { column: 'id', type: 'columnValueLengthsToBeBetween', label: 'Column Value Lengths To Be Between', - min: 3, - max: 6, + min: '3', + max: '6', description: 'New table test case for columnValueLengthsToBeBetween', }; @@ -464,6 +465,7 @@ export const service = { description: 'This is a Glue service', newDescription: 'This is updated Glue service description', Owner: 'Aaron Johnson', + serviceType: 'databaseService', }; export const SERVICE_TYPE = { @@ -474,6 +476,7 @@ export const SERVICE_TYPE = { MLModels: GlobalSettingOptions.MLMODELS, Storage: GlobalSettingOptions.STORAGES, Search: GlobalSettingOptions.SEARCH, + Metadata: GlobalSettingOptions.METADATA, StoredProcedure: GlobalSettingOptions.STORED_PROCEDURES, }; @@ -495,6 +498,10 @@ export const ENTITIES = { integerValue: '45', stringValue: 'This is string propery', markdownValue: 'This is markdown value', + enumConfig: { + values: ['enum1', 'enum2', 'enum3'], + multiSelect: false, + }, entityObj: SEARCH_ENTITY_TABLE.table_1, entityApiType: 'tables', }, @@ -504,6 +511,10 @@ export const ENTITIES = { integerValue: '23', stringValue: 'This is string propery', markdownValue: 'This is markdown value', + enumConfig: { + values: ['enum1', 'enum2', 'enum3'], + multiSelect: false, + }, entityObj: SEARCH_ENTITY_TOPIC.topic_1, entityApiType: 'topics', }, @@ -523,6 +534,10 @@ export const ENTITIES = { integerValue: '78', stringValue: 'This is string propery', markdownValue: 'This is markdown value', + enumConfig: { + values: ['enum1', 'enum2', 'enum3'], + multiSelect: true, + }, entityObj: SEARCH_ENTITY_PIPELINE.pipeline_1, entityApiType: 'pipelines', }, diff --git a/openmetadata-ui/src/main/resources/ui/cypress/constants/lineage.constants.js b/openmetadata-ui/src/main/resources/ui/cypress/constants/lineage.constants.ts similarity index 91% rename from openmetadata-ui/src/main/resources/ui/cypress/constants/lineage.constants.js rename to openmetadata-ui/src/main/resources/ui/cypress/constants/lineage.constants.ts index 17331870b7d0..3249fe7d39ee 100644 --- a/openmetadata-ui/src/main/resources/ui/cypress/constants/lineage.constants.js +++ b/openmetadata-ui/src/main/resources/ui/cypress/constants/lineage.constants.ts @@ -44,6 +44,7 @@ export const LINEAGE_ITEMS = [ fqn: 'sample_kafka.shop_products', entityType: 'Topic', searchIndex: SEARCH_INDEX.topics, + columns: ['sample_kafka.shop_products.Shop.shop_id'], }, { term: 'forecast_sales', @@ -52,14 +53,16 @@ export const LINEAGE_ITEMS = [ entityType: 'ML Model', fqn: 'mlflow_svc.forecast_sales', searchIndex: SEARCH_INDEX.mlmodels, + columns: [], }, { - term: 'media', + term: 'transactions', entity: DATA_ASSETS.containers, serviceName: 's3_storage_sample', entityType: 'Container', - fqn: 's3_storage_sample.departments.media', + fqn: 's3_storage_sample.transactions', searchIndex: SEARCH_INDEX.containers, + columns: ['s3_storage_sample.transactions.transaction_id'], }, { term: 'customers', @@ -68,6 +71,7 @@ export const LINEAGE_ITEMS = [ entityType: 'Dashboard', fqn: 'sample_looker.customers', searchIndex: SEARCH_INDEX.dashboards, + columns: ['sample_looker.chart_1'], }, ]; diff --git a/openmetadata-ui/src/main/resources/ui/cypress/constants/redirections.constants.js b/openmetadata-ui/src/main/resources/ui/cypress/constants/redirections.constants.ts similarity index 100% rename from openmetadata-ui/src/main/resources/ui/cypress/constants/redirections.constants.js rename to openmetadata-ui/src/main/resources/ui/cypress/constants/redirections.constants.ts diff --git a/openmetadata-ui/src/main/resources/ui/cypress/constants/service.constants.js b/openmetadata-ui/src/main/resources/ui/cypress/constants/service.constants.ts similarity index 100% rename from openmetadata-ui/src/main/resources/ui/cypress/constants/service.constants.js rename to openmetadata-ui/src/main/resources/ui/cypress/constants/service.constants.ts diff --git a/openmetadata-ui/src/main/resources/ui/cypress/constants/settings.constant.ts b/openmetadata-ui/src/main/resources/ui/cypress/constants/settings.constant.ts index af07531aa4dd..6fdacd4207a7 100644 --- a/openmetadata-ui/src/main/resources/ui/cypress/constants/settings.constant.ts +++ b/openmetadata-ui/src/main/resources/ui/cypress/constants/settings.constant.ts @@ -15,7 +15,7 @@ export enum GlobalSettingsMenuCategory { ACCESS = 'access', NOTIFICATIONS = 'notifications', CUSTOM_PROPERTIES = 'customProperties', - OPEN_METADATA = 'openMetadata', + PREFERENCES = 'preferences', MEMBERS = 'members', SERVICES = 'services', BOTS = 'bots', @@ -65,6 +65,7 @@ export enum GlobalSettingOptions { TOPICS = 'topics', CONTAINERS = 'containers', APPLICATIONS = 'apps', + OM_HEALTH = 'om-health', } export const SETTINGS_OPTIONS_PATH = { @@ -150,20 +151,24 @@ export const SETTINGS_OPTIONS_PATH = { // Open-metadata [GlobalSettingOptions.CUSTOMIZE_LANDING_PAGE]: [ - GlobalSettingsMenuCategory.OPEN_METADATA, - `${GlobalSettingsMenuCategory.OPEN_METADATA}.${GlobalSettingOptions.CUSTOMIZE_LANDING_PAGE}`, + GlobalSettingsMenuCategory.PREFERENCES, + `${GlobalSettingsMenuCategory.PREFERENCES}.${GlobalSettingOptions.CUSTOMIZE_LANDING_PAGE}`, ], [GlobalSettingOptions.EMAIL]: [ - GlobalSettingsMenuCategory.OPEN_METADATA, - `${GlobalSettingsMenuCategory.OPEN_METADATA}.${GlobalSettingOptions.EMAIL}`, + GlobalSettingsMenuCategory.PREFERENCES, + `${GlobalSettingsMenuCategory.PREFERENCES}.${GlobalSettingOptions.EMAIL}`, ], [GlobalSettingOptions.CUSTOM_LOGO]: [ - GlobalSettingsMenuCategory.OPEN_METADATA, - `${GlobalSettingsMenuCategory.OPEN_METADATA}.${GlobalSettingOptions.CUSTOM_LOGO}`, + GlobalSettingsMenuCategory.PREFERENCES, + `${GlobalSettingsMenuCategory.PREFERENCES}.${GlobalSettingOptions.CUSTOM_LOGO}`, ], [GlobalSettingOptions.LOGIN_CONFIGURATION]: [ - GlobalSettingsMenuCategory.OPEN_METADATA, - `${GlobalSettingsMenuCategory.OPEN_METADATA}.${GlobalSettingOptions.LOGIN_CONFIGURATION}`, + GlobalSettingsMenuCategory.PREFERENCES, + `${GlobalSettingsMenuCategory.PREFERENCES}.${GlobalSettingOptions.LOGIN_CONFIGURATION}`, + ], + [GlobalSettingOptions.OM_HEALTH]: [ + GlobalSettingsMenuCategory.PREFERENCES, + `${GlobalSettingsMenuCategory.PREFERENCES}.${GlobalSettingOptions.OM_HEALTH}`, ], [GlobalSettingOptions.GLOSSARY_TERM]: [ GlobalSettingsMenuCategory.CUSTOM_PROPERTIES, diff --git a/openmetadata-ui/src/main/resources/ui/cypress/constants/sidebar.constant.js b/openmetadata-ui/src/main/resources/ui/cypress/constants/sidebar.constant.ts similarity index 100% rename from openmetadata-ui/src/main/resources/ui/cypress/constants/sidebar.constant.js rename to openmetadata-ui/src/main/resources/ui/cypress/constants/sidebar.constant.ts diff --git a/openmetadata-ui/src/main/resources/ui/cypress/e2e/Features/ActivityFeed.spec.js b/openmetadata-ui/src/main/resources/ui/cypress/e2e/Features/ActivityFeed.spec.ts similarity index 97% rename from openmetadata-ui/src/main/resources/ui/cypress/e2e/Features/ActivityFeed.spec.js rename to openmetadata-ui/src/main/resources/ui/cypress/e2e/Features/ActivityFeed.spec.ts index e3ecda49ea4a..42722e62070e 100644 --- a/openmetadata-ui/src/main/resources/ui/cypress/e2e/Features/ActivityFeed.spec.js +++ b/openmetadata-ui/src/main/resources/ui/cypress/e2e/Features/ActivityFeed.spec.ts @@ -11,9 +11,6 @@ * limitations under the License. */ -// eslint-disable-next-line spaced-comment -/// - import { interceptURL, verifyResponseStatusCode } from '../../common/common'; import { createEntityTable, @@ -22,11 +19,12 @@ import { } from '../../common/EntityUtils'; import { createDescriptionTask } from '../../common/TaskUtils'; import { visitEntityDetailsPage } from '../../common/Utils/Entity'; -import { DATA_ASSETS } from '../../constants/constants'; +import { getToken } from '../../common/Utils/LocalStorage'; +import { EntityType } from '../../constants/Entity.interface'; import { DATABASE_SERVICE } from '../../constants/EntityConstant'; import { SERVICE_CATEGORIES } from '../../constants/service.constants'; -const reactOnFeed = (feedSelector, reaction) => { +const reactOnFeed = (feedSelector: string, reaction: string) => { cy.get(feedSelector).within(() => { cy.get('[data-testid="feed-actions"]').invoke('show'); cy.get('[data-testid="feed-actions"]').within(() => { @@ -49,7 +47,7 @@ describe('Activity feed', () => { before(() => { cy.login(); cy.getAllLocalStorage().then((data) => { - const token = Object.values(data)[0].oidcIdToken; + const token = getToken(data); createEntityTable({ token, @@ -62,7 +60,7 @@ describe('Activity feed', () => { after(() => { cy.login(); cy.getAllLocalStorage().then((data) => { - const token = Object.values(data)[0].oidcIdToken; + const token = getToken(data); hardDeleteService({ token, @@ -88,7 +86,7 @@ describe('Activity feed', () => { const value = { term: table1.name, displayName: table1.name, - entity: DATA_ASSETS.tables, + entity: EntityType.Table, serviceName: DATABASE_SERVICE.service.name, entityType: 'Table', }; @@ -374,7 +372,7 @@ describe('Activity feed', () => { const value = { term: table2.name, displayName: table2.name, - entity: DATA_ASSETS.tables, + entity: EntityType.Table, serviceName: DATABASE_SERVICE.service.name, entityType: 'Table', }; diff --git a/openmetadata-ui/src/main/resources/ui/cypress/e2e/Features/CustomMetric.spec.js b/openmetadata-ui/src/main/resources/ui/cypress/e2e/Features/CustomMetric.spec.ts similarity index 90% rename from openmetadata-ui/src/main/resources/ui/cypress/e2e/Features/CustomMetric.spec.js rename to openmetadata-ui/src/main/resources/ui/cypress/e2e/Features/CustomMetric.spec.ts index b2e99d933b81..f32b21380660 100644 --- a/openmetadata-ui/src/main/resources/ui/cypress/e2e/Features/CustomMetric.spec.js +++ b/openmetadata-ui/src/main/resources/ui/cypress/e2e/Features/CustomMetric.spec.ts @@ -17,19 +17,17 @@ import { } from '../../common/common'; import { createEntityTable, hardDeleteService } from '../../common/EntityUtils'; import { visitEntityDetailsPage } from '../../common/Utils/Entity'; +import { getToken } from '../../common/Utils/LocalStorage'; import { - DATA_ASSETS, INVALID_NAMES, NAME_MAX_LENGTH_VALIDATION_ERROR, NAME_VALIDATION_ERROR, uuid, } from '../../constants/constants'; +import { EntityType } from '../../constants/Entity.interface'; import { DATABASE_SERVICE } from '../../constants/EntityConstant'; import { SERVICE_CATEGORIES } from '../../constants/service.constants'; -// eslint-disable-next-line spaced-comment -/// - const TABLE_CUSTOM_METRIC = { name: `tableCustomMetric-${uuid()}`, expression: `SELECT * FROM ${DATABASE_SERVICE.entity.name}`, @@ -66,13 +64,25 @@ const validateForm = (isColumnMetric = false) => { cy.get('#name').clear(); }; +type CustomMetricDetails = { + term: string; + serviceName: string; + entity: EntityType.Table; + isColumnMetric?: boolean; + metric: { + name: string; + column?: string; + expression: string; + }; +}; + const createCustomMetric = ({ term, serviceName, entity, isColumnMetric = false, metric, -}) => { +}: CustomMetricDetails) => { interceptURL('PUT', '/api/v1/tables/*/customMetric', 'createCustomMetric'); interceptURL( 'GET', @@ -125,7 +135,8 @@ const createCustomMetric = ({ cy.get('#columnName').click(); cy.get(`[title="${metric.column}"]`).click(); } - cy.get('.CodeMirror-scroll').click().type(metric.expression); + metric.expression && + cy.get('.CodeMirror-scroll').click().type(metric.expression); cy.get('[data-testid="submit-button"]').click(); verifyResponseStatusCode('@createCustomMetric', 200); toastNotification(`${metric.name} created successfully.`); @@ -148,7 +159,7 @@ const editCustomMetric = ({ entity, isColumnMetric = false, metric, -}) => { +}: CustomMetricDetails) => { interceptURL( 'GET', '/api/v1/tables/name/*?fields=customMetrics%2Ccolumns&include=all', @@ -166,7 +177,8 @@ const editCustomMetric = ({ cy.get('[data-testid="profiler-tab-left-panel"]') .contains('Column Profile') .click(); - cy.get('[data-row-key="user_id"]').contains(metric.column).click(); + metric.column && + cy.get('[data-row-key="user_id"]').contains(metric.column).click(); } cy.get(`[data-testid="${metric.name}-custom-metrics"]`) .scrollIntoView() @@ -195,7 +207,7 @@ const deleteCustomMetric = ({ entity, metric, isColumnMetric = false, -}) => { +}: CustomMetricDetails) => { interceptURL( 'GET', '/api/v1/tables/name/*?fields=customMetrics%2Ccolumns&include=all', @@ -219,7 +231,8 @@ const deleteCustomMetric = ({ cy.get('[data-testid="profiler-tab-left-panel"]') .contains('Column Profile') .click(); - cy.get('[data-row-key="user_id"]').contains(metric.column).click(); + metric.column && + cy.get('[data-row-key="user_id"]').contains(metric.column).click(); } cy.get(`[data-testid="${metric.name}-custom-metrics"]`) .scrollIntoView() @@ -237,7 +250,7 @@ describe('Custom Metric', { tags: 'Observability' }, () => { before(() => { cy.login(); cy.getAllLocalStorage().then((data) => { - const token = Object.values(data)[0].oidcIdToken; + const token = getToken(data); createEntityTable({ token, @@ -250,7 +263,7 @@ describe('Custom Metric', { tags: 'Observability' }, () => { after(() => { cy.login(); cy.getAllLocalStorage().then((data) => { - const token = Object.values(data)[0].oidcIdToken; + const token = getToken(data); hardDeleteService({ token, @@ -268,7 +281,7 @@ describe('Custom Metric', { tags: 'Observability' }, () => { createCustomMetric({ term: DATABASE_SERVICE.entity.name, serviceName: DATABASE_SERVICE.service.name, - entity: DATA_ASSETS.tables, + entity: EntityType.Table, metric: TABLE_CUSTOM_METRIC, }); }); @@ -277,7 +290,7 @@ describe('Custom Metric', { tags: 'Observability' }, () => { editCustomMetric({ term: DATABASE_SERVICE.entity.name, serviceName: DATABASE_SERVICE.service.name, - entity: DATA_ASSETS.tables, + entity: EntityType.Table, metric: TABLE_CUSTOM_METRIC, }); }); @@ -286,7 +299,7 @@ describe('Custom Metric', { tags: 'Observability' }, () => { deleteCustomMetric({ term: DATABASE_SERVICE.entity.name, serviceName: DATABASE_SERVICE.service.name, - entity: DATA_ASSETS.tables, + entity: EntityType.Table, metric: TABLE_CUSTOM_METRIC, }); }); @@ -295,7 +308,7 @@ describe('Custom Metric', { tags: 'Observability' }, () => { createCustomMetric({ term: DATABASE_SERVICE.entity.name, serviceName: DATABASE_SERVICE.service.name, - entity: DATA_ASSETS.tables, + entity: EntityType.Table, metric: COLUMN_CUSTOM_METRIC, isColumnMetric: true, }); @@ -305,7 +318,7 @@ describe('Custom Metric', { tags: 'Observability' }, () => { editCustomMetric({ term: DATABASE_SERVICE.entity.name, serviceName: DATABASE_SERVICE.service.name, - entity: DATA_ASSETS.tables, + entity: EntityType.Table, metric: COLUMN_CUSTOM_METRIC, isColumnMetric: true, }); @@ -315,7 +328,7 @@ describe('Custom Metric', { tags: 'Observability' }, () => { deleteCustomMetric({ term: DATABASE_SERVICE.entity.name, serviceName: DATABASE_SERVICE.service.name, - entity: DATA_ASSETS.tables, + entity: EntityType.Table, metric: COLUMN_CUSTOM_METRIC, isColumnMetric: true, }); diff --git a/openmetadata-ui/src/main/resources/ui/cypress/e2e/Features/EntitySummaryPanel.spec.ts b/openmetadata-ui/src/main/resources/ui/cypress/e2e/Features/EntitySummaryPanel.spec.ts index 18d95be27a01..e8bfa6dd73f4 100644 --- a/openmetadata-ui/src/main/resources/ui/cypress/e2e/Features/EntitySummaryPanel.spec.ts +++ b/openmetadata-ui/src/main/resources/ui/cypress/e2e/Features/EntitySummaryPanel.spec.ts @@ -98,6 +98,7 @@ describe('Entity Summary Panel', () => { 'be.visible' ); cy.get('[data-testid="Dashboard URL-label"]').should('be.visible'); + cy.get('[data-testid="Project-label"]').should('be.visible'); cy.get('[data-testid="tags-header"]').scrollIntoView().should('be.visible'); cy.get('[data-testid="description-header"]') .scrollIntoView() diff --git a/openmetadata-ui/src/main/resources/ui/cypress/e2e/Features/IncidentManager.spec.js b/openmetadata-ui/src/main/resources/ui/cypress/e2e/Features/IncidentManager.spec.ts similarity index 98% rename from openmetadata-ui/src/main/resources/ui/cypress/e2e/Features/IncidentManager.spec.js rename to openmetadata-ui/src/main/resources/ui/cypress/e2e/Features/IncidentManager.spec.ts index a74ef926ecb1..2aab39db6ee7 100644 --- a/openmetadata-ui/src/main/resources/ui/cypress/e2e/Features/IncidentManager.spec.js +++ b/openmetadata-ui/src/main/resources/ui/cypress/e2e/Features/IncidentManager.spec.ts @@ -16,7 +16,8 @@ import { deleteEntityViaREST, visitEntityDetailsPage, } from '../../common/Utils/Entity'; -import { DATA_ASSETS, uuid } from '../../constants/constants'; +import { getToken } from '../../common/Utils/LocalStorage'; +import { uuid } from '../../constants/constants'; import { EntityType, SidebarItem } from '../../constants/Entity.interface'; import { DATABASE_SERVICE } from '../../constants/EntityConstant'; const TABLE_NAME = DATABASE_SERVICE.entity.name; @@ -41,7 +42,7 @@ const goToProfilerTab = () => { visitEntityDetailsPage({ term: TABLE_NAME, serviceName: DATABASE_SERVICE.service.name, - entity: DATA_ASSETS.tables, + entity: EntityType.Table, }); verifyResponseStatusCode('@waitForPageLoad', 200); @@ -71,7 +72,7 @@ const verifySuccessStatus = (time = 20000) => { }); }; -const acknowledgeTask = (testCase) => { +const acknowledgeTask = (testCase: string) => { goToProfilerTab(); cy.get('[data-testid="profiler-tab-left-panel"]') @@ -130,7 +131,7 @@ const triggerTestCasePipeline = () => { verifySuccessStatus(); }; -const assignIncident = (testCaseName) => { +const assignIncident = (testCaseName: string) => { cy.sidebarClick(SidebarItem.INCIDENT_MANAGER); cy.get(`[data-testid="test-case-${testCaseName}"]`).should('be.visible'); cy.get(`[data-testid="${testCaseName}-status"]`) @@ -164,7 +165,7 @@ describe('Incident Manager', { tags: 'Observability' }, () => { cy.login(); cy.getAllLocalStorage().then((data) => { - const token = Object.values(data)[0].oidcIdToken; + const token = getToken(data); createEntityTableViaREST({ token, @@ -233,7 +234,7 @@ describe('Incident Manager', { tags: 'Observability' }, () => { cy.login(); cy.getAllLocalStorage().then((data) => { - const token = Object.values(data)[0].oidcIdToken; + const token = getToken(data); deleteEntityViaREST({ token, endPoint: EntityType.DatabaseService, diff --git a/openmetadata-ui/src/main/resources/ui/cypress/e2e/Features/QueryEntity.spec.js b/openmetadata-ui/src/main/resources/ui/cypress/e2e/Features/QueryEntity.spec.ts similarity index 95% rename from openmetadata-ui/src/main/resources/ui/cypress/e2e/Features/QueryEntity.spec.js rename to openmetadata-ui/src/main/resources/ui/cypress/e2e/Features/QueryEntity.spec.ts index ed1ba0295ea5..e92d7fd33f34 100644 --- a/openmetadata-ui/src/main/resources/ui/cypress/e2e/Features/QueryEntity.spec.js +++ b/openmetadata-ui/src/main/resources/ui/cypress/e2e/Features/QueryEntity.spec.ts @@ -22,7 +22,8 @@ import { hardDeleteService, } from '../../common/EntityUtils'; import { visitEntityDetailsPage } from '../../common/Utils/Entity'; -import { DATA_ASSETS } from '../../constants/constants'; +import { getToken } from '../../common/Utils/LocalStorage'; +import { EntityType } from '../../constants/Entity.interface'; import { DATABASE_SERVICE, DATABASE_SERVICE_DETAILS, @@ -32,7 +33,7 @@ import { SERVICE_CATEGORIES } from '../../constants/service.constants'; const queryTable = { term: DATABASE_SERVICE.entity.name, displayName: DATABASE_SERVICE.entity.name, - entity: DATA_ASSETS.tables, + entity: EntityType.Table, serviceName: DATABASE_SERVICE.service.name, entityType: 'Table', }; @@ -55,7 +56,7 @@ describe('Query Entity', { tags: 'DataAssets' }, () => { before(() => { cy.login(); cy.getAllLocalStorage().then((data) => { - const token = Object.values(data)[0].oidcIdToken; + const token = getToken(data); createEntityTable({ token, @@ -70,7 +71,7 @@ describe('Query Entity', { tags: 'DataAssets' }, () => { after(() => { cy.login(); cy.getAllLocalStorage().then((data) => { - const token = Object.values(data)[0].oidcIdToken; + const token = getToken(data); hardDeleteService({ token, @@ -153,7 +154,7 @@ describe('Query Entity', { tags: 'DataAssets' }, () => { cy.get('[data-testid="owner-link"]').should('contain', DATA.owner); // Update Description - cy.get('[data-testid="edit-description-btn"]').click(); + cy.get('[data-testid="edit-description"]').filter(':visible').click(); cy.get(descriptionBox).clear().type('updated description'); cy.get('[data-testid="save"]').click(); verifyResponseStatusCode('@patchQuery', 200); @@ -184,7 +185,7 @@ describe('Query Entity', { tags: 'DataAssets' }, () => { cy.get('[data-testid="table_queries"]').click(); verifyResponseStatusCode('@fetchQuery', 200); - cy.get('[data-testid="more-option-btn"]').click(); + cy.get('[data-testid="query-btn"]').click(); cy.get('[data-menu-id*="edit-query"]').click(); cy.get('.CodeMirror-line') .click() @@ -218,7 +219,7 @@ describe('Query Entity', { tags: 'DataAssets' }, () => { cy.get('[data-testid="query-entity-expand-button"]').click(); verifyResponseStatusCode('@getQueryById', 200); - cy.get('[data-testid="more-option-btn"]').click(); + cy.get('[data-testid="query-btn"]').click(); cy.get('.ant-dropdown').should('be.visible'); cy.get('[data-menu-id*="delete-query"]').click(); cy.get('[data-testid="save-button"]').click(); diff --git a/openmetadata-ui/src/main/resources/ui/cypress/e2e/Features/RecentlyViewed.spec.js b/openmetadata-ui/src/main/resources/ui/cypress/e2e/Features/RecentlyViewed.spec.ts similarity index 95% rename from openmetadata-ui/src/main/resources/ui/cypress/e2e/Features/RecentlyViewed.spec.js rename to openmetadata-ui/src/main/resources/ui/cypress/e2e/Features/RecentlyViewed.spec.ts index a3d8eb0dd8c3..9ca58b4a4d91 100644 --- a/openmetadata-ui/src/main/resources/ui/cypress/e2e/Features/RecentlyViewed.spec.js +++ b/openmetadata-ui/src/main/resources/ui/cypress/e2e/Features/RecentlyViewed.spec.ts @@ -17,6 +17,7 @@ import { hardDeleteService, } from '../../common/EntityUtils'; import { visitEntityDetailsPage } from '../../common/Utils/Entity'; +import { getToken } from '../../common/Utils/LocalStorage'; import { DATABASE_SERVICE, SINGLE_LEVEL_SERVICE, @@ -25,9 +26,6 @@ import { } from '../../constants/EntityConstant'; import { SERVICE_CATEGORIES } from '../../constants/service.constants'; -// eslint-disable-next-line spaced-comment -/// - // Update list if we support this for other entities too const RECENTLY_VIEW_ENTITIES = [ VISIT_ENTITIES_DATA.table, @@ -43,7 +41,7 @@ describe('Recently viwed data assets', { tags: 'DataAssets' }, () => { before(() => { cy.login(); cy.getAllLocalStorage().then((data) => { - const token = Object.values(data)[0].oidcIdToken; + const token = getToken(data); createEntityTable({ token, @@ -71,7 +69,7 @@ describe('Recently viwed data assets', { tags: 'DataAssets' }, () => { after(() => { cy.login(); cy.getAllLocalStorage().then((data) => { - const token = Object.values(data)[0].oidcIdToken; + const token = getToken(data); hardDeleteService({ token, diff --git a/openmetadata-ui/src/main/resources/ui/cypress/e2e/Features/SchemaSearch.spec.js b/openmetadata-ui/src/main/resources/ui/cypress/e2e/Features/SchemaSearch.spec.ts similarity index 61% rename from openmetadata-ui/src/main/resources/ui/cypress/e2e/Features/SchemaSearch.spec.js rename to openmetadata-ui/src/main/resources/ui/cypress/e2e/Features/SchemaSearch.spec.ts index 1dbb8670c9fa..2d52befd9f08 100644 --- a/openmetadata-ui/src/main/resources/ui/cypress/e2e/Features/SchemaSearch.spec.js +++ b/openmetadata-ui/src/main/resources/ui/cypress/e2e/Features/SchemaSearch.spec.ts @@ -10,15 +10,14 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -// eslint-disable-next-line spaced-comment -/// import { interceptURL, verifyResponseStatusCode } from '../../common/common'; import { searchServiceFromSettingPage } from '../../common/serviceUtils'; +import { getToken } from '../../common/Utils/LocalStorage'; import { GlobalSettingOptions } from '../../constants/settings.constant'; const schemaNames = ['sales', 'admin', 'anonymous', 'dip', 'gsmadmin_internal']; -let serviceId; +let serviceId: string; const serviceName = 'cypress_mysql_schema_test'; describe('Schema search', { tags: 'DataAssets' }, () => { @@ -27,53 +26,55 @@ describe('Schema search', { tags: 'DataAssets' }, () => { }); it('Prerequisite', () => { - const token = localStorage.getItem('oidcIdToken'); - cy.request({ - method: 'POST', - url: `/api/v1/services/databaseServices`, - headers: { Authorization: `Bearer ${token}` }, - body: { - name: serviceName, - serviceType: 'Mysql', - connection: { - config: { - type: 'Mysql', - scheme: 'mysql+pymysql', - username: 'test', - authType: { password: 'test' }, - hostPort: 'test', - supportsMetadataExtraction: true, - supportsDBTExtraction: true, - supportsProfiler: true, - supportsQueryComment: true, - }, - }, - }, - }).then((response) => { - expect(response.status).to.eq(201); - - const service = response.body.fullyQualifiedName; - serviceId = response.body.id; - + cy.getAllLocalStorage().then((data) => { + const token = getToken(data); cy.request({ method: 'POST', - url: `/api/v1/databases`, + url: `/api/v1/services/databaseServices`, headers: { Authorization: `Bearer ${token}` }, body: { - name: 'default', - service, + name: serviceName, + serviceType: 'Mysql', + connection: { + config: { + type: 'Mysql', + scheme: 'mysql+pymysql', + username: 'test', + authType: { password: 'test' }, + hostPort: 'test', + supportsMetadataExtraction: true, + supportsDBTExtraction: true, + supportsProfiler: true, + supportsQueryComment: true, + }, + }, }, }).then((response) => { - const database = response.body.fullyQualifiedName; - schemaNames.map((schema) => { - cy.request({ - method: 'POST', - url: `/api/v1/databaseSchemas`, - headers: { Authorization: `Bearer ${token}` }, - body: { - name: schema, - database, - }, + expect(response.status).to.eq(201); + + const service = response.body.fullyQualifiedName; + serviceId = response.body.id; + + cy.request({ + method: 'POST', + url: `/api/v1/databases`, + headers: { Authorization: `Bearer ${token}` }, + body: { + name: 'default', + service, + }, + }).then((response) => { + const database = response.body.fullyQualifiedName; + schemaNames.map((schema) => { + cy.request({ + method: 'POST', + url: `/api/v1/databaseSchemas`, + headers: { Authorization: `Bearer ${token}` }, + body: { + name: schema, + database, + }, + }); }); }); }); @@ -133,14 +134,15 @@ describe('Schema search', { tags: 'DataAssets' }, () => { }); it('Cleanup', () => { - const token = localStorage.getItem('oidcIdToken'); - - cy.request({ - method: 'DELETE', - url: `/api/v1/services/databaseServices/${serviceId}?hardDelete=true&recursive=true`, - headers: { Authorization: `Bearer ${token}` }, - }).then((response) => { - expect(response.status).to.eq(200); + cy.getAllLocalStorage().then((data) => { + const token = getToken(data); + cy.request({ + method: 'DELETE', + url: `/api/v1/services/databaseServices/${serviceId}?hardDelete=true&recursive=true`, + headers: { Authorization: `Bearer ${token}` }, + }).then((response) => { + expect(response.status).to.eq(200); + }); }); }); }); diff --git a/openmetadata-ui/src/main/resources/ui/cypress/e2e/Features/TeamsDragAndDrop.spec.js b/openmetadata-ui/src/main/resources/ui/cypress/e2e/Features/TeamsDragAndDrop.spec.js deleted file mode 100644 index b4e23a332ba8..000000000000 --- a/openmetadata-ui/src/main/resources/ui/cypress/e2e/Features/TeamsDragAndDrop.spec.js +++ /dev/null @@ -1,168 +0,0 @@ -/* - * Copyright 2024 Collate. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * http://www.apache.org/licenses/LICENSE-2.0 - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -import { addTeam, interceptURL, toastNotification } from '../../common/common'; -import { - dragAndDropElement, - openDragDropDropdown, -} from '../../common/Utils/DragAndDrop'; -import { - commonTeamDetails, - confirmationDragAndDropTeam, - deleteTeamPermanently, -} from '../../common/Utils/Teams'; -import { uuid } from '../../constants/constants'; -import { SidebarItem } from '../../constants/Entity.interface'; -import { GlobalSettingOptions } from '../../constants/settings.constant'; - -const teamNameGroup = `team-ct-test-${uuid()}`; -const teamNameBusiness = `team-ct-test-${uuid()}`; -const teamNameDivision = `team-ct-test-${uuid()}`; -const teamNameDepartment = `team-ct-test-${uuid()}`; - -const TEAM_TYPE_BY_NAME = { - [teamNameBusiness]: 'BusinessUnit', - [teamNameDivision]: 'Division', - [teamNameDepartment]: 'Department', - [teamNameGroup]: 'Group', -}; - -const DRAG_AND_DROP_TEAM_DETAILS = [ - { - name: teamNameBusiness, - updatedName: `${teamNameBusiness}-updated`, - teamType: 'BusinessUnit', - description: `This is ${teamNameBusiness} description`, - ...commonTeamDetails, - }, - { - name: teamNameDivision, - updatedName: `${teamNameDivision}-updated`, - teamType: 'Division', - description: `This is ${teamNameDivision} description`, - ...commonTeamDetails, - }, - { - name: teamNameDepartment, - updatedName: `${teamNameDepartment}-updated`, - teamType: 'Department', - description: `This is ${teamNameDepartment} description`, - ...commonTeamDetails, - }, - { - name: teamNameGroup, - updatedName: `${teamNameGroup}-updated`, - teamType: 'Group', - description: `This is ${teamNameGroup} description`, - ...commonTeamDetails, - }, -]; - -describe('Teams drag and drop should work properly', () => { - beforeEach(() => { - interceptURL('GET', `/api/v1/users?fields=*`, 'getUserDetails'); - interceptURL('GET', `/api/v1/permissions/team/name/*`, 'permissions'); - cy.login(); - - cy.sidebarClick(SidebarItem.SETTINGS); - - // Clicking on teams - cy.settingClick(GlobalSettingOptions.TEAMS); - }); - - it('Add new team for drag and drop', () => { - DRAG_AND_DROP_TEAM_DETAILS.map((team) => { - addTeam(team); - cy.reload(); - // asserting the added values - cy.get(`[data-row-key="${team.name}"]`) - .scrollIntoView() - .should('be.visible'); - cy.get(`[data-row-key="${team.name}"]`).should( - 'contain', - team.description - ); - }); - }); - - it('Should fail when drop team type is Group', () => { - [teamNameBusiness, teamNameDepartment, teamNameDivision].map((team) => { - dragAndDropElement(team, teamNameGroup); - toastNotification( - `You cannot move to this team as Team Type ${TEAM_TYPE_BY_NAME[team]} can't be Group children` - ); - - cy.get('.Toastify__toast-body', { timeout: 10000 }).should('not.exist'); - }); - }); - - it('Should fail when droppable team type is Department', () => { - [teamNameBusiness, teamNameDivision].map((team) => { - dragAndDropElement(team, teamNameDepartment); - toastNotification( - `You cannot move to this team as Team Type ${TEAM_TYPE_BY_NAME[team]} can't be Department children` - ); - cy.get('.Toastify__toast-body', { timeout: 10000 }).should('not.exist'); - }); - }); - - it('Should fail when draggable team type is BusinessUnit and droppable team type is Division', () => { - dragAndDropElement(teamNameBusiness, teamNameDivision); - toastNotification( - `You cannot move to this team as Team Type BusinessUnit can't be Division children` - ); - }); - - [teamNameBusiness, teamNameDivision, teamNameDepartment].map( - (droppableTeamName, index) => { - it(`Should drag and drop on ${TEAM_TYPE_BY_NAME[droppableTeamName]} team type`, () => { - // nested team will be shown once anything is moved under it - if (index !== 0) { - openDragDropDropdown( - [teamNameBusiness, teamNameDivision, teamNameDepartment][index - 1] - ); - } - - dragAndDropElement(teamNameGroup, droppableTeamName); - - confirmationDragAndDropTeam(teamNameGroup, droppableTeamName); - - // verify the team is moved under the business team - openDragDropDropdown(droppableTeamName); - cy.get( - `.ant-table-row-level-1[data-row-key="${teamNameGroup}"]` - ).should('be.visible'); - }); - } - ); - - it(`Should drag and drop team on table level`, () => { - // open department team dropdown as it is moved under it from last test - openDragDropDropdown(teamNameDepartment); - - dragAndDropElement(teamNameGroup, '.ant-table-thead > tr', true); - confirmationDragAndDropTeam(teamNameGroup, 'Organization'); - - // verify the team is moved under the table level - cy.get(`.ant-table-row-level-0[data-row-key="${teamNameGroup}"]`).should( - 'be.visible' - ); - }); - - it('Permanently deleting a team for drag and drop', () => { - [teamNameBusiness, teamNameDivision, teamNameDepartment, teamNameGroup].map( - (teamName) => { - deleteTeamPermanently(teamName); - } - ); - }); -}); diff --git a/openmetadata-ui/src/main/resources/ui/cypress/e2e/Features/TeamsDragAndDrop.spec.ts b/openmetadata-ui/src/main/resources/ui/cypress/e2e/Features/TeamsDragAndDrop.spec.ts new file mode 100644 index 000000000000..94885345ed2f --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/cypress/e2e/Features/TeamsDragAndDrop.spec.ts @@ -0,0 +1,189 @@ +/* + * Copyright 2024 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { interceptURL, toastNotification } from '../../common/common'; +import { + dragAndDropElement, + openDragDropDropdown, +} from '../../common/Utils/DragAndDrop'; +import { + addTeam, + commonTeamDetails, + confirmationDragAndDropTeam, + deleteTeamPermanently, +} from '../../common/Utils/Teams'; +import { uuid } from '../../constants/constants'; +import { SidebarItem } from '../../constants/Entity.interface'; +import { GlobalSettingOptions } from '../../constants/settings.constant'; + +const teamNameGroup = `team-ct-test-${uuid()}`; +const teamNameBusiness = `team-ct-test-${uuid()}`; +const teamNameDivision = `team-ct-test-${uuid()}`; +const teamNameDepartment = `team-ct-test-${uuid()}`; + +const TEAM_TYPE_BY_NAME = { + [teamNameBusiness]: 'BusinessUnit', + [teamNameDivision]: 'Division', + [teamNameDepartment]: 'Department', + [teamNameGroup]: 'Group', +}; + +const DRAG_AND_DROP_TEAM_DETAILS = [ + { + name: teamNameBusiness, + updatedName: `${teamNameBusiness}-updated`, + teamType: 'BusinessUnit', + description: `This is ${teamNameBusiness} description`, + ...commonTeamDetails, + }, + { + name: teamNameDivision, + updatedName: `${teamNameDivision}-updated`, + teamType: 'Division', + description: `This is ${teamNameDivision} description`, + ...commonTeamDetails, + }, + { + name: teamNameDepartment, + updatedName: `${teamNameDepartment}-updated`, + teamType: 'Department', + description: `This is ${teamNameDepartment} description`, + ...commonTeamDetails, + }, + { + name: teamNameGroup, + updatedName: `${teamNameGroup}-updated`, + teamType: 'Group', + description: `This is ${teamNameGroup} description`, + ...commonTeamDetails, + }, +]; + +describe( + 'Teams drag and drop should work properly', + { tags: 'Settings' }, + () => { + beforeEach(() => { + interceptURL('GET', `/api/v1/users?fields=*`, 'getUserDetails'); + interceptURL('GET', `/api/v1/permissions/team/name/*`, 'permissions'); + cy.login(); + + cy.sidebarClick(SidebarItem.SETTINGS); + + // Clicking on teams + cy.settingClick(GlobalSettingOptions.TEAMS); + }); + + before(() => { + cy.login(); + cy.sidebarClick(SidebarItem.SETTINGS); + // Clicking on teams + cy.settingClick(GlobalSettingOptions.TEAMS); + + DRAG_AND_DROP_TEAM_DETAILS.map((team) => { + addTeam(team); + cy.reload(); + // asserting the added values + cy.get(`[data-row-key="${team.name}"]`) + .scrollIntoView() + .should('be.visible'); + cy.get(`[data-row-key="${team.name}"]`).should( + 'contain', + team.description + ); + }); + }); + + after(() => { + cy.login(); + cy.sidebarClick(SidebarItem.SETTINGS); + + // Clicking on teams + cy.settingClick(GlobalSettingOptions.TEAMS); + + [ + teamNameBusiness, + teamNameDivision, + teamNameDepartment, + teamNameGroup, + ].map((teamName) => { + deleteTeamPermanently(teamName); + }); + }); + + it('Should fail when drop team type is Group', () => { + [teamNameBusiness, teamNameDepartment, teamNameDivision].map((team) => { + dragAndDropElement(team, teamNameGroup); + toastNotification( + `You cannot move to this team as Team Type ${TEAM_TYPE_BY_NAME[team]} can't be Group children` + ); + + cy.get('.Toastify__toast-body', { timeout: 10000 }).should('not.exist'); + }); + }); + + it('Should fail when droppable team type is Department', () => { + [teamNameBusiness, teamNameDivision].map((team) => { + dragAndDropElement(team, teamNameDepartment); + toastNotification( + `You cannot move to this team as Team Type ${TEAM_TYPE_BY_NAME[team]} can't be Department children` + ); + cy.get('.Toastify__toast-body', { timeout: 10000 }).should('not.exist'); + }); + }); + + it('Should fail when draggable team type is BusinessUnit and droppable team type is Division', () => { + dragAndDropElement(teamNameBusiness, teamNameDivision); + toastNotification( + `You cannot move to this team as Team Type BusinessUnit can't be Division children` + ); + }); + + [teamNameBusiness, teamNameDivision, teamNameDepartment].map( + (droppableTeamName, index) => { + it(`Should drag and drop on ${TEAM_TYPE_BY_NAME[droppableTeamName]} team type`, () => { + // nested team will be shown once anything is moved under it + if (index !== 0) { + openDragDropDropdown( + [teamNameBusiness, teamNameDivision, teamNameDepartment][ + index - 1 + ] + ); + } + + dragAndDropElement(teamNameGroup, droppableTeamName); + + confirmationDragAndDropTeam(teamNameGroup, droppableTeamName); + + // verify the team is moved under the business team + openDragDropDropdown(droppableTeamName); + cy.get( + `.ant-table-row-level-1[data-row-key="${teamNameGroup}"]` + ).should('be.visible'); + }); + } + ); + + it(`Should drag and drop team on table level`, () => { + // open department team dropdown as it is moved under it from last test + openDragDropDropdown(teamNameDepartment); + + dragAndDropElement(teamNameGroup, '.ant-table-thead > tr', true); + confirmationDragAndDropTeam(teamNameGroup, 'Organization'); + + // verify the team is moved under the table level + cy.get(`.ant-table-row-level-0[data-row-key="${teamNameGroup}"]`) + .scrollIntoView() + .should('be.visible'); + }); + } +); diff --git a/openmetadata-ui/src/main/resources/ui/cypress/e2e/Features/TeamsHierarchy.spec.js b/openmetadata-ui/src/main/resources/ui/cypress/e2e/Features/TeamsHierarchy.spec.ts similarity index 96% rename from openmetadata-ui/src/main/resources/ui/cypress/e2e/Features/TeamsHierarchy.spec.js rename to openmetadata-ui/src/main/resources/ui/cypress/e2e/Features/TeamsHierarchy.spec.ts index b97cc74a49c5..b4ad6bb0f239 100644 --- a/openmetadata-ui/src/main/resources/ui/cypress/e2e/Features/TeamsHierarchy.spec.js +++ b/openmetadata-ui/src/main/resources/ui/cypress/e2e/Features/TeamsHierarchy.spec.ts @@ -12,11 +12,11 @@ */ import { - addTeam, interceptURL, uuid, verifyResponseStatusCode, } from '../../common/common'; +import { addTeam } from '../../common/Utils/Teams'; import { GlobalSettingOptions } from '../../constants/settings.constant'; const buTeamName = `bu-${uuid()}`; @@ -24,7 +24,8 @@ const divTeamName = `div-${uuid()}`; const depTeamName = `dep-${uuid()}`; const grpTeamName = `grp-${uuid()}`; const teamNames = [buTeamName, divTeamName, depTeamName, grpTeamName]; -const getTeam = (teamName) => { + +const getTeam = (teamName: string) => { return { name: teamName, displayName: teamName, diff --git a/openmetadata-ui/src/main/resources/ui/cypress/e2e/Flow/AddRoleAndAssignToUser.spec.js b/openmetadata-ui/src/main/resources/ui/cypress/e2e/Flow/AddRoleAndAssignToUser.spec.js deleted file mode 100644 index 5f97d6242b5d..000000000000 --- a/openmetadata-ui/src/main/resources/ui/cypress/e2e/Flow/AddRoleAndAssignToUser.spec.js +++ /dev/null @@ -1,117 +0,0 @@ -/* - * Copyright 2022 Collate. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * http://www.apache.org/licenses/LICENSE-2.0 - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { - descriptionBox, - interceptURL, - uuid, - verifyResponseStatusCode, -} from '../../common/common'; -import { BASE_URL } from '../../constants/constants'; -import { GlobalSettingOptions } from '../../constants/settings.constant'; - -const roleName = `Role-test-${uuid()}`; -const userName = `usercttest${uuid()}`; -const userEmail = `${userName}@gmail.com`; - -describe('Test Add role and assign it to the user', () => { - beforeEach(() => { - cy.login(); - interceptURL('GET', '*api/v1/roles*', 'getRoles'); - interceptURL('GET', '/api/v1/users?*', 'usersPage'); - }); - - it('Create role', () => { - cy.settingClick(GlobalSettingOptions.ROLES); - verifyResponseStatusCode('@getRoles', 200); - - cy.get('[data-testid="add-role"]').contains('Add Role').click(); - - // Asserting navigation - cy.get('[data-testid="inactive-link"]').should('contain', 'Add New Role'); - - // Entering name - cy.get('#name').type(roleName); - // Entering descrription - cy.get(descriptionBox).type('description'); - // Select the policies - cy.get('[data-testid="policies"]').click(); - - cy.get('[title="Data Consumer Policy"]').scrollIntoView().click(); - - cy.get('[title="Data Steward Policy"]').scrollIntoView().click(); - // Save the role - cy.get('[data-testid="submit-btn"]').scrollIntoView().click(); - - // Verify the role is added successfully - cy.url().should('eq', `${BASE_URL}/settings/access/roles/${roleName}`); - cy.get('[data-testid="inactive-link"]').should('contain', roleName); - - // Verify added description - cy.get( - '[data-testid="description"] > [data-testid="viewer-container"]' - ).should('contain', 'description'); - }); - - it('Create new user and assign new role to him', () => { - // Create user and assign newly created role to the user - cy.settingClick(GlobalSettingOptions.USERS); - verifyResponseStatusCode('@usersPage', 200); - - cy.get('[data-testid="add-user"]').contains('Add User').click(); - - cy.get('[data-testid="email"]').scrollIntoView().type(userEmail); - - cy.get('[data-testid="displayName"]').type(userName); - - cy.get(descriptionBox).type('Adding user'); - - interceptURL('GET', '/api/v1/users/generateRandomPwd', 'generatePassword'); - cy.get('[data-testid="password-generator"]').scrollIntoView().click(); - verifyResponseStatusCode('@generatePassword', 200); - - cy.get(`[data-testid="roles-dropdown"]`).type(roleName); - cy.get(`.ant-select-dropdown [title="${roleName}"]`).click(); - - cy.clickOutside(); - interceptURL('POST', '/api/v1/users', 'createUser'); - cy.get('[data-testid="save-user"]').scrollIntoView().click(); - verifyResponseStatusCode('@createUser', 201); - }); - - it('Verify assigned role to new user', () => { - cy.settingClick(GlobalSettingOptions.USERS); - verifyResponseStatusCode('@usersPage', 200); - - interceptURL( - 'GET', - '/api/v1/search/query?q=**&from=0&size=*&index=*', - 'searchUser' - ); - interceptURL('GET', '/api/v1/users/name/*', 'userDetailsPage'); - cy.get('[data-testid="searchbar"]').type(userName); - verifyResponseStatusCode('@searchUser', 200); - - cy.get(`[data-testid="${userName}"]`).click(); - verifyResponseStatusCode('@userDetailsPage', 200); - - // click the collapse button to open the other details - cy.get( - '[data-testid="user-profile"] .ant-collapse-expand-icon > .anticon' - ).click(); - - cy.get( - '[data-testid="user-profile"] [data-testid="user-profile-roles"]' - ).should('contain', roleName); - }); -}); diff --git a/openmetadata-ui/src/main/resources/ui/cypress/e2e/Flow/AddRoleAndAssignToUser.spec.ts b/openmetadata-ui/src/main/resources/ui/cypress/e2e/Flow/AddRoleAndAssignToUser.spec.ts new file mode 100644 index 000000000000..ca15d9760c7f --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/cypress/e2e/Flow/AddRoleAndAssignToUser.spec.ts @@ -0,0 +1,125 @@ +/* + * Copyright 2022 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { + descriptionBox, + interceptURL, + uuid, + verifyResponseStatusCode, +} from '../../common/common'; +import { BASE_URL } from '../../constants/constants'; +import { GlobalSettingOptions } from '../../constants/settings.constant'; + +const roleName = `Role-test-${uuid()}`; +const userName = `usercttest${uuid()}`; +const userEmail = `${userName}@gmail.com`; + +describe( + 'Test Add role and assign it to the user', + { tags: 'Settings' }, + () => { + beforeEach(() => { + cy.login(); + interceptURL('GET', '*api/v1/roles*', 'getRoles'); + interceptURL('GET', '/api/v1/users?*', 'usersPage'); + }); + + it('Create role', () => { + cy.settingClick(GlobalSettingOptions.ROLES); + verifyResponseStatusCode('@getRoles', 200); + + cy.get('[data-testid="add-role"]').contains('Add Role').click(); + + // Asserting navigation + cy.get('[data-testid="inactive-link"]').should('contain', 'Add New Role'); + + // Entering name + cy.get('#name').type(roleName); + // Entering descrription + cy.get(descriptionBox).type('description'); + // Select the policies + cy.get('[data-testid="policies"]').click(); + + cy.get('[title="Data Consumer Policy"]').scrollIntoView().click(); + + cy.get('[title="Data Steward Policy"]').scrollIntoView().click(); + // Save the role + cy.get('[data-testid="submit-btn"]').scrollIntoView().click(); + + // Verify the role is added successfully + cy.url().should('eq', `${BASE_URL}/settings/access/roles/${roleName}`); + cy.get('[data-testid="inactive-link"]').should('contain', roleName); + + // Verify added description + cy.get( + '[data-testid="asset-description-container"] [data-testid="viewer-container"]' + ).should('contain', 'description'); + }); + + it('Create new user and assign new role to him', () => { + // Create user and assign newly created role to the user + cy.settingClick(GlobalSettingOptions.USERS); + verifyResponseStatusCode('@usersPage', 200); + + cy.get('[data-testid="add-user"]').contains('Add User').click(); + + cy.get('[data-testid="email"]').scrollIntoView().type(userEmail); + + cy.get('[data-testid="displayName"]').type(userName); + + cy.get(descriptionBox).type('Adding user'); + + interceptURL( + 'GET', + '/api/v1/users/generateRandomPwd', + 'generatePassword' + ); + cy.get('[data-testid="password-generator"]').scrollIntoView().click(); + verifyResponseStatusCode('@generatePassword', 200); + + cy.get(`[data-testid="roles-dropdown"]`).type(roleName); + cy.get(`.ant-select-dropdown [title="${roleName}"]`).click(); + + cy.clickOutside(); + interceptURL('POST', '/api/v1/users', 'createUser'); + cy.get('[data-testid="save-user"]').scrollIntoView().click(); + verifyResponseStatusCode('@createUser', 201); + }); + + it('Verify assigned role to new user', () => { + cy.settingClick(GlobalSettingOptions.USERS); + verifyResponseStatusCode('@usersPage', 200); + + interceptURL( + 'GET', + '/api/v1/search/query?q=**&from=0&size=*&index=*', + 'searchUser' + ); + interceptURL('GET', '/api/v1/users/name/*', 'userDetailsPage'); + cy.get('[data-testid="searchbar"]').type(userName); + verifyResponseStatusCode('@searchUser', 200); + + cy.get(`[data-testid="${userName}"]`).click(); + verifyResponseStatusCode('@userDetailsPage', 200); + + // click the collapse button to open the other details + cy.get( + '[data-testid="user-profile"] .ant-collapse-expand-icon > .anticon' + ).click(); + + cy.get( + '[data-testid="user-profile"] [data-testid="user-profile-roles"]' + ).should('contain', roleName); + }); + } +); diff --git a/openmetadata-ui/src/main/resources/ui/cypress/e2e/Flow/AdvancedSearchQuickFilters.spec.js b/openmetadata-ui/src/main/resources/ui/cypress/e2e/Flow/AdvancedSearchQuickFilters.spec.ts similarity index 55% rename from openmetadata-ui/src/main/resources/ui/cypress/e2e/Flow/AdvancedSearchQuickFilters.spec.js rename to openmetadata-ui/src/main/resources/ui/cypress/e2e/Flow/AdvancedSearchQuickFilters.spec.ts index bf4c614cf419..683b577c62cf 100644 --- a/openmetadata-ui/src/main/resources/ui/cypress/e2e/Flow/AdvancedSearchQuickFilters.spec.js +++ b/openmetadata-ui/src/main/resources/ui/cypress/e2e/Flow/AdvancedSearchQuickFilters.spec.ts @@ -11,75 +11,90 @@ * limitations under the License. */ -import { - addOwner, - goToAdvanceSearch, - removeOwner, -} from '../../common/advancedSearch'; import { searchAndClickOnOption } from '../../common/advancedSearchQuickFilters'; import { interceptURL, verifyResponseStatusCode } from '../../common/common'; +import { goToAdvanceSearch } from '../../common/Utils/AdvancedSearch'; +import { visitEntityDetailsPage } from '../../common/Utils/Entity'; +import { addOwner, removeOwner } from '../../common/Utils/Owner'; import { QUICK_FILTERS_BY_ASSETS } from '../../constants/advancedSearchQuickFilters.constants'; import { SEARCH_ENTITY_TABLE } from '../../constants/constants'; import { SidebarItem } from '../../constants/Entity.interface'; const ownerName = 'Aaron Johnson'; -describe(`Advanced search quick filters should work properly for assets`, () => { - before(() => { - cy.login(); - addOwner({ ownerName, ...SEARCH_ENTITY_TABLE.table_1 }); - }); +describe( + `Advanced search quick filters should work properly for assets`, + { tags: 'DataAssets' }, + () => { + before(() => { + cy.login(); + + visitEntityDetailsPage({ + term: SEARCH_ENTITY_TABLE.table_1.term, + entity: SEARCH_ENTITY_TABLE.table_1.entity, + serviceName: SEARCH_ENTITY_TABLE.table_1.serviceName, + }); - after(() => { - cy.login(); - removeOwner(); - }); + addOwner(ownerName); + }); - beforeEach(() => { - cy.login(); - }); + after(() => { + cy.login(); + visitEntityDetailsPage({ + term: SEARCH_ENTITY_TABLE.table_1.term, + entity: SEARCH_ENTITY_TABLE.table_1.entity, + serviceName: SEARCH_ENTITY_TABLE.table_1.serviceName, + }); - it(`should show the quick filters for respective assets`, () => { - // Navigate to explore page - cy.sidebarClick(SidebarItem.EXPLORE); - QUICK_FILTERS_BY_ASSETS.map((asset) => { - cy.get(`[data-testid="${asset.tab}"]`).scrollIntoView().click(); + removeOwner(ownerName); + }); - asset.filters.map((filter) => { - cy.get(`[data-testid="search-dropdown-${filter.label}"]`) - .scrollIntoView() - .should('be.visible'); + beforeEach(() => { + cy.login(); + }); + + it(`should show the quick filters for respective assets`, () => { + // Navigate to explore page + cy.sidebarClick(SidebarItem.EXPLORE); + QUICK_FILTERS_BY_ASSETS.map((asset) => { + cy.get(`[data-testid="${asset.tab}"]`).scrollIntoView().click(); + + asset.filters.map((filter) => { + cy.get(`[data-testid="search-dropdown-${filter.label}"]`) + .scrollIntoView() + .should('be.visible'); + }); }); }); - }); - it('search dropdown should work properly for tables', () => { - // Table - const asset = QUICK_FILTERS_BY_ASSETS[0]; + it('search dropdown should work properly for tables', () => { + // Table + const asset = QUICK_FILTERS_BY_ASSETS[0]; - // Navigate to explore page - cy.sidebarClick(SidebarItem.EXPLORE); - cy.get(`[data-testid="${asset.tab}"]`).scrollIntoView().click(); + // Navigate to explore page + cy.sidebarClick(SidebarItem.EXPLORE); + cy.get(`[data-testid="${asset.tab}"]`).scrollIntoView().click(); - asset.filters - .filter((item) => item.select) - .map((filter) => { - cy.get(`[data-testid="search-dropdown-${filter.label}"]`).click(); - searchAndClickOnOption(asset, filter, true); + asset.filters + .filter((item) => item.select) + .map((filter) => { + cy.get(`[data-testid="search-dropdown-${filter.label}"]`).click(); + searchAndClickOnOption(asset, filter, true); - const querySearchURL = `/api/v1/search/query?*index=${ - asset.searchIndex - }*query_filter=*should*${filter.key}*${encodeURI( - Cypress._.toLower(filter.selectOption1).replace(' ', '+') - )}*`; + const querySearchURL = `/api/v1/search/query?*index=${ + asset.searchIndex + }*query_filter=*should*${filter.key}*${encodeURI( + Cypress._.toLower(filter.selectOption1).replace(' ', '+') + )}*`; - interceptURL('GET', querySearchURL, 'querySearchAPI'); + interceptURL('GET', querySearchURL, 'querySearchAPI'); - cy.get('[data-testid="update-btn"]').click(); + cy.get('[data-testid="update-btn"]').click(); - verifyResponseStatusCode('@querySearchAPI', 200); - }); - }); -}); + verifyResponseStatusCode('@querySearchAPI', 200); + }); + }); + } +); const testIsNullAndIsNotNullFilters = (operatorTitle, queryFilter, alias) => { goToAdvanceSearch(); diff --git a/openmetadata-ui/src/main/resources/ui/cypress/e2e/Flow/Collect.spec.js b/openmetadata-ui/src/main/resources/ui/cypress/e2e/Flow/Collect.spec.ts similarity index 100% rename from openmetadata-ui/src/main/resources/ui/cypress/e2e/Flow/Collect.spec.js rename to openmetadata-ui/src/main/resources/ui/cypress/e2e/Flow/Collect.spec.ts diff --git a/openmetadata-ui/src/main/resources/ui/cypress/e2e/Flow/CustomizeLandingPage.spec.js b/openmetadata-ui/src/main/resources/ui/cypress/e2e/Flow/CustomizeLandingPage.spec.ts similarity index 92% rename from openmetadata-ui/src/main/resources/ui/cypress/e2e/Flow/CustomizeLandingPage.spec.js rename to openmetadata-ui/src/main/resources/ui/cypress/e2e/Flow/CustomizeLandingPage.spec.ts index 4c54dfa04d9a..ff3f2aa0b398 100644 --- a/openmetadata-ui/src/main/resources/ui/cypress/e2e/Flow/CustomizeLandingPage.spec.js +++ b/openmetadata-ui/src/main/resources/ui/cypress/e2e/Flow/CustomizeLandingPage.spec.ts @@ -10,8 +10,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -// eslint-disable-next-line spaced-comment -/// import { compare } from 'fast-json-patch'; import { interceptURL, toastNotification } from '../../common/common'; @@ -23,14 +21,15 @@ import { removeAndCheckWidget, saveLayout, } from '../../common/CustomizeLandingPageUtils'; +import { getToken } from '../../common/Utils/LocalStorage'; import { PERSONA_DETAILS } from '../../constants/EntityConstant'; describe('Customize Landing Page Flow', { tags: 'Settings' }, () => { - let testData = {}; + const testData = {}; before(() => { cy.login(); cy.getAllLocalStorage().then((data) => { - const token = Object.values(data)[0].oidcIdToken; + const token = getToken(data); // Fetch logged in user details to get user id cy.request({ @@ -86,20 +85,22 @@ describe('Customize Landing Page Flow', { tags: 'Settings' }, () => { after(() => { cy.login(); - const token = localStorage.getItem('oidcIdToken'); + cy.getAllLocalStorage().then((data) => { + const token = getToken(data); - // Delete created user - cy.request({ - method: 'DELETE', - url: `/api/v1/personas/${testData.persona.id}`, - headers: { Authorization: `Bearer ${token}` }, - }); + // Delete created user + cy.request({ + method: 'DELETE', + url: `/api/v1/personas/${testData.persona.id}`, + headers: { Authorization: `Bearer ${token}` }, + }); - // Delete created landing page config doc - cy.request({ - method: 'DELETE', - url: `/api/v1/docStore/${testData.docStoreData.id}`, - headers: { Authorization: `Bearer ${token}` }, + // Delete created landing page config doc + cy.request({ + method: 'DELETE', + url: `/api/v1/docStore/${testData.docStoreData.id}`, + headers: { Authorization: `Bearer ${token}` }, + }); }); }); diff --git a/openmetadata-ui/src/main/resources/ui/cypress/e2e/Flow/Explore.spec.js b/openmetadata-ui/src/main/resources/ui/cypress/e2e/Flow/Explore.spec.ts similarity index 96% rename from openmetadata-ui/src/main/resources/ui/cypress/e2e/Flow/Explore.spec.js rename to openmetadata-ui/src/main/resources/ui/cypress/e2e/Flow/Explore.spec.ts index 474a568038d0..20e5ed6f6f01 100644 --- a/openmetadata-ui/src/main/resources/ui/cypress/e2e/Flow/Explore.spec.js +++ b/openmetadata-ui/src/main/resources/ui/cypress/e2e/Flow/Explore.spec.ts @@ -13,7 +13,7 @@ import { interceptURL, verifyResponseStatusCode } from '../../common/common'; import { SidebarItem } from '../../constants/Entity.interface'; -describe('Explore Page', { tags: 'DataAssets' }, () => { +describe.skip('Explore Page', { tags: 'DataAssets' }, () => { before(() => { cy.login(); }); diff --git a/openmetadata-ui/src/main/resources/ui/cypress/e2e/Flow/Lineage.spec.js b/openmetadata-ui/src/main/resources/ui/cypress/e2e/Flow/Lineage.spec.ts similarity index 82% rename from openmetadata-ui/src/main/resources/ui/cypress/e2e/Flow/Lineage.spec.js rename to openmetadata-ui/src/main/resources/ui/cypress/e2e/Flow/Lineage.spec.ts index 8f67a47cd368..fe38ed770839 100644 --- a/openmetadata-ui/src/main/resources/ui/cypress/e2e/Flow/Lineage.spec.js +++ b/openmetadata-ui/src/main/resources/ui/cypress/e2e/Flow/Lineage.spec.ts @@ -13,6 +13,7 @@ import { interceptURL, verifyResponseStatusCode } from '../../common/common'; import { visitEntityDetailsPage } from '../../common/Utils/Entity'; +import { EntityType } from '../../constants/Entity.interface'; import { LINEAGE_ITEMS, PIPELINE_ITEMS, @@ -81,12 +82,36 @@ const deleteNode = (node) => { verifyResponseStatusCode('@lineageDeleteApi', 200); }; +const deleteEdge = (fromNode, toNode) => { + interceptURL('DELETE', '/api/v1/lineage/**', 'lineageDeleteApi'); + cy.get(`[data-testid="edge-${fromNode.fqn}-${toNode.fqn}"]`).click({ + force: true, + }); + + if ( + ['Table', 'Topic'].indexOf(fromNode.entityType) > -1 && + ['Table', 'Topic'].indexOf(toNode.entityType) > -1 + ) { + // Adding force true for handles because it can be hidden behind the node + cy.get('[data-testid="add-pipeline"]').click({ force: true }); + cy.get( + '[data-testid="add-edge-modal"] [data-testid="remove-edge-button"]' + ).click(); + } else { + cy.get('[data-testid="delete-button"]').click({ force: true }); + } + cy.get( + '[data-testid="delete-edge-confirmation-modal"] .ant-btn-primary' + ).click(); + verifyResponseStatusCode('@lineageDeleteApi', 200); +}; + const applyPipelineFromModal = (fromNode, toNode, pipelineData) => { interceptURL('PUT', '/api/v1/lineage', 'lineageApi'); cy.get(`[data-testid="edge-${fromNode.fqn}-${toNode.fqn}"]`).click({ force: true, }); - cy.get('[data-testid="add-pipeline"]').click(); + cy.get('[data-testid="add-pipeline"]').click({ force: true }); cy.get('[data-testid="add-edge-modal"] [data-testid="field-input"]') .click() @@ -171,16 +196,18 @@ const expandCols = (nodeFqn, hasShowMore) => { } }; -const addColumnLineage = (fromNode, toNode) => { +const addColumnLineage = (fromNode, toNode, exitEditMode = true) => { interceptURL('PUT', '/api/v1/lineage', 'lineageApi'); expandCols(fromNode.fqn, false); - expandCols(toNode.fqn, true); + expandCols(toNode.fqn, toNode.entityType === EntityType.Table); dragConnection( `column-${fromNode.columns[0]}`, `column-${toNode.columns[0]}` ); verifyResponseStatusCode('@lineageApi', 200); - cy.get('[data-testid="edit-lineage"]').click(); + if (exitEditMode) { + cy.get('[data-testid="edit-lineage"]').click(); + } cy.get( `[data-testid="column-edge-${fromNode.columns[0]}-${toNode.columns[0]}"]` ); @@ -241,10 +268,10 @@ describe('Lineage verification', { tags: 'DataAssets' }, () => { // Delete Nodes for (let i = 0; i < LINEAGE_ITEMS.length; i++) { if (i !== index) { - deleteNode(LINEAGE_ITEMS[i]); - cy.get(`[data-testid="lineage-node-${LINEAGE_ITEMS[i].fqn}"]`).should( - 'not.exist' - ); + deleteEdge(entity, LINEAGE_ITEMS[i]); + cy.get( + `[data-testid="edge-${entity.fqn}-${LINEAGE_ITEMS[i].fqn}"]` + ).should('not.exist'); } } @@ -280,11 +307,16 @@ describe('Lineage verification', { tags: 'DataAssets' }, () => { it('Add column lineage', () => { const sourceEntity = LINEAGE_ITEMS[0]; - const targetEntity = LINEAGE_ITEMS[1]; - addPipelineBetweenNodes(sourceEntity, targetEntity); - // Add column lineage - addColumnLineage(sourceEntity, targetEntity); - cy.get('[data-testid="edit-lineage"]').click(); - deleteNode(targetEntity); + for (let i = 1; i < LINEAGE_ITEMS.length; i++) { + const targetEntity = LINEAGE_ITEMS[i]; + if (targetEntity.columns.length > 0) { + addPipelineBetweenNodes(sourceEntity, targetEntity); + // Add column lineage + addColumnLineage(sourceEntity, targetEntity); + cy.get('[data-testid="edit-lineage"]').click(); + deleteNode(targetEntity); + cy.goToHomePage(); + } + } }); }); diff --git a/openmetadata-ui/src/main/resources/ui/cypress/e2e/Flow/NotificationAlerts.spec.ts b/openmetadata-ui/src/main/resources/ui/cypress/e2e/Flow/NotificationAlerts.spec.ts index 5bf7aef93c24..cdf087603a17 100644 --- a/openmetadata-ui/src/main/resources/ui/cypress/e2e/Flow/NotificationAlerts.spec.ts +++ b/openmetadata-ui/src/main/resources/ui/cypress/e2e/Flow/NotificationAlerts.spec.ts @@ -11,18 +11,15 @@ * limitations under the License. */ -// eslint-disable-next-line spaced-comment -/// - import { addDomainFilter, addEntityFQNFilter, addEventTypeFilter, addExternalDestination, + addFilterWithUsersListInput, addGMEFilter, addInternalDestination, addOwnerFilter, - addUpdaterNameFilter, deleteAlertSteps, verifyAlertDetails, } from '../../common/AlertUtils'; @@ -36,6 +33,7 @@ import { createSingleLevelEntity, hardDeleteService, } from '../../common/EntityUtils'; +import { getToken } from '../../common/Utils/LocalStorage'; import { ALERT_DESCRIPTION, ALERT_NAME, @@ -54,380 +52,496 @@ const SOURCE_NAME_2 = 'dashboard'; const SOURCE_DISPLAY_NAME_2 = 'Dashboard'; const SOURCE_NAME_3 = 'task'; const SOURCE_DISPLAY_NAME_3 = 'Task'; - -describe('Notification Alert Flow', { tags: 'Settings' }, () => { - const data = { - user: { - displayName: '', - }, - domain: { - name: '', - }, - alertDetails: { - id: '', - }, - }; - - before(() => { - cy.login(); - cy.getAllLocalStorage().then((storageData) => { - const token = Object.values(storageData)[0].oidcIdToken; - - // Create a dashboard - createSingleLevelEntity({ - token, - ...DASHBOARD_SERVICE, +const SOURCE_NAME_4 = 'conversation'; +const SOURCE_DISPLAY_NAME_4 = 'Conversation'; + +describe( + 'Notification Alert Flow', + { tags: ['Settings', 'Observability'] }, + () => { + const data = { + user: { + displayName: '', + id: '', + }, + domain: { + name: '', + }, + alertDetails: { + id: '', + }, + }; + + before(() => { + cy.login(); + cy.getAllLocalStorage().then((storageData) => { + const token = getToken(storageData); + + // Create a dashboard + createSingleLevelEntity({ + token, + ...DASHBOARD_SERVICE, + }); + + // Create a new user + cy.request({ + method: 'POST', + url: `/api/v1/users/signup`, + headers: { Authorization: `Bearer ${token}` }, + body: USER_DETAILS, + }).then((response) => { + data.user = response.body; + }); + + // Create a domain + cy.request({ + method: 'PUT', + url: `/api/v1/domains`, + headers: { Authorization: `Bearer ${token}` }, + body: DOMAIN_CREATION_DETAILS, + }).then((response) => { + data.domain = response.body; + }); }); + }); - // Create a new user - cy.request({ - method: 'POST', - url: `/api/v1/users/signup`, - headers: { Authorization: `Bearer ${token}` }, - body: USER_DETAILS, - }).then((response) => { - data.user = response.body; + after(() => { + cy.login(); + cy.getAllLocalStorage().then((storageData) => { + const token = getToken(storageData); + + hardDeleteService({ + token, + serviceFqn: DASHBOARD_SERVICE.service.name, + serviceType: DASHBOARD_SERVICE.serviceType, + }); + + // Delete created domain + cy.request({ + method: 'DELETE', + url: `/api/v1/domains/name/${DOMAIN_CREATION_DETAILS.name}`, + headers: { Authorization: `Bearer ${token}` }, + }); + + // Delete created user + cy.request({ + method: 'DELETE', + url: `/api/v1/users/${data.user.id}?hardDelete=true&recursive=false`, + headers: { Authorization: `Bearer ${token}` }, + }); }); + }); - // Create a domain - cy.request({ - method: 'PUT', - url: `/api/v1/domains`, - headers: { Authorization: `Bearer ${token}` }, - body: DOMAIN_CREATION_DETAILS, - }).then((response) => { - data.domain = response.body; - }); + beforeEach(() => { + interceptURL('POST', '/api/v1/events/subscriptions', 'createAlert'); + interceptURL('PUT', '/api/v1/events/subscriptions', 'updateAlert'); + interceptURL( + 'GET', + '/api/v1/events/subscriptions/name/*', + 'alertDetails' + ); + cy.login(); + cy.sidebarClick(SidebarItem.SETTINGS); + interceptURL('GET', '/api/v1/events/subscriptions?*', 'alertsPage'); + cy.get('[data-testid="notifications"]') + .contains('Notifications') + .scrollIntoView() + .click(); }); - }); - after(() => { - cy.login(); - cy.getAllLocalStorage().then((storageData) => { - const token = Object.values(storageData)[0].oidcIdToken; + it('Create new alert with single filter and destination', () => { + verifyResponseStatusCode('@alertsPage', 200); - hardDeleteService({ - token, - serviceFqn: DASHBOARD_SERVICE.service.name, - serviceType: DASHBOARD_SERVICE.serviceType, - }); + cy.get('[data-testid="create-notification"]').click(); - // Delete created domain - cy.request({ - method: 'DELETE', - url: `/api/v1/domains/name/${DOMAIN_CREATION_DETAILS.name}`, - headers: { Authorization: `Bearer ${token}` }, - }); + // Enter alert name + cy.get('#name').type(ALERT_NAME); + + // Enter description + cy.get(descriptionBox).clear().type(ALERT_DESCRIPTION); + + // Select all source + cy.get('[data-testid="add-source-button"]').scrollIntoView().click(); + + cy.get( + `[data-testid="drop-down-menu"] [data-testid="${SOURCE_NAME_1}-option"]` + ) + .contains(SOURCE_DISPLAY_NAME_1) + .click(); + + cy.get('[data-testid="source-select"]').should( + 'contain', + SOURCE_DISPLAY_NAME_1 + ); + + // Select filters + cy.get('[data-testid="add-filters"]').click(); + + addOwnerFilter(0, data.user.displayName); + + // Select Destination + cy.get('[data-testid="add-destination-button"]').scrollIntoView().click(); - // Delete created user - cy.request({ - method: 'DELETE', - url: `/api/v1/users/${data.user.id}?hardDelete=true&recursive=false`, - headers: { Authorization: `Bearer ${token}` }, + addInternalDestination(0, 'Admins', 'Email'); + + // Click save + cy.get('[data-testid="save-button"]').scrollIntoView().click(); + cy.wait('@createAlert').then((interception) => { + data.alertDetails = interception?.response?.body; + + expect(interception?.response?.statusCode).equal(201); }); - }); - }); + toastNotification('Alerts created successfully.'); - beforeEach(() => { - interceptURL('POST', '/api/v1/events/subscriptions', 'createAlert'); - interceptURL('PUT', '/api/v1/events/subscriptions', 'updateAlert'); - interceptURL('GET', '/api/v1/events/subscriptions/name/*', 'alertDetails'); - cy.login(); - cy.sidebarClick(SidebarItem.SETTINGS); - interceptURL('GET', '/api/v1/events/subscriptions?*', 'alertsPage'); - cy.get('[data-testid="notifications"]') - .contains('Notifications') - .scrollIntoView() - .click(); - }); + // Check if the alert details page is visible + verifyResponseStatusCode('@alertDetails', 200); + cy.get('[data-testid="alert-details-container"]').should('exist'); + }); - it('Create new alert with single filter and destination', () => { - verifyResponseStatusCode('@alertsPage', 200); + it('Check created alert details', () => { + const { id: alertId } = data.alertDetails; + verifyResponseStatusCode('@alertsPage', 200); - cy.get('[data-testid="create-notification"]').click(); + cy.get(`[data-row-key="${alertId}"] [data-testid="alert-name"]`) + .should('contain', ALERT_NAME) + .click(); - // Enter alert name - cy.get('#name').type(ALERT_NAME); + verifyResponseStatusCode('@alertDetails', 200); - // Enter description - cy.get(descriptionBox).clear().type(ALERT_DESCRIPTION); + // Verify alert details + verifyAlertDetails(data.alertDetails); + }); - // Select all source - cy.get('[data-testid="add-source-button"]').scrollIntoView().click(); + it('Edit and check alert by adding multiple filters and internal destinations', () => { + const { id: alertId } = data.alertDetails; - cy.get( - `[data-testid="drop-down-menu"] [data-testid="${SOURCE_NAME_1}-option"]` - ) - .contains(SOURCE_DISPLAY_NAME_1) - .click(); + // Go to edit alert page + cy.get('table').should('contain', ALERT_NAME).click(); - cy.get('[data-testid="source-select"]').should( - 'contain', - SOURCE_DISPLAY_NAME_1 - ); + cy.get( + `[data-row-key="${alertId}"] [data-testid="alert-edit-${ALERT_NAME}"]` + ).click(); - // Select filters - cy.get('[data-testid="add-filters"]').click(); + // Update description + cy.get(descriptionBox).click().clear().type(ALERT_UPDATED_DESCRIPTION); - addOwnerFilter(0, data.user.displayName); + // Update source + cy.get('[data-testid="source-select"]').scrollIntoView().click(); + cy.get(`[data-testid="${SOURCE_NAME_2}-option"]`) + .contains(SOURCE_DISPLAY_NAME_2) + .click(); - // Select Destination - cy.get('[data-testid="add-destination-button"]').scrollIntoView().click(); + // Filters should reset after source change + cy.get('[data-testid="filter-select-0"]').should('not.exist'); - addInternalDestination(0, 'Admins', 'Email'); + // Add multiple filters + [...Array(6).keys()].forEach(() => { + cy.get('[data-testid="add-filters"]').scrollIntoView().click(); + }); - // Click save - cy.get('[data-testid="save-button"]').scrollIntoView().click(); - cy.wait('@createAlert').then((interception) => { - data.alertDetails = interception?.response?.body; + addOwnerFilter(0, data.user.displayName); + addEntityFQNFilter( + 1, + `${DASHBOARD_SERVICE.service.name}.${DASHBOARD_SERVICE.entity.name}`, + true + ); + addEventTypeFilter(2, 'entityCreated'); + addFilterWithUsersListInput( + 'Updater Name-filter-option', + 3, + data.user.displayName, + true + ); + addDomainFilter(4, data.domain.name); + addGMEFilter(5); + + // Add multiple destinations + [...Array(3).keys()].forEach(() => { + cy.get('[data-testid="add-destination-button"]') + .scrollIntoView() + .click(); + }); - expect(interception?.response?.statusCode).equal(201); + addInternalDestination(1, 'Owners', 'G Chat'); + addInternalDestination( + 2, + 'Teams', + 'Slack', + 'Team-select', + 'Organization' + ); + addInternalDestination( + 3, + 'Users', + 'Email', + 'User-select', + data.user.displayName + ); + + // Click save + cy.get('[data-testid="save-button"]').scrollIntoView().click(); + cy.wait('@updateAlert').then((interception) => { + data.alertDetails = interception?.response?.body; + + expect(interception?.response?.statusCode).equal(200); + + // Verify the edited alert changes + verifyAlertDetails(interception?.response?.body); + }); }); - toastNotification('Alerts created successfully.'); - // Check if the alert details page is visible - verifyResponseStatusCode('@alertDetails', 200); - cy.get('[data-testid="alert-details-container"]').should('exist'); - }); + it('Delete alert with single filter', () => { + deleteAlertSteps(ALERT_NAME); + }); - it('Check created alert details', () => { - const { id: alertId } = data.alertDetails; - verifyResponseStatusCode('@alertsPage', 200); + it('Create new alert with multiple filters and destinations', () => { + verifyResponseStatusCode('@alertsPage', 200); - cy.get(`[data-row-key="${alertId}"] [data-testid="alert-name"]`) - .should('contain', ALERT_NAME) - .click(); + cy.get('[data-testid="create-notification"]').click(); - verifyResponseStatusCode('@alertDetails', 200); + // Enter alert name + cy.get('#name').type(ALERT_NAME); - // Verify alert details - verifyAlertDetails(data.alertDetails); - }); + // Enter description + cy.get(descriptionBox).clear().type(ALERT_DESCRIPTION); - it('Edit and check alert by adding multiple filters and internal destinations', () => { - const { id: alertId } = data.alertDetails; + // Select all source + cy.get('[data-testid="add-source-button"]').scrollIntoView().click(); - // Go to edit alert page - cy.get('table').should('contain', ALERT_NAME).click(); + cy.get( + `[data-testid="drop-down-menu"] [data-testid="${SOURCE_NAME_1}-option"]` + ) + .contains(SOURCE_DISPLAY_NAME_1) + .click(); - cy.get( - `[data-row-key="${alertId}"] [data-testid="alert-edit-${ALERT_NAME}"]` - ).click(); + cy.get('[data-testid="source-select"]').should( + 'contain', + SOURCE_DISPLAY_NAME_1 + ); - // Update description - cy.get(descriptionBox).click().clear().type(ALERT_UPDATED_DESCRIPTION); + // Add multiple filters + [...Array(6).keys()].forEach(() => { + cy.get('[data-testid="add-filters"]').scrollIntoView().click(); + }); - // Update source - cy.get('[data-testid="source-select"]').scrollIntoView().click(); - cy.get(`[data-testid="${SOURCE_NAME_2}-option"]`) - .contains(SOURCE_DISPLAY_NAME_2) - .click(); + addOwnerFilter(0, data.user.displayName); + addEntityFQNFilter( + 1, + `${DASHBOARD_SERVICE.service.name}.${DASHBOARD_SERVICE.entity.name}`, + true + ); + addEventTypeFilter(2, 'entityCreated'); + addFilterWithUsersListInput( + 'Updater Name-filter-option', + 3, + data.user.displayName, + true + ); + addDomainFilter(4, data.domain.name); + addGMEFilter(5); + + // Add multiple destinations + [...Array(6).keys()].forEach(() => { + cy.get('[data-testid="add-destination-button"]') + .scrollIntoView() + .click(); + }); - // Filters should reset after source change - cy.get('[data-testid="filter-select-0"]').should('not.exist'); + addInternalDestination(0, 'Followers', 'Email'); + addExternalDestination(1, 'Email', 'test@example.com'); + addExternalDestination(2, 'G Chat', 'https://gchat.com'); + addExternalDestination(3, 'Generic', 'https://generic.com'); + addExternalDestination(4, 'Ms Teams', 'https://msteams.com'); + addExternalDestination(5, 'Slack', 'https://slack.com'); - // Add multiple filters - [...Array(6).keys()].forEach(() => { - cy.get('[data-testid="add-filters"]').scrollIntoView().click(); - }); + // Click save + cy.get('[data-testid="save-button"]').scrollIntoView().click(); + cy.wait('@createAlert').then((interception) => { + data.alertDetails = interception?.response?.body; - addOwnerFilter(0, data.user.displayName); - addEntityFQNFilter( - 1, - `${DASHBOARD_SERVICE.service.name}.${DASHBOARD_SERVICE.entity.name}`, - true - ); - addEventTypeFilter(2, 'entityCreated'); - addUpdaterNameFilter(3, data.user.displayName, true); - addDomainFilter(4, data.domain.name); - addGMEFilter(5); - - // Add multiple destinations - [...Array(3).keys()].forEach(() => { - cy.get('[data-testid="add-destination-button"]').scrollIntoView().click(); - }); + expect(interception?.response?.statusCode).equal(201); + }); + toastNotification('Alerts created successfully.'); - addInternalDestination(1, 'Owners', 'G Chat'); - addInternalDestination(2, 'Teams', 'Slack', 'Team-select', 'Organization'); - addInternalDestination( - 3, - 'Users', - 'Email', - 'User-select', - data.user.displayName - ); - - // Click save - cy.get('[data-testid="save-button"]').scrollIntoView().click(); - cy.wait('@updateAlert').then((interception) => { - data.alertDetails = interception?.response?.body; - - expect(interception?.response?.statusCode).equal(200); - - // Verify the edited alert changes - verifyAlertDetails(interception?.response?.body); + // Check if the alert details page is visible + verifyResponseStatusCode('@alertDetails', 200); + cy.get('[data-testid="alert-details-container"]').should('exist'); }); - }); - it('Delete alert with single filter', () => { - deleteAlertSteps(ALERT_NAME); - }); + it('Edit and check alert by removing added filters and internal destinations', () => { + const { id: alertId } = data.alertDetails; - it('Create new alert with multiple filters and destinations', () => { - verifyResponseStatusCode('@alertsPage', 200); + // Go to edit alert page + cy.get('table').should('contain', ALERT_NAME).click(); - cy.get('[data-testid="create-notification"]').click(); + cy.get( + `[data-row-key="${alertId}"] [data-testid="alert-edit-${ALERT_NAME}"]` + ).click(); - // Enter alert name - cy.get('#name').type(ALERT_NAME); + // Remove description + cy.get(descriptionBox).click().clear(); - // Enter description - cy.get(descriptionBox).clear().type(ALERT_DESCRIPTION); + // Remove all filters + [...Array(6).keys()].forEach(() => { + cy.get('[data-testid="remove-filter-0"]').scrollIntoView().click(); + }); - // Select all source - cy.get('[data-testid="add-source-button"]').scrollIntoView().click(); + // Remove all destinations except one + [...Array(5).keys()].forEach(() => { + cy.get('[data-testid="remove-destination-0"]').scrollIntoView().click(); + }); - cy.get( - `[data-testid="drop-down-menu"] [data-testid="${SOURCE_NAME_1}-option"]` - ) - .contains(SOURCE_DISPLAY_NAME_1) - .click(); + // Click save + cy.get('[data-testid="save-button"]').scrollIntoView().click(); + cy.wait('@updateAlert').then((interception) => { + data.alertDetails = interception?.response?.body; - cy.get('[data-testid="source-select"]').should( - 'contain', - SOURCE_DISPLAY_NAME_1 - ); + expect(interception?.response?.statusCode).equal(200); - // Add multiple filters - [...Array(6).keys()].forEach(() => { - cy.get('[data-testid="add-filters"]').scrollIntoView().click(); + // Verify the edited alert changes + verifyAlertDetails(interception?.response?.body); + }); }); - addOwnerFilter(0, data.user.displayName); - addEntityFQNFilter( - 1, - `${DASHBOARD_SERVICE.service.name}.${DASHBOARD_SERVICE.entity.name}`, - true - ); - addEventTypeFilter(2, 'entityCreated'); - addUpdaterNameFilter(3, data.user.displayName, true); - addDomainFilter(4, data.domain.name); - addGMEFilter(5); - - // Add multiple destinations - [...Array(6).keys()].forEach(() => { - cy.get('[data-testid="add-destination-button"]').scrollIntoView().click(); + it('Delete alert with multiple filters', () => { + deleteAlertSteps(ALERT_NAME); }); - addInternalDestination(0, 'Followers', 'Email'); - addExternalDestination(1, 'Email', 'test@example.com'); - addExternalDestination(2, 'G Chat', 'https://gchat.com'); - addExternalDestination(3, 'Generic', 'https://generic.com'); - addExternalDestination(4, 'Ms Teams', 'https://msteams.com'); - addExternalDestination(5, 'Slack', 'https://slack.com'); + it('Create alert for task source', () => { + verifyResponseStatusCode('@alertsPage', 200); - // Click save - cy.get('[data-testid="save-button"]').scrollIntoView().click(); - cy.wait('@createAlert').then((interception) => { - data.alertDetails = interception?.response?.body; + cy.get('[data-testid="create-notification"]').click(); - expect(interception?.response?.statusCode).equal(201); - }); - toastNotification('Alerts created successfully.'); + // Enter alert name + cy.get('#name').type(ALERT_NAME); - // Check if the alert details page is visible - verifyResponseStatusCode('@alertDetails', 200); - cy.get('[data-testid="alert-details-container"]').should('exist'); - }); + // Enter description + cy.get(descriptionBox).clear().type(ALERT_DESCRIPTION); - it('Edit and check alert by removing added filters and internal destinations', () => { - const { id: alertId } = data.alertDetails; + // Select all source + cy.get('[data-testid="add-source-button"]').scrollIntoView().click(); - // Go to edit alert page - cy.get('table').should('contain', ALERT_NAME).click(); + cy.get( + `[data-testid="drop-down-menu"] [data-testid="${SOURCE_NAME_3}-option"]` + ) + .contains(SOURCE_DISPLAY_NAME_3) + .click(); - cy.get( - `[data-row-key="${alertId}"] [data-testid="alert-edit-${ALERT_NAME}"]` - ).click(); + cy.get('[data-testid="source-select"]').should( + 'contain', + SOURCE_DISPLAY_NAME_3 + ); - // Remove description - cy.get(descriptionBox).click().clear(); + // Select Destination + cy.get('[data-testid="add-destination-button"]').scrollIntoView().click(); + + addInternalDestination(0, 'Owners', 'Email'); + + cy.get('[data-testid="add-destination-button"]').scrollIntoView().click(); + + addInternalDestination(1, 'Assignees', 'Email'); + + // Click save + cy.get('[data-testid="save-button"]').scrollIntoView().click(); + cy.wait('@createAlert').then((interception) => { + data.alertDetails = interception?.response?.body; + + expect(interception?.response?.statusCode).equal(201); + }); + toastNotification('Alerts created successfully.'); - // Remove all filters - [...Array(6).keys()].forEach(() => { - cy.get('[data-testid="remove-filter-0"]').scrollIntoView().click(); + // Check if the alert details page is visible + verifyResponseStatusCode('@alertDetails', 200); + cy.get('[data-testid="alert-details-container"]').should('exist'); }); - // Remove all destinations except one - [...Array(5).keys()].forEach(() => { - cy.get('[data-testid="remove-destination-0"]').scrollIntoView().click(); + it('Delete alert for task source', () => { + deleteAlertSteps(ALERT_NAME); }); - // Click save - cy.get('[data-testid="save-button"]').scrollIntoView().click(); - cy.wait('@updateAlert').then((interception) => { - data.alertDetails = interception?.response?.body; + it('Create alert for conversation source', () => { + verifyResponseStatusCode('@alertsPage', 200); - expect(interception?.response?.statusCode).equal(200); + cy.get('[data-testid="create-notification"]').click(); - // Verify the edited alert changes - verifyAlertDetails(interception?.response?.body); - }); - }); + // Enter alert name + cy.get('#name').type(ALERT_NAME); - it('Delete alert with multiple filters', () => { - deleteAlertSteps(ALERT_NAME); - }); + // Enter description + cy.get(descriptionBox).clear().type(ALERT_DESCRIPTION); - it('Create alert for task source', () => { - verifyResponseStatusCode('@alertsPage', 200); + // Select all source + cy.get('[data-testid="add-source-button"]').scrollIntoView().click(); - cy.get('[data-testid="create-notification"]').click(); + cy.get( + `[data-testid="drop-down-menu"] [data-testid="${SOURCE_NAME_4}-option"]` + ) + .contains(SOURCE_DISPLAY_NAME_4) + .click(); - // Enter alert name - cy.get('#name').type(ALERT_NAME); + cy.get('[data-testid="source-select"]').should( + 'contain', + SOURCE_DISPLAY_NAME_4 + ); - // Enter description - cy.get(descriptionBox).clear().type(ALERT_DESCRIPTION); + // Select Destination + cy.get('[data-testid="add-destination-button"]').scrollIntoView().click(); - // Select all source - cy.get('[data-testid="add-source-button"]').scrollIntoView().click(); + addInternalDestination(0, 'Owners', 'Email'); - cy.get( - `[data-testid="drop-down-menu"] [data-testid="${SOURCE_NAME_3}-option"]` - ) - .contains(SOURCE_DISPLAY_NAME_3) - .click(); + // Click save + cy.get('[data-testid="save-button"]').scrollIntoView().click(); + cy.wait('@createAlert').then((interception) => { + data.alertDetails = interception?.response?.body; - cy.get('[data-testid="source-select"]').should( - 'contain', - SOURCE_DISPLAY_NAME_3 - ); + expect(interception?.response?.statusCode).equal(201); + }); + toastNotification('Alerts created successfully.'); - // Select Destination - cy.get('[data-testid="add-destination-button"]').scrollIntoView().click(); + // Check if the alert details page is visible + verifyResponseStatusCode('@alertDetails', 200); + cy.get('[data-testid="alert-details-container"]').should('exist'); + }); - addInternalDestination(0, 'Owners', 'Email'); + it('Edit and check alert by adding mentions filter', () => { + const { id: alertId } = data.alertDetails; - cy.get('[data-testid="add-destination-button"]').scrollIntoView().click(); + // Go to edit alert page + cy.get('table').should('contain', ALERT_NAME).click(); - addInternalDestination(1, 'Assignees', 'Email'); + cy.get( + `[data-row-key="${alertId}"] [data-testid="alert-edit-${ALERT_NAME}"]` + ).click(); - // Click save - cy.get('[data-testid="save-button"]').scrollIntoView().click(); - cy.wait('@createAlert').then((interception) => { - data.alertDetails = interception?.response?.body; + // Add filter + cy.get('[data-testid="add-filters"]').scrollIntoView().click(); - expect(interception?.response?.statusCode).equal(201); - }); - toastNotification('Alerts created successfully.'); + addFilterWithUsersListInput( + 'Mentioned Users-filter-option', + 0, + data.user.displayName, + true + ); + + // Add mentions destination + cy.get('[data-testid="add-destination-button"]').scrollIntoView().click(); + + addInternalDestination(1, 'Mentions', 'Slack'); - // Check if the alert details page is visible - verifyResponseStatusCode('@alertDetails', 200); - cy.get('[data-testid="alert-details-container"]').should('exist'); - }); + // Click save + cy.get('[data-testid="save-button"]').scrollIntoView().click(); + cy.wait('@updateAlert').then((interception) => { + data.alertDetails = interception?.response?.body; - it('Delete alert for task source', () => { - deleteAlertSteps(ALERT_NAME); - }); -}); + expect(interception?.response?.statusCode).equal(200); + + // Verify the edited alert changes + verifyAlertDetails(interception?.response?.body); + }); + }); + + it('Delete alert for conversation source', () => { + deleteAlertSteps(ALERT_NAME); + }); + } +); diff --git a/openmetadata-ui/src/main/resources/ui/cypress/e2e/Flow/ObservabilityAlerts.spec.ts b/openmetadata-ui/src/main/resources/ui/cypress/e2e/Flow/ObservabilityAlerts.spec.ts index 17b1ec082ccb..c1489d038e34 100644 --- a/openmetadata-ui/src/main/resources/ui/cypress/e2e/Flow/ObservabilityAlerts.spec.ts +++ b/openmetadata-ui/src/main/resources/ui/cypress/e2e/Flow/ObservabilityAlerts.spec.ts @@ -11,9 +11,6 @@ * limitations under the License. */ -// eslint-disable-next-line spaced-comment -/// - import { addDomainFilter, addEntityFQNFilter, @@ -32,6 +29,7 @@ import { verifyResponseStatusCode, } from '../../common/common'; import { createEntityTable, hardDeleteService } from '../../common/EntityUtils'; +import { getToken } from '../../common/Utils/LocalStorage'; import { ALERT_DESCRIPTION, ALERT_NAME, @@ -55,473 +53,498 @@ import { SERVICE_CATEGORIES } from '../../constants/service.constants'; const SOURCE_NAME_1 = 'Container'; const SOURCE_NAME_2 = 'Pipeline'; -describe('Observability Alert Flow', { tags: 'Settings' }, () => { - const data = { - testCase: {}, - testSuite: {}, - pipelineService: {}, - ingestionPipeline: {}, - user: { - displayName: '', - }, - domain: { - name: '', - }, - alertDetails: { - id: '', - }, - }; - - before(() => { - cy.login(); - cy.getAllLocalStorage().then((storageData) => { - const token = Object.values(storageData)[0].oidcIdToken; - - // Create a table - createEntityTable({ - token, - ...DATABASE_SERVICE, - tables: [DATABASE_SERVICE.entity], - }); - - // Create a test suite and test case for table - cy.request({ - method: 'POST', - url: `/api/v1/dataQuality/testSuites/executable`, - headers: { Authorization: `Bearer ${token}` }, - body: { - name: TEST_SUITE_FQN, - executableEntityReference: TABLE_FQN, - }, - }).then((response) => { - data.testSuite = response.body; +describe( + 'Observability Alert Flow', + { tags: ['Settings', 'Observability'] }, + () => { + const data = { + testCase: {}, + testSuite: {}, + pipelineService: { + id: '', + }, + ingestionPipeline: {}, + user: { + id: '', + displayName: '', + }, + domain: { + name: '', + }, + alertDetails: { + id: '', + }, + }; + + before(() => { + cy.login(); + cy.getAllLocalStorage().then((storageData) => { + const token = getToken(storageData); + + // Create a table + createEntityTable({ + token, + ...DATABASE_SERVICE, + tables: [DATABASE_SERVICE.entity], + }); + // Create a test suite and test case for table cy.request({ method: 'POST', - url: `/api/v1/dataQuality/testCases`, + url: `/api/v1/dataQuality/testSuites/executable`, headers: { Authorization: `Bearer ${token}` }, body: { - name: TEST_CASE_NAME, - displayName: TEST_CASE_NAME, - entityLink: `<#E::table::${TABLE_FQN}>`, - parameterValues: [ - { - name: 'columnCount', - value: 7, - }, - ], - testDefinition: 'tableColumnCountToEqual', - testSuite: TEST_SUITE_FQN, + name: TEST_SUITE_FQN, + executableEntityReference: TABLE_FQN, }, - }).then((testCaseResponse) => { - data.testCase = testCaseResponse.body; + }).then((response) => { + data.testSuite = response.body; + + cy.request({ + method: 'POST', + url: `/api/v1/dataQuality/testCases`, + headers: { Authorization: `Bearer ${token}` }, + body: { + name: TEST_CASE_NAME, + displayName: TEST_CASE_NAME, + entityLink: `<#E::table::${TABLE_FQN}>`, + parameterValues: [ + { + name: 'columnCount', + value: 7, + }, + ], + testDefinition: 'tableColumnCountToEqual', + testSuite: TEST_SUITE_FQN, + }, + }).then((testCaseResponse) => { + data.testCase = testCaseResponse.body; + }); }); - }); - - // Create a pipeline - cy.request({ - method: 'POST', - url: `/api/v1/services/${PIPELINE_SERVICE.serviceType}`, - headers: { Authorization: `Bearer ${token}` }, - body: PIPELINE_SERVICE.service, - }).then((pipelineServiceResponse) => { - data.pipelineService = pipelineServiceResponse.body; + // Create a pipeline cy.request({ method: 'POST', - url: `/api/v1/${PIPELINE_SERVICE.entityType}`, + url: `/api/v1/services/${PIPELINE_SERVICE.serviceType}`, headers: { Authorization: `Bearer ${token}` }, - body: PIPELINE_SERVICE.entity, + body: PIPELINE_SERVICE.service, + }).then((pipelineServiceResponse) => { + data.pipelineService = pipelineServiceResponse.body; + + cy.request({ + method: 'POST', + url: `/api/v1/${PIPELINE_SERVICE.entityType}`, + headers: { Authorization: `Bearer ${token}` }, + body: PIPELINE_SERVICE.entity, + }); + + // Create a ingestion pipeline + cy.request({ + method: 'POST', + url: `/api/v1/services/ingestionPipelines`, + headers: { Authorization: `Bearer ${token}` }, + body: { + airflowConfig: {}, + loggerLevel: 'INFO', + name: INGESTION_PIPELINE_NAME, + pipelineType: 'metadata', + service: { + id: data.pipelineService.id, + type: 'pipelineService', + }, + sourceConfig: { + config: {}, + }, + }, + }).then((ingestionPipelineResponse) => { + data.ingestionPipeline = ingestionPipelineResponse.body; + }); }); - // Create a ingestion pipeline + // Create a new user cy.request({ method: 'POST', - url: `/api/v1/services/ingestionPipelines`, + url: `/api/v1/users/signup`, headers: { Authorization: `Bearer ${token}` }, - body: { - airflowConfig: {}, - loggerLevel: 'INFO', - name: INGESTION_PIPELINE_NAME, - pipelineType: 'metadata', - service: { - id: data.pipelineService.id, - type: 'pipelineService', - }, - sourceConfig: { - config: {}, - }, - }, - }).then((ingestionPipelineResponse) => { - data.ingestionPipeline = ingestionPipelineResponse.body; + body: USER_DETAILS, + }).then((response) => { + data.user = response.body; }); - }); - - // Create a new user - cy.request({ - method: 'POST', - url: `/api/v1/users/signup`, - headers: { Authorization: `Bearer ${token}` }, - body: USER_DETAILS, - }).then((response) => { - data.user = response.body; - }); - // Create a domain - cy.request({ - method: 'PUT', - url: `/api/v1/domains`, - headers: { Authorization: `Bearer ${token}` }, - body: DOMAIN_CREATION_DETAILS, - }).then((response) => { - data.domain = response.body; + // Create a domain + cy.request({ + method: 'PUT', + url: `/api/v1/domains`, + headers: { Authorization: `Bearer ${token}` }, + body: DOMAIN_CREATION_DETAILS, + }).then((response) => { + data.domain = response.body; + }); }); }); - }); - - after(() => { - cy.login(); - cy.getAllLocalStorage().then((storageData) => { - const token = Object.values(storageData)[0].oidcIdToken; - - // Delete created services - hardDeleteService({ - token, - serviceFqn: DATABASE_SERVICE.service.name, - serviceType: SERVICE_CATEGORIES.DATABASE_SERVICES, - }); - hardDeleteService({ - token, - serviceFqn: PIPELINE_SERVICE.service.name, - serviceType: PIPELINE_SERVICE.serviceType, - }); - // Delete created domain - cy.request({ - method: 'DELETE', - url: `/api/v1/domains/name/${DOMAIN_CREATION_DETAILS.name}`, - headers: { Authorization: `Bearer ${token}` }, - }); + after(() => { + cy.login(); + cy.getAllLocalStorage().then((storageData) => { + const token = getToken(storageData); + + // Delete created services + hardDeleteService({ + token, + serviceFqn: DATABASE_SERVICE.service.name, + serviceType: SERVICE_CATEGORIES.DATABASE_SERVICES, + }); + hardDeleteService({ + token, + serviceFqn: PIPELINE_SERVICE.service.name, + serviceType: PIPELINE_SERVICE.serviceType, + }); + + // Delete created domain + cy.request({ + method: 'DELETE', + url: `/api/v1/domains/name/${DOMAIN_CREATION_DETAILS.name}`, + headers: { Authorization: `Bearer ${token}` }, + }); - // Delete created user - cy.request({ - method: 'DELETE', - url: `/api/v1/users/${data.user.id}?hardDelete=true&recursive=false`, - headers: { Authorization: `Bearer ${token}` }, + // Delete created user + cy.request({ + method: 'DELETE', + url: `/api/v1/users/${data.user.id}?hardDelete=true&recursive=false`, + headers: { Authorization: `Bearer ${token}` }, + }); }); }); - }); - beforeEach(() => { - interceptURL('POST', '/api/v1/events/subscriptions', 'createAlert'); - interceptURL('PUT', '/api/v1/events/subscriptions', 'updateAlert'); - interceptURL('GET', '/api/v1/events/subscriptions/name/*', 'alertDetails'); - interceptURL('GET', '/api/v1/events/subscriptions?*', 'alertsPage'); - cy.login(); - cy.sidebarClick(SidebarItem.OBSERVABILITY_ALERT); - }); + beforeEach(() => { + interceptURL('POST', '/api/v1/events/subscriptions', 'createAlert'); + interceptURL('PUT', '/api/v1/events/subscriptions', 'updateAlert'); + interceptURL( + 'GET', + '/api/v1/events/subscriptions/name/*', + 'alertDetails' + ); + interceptURL('GET', '/api/v1/events/subscriptions?*', 'alertsPage'); + cy.login(); + cy.sidebarClick(SidebarItem.OBSERVABILITY_ALERT); + }); - it('Create new alert Pipeline', () => { - verifyResponseStatusCode('@alertsPage', 200); + it('Create new alert Pipeline', () => { + verifyResponseStatusCode('@alertsPage', 200); - cy.get('[data-testid="create-observability"]').click(); + cy.get('[data-testid="create-observability"]').click(); - // Enter alert name - cy.get('#name').type(ALERT_NAME); + // Enter alert name + cy.get('#name').type(ALERT_NAME); - // Enter description - cy.get(descriptionBox).clear().type(ALERT_DESCRIPTION); + // Enter description + cy.get(descriptionBox).clear().type(ALERT_DESCRIPTION); - // Select all source - cy.get('[data-testid="add-source-button"]').scrollIntoView().click(); + // Select all source + cy.get('[data-testid="add-source-button"]').scrollIntoView().click(); - cy.get('[data-testid="drop-down-menu"] [data-testid="container-option"]') - .contains(SOURCE_NAME_1) - .click(); + cy.get('[data-testid="drop-down-menu"] [data-testid="container-option"]') + .contains(SOURCE_NAME_1) + .click(); - cy.get('[data-testid="source-select"]').should('contain', SOURCE_NAME_1); + cy.get('[data-testid="source-select"]').should('contain', SOURCE_NAME_1); - // Select filters - cy.get('[data-testid="add-filters"]').click(); + // Select filters + cy.get('[data-testid="add-filters"]').click(); - addOwnerFilter(0, data.user.displayName, false, 'Owner Name'); + addOwnerFilter(0, data.user.displayName, false, 'Owner Name'); - // Select actions - cy.get('[data-testid="add-trigger"]').click(); + // Select actions + cy.get('[data-testid="add-trigger"]').click(); - addGetSchemaChangesAction(0); + addGetSchemaChangesAction(0); - // Select Destination - cy.get('[data-testid="add-destination-button"]').scrollIntoView().click(); + // Select Destination + cy.get('[data-testid="add-destination-button"]').scrollIntoView().click(); - addInternalDestination(0, 'Admins', 'Email'); + addInternalDestination(0, 'Admins', 'Email'); - // Click save - cy.get('[data-testid="save-button"]').scrollIntoView().click(); - cy.wait('@createAlert').then((interception) => { - data.alertDetails = interception?.response?.body; + // Click save + cy.get('[data-testid="save-button"]').scrollIntoView().click(); + cy.wait('@createAlert').then((interception) => { + data.alertDetails = interception?.response?.body; - expect(interception?.response?.statusCode).equal(201); - }); - toastNotification('Alerts created successfully.'); + expect(interception?.response?.statusCode).equal(201); + }); + toastNotification('Alerts created successfully.'); - // Check if the alert details page is visible - verifyResponseStatusCode('@alertDetails', 200); - cy.get('[data-testid="alert-details-container"]').should('exist'); - }); + // Check if the alert details page is visible + verifyResponseStatusCode('@alertDetails', 200); + cy.get('[data-testid="alert-details-container"]').should('exist'); + }); - it('Check created pipeline alert details', () => { - const { id: alertId } = data.alertDetails; - verifyResponseStatusCode('@alertsPage', 200); + it('Check created pipeline alert details', () => { + const { id: alertId } = data.alertDetails; + verifyResponseStatusCode('@alertsPage', 200); - cy.get(`[data-row-key="${alertId}"] [data-testid="alert-name"]`) - .should('contain', ALERT_NAME) - .click(); + cy.get(`[data-row-key="${alertId}"] [data-testid="alert-name"]`) + .should('contain', ALERT_NAME) + .click(); - verifyResponseStatusCode('@alertDetails', 200); + verifyResponseStatusCode('@alertDetails', 200); - // Verify alert details - verifyAlertDetails(data.alertDetails); - }); + // Verify alert details + verifyAlertDetails(data.alertDetails); + }); - it('Edit created alert', () => { - const { id: alertId } = data.alertDetails; + it('Edit created alert', () => { + const { id: alertId } = data.alertDetails; - // Go to edit alert page - cy.get('table').should('contain', ALERT_NAME).click(); + // Go to edit alert page + cy.get('table').should('contain', ALERT_NAME).click(); - cy.get( - `[data-row-key="${alertId}"] [data-testid="alert-edit-${ALERT_NAME}"]` - ).click(); + cy.get( + `[data-row-key="${alertId}"] [data-testid="alert-edit-${ALERT_NAME}"]` + ).click(); - // Update description - cy.get(descriptionBox).click().clear().type(ALERT_UPDATED_DESCRIPTION); + // Update description + cy.get(descriptionBox).click().clear().type(ALERT_UPDATED_DESCRIPTION); - // Update source - cy.get('[data-testid="source-select"]').scrollIntoView().click(); - cy.get('[data-testid="pipeline-option"]').contains(SOURCE_NAME_2).click(); + // Update source + cy.get('[data-testid="source-select"]').scrollIntoView().click(); + cy.get('[data-testid="pipeline-option"]').contains(SOURCE_NAME_2).click(); - // Filters should reset after source change - cy.get('[data-testid="filter-select-0"]').should('not.exist'); + // Filters should reset after source change + cy.get('[data-testid="filter-select-0"]').should('not.exist'); - // Add multiple filters - [...Array(3).keys()].forEach(() => { - cy.get('[data-testid="add-filters"]').scrollIntoView().click(); - }); + // Add multiple filters + [...Array(3).keys()].forEach(() => { + cy.get('[data-testid="add-filters"]').scrollIntoView().click(); + }); - addOwnerFilter(0, data.user.displayName, false, 'Owner Name'); - addEntityFQNFilter( - 1, - `${PIPELINE_SERVICE.service.name}.${PIPELINE_SERVICE.entity.name}`, - true, - 'Pipeline Name' - ); - addDomainFilter(2, data.domain.name); + addOwnerFilter(0, data.user.displayName, false, 'Owner Name'); + addEntityFQNFilter( + 1, + `${PIPELINE_SERVICE.service.name}.${PIPELINE_SERVICE.entity.name}`, + true, + 'Pipeline Name' + ); + addDomainFilter(2, data.domain.name); - // Add actions - cy.get('[data-testid="add-trigger"]').click(); + // Add actions + cy.get('[data-testid="add-trigger"]').click(); - addPipelineStatusUpdatesAction(0, 'Successful', true); + addPipelineStatusUpdatesAction(0, 'Successful', true); - // Add multiple destinations - [...Array(2).keys()].forEach(() => { - cy.get('[data-testid="add-destination-button"]').scrollIntoView().click(); - }); + // Add multiple destinations + [...Array(2).keys()].forEach(() => { + cy.get('[data-testid="add-destination-button"]') + .scrollIntoView() + .click(); + }); - addInternalDestination(1, 'Owners', 'G Chat'); - addInternalDestination(2, 'Teams', 'Slack', 'Team-select', 'Organization'); + addInternalDestination(1, 'Owners', 'G Chat'); + addInternalDestination( + 2, + 'Teams', + 'Slack', + 'Team-select', + 'Organization' + ); - // Click save - cy.get('[data-testid="save-button"]').scrollIntoView().click(); - cy.wait('@updateAlert').then((interception) => { - data.alertDetails = interception?.response?.body; + // Click save + cy.get('[data-testid="save-button"]').scrollIntoView().click(); + cy.wait('@updateAlert').then((interception) => { + data.alertDetails = interception?.response?.body; - expect(interception?.response?.statusCode).equal(200); + expect(interception?.response?.statusCode).equal(200); - // Verify the edited alert changes - verifyAlertDetails(interception?.response?.body); + // Verify the edited alert changes + verifyAlertDetails(interception?.response?.body); + }); }); - }); - - it('Delete created alert', () => { - deleteAlertSteps(ALERT_NAME); - }); - Object.entries(OBSERVABILITY_CREATION_DETAILS).forEach( - ([source, alertDetails]) => { - it(`Alert creation for ${source}`, () => { - verifyResponseStatusCode('@alertsPage', 200); - - cy.get('[data-testid="create-observability"]').click(); - - // Enter alert name - cy.get('#name').type(ALERT_NAME); + it('Delete created alert', () => { + deleteAlertSteps(ALERT_NAME); + }); - // Enter description - cy.get(descriptionBox).clear().type(ALERT_DESCRIPTION); + Object.entries(OBSERVABILITY_CREATION_DETAILS).forEach( + ([source, alertDetails]) => { + it(`Alert creation for ${source}`, () => { + verifyResponseStatusCode('@alertsPage', 200); - // Select source - cy.get('[data-testid="add-source-button"]').scrollIntoView().click(); + cy.get('[data-testid="create-observability"]').click(); - cy.get( - `[data-testid="drop-down-menu"] [data-testid="${source}-option"]` - ) - .contains(alertDetails.sourceDisplayName) - .click(); + // Enter alert name + cy.get('#name').type(ALERT_NAME); - cy.get('[data-testid="source-select"]').should( - 'contain', - alertDetails.sourceDisplayName - ); + // Enter description + cy.get(descriptionBox).clear().type(ALERT_DESCRIPTION); - // Add filters - alertDetails.filters.forEach((filter, filterNumber) => { - cy.get('[data-testid="add-filters"]').click(); + // Select source + cy.get('[data-testid="add-source-button"]').scrollIntoView().click(); - // Select filter - cy.get(`[data-testid="filter-select-${filterNumber}"]`).click({ - waitForAnimations: true, - }); - cy.get(`[data-testid="${filter.name}-filter-option"]`) - .filter(':visible') + cy.get( + `[data-testid="drop-down-menu"] [data-testid="${source}-option"]` + ) + .contains(alertDetails.sourceDisplayName) .click(); - // Search and select filter input value - interceptURL('GET', `/api/v1/search/query?q=*`, 'getSearchResult'); - cy.get(`[data-testid="${filter.inputSelector}"]`) - .click() - .type(filter.inputValue); - - // Adding manual wait here as as safe since debounced API is not being detected in the cypress - cy.wait(500); - verifyResponseStatusCode('@getSearchResult', 200); - cy.get(`[title="${filter.inputValue}"]`) - .filter(':visible') - .scrollIntoView() - .click(); + cy.get('[data-testid="source-select"]').should( + 'contain', + alertDetails.sourceDisplayName + ); - // Check if option is selected - cy.get( - `[title="${filter.inputValue}"] .ant-select-item-option-state` - ).should('exist'); + // Add filters + alertDetails.filters.forEach((filter, filterNumber) => { + cy.get('[data-testid="add-filters"]').click(); - if (filter.exclude) { - // Change filter effect - cy.get(`[data-testid="filter-switch-${filterNumber}"]`) - .scrollIntoView() + // Select filter + cy.get(`[data-testid="filter-select-${filterNumber}"]`).click({ + waitForAnimations: true, + }); + cy.get(`[data-testid="${filter.name}-filter-option"]`) + .filter(':visible') .click(); - } - }); - // Add actions - alertDetails.actions.forEach((action, actionNumber) => { - cy.get('[data-testid="add-trigger"]').click(); + // Search and select filter input value + interceptURL('GET', `/api/v1/search/query?q=*`, 'getSearchResult'); + cy.get(`[data-testid="${filter.inputSelector}"]`) + .click() + .type(filter.inputValue); + + // Adding manual wait here as as safe since debounced API is not being detected in the cypress + cy.wait(500); + verifyResponseStatusCode('@getSearchResult', 200); + cy.get(`[title="${filter.inputValue}"]`) + .filter(':visible') + .scrollIntoView() + .click(); - // Select action - cy.get(`[data-testid="trigger-select-${actionNumber}"]`).click({ - waitForAnimations: true, - }); - cy.get(`[data-testid="${action.name}-filter-option"]`) - .filter(':visible') - .click(); + // Check if option is selected + cy.get( + `[title="${filter.inputValue}"] .ant-select-item-option-state` + ).should('exist'); - if (action.inputs && action.inputs.length > 0) { - action.inputs.forEach((input) => { - // Search and select domain - interceptURL( - 'GET', - `/api/v1/search/query?q=*`, - 'getSearchResult' - ); - cy.get(`[data-testid="${input.inputSelector}"]`) - .click() - .type(input.inputValue); - if (input.waitForAPI) { - verifyResponseStatusCode('@getSearchResult', 200); - } - cy.get(`[title="${input.inputValue}"]`) - .filter(':visible') + if (filter.exclude) { + // Change filter effect + cy.get(`[data-testid="filter-switch-${filterNumber}"]`) .scrollIntoView() .click(); - cy.get(`[data-testid="${input.inputSelector}"]`).should( - 'contain', - input.inputValue - ); - cy.clickOutside(); - }); - } - - if (action.exclude) { - // Change filter effect - cy.get(`[data-testid="trigger-switch-${actionNumber}"]`) - .scrollIntoView() - .click(); - } - }); + } + }); - // Add Destinations - alertDetails.destinations.forEach((destination, destinationNumber) => { - cy.get('[data-testid="add-destination-button"]') - .scrollIntoView() - .click(); + // Add actions + alertDetails.actions.forEach((action, actionNumber) => { + cy.get('[data-testid="add-trigger"]').click(); - if (destination.mode === 'internal') { - addInternalDestination( - destinationNumber, - destination.category, - destination.type, - destination.inputSelector, - destination.inputValue - ); - } else { - addExternalDestination( - destinationNumber, - destination.category, - destination.inputValue - ); - } - }); + // Select action + cy.get(`[data-testid="trigger-select-${actionNumber}"]`).click({ + waitForAnimations: true, + }); + cy.get(`[data-testid="${action.name}-filter-option"]`) + .filter(':visible') + .click(); - // Click save - cy.get('[data-testid="save-button"]').scrollIntoView().click(); - cy.wait('@createAlert').then((interception) => { - data.alertDetails = interception?.response?.body; + if (action.inputs && action.inputs.length > 0) { + action.inputs.forEach((input) => { + // Search and select domain + interceptURL( + 'GET', + `/api/v1/search/query?q=*`, + 'getSearchResult' + ); + cy.get(`[data-testid="${input.inputSelector}"]`) + .click() + .type(input.inputValue); + if (input.waitForAPI) { + verifyResponseStatusCode('@getSearchResult', 200); + } + cy.get(`[title="${input.inputValue}"]`) + .filter(':visible') + .scrollIntoView() + .click(); + cy.get(`[data-testid="${input.inputSelector}"]`).should( + 'contain', + input.inputValue + ); + cy.clickOutside(); + }); + } + + if (action.exclude) { + // Change filter effect + cy.get(`[data-testid="trigger-switch-${actionNumber}"]`) + .scrollIntoView() + .click(); + } + }); - expect(interception?.response?.statusCode).equal(201); - }); - toastNotification('Alerts created successfully.'); + // Add Destinations + alertDetails.destinations.forEach( + (destination, destinationNumber) => { + cy.get('[data-testid="add-destination-button"]') + .scrollIntoView() + .click(); - // Check if the alert details page is visible - verifyResponseStatusCode('@alertDetails', 200); - cy.get('[data-testid="alert-details-container"]').should('exist'); - }); + if (destination.mode === 'internal') { + addInternalDestination( + destinationNumber, + destination.category, + destination.type, + destination.inputSelector, + destination.inputValue + ); + } else { + addExternalDestination( + destinationNumber, + destination.category, + destination.inputValue + ); + } + } + ); - it(`Verify created ${source} alert details and delete alert`, () => { - const { id: alertId } = data.alertDetails; - verifyResponseStatusCode('@alertsPage', 200); + // Click save + cy.get('[data-testid="save-button"]').scrollIntoView().click(); + cy.wait('@createAlert').then((interception) => { + data.alertDetails = interception?.response?.body; - cy.get(`[data-row-key="${alertId}"] [data-testid="alert-name"]`) - .should('contain', ALERT_NAME) - .click(); + expect(interception?.response?.statusCode).equal(201); + }); + toastNotification('Alerts created successfully.'); - verifyResponseStatusCode('@alertDetails', 200); + // Check if the alert details page is visible + verifyResponseStatusCode('@alertDetails', 200); + cy.get('[data-testid="alert-details-container"]').should('exist'); + }); - // Verify alert details - verifyAlertDetails(data.alertDetails); + it(`Verify created ${source} alert details and delete alert`, () => { + const { id: alertId } = data.alertDetails; + verifyResponseStatusCode('@alertsPage', 200); - // Delete alert - cy.get('[data-testid="delete-button"]').scrollIntoView().click(); - cy.get('.ant-modal-header').should( - 'contain', - `Delete subscription "${ALERT_NAME}"` - ); - cy.get('[data-testid="confirmation-text-input"]').type(DELETE_TERM); - interceptURL('DELETE', '/api/v1/events/subscriptions/*', 'deleteAlert'); - cy.get('[data-testid="confirm-button"]').click(); - verifyResponseStatusCode('@deleteAlert', 200); + cy.get(`[data-row-key="${alertId}"] [data-testid="alert-name"]`) + .should('contain', ALERT_NAME) + .click(); - toastNotification(`"${ALERT_NAME}" deleted successfully!`); - }); - } - ); -}); + verifyResponseStatusCode('@alertDetails', 200); + + // Verify alert details + verifyAlertDetails(data.alertDetails); + + // Delete alert + cy.get('[data-testid="delete-button"]').scrollIntoView().click(); + cy.get('.ant-modal-header').should( + 'contain', + `Delete subscription "${ALERT_NAME}"` + ); + cy.get('[data-testid="confirmation-text-input"]').type(DELETE_TERM); + interceptURL( + 'DELETE', + '/api/v1/events/subscriptions/*', + 'deleteAlert' + ); + cy.get('[data-testid="confirm-button"]').click(); + verifyResponseStatusCode('@deleteAlert', 200); + + toastNotification(`"${ALERT_NAME}" deleted successfully!`); + }); + } + ); + } +); diff --git a/openmetadata-ui/src/main/resources/ui/cypress/e2e/Flow/PersonaFlow.spec.js b/openmetadata-ui/src/main/resources/ui/cypress/e2e/Flow/PersonaFlow.spec.ts similarity index 97% rename from openmetadata-ui/src/main/resources/ui/cypress/e2e/Flow/PersonaFlow.spec.js rename to openmetadata-ui/src/main/resources/ui/cypress/e2e/Flow/PersonaFlow.spec.ts index f033ea1d57c6..07b5fa9c4799 100644 --- a/openmetadata-ui/src/main/resources/ui/cypress/e2e/Flow/PersonaFlow.spec.js +++ b/openmetadata-ui/src/main/resources/ui/cypress/e2e/Flow/PersonaFlow.spec.ts @@ -10,8 +10,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -// eslint-disable-next-line spaced-comment -/// import { descriptionBox, @@ -19,6 +17,7 @@ import { toastNotification, verifyResponseStatusCode, } from '../../common/common'; +import { getToken } from '../../common/Utils/LocalStorage'; import { DELETE_TERM } from '../../constants/constants'; import { PERSONA_DETAILS, USER_DETAILS } from '../../constants/EntityConstant'; import { GlobalSettingOptions } from '../../constants/settings.constant'; @@ -42,12 +41,12 @@ const updatePersonaDisplayName = (displayName) => { }; describe('Persona operations', { tags: 'Settings' }, () => { - let user = {}; + const user = {}; const userSearchText = `${USER_DETAILS.firstName}${USER_DETAILS.lastName}`; before(() => { cy.login(); cy.getAllLocalStorage().then((data) => { - const token = Object.values(data)[0].oidcIdToken; + const token = getToken(data); // Create a new user cy.request({ @@ -64,7 +63,7 @@ describe('Persona operations', { tags: 'Settings' }, () => { after(() => { cy.login(); cy.getAllLocalStorage().then((data) => { - const token = Object.values(data)[0].oidcIdToken; + const token = getToken(data); // Delete created user cy.request({ diff --git a/openmetadata-ui/src/main/resources/ui/cypress/e2e/Flow/SearchFlow.spec.js b/openmetadata-ui/src/main/resources/ui/cypress/e2e/Flow/SearchFlow.spec.js deleted file mode 100644 index c91099d9bf8f..000000000000 --- a/openmetadata-ui/src/main/resources/ui/cypress/e2e/Flow/SearchFlow.spec.js +++ /dev/null @@ -1,274 +0,0 @@ -/* - * Copyright 2023 Collate. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * http://www.apache.org/licenses/LICENSE-2.0 - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// The spec is related to advance search feature - -import { - advanceSearchPreRequests, - ADVANCE_SEARCH_DATABASE_SERVICE, - checkAddGroupWithOperator, - checkAddRuleWithOperator, - checkmustPaths, - checkmust_notPaths, - CONDITIONS_MUST, - CONDITIONS_MUST_NOT, - FIELDS, - OPERATOR, -} from '../../common/advancedSearch'; -import { hardDeleteService } from '../../common/EntityUtils'; -import { USER_CREDENTIALS } from '../../constants/SearchIndexDetails.constants'; -import { SERVICE_CATEGORIES } from '../../constants/service.constants'; - -describe('Advance search', () => { - before(() => { - cy.login(); - cy.getAllLocalStorage().then((data) => { - const token = Object.values(data)[0].oidcIdToken; - advanceSearchPreRequests(token); - }); - }); - - after(() => { - cy.login(); - cy.getAllLocalStorage().then((data) => { - const token = Object.values(data)[0].oidcIdToken; - - hardDeleteService({ - token, - serviceFqn: ADVANCE_SEARCH_DATABASE_SERVICE.service.name, - serviceType: SERVICE_CATEGORIES.DATABASE_SERVICES, - }); - // Delete created user - cy.request({ - method: 'DELETE', - url: `/api/v1/users/${USER_CREDENTIALS.id}?hardDelete=true&recursive=false`, - headers: { Authorization: `Bearer ${token}` }, - }); - }); - }); - - describe('Single filed search', () => { - beforeEach(() => { - cy.login(); - }); - - Object.values(FIELDS).forEach((field) => { - it(`Verify advance search results for ${field.name} field and all condition`, () => { - Object.values(CONDITIONS_MUST).forEach((condition) => { - checkmustPaths( - condition.name, - field.testid, - field.isLocalSearch - ? field.searchCriteriaFirstGroup - : Cypress._.toLower(field.searchCriteriaFirstGroup), - 0, - field.responseValueFirstGroup, - field.isLocalSearch - ); - }); - - Object.values(CONDITIONS_MUST_NOT).forEach((condition) => { - checkmust_notPaths( - condition.name, - field.testid, - field.isLocalSearch - ? field.searchCriteriaFirstGroup - : Cypress._.toLower(field.searchCriteriaFirstGroup), - 0, - field.responseValueFirstGroup, - field.isLocalSearch - ); - }); - }); - }); - - after(() => { - cy.logout(); - Cypress.session.clearAllSavedSessions(); - }); - }); - - describe('Group search', () => { - beforeEach(() => { - cy.login(); - }); - - Object.values(OPERATOR).forEach((operator) => { - it(`Verify Add group functionality for All with ${operator.name} operator & condition ${CONDITIONS_MUST.equalTo.name} and ${CONDITIONS_MUST_NOT.notEqualTo.name} `, () => { - Object.values(FIELDS).forEach((field) => { - let val = field.searchCriteriaSecondGroup; - if (field.owner) { - val = field.responseValueSecondGroup; - } - checkAddGroupWithOperator( - CONDITIONS_MUST.equalTo.name, - CONDITIONS_MUST_NOT.notEqualTo.name, - field.testid, - field.isLocalSearch - ? field.searchCriteriaFirstGroup - : Cypress._.toLower(field.searchCriteriaFirstGroup), - field.isLocalSearch - ? field.searchCriteriaSecondGroup - : Cypress._.toLower(field.searchCriteriaSecondGroup), - 0, - 1, - operator.index, - CONDITIONS_MUST.equalTo.filter, - CONDITIONS_MUST_NOT.notEqualTo.filter, - field.responseValueFirstGroup, - val, - field.isLocalSearch - ); - }); - }); - - it(`Verify Add group functionality for All with ${operator.name} operator & condition ${CONDITIONS_MUST.anyIn.name} and ${CONDITIONS_MUST_NOT.notIn.name} `, () => { - Object.values(FIELDS).forEach((field) => { - let val = field.searchCriteriaSecondGroup; - if (field.owner) { - val = field.responseValueSecondGroup; - } - checkAddGroupWithOperator( - CONDITIONS_MUST.anyIn.name, - CONDITIONS_MUST_NOT.notIn.name, - field.testid, - field.isLocalSearch - ? field.searchCriteriaFirstGroup - : Cypress._.toLower(field.searchCriteriaFirstGroup), - field.isLocalSearch - ? field.searchCriteriaSecondGroup - : Cypress._.toLower(field.searchCriteriaSecondGroup), - 0, - 1, - operator.index, - CONDITIONS_MUST.anyIn.filter, - CONDITIONS_MUST_NOT.notIn.filter, - field.responseValueFirstGroup, - val, - field.isLocalSearch - ); - }); - }); - - it(`Verify Add group functionality for All with ${operator.name} operator & condition ${CONDITIONS_MUST.contains.name} and ${CONDITIONS_MUST_NOT.notContains.name} `, () => { - Object.values(FIELDS).forEach((field) => { - let val = field.searchCriteriaSecondGroup; - - checkAddGroupWithOperator( - CONDITIONS_MUST.contains.name, - CONDITIONS_MUST_NOT.notContains.name, - field.testid, - field.isLocalSearch - ? field.searchCriteriaFirstGroup - : Cypress._.toLower(field.searchCriteriaFirstGroup), - field.isLocalSearch - ? field.searchCriteriaSecondGroup - : Cypress._.toLower(field.searchCriteriaSecondGroup), - 0, - 1, - operator.index, - CONDITIONS_MUST.contains.filter, - CONDITIONS_MUST_NOT.notContains.filter, - field.responseValueFirstGroup, - val, - field.isLocalSearch - ); - }); - }); - }); - - after(() => { - cy.logout(); - Cypress.session.clearAllSavedSessions(); - }); - }); - - describe.skip('Search with additional rule', () => { - beforeEach(() => { - cy.login(); - }); - - Object.values(OPERATOR).forEach((operator) => { - it(`Verify Add Rule functionality for All with ${operator.name} operator & condition ${CONDITIONS_MUST.equalTo.name} and ${CONDITIONS_MUST_NOT.notEqualTo.name} `, () => { - Object.values(FIELDS).forEach((field) => { - let val = field.searchCriteriaSecondGroup; - if (field.owner) { - val = field.responseValueSecondGroup; - } - checkAddRuleWithOperator( - CONDITIONS_MUST.equalTo.name, - CONDITIONS_MUST_NOT.notEqualTo.name, - field.testid, - Cypress._.toLower(field.searchCriteriaFirstGroup), - Cypress._.toLower(field.searchCriteriaSecondGroup), - 0, - 1, - operator.index, - CONDITIONS_MUST.equalTo.filter, - CONDITIONS_MUST_NOT.notEqualTo.filter, - field.responseValueFirstGroup, - Cypress._.toLower(val) - ); - }); - }); - - it(`Verify Add Rule functionality for All with ${operator.name} operator & condition ${CONDITIONS_MUST.anyIn.name} and ${CONDITIONS_MUST_NOT.notIn.name} `, () => { - Object.values(FIELDS).forEach((field) => { - let val = field.searchCriteriaSecondGroup; - if (field.owner) { - val = field.responseValueSecondGroup; - } - checkAddRuleWithOperator( - CONDITIONS_MUST.anyIn.name, - CONDITIONS_MUST_NOT.notIn.name, - field.testid, - Cypress._.toLower(field.searchCriteriaFirstGroup), - Cypress._.toLower(field.searchCriteriaSecondGroup), - 0, - 1, - operator.index, - CONDITIONS_MUST.anyIn.filter, - CONDITIONS_MUST_NOT.notIn.filter, - field.responseValueFirstGroup, - Cypress._.toLower(val) - ); - }); - }); - - it(`Verify Add Rule functionality for All with ${operator.name} operator & condition ${CONDITIONS_MUST.contains.name} and ${CONDITIONS_MUST_NOT.notContains.name} `, () => { - Object.values(FIELDS).forEach((field) => { - let val = field.searchCriteriaSecondGroup; - checkAddRuleWithOperator( - CONDITIONS_MUST.contains.name, - CONDITIONS_MUST_NOT.notContains.name, - field.testid, - Cypress._.toLower(field.searchCriteriaFirstGroup), - Cypress._.toLower(field.searchCriteriaSecondGroup), - 0, - 1, - operator.index, - CONDITIONS_MUST.contains.filter, - CONDITIONS_MUST_NOT.notContains.filter, - field.responseValueFirstGroup, - Cypress._.toLower(val) - ); - }); - }); - }); - - after(() => { - cy.logout(); - Cypress.session.clearAllSavedSessions(); - }); - }); -}); diff --git a/openmetadata-ui/src/main/resources/ui/cypress/e2e/Flow/SearchFlow/AdditionalRuleSearch.spec.ts b/openmetadata-ui/src/main/resources/ui/cypress/e2e/Flow/SearchFlow/AdditionalRuleSearch.spec.ts new file mode 100644 index 000000000000..7c940d557e89 --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/cypress/e2e/Flow/SearchFlow/AdditionalRuleSearch.spec.ts @@ -0,0 +1,135 @@ +/* + * Copyright 2023 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// The spec is related to advance search feature + +import { + advancedSearchFlowCleanup, + advanceSearchPreRequests, + checkAddRuleWithOperator, + CONDITIONS_MUST, + CONDITIONS_MUST_NOT, + FIELDS, + OPERATOR, +} from '../../../common/Utils/AdvancedSearch'; +import { getToken } from '../../../common/Utils/LocalStorage'; + +describe('Search with additional rule', () => { + const testData = { + user_1: { + id: '', + }, + user_2: { + id: '', + }, + }; + + before(() => { + cy.login(); + cy.getAllLocalStorage().then((data) => { + const token = getToken(data); + advanceSearchPreRequests(testData, token); + }); + }); + + after(() => { + cy.login(); + cy.getAllLocalStorage().then((data) => { + const token = getToken(data); + + advancedSearchFlowCleanup(token); + }); + Cypress.session.clearAllSavedSessions(); + }); + + beforeEach(() => { + cy.login(); + }); + + Object.values(OPERATOR).forEach((operator) => { + it(`Verify Add Rule functionality for All with ${operator.name} operator & condition ${CONDITIONS_MUST.equalTo.name} and ${CONDITIONS_MUST_NOT.notEqualTo.name} `, () => { + Object.values(FIELDS).forEach((field) => { + let val = field.searchCriteriaSecondGroup; + if (field.owner) { + val = field.responseValueSecondGroup; + } + checkAddRuleWithOperator({ + condition_1: CONDITIONS_MUST.equalTo.name, + condition_2: CONDITIONS_MUST_NOT.notEqualTo.name, + fieldId: field.testId, + searchCriteria_1: field.isLocalSearch + ? field.searchCriteriaFirstGroup + : Cypress._.toLower(field.searchCriteriaFirstGroup), + searchCriteria_2: field.isLocalSearch + ? field.searchCriteriaSecondGroup + : Cypress._.toLower(field.searchCriteriaSecondGroup), + index_1: 0, + index_2: 1, + operatorIndex: operator.index, + filter_1: CONDITIONS_MUST.equalTo.filter, + filter_2: CONDITIONS_MUST_NOT.notEqualTo.filter, + response: field.isLocalSearch ? val : Cypress._.toLower(val), + }); + }); + }); + + it(`Verify Add Rule functionality for All with ${operator.name} operator & condition ${CONDITIONS_MUST.anyIn.name} and ${CONDITIONS_MUST_NOT.notIn.name} `, () => { + Object.values(FIELDS).forEach((field) => { + let val = field.searchCriteriaSecondGroup; + if (field.owner) { + val = field.responseValueSecondGroup; + } + checkAddRuleWithOperator({ + condition_1: CONDITIONS_MUST.anyIn.name, + condition_2: CONDITIONS_MUST_NOT.notIn.name, + fieldId: field.testId, + searchCriteria_1: field.isLocalSearch + ? field.searchCriteriaFirstGroup + : Cypress._.toLower(field.searchCriteriaFirstGroup), + searchCriteria_2: field.isLocalSearch + ? field.searchCriteriaSecondGroup + : Cypress._.toLower(field.searchCriteriaSecondGroup), + index_1: 0, + index_2: 1, + operatorIndex: operator.index, + filter_1: CONDITIONS_MUST.anyIn.filter, + filter_2: CONDITIONS_MUST_NOT.notIn.filter, + response: field.isLocalSearch ? val : Cypress._.toLower(val), + }); + }); + }); + + it(`Verify Add Rule functionality for All with ${operator.name} operator & condition ${CONDITIONS_MUST.contains.name} and ${CONDITIONS_MUST_NOT.notContains.name} `, () => { + Object.values(FIELDS).forEach((field) => { + const val = field.searchCriteriaSecondGroup; + checkAddRuleWithOperator({ + condition_1: CONDITIONS_MUST.contains.name, + condition_2: CONDITIONS_MUST_NOT.notContains.name, + fieldId: field.testId, + searchCriteria_1: field.isLocalSearch + ? field.searchCriteriaFirstGroup + : Cypress._.toLower(field.searchCriteriaFirstGroup), + searchCriteria_2: field.isLocalSearch + ? field.searchCriteriaSecondGroup + : Cypress._.toLower(field.searchCriteriaSecondGroup), + index_1: 0, + index_2: 1, + operatorIndex: operator.index, + filter_1: CONDITIONS_MUST.contains.filter, + filter_2: CONDITIONS_MUST_NOT.notContains.filter, + response: field.isLocalSearch ? val : Cypress._.toLower(val), + }); + }); + }); + }); +}); diff --git a/openmetadata-ui/src/main/resources/ui/cypress/e2e/Flow/SearchFlow/GroupSearch.spec.ts b/openmetadata-ui/src/main/resources/ui/cypress/e2e/Flow/SearchFlow/GroupSearch.spec.ts new file mode 100644 index 000000000000..f7a0b805f3fe --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/cypress/e2e/Flow/SearchFlow/GroupSearch.spec.ts @@ -0,0 +1,120 @@ +/* + * Copyright 2023 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// The spec is related to advance search feature + +import { + advancedSearchFlowCleanup, + advanceSearchPreRequests, + checkAddGroupWithOperator, + CONDITIONS_MUST, + CONDITIONS_MUST_NOT, + FIELDS, + OPERATOR, +} from '../../../common/Utils/AdvancedSearch'; +import { getToken } from '../../../common/Utils/LocalStorage'; + +describe('Group search', () => { + const testData = { + user_1: { + id: '', + }, + user_2: { + id: '', + }, + }; + + before(() => { + cy.login(); + cy.getAllLocalStorage().then((data) => { + const token = getToken(data); + advanceSearchPreRequests(testData, token); + }); + }); + + after(() => { + cy.login(); + cy.getAllLocalStorage().then((data) => { + const token = getToken(data); + + advancedSearchFlowCleanup(token); + }); + Cypress.session.clearAllSavedSessions(); + }); + + beforeEach(() => { + cy.login(); + }); + + Object.values(OPERATOR).forEach((operator) => { + it(`Verify Add group functionality for All with ${operator.name} operator & condition ${CONDITIONS_MUST.equalTo.name} and ${CONDITIONS_MUST_NOT.notEqualTo.name} `, () => { + Object.values(FIELDS).forEach((field) => { + checkAddGroupWithOperator({ + condition_1: CONDITIONS_MUST.equalTo.name, + condition_2: CONDITIONS_MUST_NOT.notEqualTo.name, + fieldId: field.testId, + searchCriteria_1: field.isLocalSearch + ? field.searchCriteriaFirstGroup + : Cypress._.toLower(field.searchCriteriaFirstGroup), + searchCriteria_2: field.isLocalSearch + ? field.searchCriteriaSecondGroup + : Cypress._.toLower(field.searchCriteriaSecondGroup), + index_1: 0, + index_2: 1, + operatorIndex: operator.index, + isLocalSearch: field.isLocalSearch, + }); + }); + }); + + it(`Verify Add group functionality for All with ${operator.name} operator & condition ${CONDITIONS_MUST.anyIn.name} and ${CONDITIONS_MUST_NOT.notIn.name} `, () => { + Object.values(FIELDS).forEach((field) => { + checkAddGroupWithOperator({ + condition_1: CONDITIONS_MUST.anyIn.name, + condition_2: CONDITIONS_MUST_NOT.notIn.name, + fieldId: field.testId, + searchCriteria_1: field.isLocalSearch + ? field.searchCriteriaFirstGroup + : Cypress._.toLower(field.searchCriteriaFirstGroup), + searchCriteria_2: field.isLocalSearch + ? field.searchCriteriaSecondGroup + : Cypress._.toLower(field.searchCriteriaSecondGroup), + index_1: 0, + index_2: 1, + operatorIndex: operator.index, + isLocalSearch: field.isLocalSearch, + }); + }); + }); + + it(`Verify Add group functionality for All with ${operator.name} operator & condition ${CONDITIONS_MUST.contains.name} and ${CONDITIONS_MUST_NOT.notContains.name} `, () => { + Object.values(FIELDS).forEach((field) => { + checkAddGroupWithOperator({ + condition_1: CONDITIONS_MUST.contains.name, + condition_2: CONDITIONS_MUST_NOT.notContains.name, + fieldId: field.testId, + searchCriteria_1: field.isLocalSearch + ? field.searchCriteriaFirstGroup + : Cypress._.toLower(field.searchCriteriaFirstGroup), + searchCriteria_2: field.isLocalSearch + ? field.searchCriteriaSecondGroup + : Cypress._.toLower(field.searchCriteriaSecondGroup), + index_1: 0, + index_2: 1, + operatorIndex: operator.index, + isLocalSearch: field.isLocalSearch, + }); + }); + }); + }); +}); diff --git a/openmetadata-ui/src/main/resources/ui/cypress/e2e/Flow/SearchFlow/SingleFiledSearch.spec.ts b/openmetadata-ui/src/main/resources/ui/cypress/e2e/Flow/SearchFlow/SingleFiledSearch.spec.ts new file mode 100644 index 000000000000..d241e1eb0952 --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/cypress/e2e/Flow/SearchFlow/SingleFiledSearch.spec.ts @@ -0,0 +1,88 @@ +/* + * Copyright 2023 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// The spec is related to advance search feature + +import { + advancedSearchFlowCleanup, + advanceSearchPreRequests, + checkMustPaths, + checkMust_notPaths, + CONDITIONS_MUST, + CONDITIONS_MUST_NOT, + FIELDS, +} from '../../../common/Utils/AdvancedSearch'; +import { getToken } from '../../../common/Utils/LocalStorage'; + +describe('Single filed search', () => { + const testData = { + user_1: { + id: '', + }, + user_2: { + id: '', + }, + }; + + before(() => { + cy.login(); + cy.getAllLocalStorage().then((data) => { + const token = getToken(data); + advanceSearchPreRequests(testData, token); + }); + }); + + after(() => { + cy.login(); + cy.getAllLocalStorage().then((data) => { + const token = getToken(data); + + advancedSearchFlowCleanup(token); + }); + Cypress.session.clearAllSavedSessions(); + }); + + beforeEach(() => { + cy.login(); + }); + + Object.values(FIELDS).forEach((field) => { + it(`Verify advance search results for ${field.name} field and all conditions`, () => { + Object.values(CONDITIONS_MUST).forEach((condition) => { + checkMustPaths( + condition.name, + field.testId, + field.isLocalSearch + ? field.searchCriteriaFirstGroup + : Cypress._.toLower(field.searchCriteriaFirstGroup), + 0, + field.responseValueFirstGroup, + field.isLocalSearch + ); + }); + + Object.values(CONDITIONS_MUST_NOT).forEach((condition) => { + checkMust_notPaths( + condition.name, + field.testId, + field.isLocalSearch + ? field.searchCriteriaFirstGroup + : Cypress._.toLower(field.searchCriteriaFirstGroup), + 0, + field.responseValueFirstGroup, + field.isLocalSearch + ); + }); + }); + }); +}); diff --git a/openmetadata-ui/src/main/resources/ui/cypress/e2e/Flow/Task.spec.js b/openmetadata-ui/src/main/resources/ui/cypress/e2e/Flow/Task.spec.ts similarity index 53% rename from openmetadata-ui/src/main/resources/ui/cypress/e2e/Flow/Task.spec.js rename to openmetadata-ui/src/main/resources/ui/cypress/e2e/Flow/Task.spec.ts index 9a87e3c1971e..e3025710636f 100644 --- a/openmetadata-ui/src/main/resources/ui/cypress/e2e/Flow/Task.spec.js +++ b/openmetadata-ui/src/main/resources/ui/cypress/e2e/Flow/Task.spec.ts @@ -10,15 +10,17 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -// eslint-disable-next-line spaced-comment -/// import { interceptURL, toastNotification, verifyResponseStatusCode, } from '../../common/common'; -import { createEntityTable, hardDeleteService } from '../../common/EntityUtils'; +import { + createEntityTable, + deleteUserEntity, + hardDeleteService, +} from '../../common/EntityUtils'; import { createAndUpdateDescriptionTask, createDescriptionTask, @@ -26,9 +28,14 @@ import { verifyTaskDetails, } from '../../common/TaskUtils'; import { visitEntityDetailsPage } from '../../common/Utils/Entity'; +import { getToken } from '../../common/Utils/LocalStorage'; import { addOwner } from '../../common/Utils/Owner'; -import { DATA_ASSETS } from '../../constants/constants'; -import { DATABASE_SERVICE } from '../../constants/EntityConstant'; +import { DATA_ASSETS, uuid } from '../../constants/constants'; +import { + DATABASE_SERVICE, + USER_DETAILS, + USER_NAME, +} from '../../constants/EntityConstant'; import { SERVICE_CATEGORIES } from '../../constants/service.constants'; const ENTITY_TABLE = { @@ -40,30 +47,137 @@ const ENTITY_TABLE = { entityType: 'Table', }; +const POLICY_DETAILS = { + name: `cy-data-viewAll-policy-${uuid()}`, + rules: [ + { + name: 'viewRuleAllowed', + resources: ['All'], + operations: ['ViewAll'], + effect: 'allow', + }, + { + effect: 'deny', + name: 'editNotAllowed', + operations: ['EditAll'], + resources: ['All'], + }, + ], +}; +const ROLE_DETAILS = { + name: `cy-data-viewAll-role-${uuid()}`, + policies: [POLICY_DETAILS.name], +}; + +const TEAM_DETAILS = { + name: 'viewAllTeam', + displayName: 'viewAllTeam', + teamType: 'Group', +}; + describe('Task flow should work', { tags: 'DataAssets' }, () => { + const data = { + user: { id: '' }, + policy: { id: '' }, + role: { id: '' }, + team: { id: '' }, + }; + before(() => { cy.login(); - cy.getAllLocalStorage().then((data) => { - const token = Object.values(data)[0].oidcIdToken; + cy.getAllLocalStorage().then((storageData) => { + const token = getToken(storageData); createEntityTable({ token, ...DATABASE_SERVICE, tables: [DATABASE_SERVICE.entity], }); + + // Create ViewAll Policy + cy.request({ + method: 'POST', + url: `/api/v1/policies`, + headers: { Authorization: `Bearer ${token}` }, + body: POLICY_DETAILS, + }).then((policyResponse) => { + data.policy = policyResponse.body; + + // Create ViewAll Role + cy.request({ + method: 'POST', + url: `/api/v1/roles`, + headers: { Authorization: `Bearer ${token}` }, + body: ROLE_DETAILS, + }).then((roleResponse) => { + data.role = roleResponse.body; + + // Create a new user + cy.request({ + method: 'POST', + url: `/api/v1/users/signup`, + headers: { Authorization: `Bearer ${token}` }, + body: USER_DETAILS, + }).then((userResponse) => { + data.user = userResponse.body; + + // create team + cy.request({ + method: 'GET', + url: `/api/v1/teams/name/Organization`, + headers: { Authorization: `Bearer ${token}` }, + }).then((teamResponse) => { + cy.request({ + method: 'POST', + url: `/api/v1/teams`, + headers: { Authorization: `Bearer ${token}` }, + body: { + ...TEAM_DETAILS, + parents: [teamResponse.body.id], + users: [userResponse.body.id], + defaultRoles: [roleResponse.body.id], + }, + }).then((teamResponse) => { + data.team = teamResponse.body; + }); + }); + }); + }); + }); }); }); after(() => { cy.login(); - cy.getAllLocalStorage().then((data) => { - const token = Object.values(data)[0].oidcIdToken; + cy.getAllLocalStorage().then((storageData) => { + const token = getToken(storageData); hardDeleteService({ token, serviceFqn: ENTITY_TABLE.serviceName, serviceType: SERVICE_CATEGORIES.DATABASE_SERVICES, }); + + // Clean up for the created data + deleteUserEntity({ token, id: data.user.id }); + + cy.request({ + method: 'DELETE', + url: `/api/v1/teams/${data.team.id}?hardDelete=true&recursive=true`, + headers: { Authorization: `Bearer ${token}` }, + }); + + cy.request({ + method: 'DELETE', + url: `/api/v1/policies/${data.policy.id}?hardDelete=true&recursive=true`, + headers: { Authorization: `Bearer ${token}` }, + }); + + cy.request({ + method: 'DELETE', + url: `/api/v1/roles/${data.role.id}?hardDelete=true&recursive=true`, + headers: { Authorization: `Bearer ${token}` }, + }); }); }); @@ -194,7 +308,7 @@ describe('Task flow should work', { tags: 'DataAssets' }, () => { }); }); - it('Asignee field should be disabled for owned entity tasks', () => { + it('Assignee field should not be disabled for owned entity tasks', () => { interceptURL( 'GET', `/api/v1/${ENTITY_TABLE.entity}/name/*`, @@ -214,17 +328,49 @@ describe('Task flow should work', { tags: 'DataAssets' }, () => { cy.wait('@getEntityDetails').then((res) => { const entity = res.response.body; - // create description task and verify asignee field to have owner - // and should be disbaled - - createDescriptionTask( - { - ...ENTITY_TABLE, - assignee: 'Adam Rodriguez', - term: entity.displayName ?? entity.name, - }, - true - ); + createDescriptionTask({ + ...ENTITY_TABLE, + assignee: USER_NAME, + term: entity.displayName ?? entity.name, + }); + }); + }); + + it(`should throw error for not having edit permission for viewAll user`, () => { + // logout for the admin user + cy.logout(); + + // login to viewAll user + cy.login(USER_DETAILS.email, USER_DETAILS.password); + + interceptURL( + 'GET', + `/api/v1/${ENTITY_TABLE.entity}/name/*`, + 'getEntityDetails' + ); + + visitEntityDetailsPage({ + term: ENTITY_TABLE.term, + serviceName: ENTITY_TABLE.serviceName, + entity: ENTITY_TABLE.entity, }); + + cy.get('[data-testid="activity_feed"]').click(); + + cy.get('[data-menu-id*="tasks"]').click(); + + // verify the task details + verifyTaskDetails(/#(\d+) Request to update description for/, USER_NAME); + + cy.get(`[data-testid="${USER_NAME}"]`).should('be.visible'); + + // Accept the description suggestion which is created + cy.get('.ant-btn-compact-first-item').contains('Accept Suggestion').click(); + + verifyResponseStatusCode('@taskResolve', 403); + + toastNotification( + `Principal: CatalogPrincipal{name='${USER_NAME}'} operation EditDescription denied by role ${ROLE_DETAILS.name}, policy ${POLICY_DETAILS.name}, rule editNotAllowed` + ); }); }); diff --git a/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/Bots.spec.js b/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/Bots.spec.ts similarity index 100% rename from openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/Bots.spec.js rename to openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/Bots.spec.ts diff --git a/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/ClassificationVersionPage.spec.js b/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/ClassificationVersionPage.spec.ts similarity index 82% rename from openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/ClassificationVersionPage.spec.js rename to openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/ClassificationVersionPage.spec.ts index 5eb9301a88ae..5cc241c8962c 100644 --- a/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/ClassificationVersionPage.spec.js +++ b/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/ClassificationVersionPage.spec.ts @@ -10,11 +10,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -// eslint-disable-next-line spaced-comment -/// import { interceptURL, verifyResponseStatusCode } from '../../common/common'; import { visitClassificationPage } from '../../common/TagUtils'; +import { getToken } from '../../common/Utils/LocalStorage'; import { NEW_CLASSIFICATION_FOR_VERSION_TEST, NEW_CLASSIFICATION_PATCH_PAYLOAD, @@ -47,28 +46,30 @@ describe( }); it('Prerequisites for classification version page tests', () => { - const token = localStorage.getItem('oidcIdToken'); - - cy.request({ - method: 'PUT', - url: `/api/v1/classifications`, - headers: { Authorization: `Bearer ${token}` }, - body: NEW_CLASSIFICATION_FOR_VERSION_TEST, - }).then((response) => { - expect(response.status).to.eq(201); - - classificationId = response.body.id; + cy.getAllLocalStorage().then((data) => { + const token = getToken(data); cy.request({ - method: 'PATCH', - url: `/api/v1/classifications/${classificationId}`, - headers: { - Authorization: `Bearer ${token}`, - 'Content-Type': 'application/json-patch+json', - }, - body: NEW_CLASSIFICATION_PATCH_PAYLOAD, + method: 'PUT', + url: `/api/v1/classifications`, + headers: { Authorization: `Bearer ${token}` }, + body: NEW_CLASSIFICATION_FOR_VERSION_TEST, }).then((response) => { - expect(response.status).to.eq(200); + expect(response.status).to.eq(201); + + classificationId = response.body.id; + + cy.request({ + method: 'PATCH', + url: `/api/v1/classifications/${classificationId}`, + headers: { + Authorization: `Bearer ${token}`, + 'Content-Type': 'application/json-patch+json', + }, + body: NEW_CLASSIFICATION_PATCH_PAYLOAD, + }).then((response) => { + expect(response.status).to.eq(200); + }); }); }); }); @@ -103,12 +104,12 @@ describe( verifyResponseStatusCode('@getSelectedVersionDetails', 200); cy.get( - `[data-testid="description"] [data-testid="diff-added"]` + `[data-testid="description-container"] [data-testid="diff-added"]` ).scrollIntoView(); - cy.get(`[data-testid="description"] [data-testid="diff-added"]`).should( - 'be.visible' - ); + cy.get( + `[data-testid="description-container"] [data-testid="diff-added"]` + ).should('be.visible'); cy.get('[data-testid="mutually-exclusive-container"]').as( 'mutuallyExclusiveContainer' diff --git a/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/CustomLogoConfig.spec.js b/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/CustomLogoConfig.spec.ts similarity index 97% rename from openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/CustomLogoConfig.spec.js rename to openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/CustomLogoConfig.spec.ts index 15f4841ac8c1..d7f76c5e6fcb 100644 --- a/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/CustomLogoConfig.spec.js +++ b/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/CustomLogoConfig.spec.ts @@ -10,8 +10,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -// eslint-disable-next-line spaced-comment -/// import { interceptURL, verifyResponseStatusCode } from '../../common/common'; import { GlobalSettingOptions } from '../../constants/settings.constant'; diff --git a/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/Customproperties.spec.js b/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/Customproperties.spec.ts similarity index 90% rename from openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/Customproperties.spec.js rename to openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/Customproperties.spec.ts index 9cfdd069b27b..0da437cfa0f5 100644 --- a/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/Customproperties.spec.js +++ b/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/Customproperties.spec.ts @@ -13,19 +13,19 @@ import { lowerCase } from 'lodash'; import { - addCustomPropertiesForEntity, - deleteCreatedProperty, descriptionBox, - editCreatedProperty, interceptURL, verifyResponseStatusCode, } from '../../common/common'; import { deleteGlossary } from '../../common/GlossaryUtils'; import { + addCustomPropertiesForEntity, customPropertiesArray, CustomPropertyType, + deleteCreatedProperty, deleteCustomProperties, deleteCustomPropertyForEntity, + editCreatedProperty, generateCustomProperty, setValueForProperty, validateValueForProperty, @@ -35,6 +35,7 @@ import { createEntityTableViaREST, visitEntityDetailsPage, } from '../../common/Utils/Entity'; +import { getToken } from '../../common/Utils/LocalStorage'; import { DATA_ASSETS, ENTITIES, @@ -459,6 +460,64 @@ describe('Custom Properties should work properly', { tags: 'Settings' }, () => { }); }); + describe('Add update and delete Enum custom properties', () => { + Object.values(ENTITIES).forEach((entity) => { + const propertyName = `addcyentity${entity.name}test${uuid()}`; + + it(`Add/Update/Delete Enum custom property for ${entity.name} Entities`, () => { + interceptURL( + 'GET', + `/api/v1/metadata/types/name/${entity.name}*`, + 'getEntity' + ); + + // Selecting the entity + cy.settingClick(entity.entityApiType, true); + + verifyResponseStatusCode('@getEntity', 200); + + addCustomPropertiesForEntity( + propertyName, + entity, + 'Enum', + entity.enumConfig, + entity.entityObj + ); + + // Navigating back to custom properties page + cy.settingClick(entity.entityApiType, true); + + verifyResponseStatusCode('@getEntity', 200); + + // `Edit created property for ${entity.name} entity` + interceptURL( + 'GET', + `/api/v1/metadata/types/name/${entity.name}*`, + 'getEntity' + ); + + // Selecting the entity + cy.settingClick(entity.entityApiType, true); + + verifyResponseStatusCode('@getEntity', 200); + editCreatedProperty(propertyName, 'Enum'); + + // `Delete created property for ${entity.name} entity` + interceptURL( + 'GET', + `/api/v1/metadata/types/name/${entity.name}*`, + 'getEntity' + ); + + // Selecting the entity + cy.settingClick(entity.entityApiType, true); + + verifyResponseStatusCode('@getEntity', 200); + deleteCreatedProperty(propertyName); + }); + }); + }); + describe('Custom properties for glossary and glossary terms', () => { const propertyName = `addcyentity${glossaryTerm.name}test${uuid()}`; const properties = Object.values(CustomPropertyType).join(', '); @@ -481,7 +540,9 @@ describe('Custom Properties should work properly', { tags: 'Settings' }, () => { `/api/v1/metadata/types/name/glossaryTerm*`, 'getEntity' ); - cy.get('[data-testid="glossaries-tab"]').click(); + cy.get( + `[data-testid=${Cypress.$.escapeSelector('glossary terms-tab')}]` + ).click(); cy.get('[data-testid="advance-search-button"]').click(); verifyResponseStatusCode('@getEntity', 200); @@ -592,7 +653,7 @@ describe('Custom Properties should work properly', { tags: 'Settings' }, () => { cy.login(); cy.getAllLocalStorage().then((data) => { - token = Object.values(data)[0].oidcIdToken; + token = getToken(data); createEntityTableViaREST({ token, ...DATABASE_SERVICE, diff --git a/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/DataInsight.spec.js b/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/DataInsight.spec.ts similarity index 94% rename from openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/DataInsight.spec.js rename to openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/DataInsight.spec.ts index 4381d19593e8..ee108d491da1 100644 --- a/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/DataInsight.spec.js +++ b/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/DataInsight.spec.ts @@ -10,8 +10,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -// eslint-disable-next-line spaced-comment -/// + import { customFormatDateTime, getCurrentMillis, @@ -23,6 +22,7 @@ import { verifyResponseStatusCode, } from '../../common/common'; import { verifyKpiChart } from '../../common/DataInsightUtils'; +import { getToken } from '../../common/Utils/LocalStorage'; import { SidebarItem } from '../../constants/Entity.interface'; import { GlobalSettingOptions } from '../../constants/settings.constant'; @@ -44,14 +44,17 @@ const deleteKpiRequest = () => { cy.wait('@getKpi').then(({ response }) => { const data = response.body.data; if (data.length > 0) { - const token = localStorage.getItem('oidcIdToken'); - data.forEach((element) => { - cy.request({ - method: 'DELETE', - url: `/api/v1/kpi/${element.id}?hardDelete=true&recursive=false`, - headers: { Authorization: `Bearer ${token}` }, - }).then((response) => { - expect(response.status).to.eq(200); + cy.getAllLocalStorage().then((storageData) => { + const token = getToken(storageData); + + data.forEach((element) => { + cy.request({ + method: 'DELETE', + url: `/api/v1/kpi/${element.id}?hardDelete=true&recursive=false`, + headers: { Authorization: `Bearer ${token}` }, + }).then((response) => { + expect(response.status).to.eq(200); + }); }); }); cy.reload(); diff --git a/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/DataInsightAlert.spec.js b/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/DataInsightAlert.spec.ts similarity index 100% rename from openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/DataInsightAlert.spec.js rename to openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/DataInsightAlert.spec.ts diff --git a/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/DataInsightReportApplication.spec.ts b/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/DataInsightReportApplication.spec.ts new file mode 100644 index 000000000000..d21b3c3e9843 --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/DataInsightReportApplication.spec.ts @@ -0,0 +1,151 @@ +/* + * Copyright 2024 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { interceptURL, verifyResponseStatusCode } from '../../common/common'; +import { checkAndDeleteApp } from '../../common/Utils/Apps'; +import { getToken } from '../../common/Utils/LocalStorage'; +import { GlobalSettingOptions } from '../../constants/settings.constant'; + +const visitDataInsightReportApplicationPage = () => { + interceptURL( + 'GET', + '/api/v1/apps/name/DataInsightsReportApplication?fields=*', + 'getDataInsightsReportApplication' + ); + cy.get( + '[data-testid="data-insights-report-application-card"] [data-testid="config-btn"]' + ).click(); + verifyResponseStatusCode('@getDataInsightsReportApplication', 200); +}; + +const logButton = () => { + // check if the button is disabled to perform the click action + cy.get('[data-testid="logs"]').then(($logButton) => { + if ($logButton.is(':enabled')) { + cy.get('[data-testid="logs"]').click(); + cy.get('[data-testid="logs"]').should('be.visible'); + } else { + cy.reload(); + logButton(); + } + }); +}; + +describe('Data Insight Report Application', { tags: 'Settings' }, () => { + before(() => { + cy.login(); + cy.getAllLocalStorage().then((data) => { + const token = getToken(data); + + checkAndDeleteApp({ + token, + applicationName: 'DataInsightsReportApplication', + }); + }); + }); + + beforeEach(() => { + cy.login(); + + interceptURL('GET', '/api/v1/apps?limit=*', 'getApplications'); + + cy.settingClick(GlobalSettingOptions.APPLICATIONS); + + verifyResponseStatusCode('@getApplications', 200); + }); + + it('Install application', () => { + interceptURL('GET', '/api/v1/apps/marketplace?limit=*', 'getMarketPlace'); + interceptURL('POST', '/api/v1/apps', 'installApplication'); + cy.get('[data-testid="add-application"]').click(); + verifyResponseStatusCode('@getMarketPlace', 200); + cy.get( + '[data-testid="data-insights-report-application-card"] [data-testid="config-btn"]' + ).click(); + cy.get('[data-testid="install-application"]').click(); + cy.get('[data-testid="save-button"]').scrollIntoView().click(); + cy.get('[data-testid="submit-btn"]').scrollIntoView().click(); + cy.get('[data-testid="cron-type"]').click(); + // selecting day in week + cy.get('[data-value="5"]').click(); + cy.get('[data-testid="deploy-button"]').click(); + verifyResponseStatusCode('@installApplication', 201); + verifyResponseStatusCode('@getApplications', 200); + cy.get('[data-testid="data-insights-report-application-card"]').should( + 'be.visible' + ); + }); + + it('Edit application', () => { + interceptURL('PATCH', '/api/v1/apps/*', 'updateApplication'); + visitDataInsightReportApplicationPage(); + cy.get('[data-testid="edit-button"]').click(); + cy.get('[data-testid="cron-type"]').click(); + // selecting day in week + cy.get('[data-value="3"]').click(); + cy.get('[data-testid="hour-options"]').click(); + cy.get('[title="01"]').click(); + cy.get('.ant-modal-body [data-testid="deploy-button"]').click(); + verifyResponseStatusCode('@updateApplication', 200); + cy.get('[data-testid="cron-string"]').should('contain', 'At 01:00 AM'); + + cy.get('[data-testid="configuration"]').click(); + + cy.get('#root\\/sendToAdmins').click(); + cy.get('#root\\/sendToTeams').click(); + + cy.get('[data-testid="submit-btn"]').click(); + verifyResponseStatusCode('@updateApplication', 200); + }); + + it('Run application', () => { + interceptURL( + 'GET', + '/api/v1/apps/name/DataInsightsReportApplication?fields=*', + 'getDataInsightReportApplication' + ); + interceptURL( + 'POST', + '/api/v1/apps/trigger/DataInsightsReportApplication', + 'triggerPipeline' + ); + cy.get( + '[data-testid="data-insights-report-application-card"] [data-testid="config-btn"]' + ).click(); + verifyResponseStatusCode('@getDataInsightReportApplication', 200); + cy.get('[data-testid="run-now-button"]').click(); + verifyResponseStatusCode('@triggerPipeline', 200); + + // check the logs in the history table + cy.get('[data-testid="history"]').click(); + + logButton(); + }); + + it('Uninstall application', () => { + interceptURL('GET', '/api/v1/apps?limit=*', 'getApplications'); + interceptURL( + 'DELETE', + '/api/v1/apps/name/DataInsightsReportApplication?hardDelete=true', + 'deleteApplication' + ); + visitDataInsightReportApplicationPage(); + cy.get('[data-testid="manage-button"]').click(); + cy.get('[data-testid="uninstall-button-title"]').click(); + cy.get('[data-testid="save-button"]').click(); + verifyResponseStatusCode('@deleteApplication', 200); + verifyResponseStatusCode('@getApplications', 200); + cy.get('[data-testid="data-insights-report-application-card"]').should( + 'not.exist' + ); + }); +}); diff --git a/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/DataInsightSettings.spec.js b/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/DataInsightSettings.spec.ts similarity index 96% rename from openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/DataInsightSettings.spec.js rename to openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/DataInsightSettings.spec.ts index 958e7af0b8a4..437886852ab5 100644 --- a/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/DataInsightSettings.spec.js +++ b/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/DataInsightSettings.spec.ts @@ -12,6 +12,7 @@ */ import { interceptURL, verifyResponseStatusCode } from '../../common/common'; +import { BASE_URL } from '../../constants/constants'; import { GlobalSettingOptions } from '../../constants/settings.constant'; describe( @@ -122,6 +123,9 @@ describe( verifyResponseStatusCode('@getDataInsightDetails', 200); cy.get('[data-testid="run-now-button"]').click(); verifyResponseStatusCode('@triggerPipeline', 200); + + cy.get('[data-testid="logs"]').click(); + cy.url().should('eq', `${BASE_URL}/apps/DataInsightsApplication/logs`); }); } ); diff --git a/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/DataModelVersionPage.spec.js b/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/DataModelVersionPage.spec.ts similarity index 65% rename from openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/DataModelVersionPage.spec.js rename to openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/DataModelVersionPage.spec.ts index d325c7353e1c..bdca192fd5bb 100644 --- a/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/DataModelVersionPage.spec.js +++ b/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/DataModelVersionPage.spec.ts @@ -10,8 +10,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -// eslint-disable-next-line spaced-comment -/// import { interceptURL, @@ -19,15 +17,19 @@ import { verifyResponseStatusCode, visitDataModelPage, } from '../../common/common'; +import { hardDeleteService } from '../../common/EntityUtils'; +import { getToken } from '../../common/Utils/LocalStorage'; import { addOwner } from '../../common/Utils/Owner'; import { addTier } from '../../common/Utils/Tier'; import { visitDataModelVersionPage } from '../../common/VersionUtils'; import { DELETE_TERM } from '../../constants/constants'; +import { DASHBOARD_SERVICE_DETAILS } from '../../constants/EntityConstant'; +import { SERVICE_CATEGORIES } from '../../constants/service.constants'; import { DATA_MODEL_DETAILS, DATA_MODEL_DETAILS_FOR_VERSION_TEST, DATA_MODEL_PATCH_PAYLOAD, - OWNER, + OWNER_DETAILS, TIER, } from '../../constants/Version.constants'; @@ -35,34 +37,70 @@ describe( 'Data model version page should work properly', { tags: 'DataAssets' }, () => { - const dataModelName = DATA_MODEL_DETAILS.name; - let dataModelId; - let dataModelFQN; + const data = { + user: { id: '', displayName: '' }, + dataModel: { id: '', fullyQualifiedName: '', name: '' }, + }; before(() => { cy.login(); - cy.getAllLocalStorage().then((data) => { - const token = Object.values(data)[0].oidcIdToken; - + cy.getAllLocalStorage().then((responseData) => { + const token = getToken(responseData); cy.request({ - method: 'PUT', - url: `/api/v1/dashboard/datamodels`, + method: 'POST', + url: `/api/v1/services/dashboardServices`, headers: { Authorization: `Bearer ${token}` }, - body: DATA_MODEL_DETAILS_FOR_VERSION_TEST, + body: DASHBOARD_SERVICE_DETAILS, }).then((response) => { - dataModelId = response.body.id; - dataModelFQN = response.body.fullyQualifiedName; - cy.request({ - method: 'PATCH', - url: `/api/v1/dashboard/datamodels/${dataModelId}`, - headers: { - Authorization: `Bearer ${token}`, - 'Content-Type': 'application/json-patch+json', - }, - body: DATA_MODEL_PATCH_PAYLOAD, + method: 'PUT', + url: `/api/v1/dashboard/datamodels`, + headers: { Authorization: `Bearer ${token}` }, + body: DATA_MODEL_DETAILS_FOR_VERSION_TEST, + }).then((response) => { + data.dataModel = response.body; + + cy.request({ + method: 'PATCH', + url: `/api/v1/dashboard/datamodels/${data.dataModel.id}`, + headers: { + Authorization: `Bearer ${token}`, + 'Content-Type': 'application/json-patch+json', + }, + body: DATA_MODEL_PATCH_PAYLOAD, + }); }); }); + + // Create user + cy.request({ + method: 'POST', + url: `/api/v1/users/signup`, + headers: { Authorization: `Bearer ${token}` }, + body: OWNER_DETAILS, + }).then((response) => { + data.user = response.body; + }); + }); + }); + + after(() => { + cy.login(); + cy.getAllLocalStorage().then((responseData) => { + const token = getToken(responseData); + + // Delete created user + cy.request({ + method: 'DELETE', + url: `/api/v1/users/${data.user.id}?hardDelete=true&recursive=false`, + headers: { Authorization: `Bearer ${token}` }, + }); + + hardDeleteService({ + token, + serviceFqn: DASHBOARD_SERVICE_DETAILS.name, + serviceType: SERVICE_CATEGORIES.DASHBOARD_SERVICES, + }); }); }); @@ -72,9 +110,10 @@ describe( it('Data model version page should show description and tag changes properly', () => { visitDataModelVersionPage( - dataModelFQN, - dataModelId, - dataModelName, + data.dataModel.fullyQualifiedName, + data.dataModel.id, + data.dataModel.name, + DASHBOARD_SERVICE_DETAILS.name, '0.2' ); @@ -110,27 +149,31 @@ describe( }); it(`Data model version page should show owner changes properly`, () => { - visitDataModelPage(dataModelFQN, dataModelName); + visitDataModelPage( + data.dataModel.fullyQualifiedName, + data.dataModel.name, + DASHBOARD_SERVICE_DETAILS.name + ); cy.get('[data-testid="version-button"]').as('versionButton'); cy.get('@versionButton').contains('0.2'); - addOwner(OWNER); + addOwner(data.user.displayName); interceptURL( 'GET', - `/api/v1/dashboard/datamodels/name/${dataModelFQN}*`, + `/api/v1/dashboard/datamodels/name/${data.dataModel.fullyQualifiedName}*`, `getDataModelDetails` ); interceptURL( 'GET', - `/api/v1/dashboard/datamodels/${dataModelId}/versions`, + `/api/v1/dashboard/datamodels/${data.dataModel.id}/versions`, 'getVersionsList' ); interceptURL( 'GET', - `/api/v1/dashboard/datamodels/${dataModelId}/versions/0.2`, + `/api/v1/dashboard/datamodels/${data.dataModel.id}/versions/0.2`, 'getSelectedVersionDetails' ); @@ -146,7 +189,11 @@ describe( }); it(`Data model version page should show tier changes properly`, () => { - visitDataModelPage(dataModelFQN, dataModelName); + visitDataModelPage( + data.dataModel.fullyQualifiedName, + data.dataModel.name, + DASHBOARD_SERVICE_DETAILS.name + ); cy.get('[data-testid="version-button"]').as('versionButton'); @@ -156,17 +203,17 @@ describe( interceptURL( 'GET', - `/api/v1/dashboard/datamodels/name/${dataModelFQN}*`, + `/api/v1/dashboard/datamodels/name/${data.dataModel.fullyQualifiedName}*`, `getDataModelDetails` ); interceptURL( 'GET', - `/api/v1/dashboard/datamodels/${dataModelId}/versions`, + `/api/v1/dashboard/datamodels/${data.dataModel.id}/versions`, 'getVersionsList' ); interceptURL( 'GET', - `/api/v1/dashboard/datamodels/${dataModelId}/versions/0.2`, + `/api/v1/dashboard/datamodels/${data.dataModel.id}/versions/0.2`, 'getSelectedVersionDetails' ); @@ -182,7 +229,11 @@ describe( }); it('Data model version page should show version details after soft deleted', () => { - visitDataModelPage(dataModelFQN, dataModelName); + visitDataModelPage( + data.dataModel.fullyQualifiedName, + data.dataModel.name, + DASHBOARD_SERVICE_DETAILS.name + ); cy.get('[data-testid="manage-button"]').click(); @@ -211,17 +262,17 @@ describe( interceptURL( 'GET', - `/api/v1/dashboard/datamodels/name/${dataModelFQN}*`, + `/api/v1/dashboard/datamodels/name/${data.dataModel.fullyQualifiedName}*`, `getDataModelDetails` ); interceptURL( 'GET', - `/api/v1/dashboard/datamodels/${dataModelId}/versions`, + `/api/v1/dashboard/datamodels/${data.dataModel.id}/versions`, 'getVersionsList' ); interceptURL( 'GET', - `/api/v1/dashboard/datamodels/${dataModelId}/versions/0.3`, + `/api/v1/dashboard/datamodels/${data.dataModel.id}/versions/0.3`, 'getSelectedVersionDetails' ); @@ -261,31 +312,5 @@ describe( cy.get('@versionButton').should('contain', '0.4'); }); - - it(`Cleanup for data model version page test`, () => { - visitDataModelPage(dataModelFQN, dataModelName); - - cy.get('[data-testid="manage-button"]').click(); - - cy.get('[data-testid="delete-button-title"]').click(); - - cy.get('.ant-modal-header').should('contain', dataModelName); - - cy.get(`[data-testid="hard-delete-option"]`).click(); - - cy.get('[data-testid="confirm-button"]').should('be.disabled'); - cy.get('[data-testid="confirmation-text-input"]').type(DELETE_TERM); - - interceptURL( - 'DELETE', - `api/v1/dashboard/datamodels/*?hardDelete=true&recursive=true`, - `hardDeleteDataModel` - ); - cy.get('[data-testid="confirm-button"]').should('not.be.disabled'); - cy.get('[data-testid="confirm-button"]').click(); - verifyResponseStatusCode(`@hardDeleteDataModel`, 200); - - toastNotification(`"${dataModelName}" deleted successfully!`, false); - }); } ); diff --git a/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/DataQualityAndProfiler.spec.js b/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/DataQualityAndProfiler.spec.ts similarity index 94% rename from openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/DataQualityAndProfiler.spec.js rename to openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/DataQualityAndProfiler.spec.ts index c7ea136df4ce..301b562d171e 100644 --- a/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/DataQualityAndProfiler.spec.js +++ b/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/DataQualityAndProfiler.spec.ts @@ -11,46 +11,39 @@ * limitations under the License. */ -// eslint-disable-next-line spaced-comment -/// - import { - deleteCreatedService, descriptionBox, - goToAddNewServicePage, - handleIngestionRetry, interceptURL, - mySqlConnectionInput, - scheduleIngestion, - testServiceCreationAndIngestion, toastNotification, uuid, verifyResponseStatusCode, } from '../../common/common'; import { createEntityTable, hardDeleteService } from '../../common/EntityUtils'; +import MysqlIngestionClass from '../../common/Services/MysqlIngestionClass'; import { searchServiceFromSettingPage } from '../../common/serviceUtils'; import { visitEntityDetailsPage } from '../../common/Utils/Entity'; +import { + handleIngestionRetry, + scheduleIngestion, +} from '../../common/Utils/Ingestion'; +import { getToken } from '../../common/Utils/LocalStorage'; import { addOwner, removeOwner, updateOwner } from '../../common/Utils/Owner'; +import { goToServiceListingPage, Services } from '../../common/Utils/Services'; import { - API_SERVICE, - DATA_ASSETS, DATA_QUALITY_SAMPLE_DATA_TABLE, DELETE_TERM, - ENTITY_SERVICE_TYPE, NEW_COLUMN_TEST_CASE, NEW_COLUMN_TEST_CASE_WITH_NULL_TYPE, NEW_TABLE_TEST_CASE, NEW_TEST_SUITE, - SERVICE_TYPE, TEAM_ENTITY, } from '../../constants/constants'; -import { SidebarItem } from '../../constants/Entity.interface'; +import { EntityType, SidebarItem } from '../../constants/Entity.interface'; import { DATABASE_SERVICE } from '../../constants/EntityConstant'; import { SERVICE_CATEGORIES } from '../../constants/service.constants'; import { GlobalSettingOptions } from '../../constants/settings.constant'; -const serviceType = 'Mysql'; -const serviceName = `${serviceType}-ct-test-${uuid()}`; +const serviceName = `cypress-mysql`; const tableFqn = `${DATABASE_SERVICE.entity.databaseSchema}.${DATABASE_SERVICE.entity.name}`; const testSuite = { name: `${tableFqn}.testSuite`, @@ -88,7 +81,7 @@ const goToProfilerTab = () => { visitEntityDetailsPage({ term: TEAM_ENTITY, serviceName, - entity: DATA_ASSETS.tables, + entity: EntityType.Table, }); verifyResponseStatusCode('@waitForPageLoad', 200); @@ -128,10 +121,11 @@ describe( 'Data Quality and Profiler should work properly', { tags: 'Observability' }, () => { + const mySql = new MysqlIngestionClass(); before(() => { cy.login(); cy.getAllLocalStorage().then((data) => { - const token = Object.values(data)[0].oidcIdToken; + const token = getToken(data); createEntityTable({ token, @@ -166,7 +160,7 @@ describe( after(() => { cy.login(); cy.getAllLocalStorage().then((data) => { - const token = Object.values(data)[0].oidcIdToken; + const token = getToken(data); cy.request({ method: 'DELETE', url: `/api/v1/dataQuality/testCases/${testCaseId}?hardDelete=true&recursive=false`, @@ -187,21 +181,9 @@ describe( }); it('Add and ingest mysql data', () => { - goToAddNewServicePage(SERVICE_TYPE.Database); - - const addIngestionInput = () => { - cy.get('#root\\/schemaFilterPattern\\/includes') - .scrollIntoView() - .type(`${Cypress.env('mysqlDatabaseSchema')}{enter}`); - }; + goToServiceListingPage(Services.Database); - testServiceCreationAndIngestion({ - serviceType, - connectionInput: mySqlConnectionInput, - addIngestionInput, - serviceName, - serviceCategory: ENTITY_SERVICE_TYPE.Database, - }); + mySql.createService(); }); it('Add Profiler ingestion', () => { @@ -247,7 +229,7 @@ describe( .scrollIntoView() .should('be.visible') .and('not.be.disabled') - .type(10); + .type('10'); cy.get('[data-testid="submit-btn"]') .scrollIntoView() .should('be.visible') @@ -261,7 +243,7 @@ describe( .should('be.visible') .click(); - handleIngestionRetry('database', true, 0, 'profiler'); + handleIngestionRetry(0, 'profiler'); }); }); @@ -462,13 +444,9 @@ describe( interceptURL('GET', '/api/v1/dataQuality/testCases?*', 'testCase'); goToProfilerTab(); cy.get('[data-testid="profiler-tab-left-panel"]') - .contains('Column Profile') + .contains('Data Quality') .click(); verifyResponseStatusCode('@testCase', 200); - cy.get('[data-testid="id-test-count"]') - .scrollIntoView() - .should('be.visible') - .click(); cy.get(`[data-testid="${NEW_COLUMN_TEST_CASE.name}"]`).should( 'be.visible' ); @@ -480,7 +458,7 @@ describe( .scrollIntoView() .should('be.visible') .clear() - .type(4); + .type('4'); interceptURL('PATCH', '/api/v1/dataQuality/testCases/*', 'updateTest'); cy.get('.ant-modal-footer').contains('Submit').click(); verifyResponseStatusCode('@updateTest', 200); @@ -507,15 +485,10 @@ describe( interceptURL('GET', '/api/v1/dataQuality/testCases?*', 'testCase'); goToProfilerTab(); cy.get('[data-testid="profiler-tab-left-panel"]') - .contains('Column Profile') + .contains('Data Quality') .should('be.visible') .click(); verifyResponseStatusCode('@testCase', 200); - cy.get('[data-testid="id-test-count"]') - .scrollIntoView() - .should('be.visible') - .click(); - [NEW_COLUMN_TEST_CASE.name, NEW_COLUMN_TEST_CASE_WITH_NULL_TYPE.name].map( (test) => { cy.get(`[data-testid="${test}"]`) @@ -683,11 +656,8 @@ describe( }); it('delete created service', () => { - deleteCreatedService( - SERVICE_TYPE.Database, - serviceName, - API_SERVICE.databaseServices - ); + goToServiceListingPage(Services.Database); + mySql.deleteService(); }); it('Profiler matrix and test case graph should visible', () => { @@ -709,12 +679,13 @@ describe( cy.get('[data-testid="profiler-tab-left-panel"]') .contains('Column Profile') .click(); - verifyResponseStatusCode('@getTestCaseInfo', 200); + cy.get('[data-row-key="shop_id"]') .contains('shop_id') .scrollIntoView() .click(); verifyResponseStatusCode('@getProfilerInfo', 200); + verifyResponseStatusCode('@getTestCaseInfo', 200); cy.get('#count_graph').scrollIntoView().should('be.visible'); cy.get('#proportion_graph').scrollIntoView().should('be.visible'); @@ -836,7 +807,7 @@ describe( visitEntityDetailsPage({ term: tableName, serviceName: DATABASE_SERVICE.service.name, - entity: DATA_ASSETS.tables, + entity: EntityType.Table, }); verifyResponseStatusCode('@waitForPageLoad', 200); cy.get('[data-testid="entity-header-display-name"]').should( @@ -909,8 +880,8 @@ describe( it('Update profiler setting modal', () => { const profilerSetting = { - profileSample: 60, - sampleDataCount: 100, + profileSample: '60', + sampleDataCount: '100', profileQuery: 'select * from table', excludeColumns: 'user_id', includeColumns: 'shop_id', @@ -918,24 +889,28 @@ describe( partitionIntervalType: 'COLUMN-VALUE', partitionValues: 'test', }; - interceptURL('GET', '/api/v1/tables/*/tableProfile?*', 'tableProfiler'); + interceptURL( + 'GET', + '/api/v1/tables/*/tableProfile?startTs=*', + 'tableProfiler' + ); interceptURL('GET', '/api/v1/tables/*/systemProfile?*', 'systemProfiler'); interceptURL( 'GET', - '/api/v1/tables/*/tableProfilerConfig*', + '/api/v1/tables/*/tableProfilerConfig', 'tableProfilerConfig' ); visitEntityDetailsPage({ term: DATABASE_SERVICE.entity.name, serviceName: DATABASE_SERVICE.service.name, - entity: DATA_ASSETS.tables, + entity: EntityType.Table, }); cy.get('[data-testid="profiler"]').should('be.visible').click(); verifyResponseStatusCode('@tableProfiler', 200); verifyResponseStatusCode('@systemProfiler', 200); cy.get('[data-testid="profiler-setting-btn"]').click(); - verifyResponseStatusCode('@tableProfilerConfig', 200); cy.get('.ant-modal-body').should('be.visible'); + verifyResponseStatusCode('@tableProfilerConfig', 200); cy.get('[data-testid="slider-input"]') .clear() .type(profilerSetting.profileSample); diff --git a/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/DatabaseSchemaVersionPage.spec.js b/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/DatabaseSchemaVersionPage.spec.ts similarity index 59% rename from openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/DatabaseSchemaVersionPage.spec.js rename to openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/DatabaseSchemaVersionPage.spec.ts index dd8c7a60cd9d..1ce7f6730f2e 100644 --- a/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/DatabaseSchemaVersionPage.spec.js +++ b/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/DatabaseSchemaVersionPage.spec.ts @@ -10,8 +10,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -// eslint-disable-next-line spaced-comment -/// import { interceptURL, @@ -19,106 +17,49 @@ import { verifyResponseStatusCode, visitDatabaseSchemaDetailsPage, } from '../../common/common'; +import { getToken } from '../../common/Utils/LocalStorage'; import { addOwner } from '../../common/Utils/Owner'; import { addTier } from '../../common/Utils/Tier'; +import { + commonTestCleanup, + databaseSchemaVersionPrerequisites, +} from '../../common/Utils/Versions'; import { DELETE_TERM } from '../../constants/constants'; -import { DOMAIN_CREATION_DETAILS } from '../../constants/EntityConstant'; import { - COMMON_PATCH_PAYLOAD, DATABASE_DETAILS_FOR_VERSION_TEST, DATABASE_SCHEMA_DETAILS_FOR_VERSION_TEST, - OWNER, SERVICE_DETAILS_FOR_VERSION_TEST, TIER, } from '../../constants/Version.constants'; const serviceDetails = SERVICE_DETAILS_FOR_VERSION_TEST.Database; -let domainId; - describe( `Database schema version page should work properly`, { tags: 'DataAssets' }, () => { - let databaseId; - let databaseSchemaId; - let databaseSchemaFQN; + const data = { + user: { displayName: '', name: '' }, + domain: { id: '' }, + database: { id: '' }, + schema: { id: '', fullyQualifiedName: '' }, + }; before(() => { cy.login(); - cy.getAllLocalStorage().then((data) => { - const token = Object.values(data)[0].oidcIdToken; - cy.request({ - method: 'PUT', - url: `/api/v1/domains`, - headers: { Authorization: `Bearer ${token}` }, - body: DOMAIN_CREATION_DETAILS, - }).then((response) => { - domainId = response.body.id; - }); - - // Create service - cy.request({ - method: 'POST', - url: `/api/v1/services/${serviceDetails.serviceCategory}`, - headers: { Authorization: `Bearer ${token}` }, - body: serviceDetails.entityCreationDetails, - }); - - // Create Database - cy.request({ - method: 'POST', - url: `/api/v1/databases`, - headers: { Authorization: `Bearer ${token}` }, - body: DATABASE_DETAILS_FOR_VERSION_TEST, - }).then((response) => { - databaseId = response.body.id; - }); - - // Create Database Schema - cy.request({ - method: 'PUT', - url: `/api/v1/databaseSchemas`, - headers: { Authorization: `Bearer ${token}` }, - body: DATABASE_SCHEMA_DETAILS_FOR_VERSION_TEST, - }).then((response) => { - databaseSchemaId = response.body.id; - databaseSchemaFQN = response.body.fullyQualifiedName; - - cy.request({ - method: 'PATCH', - url: `/api/v1/databaseSchemas/${databaseSchemaId}`, - headers: { - Authorization: `Bearer ${token}`, - 'Content-Type': 'application/json-patch+json', - }, - body: [ - ...COMMON_PATCH_PAYLOAD, - { - op: 'add', - path: '/domain', - value: { - id: domainId, - type: 'domain', - name: DOMAIN_CREATION_DETAILS.name, - description: DOMAIN_CREATION_DETAILS.description, - }, - }, - ], - }); - }); + cy.getAllLocalStorage().then((responseData) => { + const token = getToken(responseData); + + databaseSchemaVersionPrerequisites(token, data); }); }); after(() => { cy.login(); - cy.getAllLocalStorage().then((data) => { - const token = Object.values(data)[0].oidcIdToken; - cy.request({ - method: 'DELETE', - url: `/api/v1/domains/name/${DOMAIN_CREATION_DETAILS.name}`, - headers: { Authorization: `Bearer ${token}` }, - }); + cy.getAllLocalStorage().then((responseData) => { + const token = getToken(responseData); + + commonTestCleanup(token, data); }); }); @@ -131,25 +72,25 @@ describe( settingsMenuId: serviceDetails.settingsMenuId, serviceCategory: serviceDetails.serviceCategory, serviceName: serviceDetails.serviceName, - databaseRowKey: databaseId, + databaseRowKey: data.database.id, databaseName: DATABASE_DETAILS_FOR_VERSION_TEST.name, - databaseSchemaRowKey: databaseSchemaId, + databaseSchemaRowKey: data.schema.id, databaseSchemaName: DATABASE_SCHEMA_DETAILS_FOR_VERSION_TEST.name, }); interceptURL( 'GET', - `/api/v1/databaseSchemas/name/${databaseSchemaFQN}*`, + `/api/v1/databaseSchemas/name/${data.schema.fullyQualifiedName}*`, `getDatabaseSchemaDetails` ); interceptURL( 'GET', - `/api/v1/databaseSchemas/${databaseSchemaId}/versions`, + `/api/v1/databaseSchemas/${data.schema.id}/versions`, 'getVersionsList' ); interceptURL( 'GET', - `/api/v1/databaseSchemas/${databaseSchemaId}/versions/0.2`, + `/api/v1/databaseSchemas/${data.schema.id}/versions/0.2`, 'getSelectedVersionDetails' ); @@ -187,9 +128,9 @@ describe( settingsMenuId: serviceDetails.settingsMenuId, serviceCategory: serviceDetails.serviceCategory, serviceName: serviceDetails.serviceName, - databaseRowKey: databaseId, + databaseRowKey: data.database.id, databaseName: DATABASE_DETAILS_FOR_VERSION_TEST.name, - databaseSchemaRowKey: databaseSchemaId, + databaseSchemaRowKey: data.schema.id, databaseSchemaName: DATABASE_SCHEMA_DETAILS_FOR_VERSION_TEST.name, }); @@ -197,21 +138,21 @@ describe( cy.get('@versionButton').contains('0.2'); - addOwner(OWNER); + addOwner(data.user.displayName); interceptURL( 'GET', - `/api/v1/databaseSchemas/name/${databaseSchemaFQN}*`, + `/api/v1/databaseSchemas/name/${data.schema.fullyQualifiedName}*`, `getDatabaseSchemaDetails` ); interceptURL( 'GET', - `/api/v1/databaseSchemas/${databaseSchemaId}/versions`, + `/api/v1/databaseSchemas/${data.schema.id}/versions`, 'getVersionsList' ); interceptURL( 'GET', - `/api/v1/databaseSchemas/${databaseSchemaId}/versions/0.2`, + `/api/v1/databaseSchemas/${data.schema.id}/versions/0.2`, 'getSelectedVersionDetails' ); @@ -231,9 +172,9 @@ describe( settingsMenuId: serviceDetails.settingsMenuId, serviceCategory: serviceDetails.serviceCategory, serviceName: serviceDetails.serviceName, - databaseRowKey: databaseId, + databaseRowKey: data.database.id, databaseName: DATABASE_DETAILS_FOR_VERSION_TEST.name, - databaseSchemaRowKey: databaseSchemaId, + databaseSchemaRowKey: data.schema.id, databaseSchemaName: DATABASE_SCHEMA_DETAILS_FOR_VERSION_TEST.name, }); @@ -245,17 +186,17 @@ describe( interceptURL( 'GET', - `/api/v1/databaseSchemas/name/${databaseSchemaFQN}*`, + `/api/v1/databaseSchemas/name/${data.schema.fullyQualifiedName}*`, `getDatabaseSchemaDetails` ); interceptURL( 'GET', - `/api/v1/databaseSchemas/${databaseSchemaId}/versions`, + `/api/v1/databaseSchemas/${data.schema.id}/versions`, 'getVersionsList' ); interceptURL( 'GET', - `/api/v1/databaseSchemas/${databaseSchemaId}/versions/0.2`, + `/api/v1/databaseSchemas/${data.schema.id}/versions/0.2`, 'getSelectedVersionDetails' ); @@ -275,9 +216,9 @@ describe( settingsMenuId: serviceDetails.settingsMenuId, serviceCategory: serviceDetails.serviceCategory, serviceName: serviceDetails.serviceName, - databaseRowKey: databaseId, + databaseRowKey: data.database.id, databaseName: DATABASE_DETAILS_FOR_VERSION_TEST.name, - databaseSchemaRowKey: databaseSchemaId, + databaseSchemaRowKey: data.schema.id, databaseSchemaName: DATABASE_SCHEMA_DETAILS_FOR_VERSION_TEST.name, }); @@ -306,17 +247,17 @@ describe( interceptURL( 'GET', - `/api/v1/databaseSchemas/name/${databaseSchemaFQN}*`, + `/api/v1/databaseSchemas/name/${data.schema.fullyQualifiedName}*`, `getDatabaseSchemaDetails` ); interceptURL( 'GET', - `/api/v1/databaseSchemas/${databaseSchemaId}/versions`, + `/api/v1/databaseSchemas/${data.schema.id}/versions`, 'getVersionsList' ); interceptURL( 'GET', - `/api/v1/databaseSchemas/${databaseSchemaId}/versions/0.3`, + `/api/v1/databaseSchemas/${data.schema.id}/versions/0.3`, 'getSelectedVersionDetails' ); @@ -352,50 +293,5 @@ describe( cy.get('@versionButton').should('contain', '0.4'); }); - - it(`Cleanup for Database Schema version page tests`, () => { - visitDatabaseSchemaDetailsPage({ - settingsMenuId: serviceDetails.settingsMenuId, - serviceCategory: serviceDetails.serviceCategory, - serviceName: serviceDetails.serviceName, - databaseRowKey: databaseId, - databaseName: DATABASE_DETAILS_FOR_VERSION_TEST.name, - databaseSchemaRowKey: databaseSchemaId, - databaseSchemaName: DATABASE_SCHEMA_DETAILS_FOR_VERSION_TEST.name, - }); - - // Clicking on permanent delete radio button and checking the service name - cy.get('[data-testid="manage-button"]').click(); - - cy.get('[data-menu-id*="delete-button"]').should('be.visible'); - cy.get('[data-testid="delete-button-title"]').click(); - - // Clicking on permanent delete radio button and checking the service name - cy.get('[data-testid="hard-delete-option"]') - .contains(DATABASE_SCHEMA_DETAILS_FOR_VERSION_TEST.name) - .click(); - - cy.get('[data-testid="confirmation-text-input"]') - .should('be.visible') - .type(DELETE_TERM); - interceptURL('DELETE', `/api/v1/databaseSchemas/*`, 'deleteService'); - interceptURL( - 'GET', - '/api/v1/services/*/name/*?fields=owner', - 'serviceDetails' - ); - - cy.get('[data-testid="confirm-button"]').should('be.visible').click(); - verifyResponseStatusCode('@deleteService', 200); - - // Closing the toast notification - toastNotification( - `"${DATABASE_SCHEMA_DETAILS_FOR_VERSION_TEST.name}" deleted successfully!` - ); - - cy.get( - `[data-testid="service-name-${DATABASE_SCHEMA_DETAILS_FOR_VERSION_TEST.name}"]` - ).should('not.exist'); - }); } ); diff --git a/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/DatabaseVersionPage.spec.js b/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/DatabaseVersionPage.spec.ts similarity index 61% rename from openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/DatabaseVersionPage.spec.js rename to openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/DatabaseVersionPage.spec.ts index 4330fdd41faa..292cfefe1fcf 100644 --- a/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/DatabaseVersionPage.spec.js +++ b/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/DatabaseVersionPage.spec.ts @@ -11,103 +11,53 @@ * limitations under the License. */ -// eslint-disable-next-line spaced-comment -/// - import { interceptURL, toastNotification, verifyResponseStatusCode, visitDatabaseDetailsPage, } from '../../common/common'; +import { getToken } from '../../common/Utils/LocalStorage'; import { addOwner } from '../../common/Utils/Owner'; import { addTier } from '../../common/Utils/Tier'; +import { + commonTestCleanup, + databaseVersionPrerequisites, +} from '../../common/Utils/Versions'; import { DELETE_TERM } from '../../constants/constants'; -import { DOMAIN_CREATION_DETAILS } from '../../constants/EntityConstant'; import { - COMMON_PATCH_PAYLOAD, DATABASE_DETAILS_FOR_VERSION_TEST, - OWNER, SERVICE_DETAILS_FOR_VERSION_TEST, TIER, } from '../../constants/Version.constants'; const serviceDetails = SERVICE_DETAILS_FOR_VERSION_TEST.Database; -let domainId; - describe( `Database version page should work properly`, { tags: 'DataAssets' }, () => { - let databaseId; - let databaseFQN; + const data = { + user: { displayName: '', name: '' }, + domain: { id: '' }, + database: { id: '', fullyQualifiedName: '' }, + }; before(() => { cy.login(); - cy.getAllLocalStorage().then((data) => { - const token = Object.values(data)[0].oidcIdToken; - cy.request({ - method: 'PUT', - url: `/api/v1/domains`, - headers: { Authorization: `Bearer ${token}` }, - body: DOMAIN_CREATION_DETAILS, - }).then((response) => { - domainId = response.body.id; - }); - - // Create service - cy.request({ - method: 'POST', - url: `/api/v1/services/${serviceDetails.serviceCategory}`, - headers: { Authorization: `Bearer ${token}` }, - body: serviceDetails.entityCreationDetails, - }); - - // Create Database - cy.request({ - method: 'POST', - url: `/api/v1/databases`, - headers: { Authorization: `Bearer ${token}` }, - body: DATABASE_DETAILS_FOR_VERSION_TEST, - }).then((response) => { - databaseId = response.body.id; - databaseFQN = response.body.fullyQualifiedName; - - cy.request({ - method: 'PATCH', - url: `/api/v1/databases/${databaseId}`, - headers: { - Authorization: `Bearer ${token}`, - 'Content-Type': 'application/json-patch+json', - }, - body: [ - ...COMMON_PATCH_PAYLOAD, - { - op: 'add', - path: '/domain', - value: { - id: domainId, - type: 'domain', - name: DOMAIN_CREATION_DETAILS.name, - description: DOMAIN_CREATION_DETAILS.description, - }, - }, - ], - }); - }); + cy.getAllLocalStorage().then((responseData) => { + const token = getToken(responseData); + + databaseVersionPrerequisites(token, data); }); }); after(() => { cy.login(); - cy.getAllLocalStorage().then((data) => { - const token = Object.values(data)[0].oidcIdToken; - cy.request({ - method: 'DELETE', - url: `/api/v1/domains/name/${DOMAIN_CREATION_DETAILS.name}`, - headers: { Authorization: `Bearer ${token}` }, - }); + cy.getAllLocalStorage().then((responseData) => { + const token = getToken(responseData); + + commonTestCleanup(token, data); }); }); @@ -120,23 +70,23 @@ describe( settingsMenuId: serviceDetails.settingsMenuId, serviceCategory: serviceDetails.serviceCategory, serviceName: serviceDetails.serviceName, - databaseRowKey: databaseId, + databaseRowKey: data.database.id, databaseName: DATABASE_DETAILS_FOR_VERSION_TEST.name, }); interceptURL( 'GET', - `/api/v1/databases/name/${databaseFQN}?include=all`, + `/api/v1/databases/name/${data.database.fullyQualifiedName}?include=all`, `getDatabaseDetails` ); interceptURL( 'GET', - `/api/v1/databases/${databaseId}/versions`, + `/api/v1/databases/${data.database.id}/versions`, 'getVersionsList' ); interceptURL( 'GET', - `/api/v1/databases/${databaseId}/versions/0.2`, + `/api/v1/databases/${data.database.id}/versions/0.2`, 'getSelectedVersionDetails' ); @@ -174,7 +124,7 @@ describe( settingsMenuId: serviceDetails.settingsMenuId, serviceCategory: serviceDetails.serviceCategory, serviceName: serviceDetails.serviceName, - databaseRowKey: databaseId, + databaseRowKey: data.database.id, databaseName: DATABASE_DETAILS_FOR_VERSION_TEST.name, }); @@ -182,21 +132,21 @@ describe( cy.get('@versionButton').contains('0.2'); - addOwner(OWNER); + addOwner(data.user.displayName); interceptURL( 'GET', - `/api/v1/databases/name/${databaseFQN}?include=all`, + `/api/v1/databases/name/${data.database.fullyQualifiedName}?include=all`, `getDatabaseDetails` ); interceptURL( 'GET', - `/api/v1/databases/${databaseId}/versions`, + `/api/v1/databases/${data.database.id}/versions`, 'getVersionsList' ); interceptURL( 'GET', - `/api/v1/databases/${databaseId}/versions/0.2`, + `/api/v1/databases/${data.database.id}/versions/0.2`, 'getSelectedVersionDetails' ); @@ -216,7 +166,7 @@ describe( settingsMenuId: serviceDetails.settingsMenuId, serviceCategory: serviceDetails.serviceCategory, serviceName: serviceDetails.serviceName, - databaseRowKey: databaseId, + databaseRowKey: data.database.id, databaseName: DATABASE_DETAILS_FOR_VERSION_TEST.name, }); @@ -228,17 +178,17 @@ describe( interceptURL( 'GET', - `/api/v1/databases/name/${databaseFQN}?include=all`, + `/api/v1/databases/name/${data.database.fullyQualifiedName}?include=all`, `getDatabaseDetails` ); interceptURL( 'GET', - `/api/v1/databases/${databaseId}/versions`, + `/api/v1/databases/${data.database.id}/versions`, 'getVersionsList' ); interceptURL( 'GET', - `/api/v1/databases/${databaseId}/versions/0.2`, + `/api/v1/databases/${data.database.id}/versions/0.2`, 'getSelectedVersionDetails' ); @@ -258,7 +208,7 @@ describe( settingsMenuId: serviceDetails.settingsMenuId, serviceCategory: serviceDetails.serviceCategory, serviceName: serviceDetails.serviceName, - databaseRowKey: databaseId, + databaseRowKey: data.database.id, databaseName: DATABASE_DETAILS_FOR_VERSION_TEST.name, }); @@ -290,17 +240,17 @@ describe( interceptURL( 'GET', - `/api/v1/databases/name/${databaseFQN}?include=all`, + `/api/v1/databases/name/${data.database.fullyQualifiedName}?include=all`, `getDatabaseDetails` ); interceptURL( 'GET', - `/api/v1/databases/${databaseId}/versions`, + `/api/v1/databases/${data.database.id}/versions`, 'getVersionsList' ); interceptURL( 'GET', - `/api/v1/databases/${databaseId}/versions/0.3`, + `/api/v1/databases/${data.database.id}/versions/0.3`, 'getSelectedVersionDetails' ); @@ -336,57 +286,5 @@ describe( cy.get('@versionButton').should('contain', '0.4'); }); - - it(`Cleanup for Database version page tests`, () => { - visitDatabaseDetailsPage({ - settingsMenuId: serviceDetails.settingsMenuId, - serviceCategory: serviceDetails.serviceCategory, - serviceName: serviceDetails.serviceName, - databaseRowKey: databaseId, - databaseName: DATABASE_DETAILS_FOR_VERSION_TEST.name, - }); - - // Clicking on permanent delete radio button and checking the service name - cy.get('[data-testid="manage-button"]') - .should('exist') - .should('be.visible') - .click(); - - cy.get('[data-menu-id*="delete-button"]') - .should('exist') - .should('be.visible'); - cy.get('[data-testid="delete-button-title"]') - .should('be.visible') - .click() - .as('deleteBtn'); - - // Clicking on permanent delete radio button and checking the service name - cy.get('[data-testid="hard-delete-option"]') - .contains(DATABASE_DETAILS_FOR_VERSION_TEST.name) - .should('be.visible') - .click(); - - cy.get('[data-testid="confirmation-text-input"]') - .should('be.visible') - .type(DELETE_TERM); - interceptURL('DELETE', `/api/v1/databases/*`, 'deleteService'); - interceptURL( - 'GET', - '/api/v1/services/*/name/*?fields=owner', - 'serviceDetails' - ); - - cy.get('[data-testid="confirm-button"]').should('be.visible').click(); - verifyResponseStatusCode('@deleteService', 200); - - // Closing the toast notification - toastNotification( - `"${DATABASE_DETAILS_FOR_VERSION_TEST.name}" deleted successfully!` - ); - - cy.get( - `[data-testid="service-name-${DATABASE_DETAILS_FOR_VERSION_TEST.name}"]` - ).should('not.exist'); - }); } ); diff --git a/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/Domains.spec.js b/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/Domains.spec.ts similarity index 98% rename from openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/Domains.spec.js rename to openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/Domains.spec.ts index 4f675488d85d..06adb2550feb 100644 --- a/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/Domains.spec.js +++ b/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/Domains.spec.ts @@ -10,8 +10,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -// eslint-disable-next-line spaced-comment -/// import { interceptURL, verifyResponseStatusCode } from '../../common/common'; import { diff --git a/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/EntityVersionPages.spec.js b/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/EntityVersionPages.spec.ts similarity index 90% rename from openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/EntityVersionPages.spec.js rename to openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/EntityVersionPages.spec.ts index caa6cdc07f3f..f3b7053dd371 100644 --- a/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/EntityVersionPages.spec.js +++ b/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/EntityVersionPages.spec.ts @@ -11,9 +11,6 @@ * limitations under the License. */ -// eslint-disable-next-line spaced-comment -/// - import { isEmpty } from 'lodash'; import { deleteEntity, @@ -21,38 +18,60 @@ import { verifyResponseStatusCode, } from '../../common/common'; import { visitEntityDetailsPage } from '../../common/Utils/Entity'; +import { getToken } from '../../common/Utils/LocalStorage'; import { addOwner } from '../../common/Utils/Owner'; import { addTier } from '../../common/Utils/Tier'; import { visitEntityDetailsVersionPage } from '../../common/VersionUtils'; import { DOMAIN_CREATION_DETAILS } from '../../constants/EntityConstant'; import { ENTITY_DETAILS_FOR_VERSION_TEST, - OWNER, + OWNER_DETAILS, TIER, } from '../../constants/Version.constants'; -let domainId; - describe('Version page tests for data assets', { tags: 'DataAssets' }, () => { + const data = { + user: { id: '', displayName: '' }, + domain: { id: '' }, + }; before(() => { cy.login(); - cy.getAllLocalStorage().then((data) => { - const token = Object.values(data)[0].oidcIdToken; + cy.getAllLocalStorage().then((responseData) => { + const token = getToken(responseData); + + // Create user + cy.request({ + method: 'POST', + url: `/api/v1/users/signup`, + headers: { Authorization: `Bearer ${token}` }, + body: OWNER_DETAILS, + }).then((response) => { + data.user = response.body; + }); + cy.request({ method: 'PUT', url: `/api/v1/domains`, headers: { Authorization: `Bearer ${token}` }, body: DOMAIN_CREATION_DETAILS, }).then((response) => { - domainId = response.body.id; + data.domain = response.body; }); }); }); after(() => { cy.login(); - cy.getAllLocalStorage().then((data) => { - const token = Object.values(data)[0].oidcIdToken; + cy.getAllLocalStorage().then((responseData) => { + const token = getToken(responseData); + + // Delete created user + cy.request({ + method: 'DELETE', + url: `/api/v1/users/${data.user.id}?hardDelete=true&recursive=false`, + headers: { Authorization: `Bearer ${token}` }, + }); + cy.request({ method: 'DELETE', url: `/api/v1/domains/name/${DOMAIN_CREATION_DETAILS.name}`, @@ -66,13 +85,13 @@ describe('Version page tests for data assets', { tags: 'DataAssets' }, () => { describe(`${entityType} version page should work properly`, () => { const successMessageEntityName = entityType === 'ML Model' ? 'Mlmodel' : entityType; - let entityId; - let entityFQN; + let entityId: string; + let entityFQN: string; before(() => { cy.login(); - cy.getAllLocalStorage().then((data) => { - const token = Object.values(data)[0].oidcIdToken; + cy.getAllLocalStorage().then((responseData) => { + const token = getToken(responseData); cy.request({ method: 'PUT', url: `/api/v1/${entityDetails.entity}`, @@ -95,7 +114,7 @@ describe('Version page tests for data assets', { tags: 'DataAssets' }, () => { op: 'add', path: '/domain', value: { - id: domainId, + id: data.domain.id, type: 'domain', name: DOMAIN_CREATION_DETAILS.name, description: DOMAIN_CREATION_DETAILS.description, @@ -232,7 +251,7 @@ describe('Version page tests for data assets', { tags: 'DataAssets' }, () => { cy.get('@versionButton').contains('0.2'); - addOwner(OWNER); + addOwner(data.user.displayName); interceptURL( 'GET', @@ -345,7 +364,7 @@ describe('Version page tests for data assets', { tags: 'DataAssets' }, () => { after(() => { cy.getAllLocalStorage().then((data) => { - const token = Object.values(data)[0].oidcIdToken; + const token = getToken(data); cy.request({ method: 'DELETE', url: `/api/v1/${entityDetails.entity}/${entityId}`, diff --git a/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/Glossary.spec.js b/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/Glossary.spec.ts similarity index 98% rename from openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/Glossary.spec.js rename to openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/Glossary.spec.ts index ef467843b417..7dab2e25920d 100644 --- a/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/Glossary.spec.js +++ b/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/Glossary.spec.ts @@ -11,13 +11,9 @@ * limitations under the License. */ -// eslint-disable-next-line spaced-comment -/// - import { descriptionBox, interceptURL, - login, signupAndLogin, toastNotification, uuid, @@ -28,6 +24,7 @@ import { deleteGlossary } from '../../common/GlossaryUtils'; import { dragAndDropElement } from '../../common/Utils/DragAndDrop'; import { visitEntityDetailsPage } from '../../common/Utils/Entity'; import { confirmationDragAndDropGlossary } from '../../common/Utils/Glossary'; +import { getToken } from '../../common/Utils/LocalStorage'; import { addOwner, removeOwner } from '../../common/Utils/Owner'; import { COLUMN_NAME_FOR_APPLY_GLOSSARY_TERM, @@ -688,14 +685,16 @@ const checkSummaryListItemSorting = ({ termFQN, columnName }) => { }; const deleteUser = () => { - const token = localStorage.getItem('oidcIdToken'); - - cy.request({ - method: 'DELETE', - url: `/api/v1/users/${createdUserId}?hardDelete=true&recursive=false`, - headers: { Authorization: `Bearer ${token}` }, - }).then((response) => { - expect(response.status).to.eq(200); + cy.getAllLocalStorage().then((storageData) => { + const token = getToken(storageData); + + cy.request({ + method: 'DELETE', + url: `/api/v1/users/${createdUserId}?hardDelete=true&recursive=false`, + headers: { Authorization: `Bearer ${token}` }, + }).then((response) => { + expect(response.status).to.eq(200); + }); }); }; @@ -834,7 +833,7 @@ describe('Glossary page should work properly', { tags: 'Glossary' }, () => { it('Approval Workflow for Glossary Term', () => { cy.logout(); - login(CREDENTIALS.email, CREDENTIALS.password); + cy.login(CREDENTIALS.email, CREDENTIALS.password); approveGlossaryTermWorkflow({ glossary: NEW_GLOSSARY, glossaryTerm: NEW_GLOSSARY_TERMS.term_1, @@ -919,14 +918,10 @@ describe('Glossary page should work properly', { tags: 'Glossary' }, () => { cy.get('[data-testid="request-entity-tags"]').should('exist').click(); - // set assignees for task + // check assignees for task which will be owner of the glossary term cy.get( '[data-testid="select-assignee"] > .ant-select-selector > .ant-select-selection-overflow' - ) - .click() - .type(userName); - cy.get(`[data-testid="${userName}"]`).click(); - cy.clickOutside(); + ).should('contain', 'admin'); cy.get('[data-testid="tag-selector"]') .click() diff --git a/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/GlossaryVersionPage.spec.js b/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/GlossaryVersionPage.spec.ts similarity index 89% rename from openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/GlossaryVersionPage.spec.js rename to openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/GlossaryVersionPage.spec.ts index ecd6e8a737a4..7b57ade60919 100644 --- a/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/GlossaryVersionPage.spec.js +++ b/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/GlossaryVersionPage.spec.ts @@ -10,16 +10,14 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -// eslint-disable-next-line spaced-comment -/// import { interceptURL, verifyResponseStatusCode } from '../../common/common'; import { addReviewer, - deleteGlossary, removeReviewer, visitGlossaryPage, } from '../../common/GlossaryUtils'; +import { getToken } from '../../common/Utils/LocalStorage'; import { addOwner, removeOwner } from '../../common/Utils/Owner'; import { USER_DETAILS } from '../../constants/EntityConstant'; import { GLOSSARY_OWNER_LINK_TEST_ID } from '../../constants/glossary.constant'; @@ -31,19 +29,37 @@ import { GLOSSARY_TERM_NAME_FOR_VERSION_TEST1, GLOSSARY_TERM_NAME_FOR_VERSION_TEST2, GLOSSARY_TERM_PATCH_PAYLOAD2, - REVIEWER, + REVIEWER_DETAILS, } from '../../constants/Version.constants'; describe( 'Glossary and glossary term version pages should work properly', { tags: 'Glossary' }, () => { - let data = {}; + const data = { + user: { + id: '', + displayName: '', + }, + reviewer: { + id: '', + displayName: '', + }, + glossary: { + id: '', + }, + glossaryTerm1: { + id: '', + }, + glossaryTerm2: { + id: '', + }, + }; before(() => { cy.login(); cy.getAllLocalStorage().then((storageData) => { - const token = Object.values(storageData)[0].oidcIdToken; + const token = getToken(storageData); // Create a new user cy.request({ method: 'POST', @@ -54,6 +70,16 @@ describe( data.user = response.body; }); + // Create a new reviewer + cy.request({ + method: 'POST', + url: `/api/v1/users/signup`, + headers: { Authorization: `Bearer ${token}` }, + body: REVIEWER_DETAILS, + }).then((response) => { + data.reviewer = response.body; + }); + // Create Glossary cy.request({ method: 'PUT', @@ -131,7 +157,7 @@ describe( after(() => { cy.login(); cy.getAllLocalStorage().then((storageData) => { - const token = Object.values(storageData)[0].oidcIdToken; + const token = getToken(storageData); // Delete created user cy.request({ @@ -139,6 +165,20 @@ describe( url: `/api/v1/users/${data.user.id}?hardDelete=true&recursive=false`, headers: { Authorization: `Bearer ${token}` }, }); + + // Delete created user + cy.request({ + method: 'DELETE', + url: `/api/v1/users/${data.reviewer.id}?hardDelete=true&recursive=false`, + headers: { Authorization: `Bearer ${token}` }, + }); + + // Delete created user + cy.request({ + method: 'DELETE', + url: `/api/v1/glossaries/${data.glossary.id}?hardDelete=true&recursive=true`, + headers: { Authorization: `Bearer ${token}` }, + }); }); }); @@ -202,7 +242,7 @@ describe( removeOwner(data.user.displayName, GLOSSARY_OWNER_LINK_TEST_ID); - addReviewer(REVIEWER, 'glossaries'); + addReviewer(data.reviewer.displayName, 'glossaries'); interceptURL( 'GET', @@ -349,7 +389,7 @@ describe( removeOwner(data.user.displayName, GLOSSARY_OWNER_LINK_TEST_ID); - addReviewer(REVIEWER, 'glossaryTerms'); + addReviewer(data.reviewer.displayName, 'glossaryTerms'); interceptURL( 'GET', @@ -374,9 +414,5 @@ describe( removeReviewer('glossaryTerms'); }); - - it('Cleanup for glossary and glossary term version page tests', () => { - deleteGlossary(GLOSSARY_FOR_VERSION_TEST.name); - }); } ); diff --git a/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/Login.spec.js b/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/Login.spec.ts similarity index 86% rename from openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/Login.spec.js rename to openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/Login.spec.ts index c35b3d627401..4edb973cb3e6 100644 --- a/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/Login.spec.js +++ b/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/Login.spec.ts @@ -11,11 +11,9 @@ * limitations under the License. */ -import { - interceptURL, - login, - verifyResponseStatusCode, -} from '../../common/common'; +import { interceptURL, verifyResponseStatusCode } from '../../common/common'; +import { getToken } from '../../common/Utils/LocalStorage'; +import { performLogin } from '../../common/Utils/Login'; import { BASE_URL, LOGIN_ERROR_MESSAGE } from '../../constants/constants'; const CREDENTIALS = { @@ -29,6 +27,19 @@ const invalidEmail = 'userTest@openmetadata.org'; const invalidPassword = 'testUsers@123'; describe('Login flow should work properly', { tags: 'Settings' }, () => { + after(() => { + cy.login(); + + cy.getAllLocalStorage().then((data) => { + const token = getToken(data); + cy.request({ + method: 'DELETE', + url: `/api/v1/users/${CREDENTIALS.id}?hardDelete=true&recursive=false`, + headers: { Authorization: `Bearer ${token}` }, + }); + }); + }); + it('Signup and Login with signed up credentials', () => { interceptURL('GET', 'api/v1/system/config/auth', 'getLoginPage'); interceptURL('POST', '/api/v1/users/checkEmailInUse', 'createUser'); @@ -65,7 +76,7 @@ describe('Login flow should work properly', { tags: 'Settings' }, () => { // Login with the created user - login(CREDENTIALS.email, CREDENTIALS.password); + performLogin(CREDENTIALS.email, CREDENTIALS.password); cy.url().should('eq', `${BASE_URL}/my-data`); // Verify user profile @@ -84,7 +95,7 @@ describe('Login flow should work properly', { tags: 'Settings' }, () => { .should('be.visible') .click({ force: true }); cy.wait('@getUser').then((response) => { - CREDENTIALS.id = response.response.body.id; + CREDENTIALS.id = response.response?.body.id; }); cy.get( '[data-testid="user-profile"] [data-testid="user-profile-details"]' @@ -93,14 +104,14 @@ describe('Login flow should work properly', { tags: 'Settings' }, () => { it('Signin using invalid credentials', () => { // Login with invalid email - login(invalidEmail, CREDENTIALS.password); + performLogin(invalidEmail, CREDENTIALS.password); cy.get('[data-testid="login-error-container"]') .should('be.visible') .invoke('text') .should('eq', LOGIN_ERROR_MESSAGE); // Login with invalid password - login(CREDENTIALS.email, invalidPassword); + performLogin(CREDENTIALS.email, invalidPassword); cy.get('[data-testid="login-error-container"]') .should('be.visible') .invoke('text') @@ -126,21 +137,3 @@ describe('Login flow should work properly', { tags: 'Settings' }, () => { cy.get('.ant-btn').contains('Submit').click(); }); }); - -describe('Cleanup', () => { - beforeEach(() => { - cy.login(); - }); - - it('delete user', () => { - const token = localStorage.getItem('oidcIdToken'); - - cy.request({ - method: 'DELETE', - url: `/api/v1/users/${CREDENTIALS.id}?hardDelete=true&recursive=false`, - headers: { Authorization: `Bearer ${token}` }, - }).then((response) => { - expect(response.status).to.eq(200); - }); - }); -}); diff --git a/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/LoginConfiguration.ts b/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/LoginConfiguration.ts index 6a9cac3b03f5..7e63515c9199 100644 --- a/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/LoginConfiguration.ts +++ b/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/LoginConfiguration.ts @@ -34,7 +34,7 @@ describe('Login configuration', { tags: 'Settings' }, () => { cy.get('[data-testid="access-block-time"]').should('have.text', '500'); cy.get('[data-testid="jwt-token-expiry-time"]').should( 'have.text', - '5000 Milliseconds' + '5000 Seconds' ); /* ==== End Cypress Studio ==== */ }); diff --git a/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/MyData.spec.ts b/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/MyData.spec.ts index 9e0987d2a454..47e2e8a02d23 100644 --- a/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/MyData.spec.ts +++ b/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/MyData.spec.ts @@ -12,7 +12,6 @@ */ import { interceptURL, - login, uuid, verifyResponseStatusCode, } from '../../common/common'; @@ -27,6 +26,7 @@ import { hardDeleteService, } from '../../common/EntityUtils'; import { createEntityTableViaREST } from '../../common/Utils/Entity'; +import { getToken } from '../../common/Utils/LocalStorage'; import { generateRandomUser } from '../../common/Utils/Owner'; import { DATABASE_SERVICE, @@ -133,7 +133,7 @@ const updateOwnerAndVerify = ({ url, body, type, entityName, newOwner }) => { 'feedData' ); cy.getAllLocalStorage().then((data) => { - const token = Object.values(data)[0].oidcIdToken; + const token = getToken(data); cy.request({ method: 'PATCH', url, @@ -166,7 +166,7 @@ const updateOwnerAndVerify = ({ url, body, type, entityName, newOwner }) => { const prepareData = () => { cy.login(); cy.getAllLocalStorage().then((data) => { - const token = Object.values(data)[0].oidcIdToken; + const token = getToken(data); SINGLE_LEVEL_SERVICE.forEach((data) => { createSingleLevelEntity({ token, @@ -267,7 +267,7 @@ const prepareData = () => { const cleanUp = () => { cy.login(); cy.getAllLocalStorage().then((data) => { - const token = Object.values(data)[0].oidcIdToken; + const token = getToken(data); hardDeleteService({ token, serviceFqn: DATABASE_SERVICE.service.name, @@ -301,7 +301,7 @@ describe('My Data page', { tags: 'DataAssets' }, () => { it('Verify my data widget', () => { // login with newly created user - login(user1.email, user1.password); + cy.login(user1.email, user1.password); cy.get('[data-testid="my-data-widget"]').scrollIntoView(); // verify total count @@ -320,7 +320,7 @@ describe('My Data page', { tags: 'DataAssets' }, () => { it('Verify following widget', () => { // login with newly created user - login(user1.email, user1.password); + cy.login(user1.email, user1.password); cy.get('[data-testid="following-widget"]').scrollIntoView(); // verify total count @@ -336,7 +336,7 @@ describe('My Data page', { tags: 'DataAssets' }, () => { it('Verify user as owner feed widget', () => { // login with newly created user - login(user2.email, user2.password); + cy.login(user2.email, user2.password); cy.get('[data-testid="no-data-placeholder-container"]') .scrollIntoView() .should( @@ -365,7 +365,7 @@ describe('My Data page', { tags: 'DataAssets' }, () => { it('Verify team as owner feed widget', () => { // login with newly created user - login(user1.email, user1.password); + cy.login(user1.email, user1.password); Object.entries(entities).forEach(([key, value]) => { updateOwnerAndVerify({ diff --git a/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/Permission.spec.ts b/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/Permission.spec.ts index 3a44ab3e5c3e..3c17c5a0cf28 100644 --- a/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/Permission.spec.ts +++ b/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/Permission.spec.ts @@ -12,7 +12,6 @@ */ import { interceptURL, - login, uuid, verifyResponseStatusCode, } from '../../common/common'; @@ -22,6 +21,7 @@ import { createEntityTableViaREST, visitEntityDetailsPage, } from '../../common/Utils/Entity'; +import { getToken } from '../../common/Utils/LocalStorage'; import { EntityType } from '../../constants/Entity.interface'; import { DATABASE_SERVICE, USER_DETAILS } from '../../constants/EntityConstant'; import { SERVICE_CATEGORIES } from '../../constants/service.constants'; @@ -179,7 +179,7 @@ const createViewBasicRoleViaREST = ({ token }) => { const preRequisite = () => { cy.login(); cy.getAllLocalStorage().then((data) => { - const token = Object.values(data)[0].oidcIdToken; + const token = getToken(data); createViewBasicRoleViaREST({ token, }); @@ -234,7 +234,7 @@ const preRequisite = () => { const cleanUp = () => { cy.login(); cy.getAllLocalStorage().then((data) => { - const token = Object.values(data)[0].oidcIdToken; + const token = getToken(data); hardDeleteService({ token, serviceFqn: DATABASE_SERVICE.service.name, @@ -293,7 +293,7 @@ const checkPermission = (permission?: { viewTests?: boolean; editDisplayName?: boolean; }) => { - login(USER_DETAILS.email, USER_DETAILS.password); + cy.login(USER_DETAILS.email, USER_DETAILS.password); visitEntityDetailsPage({ term: DATABASE_SERVICE.entity.name, serviceName: DATABASE_SERVICE.service.name, @@ -307,7 +307,7 @@ const updatePolicy = ( ) => { cy.login(); cy.getAllLocalStorage().then((data) => { - const token = Object.values(data)[0].oidcIdToken; + const token = getToken(data); cy.request({ method: 'PATCH', url: `/api/v1/policies/${policy.id}`, @@ -352,7 +352,7 @@ describe('Permissions', { tags: 'Settings' }, () => { { op: 'add', path: '/rules/0/operations/5', value: 'EditQueries' }, ]); - login(USER_DETAILS.email, USER_DETAILS.password); + cy.login(USER_DETAILS.email, USER_DETAILS.password); visitEntityDetailsPage({ term: DATABASE_SERVICE.entity.name, serviceName: DATABASE_SERVICE.service.name, @@ -361,7 +361,7 @@ describe('Permissions', { tags: 'Settings' }, () => { interceptURL('GET', '/api/v1/queries?*', 'getQueries'); cy.get('[data-testid="table_queries"]').click(); verifyResponseStatusCode('@getQueries', 200); - cy.get('[data-testid="more-option-btn"]').click(); + cy.get('[data-testid="query-btn"]').click(); cy.get('[data-menu-id*="edit-query"]').click(); interceptURL('PATCH', '/api/v1/queries/*', 'updateQuery'); cy.get('.CodeMirror-line').click().type('updated'); @@ -385,7 +385,7 @@ describe('Permissions', { tags: 'Settings' }, () => { }, ]); - login(USER_DETAILS.email, USER_DETAILS.password); + cy.login(USER_DETAILS.email, USER_DETAILS.password); visitEntityDetailsPage({ term: DATABASE_SERVICE.entity.name, serviceName: DATABASE_SERVICE.service.name, diff --git a/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/Policies.spec.js b/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/Policies.spec.ts similarity index 98% rename from openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/Policies.spec.js rename to openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/Policies.spec.ts index 4d064f40b616..518a0f0ed3c9 100644 --- a/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/Policies.spec.js +++ b/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/Policies.spec.ts @@ -144,7 +144,9 @@ describe('Policy page should work properly', { tags: 'Settings' }, () => { .should('contain', ruleName); // Verify policy description - cy.get('[data-testid="description"] > [data-testid="viewer-container"]') + cy.get( + '[data-testid="asset-description-container"] [data-testid="viewer-container"]' + ) .eq(0) .should('be.visible') .should('contain', description); @@ -193,7 +195,9 @@ describe('Policy page should work properly', { tags: 'Settings' }, () => { cy.get('[data-testid="save"]').should('be.visible').click(); // Validate added description - cy.get('[data-testid="description"] > [data-testid="viewer-container"]') + cy.get( + '[data-testid="asset-description-container"] [data-testid="viewer-container"]' + ) .should('be.visible') .should('contain', `${updatedDescription}-${policyName}`); }); diff --git a/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/Roles.spec.js b/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/Roles.spec.ts similarity index 97% rename from openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/Roles.spec.js rename to openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/Roles.spec.ts index e0d24b3ce0f7..4d5613c8610e 100644 --- a/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/Roles.spec.js +++ b/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/Roles.spec.ts @@ -124,7 +124,9 @@ describe('Roles page should work properly', { tags: 'Settings' }, () => { cy.get('[data-testid="inactive-link"]').should('contain', roleName); // Verify added description - cy.get('[data-testid="description"] > [data-testid="viewer-container"]') + cy.get( + '[data-testid="asset-description-container"] [data-testid="viewer-container"]' + ) .should('be.visible') .should('contain', description); @@ -206,7 +208,9 @@ describe('Roles page should work properly', { tags: 'Settings' }, () => { cy.get('[data-testid="inactive-link"]').should('be.visible'); // Asserting updated description - cy.get('[data-testid="description"] > [data-testid="viewer-container"]') + cy.get( + '[data-testid="asset-description-container"] [data-testid="viewer-container"]' + ) .should('be.visible') .should('contain', `${description}-updated`); }); diff --git a/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/SearchIndexApplication.spec.ts b/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/SearchIndexApplication.spec.ts index 3a4c16e0fa23..192a2a318416 100644 --- a/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/SearchIndexApplication.spec.ts +++ b/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/SearchIndexApplication.spec.ts @@ -25,6 +25,19 @@ const visitSearchApplicationPage = () => { verifyResponseStatusCode('@getSearchIndexingApplication', 200); }; +const verifyLastExecutionRun = (interceptedUrlLabel: string) => { + cy.wait(`@${interceptedUrlLabel}`).then((interception) => { + expect(interception.response.statusCode).to.eq(200); + + // Validate the last execution run response + const responseData = interception.response.body; + if (responseData.data.length > 0) { + expect(responseData.data).to.have.length(1); + expect(responseData.data[0].status).to.equal('success'); + } + }); +}; + describe('Search Index Application', { tags: 'Settings' }, () => { beforeEach(() => { cy.login(); @@ -36,6 +49,16 @@ describe('Search Index Application', { tags: 'Settings' }, () => { verifyResponseStatusCode('@getApplications', 200); }); + it('Verify last execution run', () => { + interceptURL( + 'GET', + '/api/v1/apps/name/SearchIndexingApplication/status?offset=0&limit=1', + 'lastExecutionRun' + ); + visitSearchApplicationPage(); + verifyLastExecutionRun('lastExecutionRun'); + }); + it('Edit application', () => { interceptURL('PATCH', '/api/v1/apps/*', 'updateApplication'); visitSearchApplicationPage(); @@ -114,5 +137,16 @@ describe('Search Index Application', { tags: 'Settings' }, () => { verifyResponseStatusCode('@getSearchIndexingApplication', 200); cy.get('[data-testid="run-now-button"]').click(); verifyResponseStatusCode('@triggerPipeline', 200); + + cy.wait(120000); // waiting for 2 minutes before we check if reindex was success + + interceptURL( + 'GET', + '/api/v1/apps/name/SearchIndexingApplication/status?offset=0&limit=1', + 'lastExecutionRun' + ); + + cy.reload(); + verifyLastExecutionRun('lastExecutionRun'); }); }); diff --git a/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/SearchIndexDetails.spec.js b/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/SearchIndexDetails.spec.ts similarity index 93% rename from openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/SearchIndexDetails.spec.js rename to openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/SearchIndexDetails.spec.ts index 3b0d9212b6be..9e5fb1f6da38 100644 --- a/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/SearchIndexDetails.spec.js +++ b/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/SearchIndexDetails.spec.ts @@ -10,14 +10,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -// eslint-disable-next-line spaced-comment -/// import { addTableFieldTags, deleteEntity, interceptURL, - login, removeTableFieldTags, updateTableFieldDescription, verifyResponseStatusCode, @@ -27,7 +24,9 @@ import { hardDeleteService, } from '../../common/EntityUtils'; import { visitEntityDetailsPage } from '../../common/Utils/Entity'; +import { getToken } from '../../common/Utils/LocalStorage'; import { BASE_URL } from '../../constants/constants'; +import { EntityType } from '../../constants/Entity.interface'; import { POLICY_DETAILS, ROLE_DETAILS, @@ -80,12 +79,12 @@ describe( 'SearchIndexDetails page should work properly for data consumer role', { tags: 'DataAssets' }, () => { - let data = {}; + const data = { user: { id: '' } }; before(() => { cy.login(); cy.getAllLocalStorage().then((storageData) => { - const token = Object.values(storageData)[0].oidcIdToken; + const token = getToken(storageData); // Create search index entity createSingleLevelEntity({ @@ -111,7 +110,7 @@ describe( cy.login(); cy.getAllLocalStorage().then((storageData) => { - const token = Object.values(storageData)[0].oidcIdToken; + const token = getToken(storageData); // Delete search index hardDeleteService({ @@ -140,7 +139,7 @@ describe( visitEntityDetailsPage({ term: SEARCH_INDEX_DETAILS_FOR_DETAILS_PAGE_TEST.name, serviceName: SEARCH_INDEX_DETAILS_FOR_DETAILS_PAGE_TEST.service, - entity: 'searchIndexes', + entity: EntityType.SearchIndex, }); // Edit domain option should not be available @@ -161,12 +160,16 @@ describe( ); describe('SearchIndexDetails page should work properly for data steward role', () => { - let data = {}; + const data = { + user: { id: '' }, + policy: { id: '' }, + role: { id: '', name: '' }, + }; before(() => { cy.login(); cy.getAllLocalStorage().then((storageData) => { - const token = Object.values(storageData)[0].oidcIdToken; + const token = getToken(storageData); // Create search index entity createSingleLevelEntity({ @@ -233,7 +236,7 @@ describe('SearchIndexDetails page should work properly for data steward role', ( cy.login(); cy.getAllLocalStorage().then((storageData) => { - const token = Object.values(storageData)[0].oidcIdToken; + const token = getToken(storageData); // Delete created user cy.request({ @@ -267,7 +270,7 @@ describe('SearchIndexDetails page should work properly for data steward role', ( beforeEach(() => { // Login with the created user - login(USER_CREDENTIALS.email, USER_CREDENTIALS.password); + cy.login(USER_CREDENTIALS.email, USER_CREDENTIALS.password); cy.url().should('eq', `${BASE_URL}/my-data`); }); @@ -276,7 +279,7 @@ describe('SearchIndexDetails page should work properly for data steward role', ( visitEntityDetailsPage({ term: SEARCH_INDEX_DETAILS_FOR_DETAILS_PAGE_TEST.name, serviceName: SEARCH_INDEX_DETAILS_FOR_DETAILS_PAGE_TEST.service, - entity: 'searchIndexes', + entity: EntityType.SearchIndex, }); // Edit domain option should not be available @@ -323,7 +326,7 @@ describe('SearchIndexDetails page should work properly for admin role', () => { before(() => { cy.login(); cy.getAllLocalStorage().then((storageData) => { - const token = Object.values(storageData)[0].oidcIdToken; + const token = getToken(storageData); // Create search index entity createSingleLevelEntity({ @@ -337,7 +340,7 @@ describe('SearchIndexDetails page should work properly for admin role', () => { cy.login(); cy.getAllLocalStorage().then((storageData) => { - const token = Object.values(storageData)[0].oidcIdToken; + const token = getToken(storageData); // Delete search index hardDeleteService({ @@ -356,7 +359,7 @@ describe('SearchIndexDetails page should work properly for admin role', () => { visitEntityDetailsPage({ term: SEARCH_INDEX_DETAILS_FOR_DETAILS_PAGE_TEST.name, serviceName: SEARCH_INDEX_DETAILS_FOR_DETAILS_PAGE_TEST.service, - entity: 'searchIndexes', + entity: EntityType.SearchIndex, }); performCommonOperations(); }); @@ -365,12 +368,12 @@ describe('SearchIndexDetails page should work properly for admin role', () => { visitEntityDetailsPage({ term: SEARCH_INDEX_DETAILS_FOR_DETAILS_PAGE_TEST.name, serviceName: SEARCH_INDEX_DETAILS_FOR_DETAILS_PAGE_TEST.service, - entity: 'searchIndexes', + entity: EntityType.SearchIndex, }); deleteEntity( SEARCH_INDEX_DETAILS_FOR_DETAILS_PAGE_TEST.name, SEARCH_INDEX_DETAILS_FOR_DETAILS_PAGE_TEST.service, - 'searchIndexes', + EntityType.SearchIndex, 'Search Index', 'soft' ); @@ -427,12 +430,12 @@ describe('SearchIndexDetails page should work properly for admin role', () => { visitEntityDetailsPage({ term: SEARCH_INDEX_DETAILS_FOR_DETAILS_PAGE_TEST.name, serviceName: SEARCH_INDEX_DETAILS_FOR_DETAILS_PAGE_TEST.service, - entity: 'searchIndexes', + entity: EntityType.SearchIndex, }); deleteEntity( SEARCH_INDEX_DETAILS_FOR_DETAILS_PAGE_TEST.name, SEARCH_INDEX_DETAILS_FOR_DETAILS_PAGE_TEST.service, - 'searchIndexes', + EntityType.SearchIndex, 'Search Index' ); }); diff --git a/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/Service.spec.js b/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/Service.spec.js deleted file mode 100644 index 6f59cee72873..000000000000 --- a/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/Service.spec.js +++ /dev/null @@ -1,161 +0,0 @@ -/* - * Copyright 2022 Collate. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * http://www.apache.org/licenses/LICENSE-2.0 - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { - descriptionBox, - interceptURL, - verifyResponseStatusCode, -} from '../../common/common'; -import { searchServiceFromSettingPage } from '../../common/serviceUtils'; -import { service } from '../../constants/constants'; -import { GlobalSettingOptions } from '../../constants/settings.constant'; - -describe('Services page should work properly', { tags: 'Integration' }, () => { - beforeEach(() => { - interceptURL( - 'GET', - '/api/v1/system/config/pipeline-service-client', - 'pipelineServiceClient' - ); - interceptURL( - 'GET', - `/api/v1/*?service=${service.name}&fields=*`, - 'serviceDetails' - ); - interceptURL( - 'GET', - `/api/v1/services/ingestionPipelines?fields=*&service=${service.name}*`, - 'ingestionPipelines' - ); - cy.login(); - // redirecting to services page - - cy.settingClick(GlobalSettingOptions.DATABASES); - }); - - it('Update service description', () => { - searchServiceFromSettingPage(service.name); - cy.get(`[data-testid="service-name-${service.name}"]`) - .should('be.visible') - .click(); - verifyResponseStatusCode('@serviceDetails', 200); - verifyResponseStatusCode('@ingestionPipelines', 200); - verifyResponseStatusCode('@pipelineServiceClient', 200); - // need wait here - cy.get('[data-testid="edit-description"]') - .should('exist') - .should('be.visible') - .click({ force: true }); - cy.get(descriptionBox).clear().type(service.newDescription); - cy.get('[data-testid="save"]').click(); - cy.get( - '[data-testid="description-container"] [data-testid="viewer-container"] [data-testid="markdown-parser"] :nth-child(1) .toastui-editor-contents p' - ).contains(service.newDescription); - cy.get(':nth-child(1) > .link-title').click(); - searchServiceFromSettingPage(service.name); - cy.get('.toastui-editor-contents > p').contains(service.newDescription); - }); - - it('Update owner and check description', () => { - searchServiceFromSettingPage(service.name); - cy.get(`[data-testid="service-name-${service.name}"]`) - .should('be.visible') - .click(); - verifyResponseStatusCode('@serviceDetails', 200); - verifyResponseStatusCode('@ingestionPipelines', 200); - verifyResponseStatusCode('@pipelineServiceClient', 200); - interceptURL( - 'GET', - '/api/v1/search/query?q=*%20AND%20teamType:Group&from=0&size=*&index=team_search_index&sort_field=displayName.keyword&sort_order=asc', - 'editOwner' - ); - cy.get('[data-testid="edit-owner"]') - .should('exist') - .should('be.visible') - .click(); - verifyResponseStatusCode('@editOwner', 200); - - cy.get( - '.ant-popover-inner-content > .ant-tabs > .ant-tabs-nav > .ant-tabs-nav-wrap' - ) - .contains('Users') - .click(); - - interceptURL( - 'PATCH', - '/api/v1/services/databaseServices/*', - 'updateService' - ); - interceptURL( - 'GET', - '/api/v1/search/query?q=*%20AND%20isBot:false*&index=user_search_index', - 'searchApi' - ); - - cy.get('[data-testid="owner-select-users-search-bar"]').type(service.Owner); - verifyResponseStatusCode('@searchApi', 200); - cy.get('[data-testid="selectable-list"]') - .contains(service.Owner) - .scrollIntoView() - .click(); - - verifyResponseStatusCode('@updateService', 200); - - // Checking if description exists after assigning the owner - cy.get(':nth-child(1) > .link-title').click(); - // need wait here - searchServiceFromSettingPage(service.name); - cy.get('[data-testid="viewer-container"]').contains(service.newDescription); - }); - - it('Remove owner from service', () => { - interceptURL( - 'GET', - '/api/v1/system/config/pipeline-service-client', - 'getService' - ); - - interceptURL('GET', '/api/v1/users?*', 'waitForUsers'); - searchServiceFromSettingPage(service.name); - cy.get(`[data-testid="service-name-${service.name}"]`) - .should('be.visible') - .click(); - verifyResponseStatusCode('@serviceDetails', 200); - verifyResponseStatusCode('@ingestionPipelines', 200); - verifyResponseStatusCode('@pipelineServiceClient', 200); - - cy.get('[data-testid="edit-owner"]') - .should('exist') - .should('be.visible') - .click(); - verifyResponseStatusCode('@waitForUsers', 200); - - interceptURL('PATCH', '/api/v1/services/databaseServices/*', 'removeOwner'); - cy.get('[data-testid="selectable-list"]') - .contains(service.Owner) - .should('be.visible'); - - cy.get('[data-testid="remove-owner"]') - .should('exist') - .should('be.visible') - .click(); - - verifyResponseStatusCode('@removeOwner', 200); - - // Check if Owner exist - cy.get('[data-testid="owner-link"]') - .scrollIntoView() - .should('exist') - .contains('No Owner'); - }); -}); diff --git a/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/ServiceVersionPage.spec.js b/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/ServiceVersionPage.spec.ts similarity index 88% rename from openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/ServiceVersionPage.spec.js rename to openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/ServiceVersionPage.spec.ts index 3d8849880bd7..576b485d132f 100644 --- a/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/ServiceVersionPage.spec.js +++ b/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/ServiceVersionPage.spec.ts @@ -10,8 +10,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -// eslint-disable-next-line spaced-comment -/// import { interceptURL, @@ -20,18 +18,17 @@ import { visitServiceDetailsPage, } from '../../common/common'; import { hardDeleteService } from '../../common/EntityUtils'; +import { getToken } from '../../common/Utils/LocalStorage'; import { addOwner } from '../../common/Utils/Owner'; import { addTier } from '../../common/Utils/Tier'; import { DELETE_TERM } from '../../constants/constants'; import { DOMAIN_CREATION_DETAILS } from '../../constants/EntityConstant'; import { - OWNER, + OWNER_DETAILS, SERVICE_DETAILS_FOR_VERSION_TEST, TIER, } from '../../constants/Version.constants'; -let domainId; - const navigateToVersionPageFromServicePage = ( serviceCategory, serviceName, @@ -65,30 +62,49 @@ describe( 'Common prerequisite for service version test', { tags: 'Integration' }, () => { + const data = { user: { id: '', displayName: '' }, domain: { id: '' } }; + before(() => { cy.login(); - cy.getAllLocalStorage().then((data) => { - const token = Object.values(data)[0].oidcIdToken; + cy.getAllLocalStorage().then((responseData) => { + const token = getToken(responseData); cy.request({ method: 'PUT', url: `/api/v1/domains`, headers: { Authorization: `Bearer ${token}` }, body: DOMAIN_CREATION_DETAILS, }).then((response) => { - domainId = response.body.id; + data.domain = response.body; + }); + + // Create user + cy.request({ + method: 'POST', + url: `/api/v1/users/signup`, + headers: { Authorization: `Bearer ${token}` }, + body: OWNER_DETAILS, + }).then((response) => { + data.user = response.body; }); }); }); after(() => { cy.login(); - cy.getAllLocalStorage().then((data) => { - const token = Object.values(data)[0].oidcIdToken; + cy.getAllLocalStorage().then((responseData) => { + const token = getToken(responseData); cy.request({ method: 'DELETE', url: `/api/v1/domains/name/${DOMAIN_CREATION_DETAILS.name}`, headers: { Authorization: `Bearer ${token}` }, }); + + // Delete created user + cy.request({ + method: 'DELETE', + url: `/api/v1/users/${data.user.id}?hardDelete=true&recursive=false`, + headers: { Authorization: `Bearer ${token}` }, + }); }); }); @@ -106,8 +122,8 @@ describe( before(() => { cy.login(); - cy.getAllLocalStorage().then((data) => { - const token = Object.values(data)[0].oidcIdToken; + cy.getAllLocalStorage().then((responseData) => { + const token = getToken(responseData); cy.request({ method: 'POST', url: `/api/v1/services/${serviceCategory}`, @@ -129,7 +145,7 @@ describe( op: 'add', path: '/domain', value: { - id: domainId, + id: data.domain.id, type: 'domain', name: DOMAIN_CREATION_DETAILS.name, description: DOMAIN_CREATION_DETAILS.description, @@ -148,7 +164,7 @@ describe( after(() => { cy.login(); cy.getAllLocalStorage().then((data) => { - const token = Object.values(data)[0].oidcIdToken; + const token = getToken(data); hardDeleteService({ token, @@ -207,7 +223,7 @@ describe( cy.get('@versionButton').contains('0.2'); - addOwner(OWNER); + addOwner(data.user.displayName); navigateToVersionPageFromServicePage( serviceCategory, @@ -232,7 +248,7 @@ describe( cy.get('@versionButton').contains('0.2'); - addTier(TIER, `services/${serviceCategory}`); + addTier(TIER); navigateToVersionPageFromServicePage( serviceCategory, diff --git a/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/Tags.spec.js b/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/Tags.spec.ts similarity index 98% rename from openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/Tags.spec.js rename to openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/Tags.spec.ts index 3ee00ad14fab..498b07b1f206 100644 --- a/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/Tags.spec.js +++ b/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/Tags.spec.ts @@ -11,9 +11,6 @@ * limitations under the License. */ -// eslint-disable-next-line spaced-comment -/// - import { addNewTagToEntity, descriptionBox, @@ -75,7 +72,7 @@ describe('Classification Page', { tags: 'Governance' }, () => { cy.get('[data-testid="add-classification"]').should('be.visible'); cy.get('[data-testid="add-new-tag-button"]').should('be.visible'); cy.get('[data-testid="manage-button"]').should('be.visible'); - cy.get('[data-testid="description"]').should('be.visible'); + cy.get('[data-testid="description-container"]').should('be.visible'); cy.get('[data-testid="table"]').should('be.visible'); cy.get('.ant-table-thead > tr > .ant-table-cell') @@ -191,7 +188,7 @@ describe('Classification Page', { tags: 'Governance' }, () => { serviceName: entity.serviceName, entity: entity.entity, }); - addNewTagToEntity(entity, NEW_TAG); + addNewTagToEntity(NEW_TAG); }); it('Assign tag to DatabaseSchema', () => { diff --git a/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/Teams.spec.js b/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/Teams.spec.ts similarity index 92% rename from openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/Teams.spec.js rename to openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/Teams.spec.ts index f0dcd2c1eadf..7476e63682f3 100644 --- a/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/Teams.spec.js +++ b/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/Teams.spec.ts @@ -11,19 +11,14 @@ * limitations under the License. */ -// eslint-disable-next-line spaced-comment -/// - import { - addTeam, descriptionBox, interceptURL, toastNotification, - updateOwner, uuid, verifyResponseStatusCode, } from '../../common/common'; -import { deleteTeamPermanently } from '../../common/Utils/Teams'; +import { addTeam, deleteTeamPermanently } from '../../common/Utils/Teams'; import { SidebarItem } from '../../constants/Entity.interface'; import { GlobalSettingOptions } from '../../constants/settings.constant'; @@ -50,7 +45,7 @@ const HARD_DELETE_TEAM_DETAILS = { email: 'team@gmail.com', }; -describe('Teams flow should work properly', () => { +describe('Teams flow should work properly', { tags: 'Settings' }, () => { beforeEach(() => { interceptURL('GET', `/api/v1/users?fields=*`, 'getUserDetails'); interceptURL('GET', `/api/v1/permissions/team/name/*`, 'permissions'); @@ -83,7 +78,25 @@ describe('Teams flow should work properly', () => { .contains(TEAM_DETAILS.name) .click(); - updateOwner(); + cy.get('[data-testid="avatar"]').click(); + cy.get('[data-testid="user-name"]') + .should('exist') + .invoke('text') + .then((text) => { + interceptURL('GET', '/api/v1/users?limit=15', 'getUsers'); + // Clicking on edit owner button + cy.get('[data-testid="edit-owner"]').click(); + + cy.get('.user-team-select-popover').contains('Users').click(); + cy.get('[data-testid="owner-select-users-search-bar"]').type(text); + cy.get('[data-testid="selectable-list"]') + .eq(1) + .find(`[title="${text.trim()}"]`) + .click(); + + // Asserting the added name + cy.get('[data-testid="owner-link"]').should('contain', text.trim()); + }); }); it('Update email of created team', () => { @@ -254,7 +267,7 @@ describe('Teams flow should work properly', () => { verifyResponseStatusCode('@patchDescription', 200); // Validating the updated description - cy.get('[data-testid="description"] p').should( + cy.get('[data-testid="asset-description-container"] p').should( 'contain', updatedDescription ); diff --git a/openmetadata-ui/src/main/resources/ui/cypress/plugins/index.js b/openmetadata-ui/src/main/resources/ui/cypress/plugins/index.js index 31e155650ad0..9b2b42e1f8e6 100644 --- a/openmetadata-ui/src/main/resources/ui/cypress/plugins/index.js +++ b/openmetadata-ui/src/main/resources/ui/cypress/plugins/index.js @@ -1,5 +1,3 @@ -// eslint-disable-next-line spaced-comment -/// // *********************************************************** // This example plugins/index.js can be used to load plugins // diff --git a/openmetadata-ui/src/main/resources/ui/cypress/support/commands.js b/openmetadata-ui/src/main/resources/ui/cypress/support/commands.ts similarity index 82% rename from openmetadata-ui/src/main/resources/ui/cypress/support/commands.js rename to openmetadata-ui/src/main/resources/ui/cypress/support/commands.ts index 35ae85c372c1..6bcd6a52b2cb 100644 --- a/openmetadata-ui/src/main/resources/ui/cypress/support/commands.js +++ b/openmetadata-ui/src/main/resources/ui/cypress/support/commands.ts @@ -45,44 +45,6 @@ import { } from '../constants/settings.constant'; import { SIDEBAR_LIST_ITEMS } from '../constants/sidebar.constant'; -Cypress.Commands.add('loginByGoogleApi', () => { - cy.log('Logging in to Google'); - cy.request({ - method: 'POST', - url: 'https://www.googleapis.com/oauth2/v4/token', - body: { - grant_type: 'refresh_token', - client_id: Cypress.env('googleClientId'), - client_secret: Cypress.env('googleClientSecret'), - refresh_token: Cypress.env('googleRefreshToken'), - }, - }).then(({ body }) => { - const { access_token, id_token } = body; - - cy.request({ - method: 'GET', - url: 'https://www.googleapis.com/oauth2/v3/userinfo', - headers: { Authorization: `Bearer ${access_token}` }, - }).then(({ body }) => { - cy.log(body); - const userItem = { - token: id_token, - user: { - googleId: body.sub, - email: body.email, - givenName: body.given_name, - familyName: body.family_name, - imageUrl: body.picture, - }, - }; - - window.localStorage.setItem('googleCypress', JSON.stringify(userItem)); - window.localStorage.setItem('oidcIdToken', id_token); - cy.visit('/'); - }); - }); -}); - Cypress.Commands.add('goToHomePage', (doNotNavigate) => { interceptURL('GET', '/api/v1/users/loggedInUser?fields=*', 'userProfile'); !doNotNavigate && cy.visit('/'); @@ -119,7 +81,7 @@ Cypress.Commands.add('storeSession', (username, password) => { cy.url().should('not.eq', `${BASE_URL}/signin`); // Don't want to show any popup in the tests - cy.setCookie(`STAR_OMD_USER_admin`, 'true'); + cy.setCookie(`STAR_OMD_USER_${username.split('@')[0]}`, 'true'); // Get version and set cookie to hide version banner cy.request({ @@ -132,7 +94,7 @@ Cypress.Commands.add('storeSession', (username, password) => { .replaceAll('.', '_')}`; cy.setCookie(versionCookie, 'true'); - window.localStorage.setItem('loggedInUsers', 'admin'); + window.localStorage.setItem('loggedInUsers', username.split('@')[0]); }); }); }); @@ -207,7 +169,7 @@ Cypress.Commands.add('settingClick', (dataTestId, isCustomProperty) => { cy.sidebarClick(SidebarItem.SETTINGS); - (paths ?? []).forEach((path) => { + (paths ?? []).forEach((path: string) => { cy.get(`[data-testid="${path}"]`).scrollIntoView().click(); }); }); diff --git a/openmetadata-ui/src/main/resources/ui/cypress/tsconfig.json b/openmetadata-ui/src/main/resources/ui/cypress/tsconfig.json new file mode 100644 index 000000000000..396fd27f32d6 --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/cypress/tsconfig.json @@ -0,0 +1,10 @@ +{ + "compilerOptions": { + "rootDir": "./", + "downlevelIteration": true, + "target": "ES5", + "lib": ["dom", "dom.iterable", "ES2020.Promise", "es2021"], + "types": ["cypress", "node", "@cypress/grep"] + }, + "include": ["./**/*.ts"] +} diff --git a/openmetadata-ui/src/main/resources/ui/jest.config.js b/openmetadata-ui/src/main/resources/ui/jest.config.js index 859a6c5ba8f8..67d0bb886b0a 100644 --- a/openmetadata-ui/src/main/resources/ui/jest.config.js +++ b/openmetadata-ui/src/main/resources/ui/jest.config.js @@ -58,8 +58,14 @@ module.exports = { '/src/test/unit/mocks/file.mock.js', '\\.json': '/src/test/unit/mocks/json.mock.js', '@github/g-emoji-element': '/src/test/unit/mocks/gemoji.mock.js', + 'quilljs-markdown': '/src/test/unit/mocks/gemoji.mock.js', + '@azure/msal-browser': + '/node_modules/@azure/msal-browser/lib/msal-browser.cjs', + '@azure/msal-react': + '/node_modules/@azure/msal-react/dist/index.js', axios: 'axios/dist/node/axios.cjs', }, + transformIgnorePatterns: ['node_modules/(?!@azure/msal-react)'], // TypeScript preset: 'ts-jest', diff --git a/openmetadata-ui/src/main/resources/ui/package.json b/openmetadata-ui/src/main/resources/ui/package.json index c0be71736dc2..f6800d8dbbd7 100644 --- a/openmetadata-ui/src/main/resources/ui/package.json +++ b/openmetadata-ui/src/main/resources/ui/package.json @@ -46,8 +46,8 @@ "@ant-design/icons": "^4.7.0", "@apidevtools/json-schema-ref-parser": "^9.0.9", "@auth0/auth0-react": "^1.9.0", - "@azure/msal-browser": "^2.37.0", - "@azure/msal-react": "^1.5.11", + "@azure/msal-browser": "^3.10.0", + "@azure/msal-react": "^2.0.12", "@deuex-solutions/react-tour": "^1.2.6", "@fontsource/poppins": "^5.0.0", "@fontsource/source-code-pro": "^5.0.0", @@ -84,6 +84,7 @@ "crypto-random-string-with-promisify-polyfill": "^5.0.0", "dagre": "^0.8.5", "diff": "^5.0.0", + "eventemitter3": "^5.0.1", "fast-json-patch": "^3.1.1", "history": "4.5.1", "html-react-parser": "^1.4.14", @@ -128,7 +129,8 @@ "socket.io-client": "^4.5.1", "styled-components": "^6.1.8", "turndown": "^7.1.2", - "use-analytics": "^0.0.5" + "use-analytics": "^0.0.5", + "zustand": "^4.5.0" }, "browserslist": { "production": [ @@ -238,4 +240,4 @@ "prosemirror-view": "1.28.2", "axios": "1.6.4" } -} \ No newline at end of file +} diff --git a/openmetadata-ui/src/main/resources/ui/public/locales/en-US/Database/Oracle.md b/openmetadata-ui/src/main/resources/ui/public/locales/en-US/Database/Oracle.md index 8cd71b0f8209..58d9a3e3d2ca 100644 --- a/openmetadata-ui/src/main/resources/ui/public/locales/en-US/Database/Oracle.md +++ b/openmetadata-ui/src/main/resources/ui/public/locales/en-US/Database/Oracle.md @@ -23,6 +23,9 @@ GRANT new_role TO user_name; -- GRANT CREATE SESSION PRIVILEGE TO USER GRANT CREATE SESSION TO new_role; + +-- GRANT SELECT CATALOG ROLE PRIVILEGE TO FETCH METADATA TO ROLE / USER +GRANT SELECT_CATALOG_ROLE TO new_role; ``` - `GRANT SELECT` on the relevant tables which are to be ingested into OpenMetadata to the user diff --git a/openmetadata-ui/src/main/resources/ui/public/locales/en-US/OpenMetadata/CustomProperty.md b/openmetadata-ui/src/main/resources/ui/public/locales/en-US/OpenMetadata/CustomProperty.md index 1e232908f642..5a04ea91ef68 100644 --- a/openmetadata-ui/src/main/resources/ui/public/locales/en-US/OpenMetadata/CustomProperty.md +++ b/openmetadata-ui/src/main/resources/ui/public/locales/en-US/OpenMetadata/CustomProperty.md @@ -18,4 +18,16 @@ $$section ### Description $(id="description") Describe your custom property to provide more information to your team. +$$ + +$$section +### Enum Values $(id="customPropertyConfig") + +Add the list of values for enum property. +$$ + +$$section +### Multi Select $(id="multiSelect") + +Enable multi select of values for enum property. $$ \ No newline at end of file diff --git a/openmetadata-ui/src/main/resources/ui/public/locales/fr-FR/Database/Oracle.md b/openmetadata-ui/src/main/resources/ui/public/locales/fr-FR/Database/Oracle.md index 6d35a6d9c43a..d35ea9aad355 100644 --- a/openmetadata-ui/src/main/resources/ui/public/locales/fr-FR/Database/Oracle.md +++ b/openmetadata-ui/src/main/resources/ui/public/locales/fr-FR/Database/Oracle.md @@ -17,6 +17,9 @@ GRANT new_role TO user_name; -- GRANT CREATE SESSION PRIVILEGE TO USER GRANT CREATE SESSION TO new_role; + +-- GRANT SELECT CATALOG ROLE PRIVILEGE TO FETCH METADATA TO ROLE / USER +GRANT SELECT_CATALOG_ROLE TO new_role; ``` **Important:** OpenMetadata utilise `python-oracledb` qui supoorte seulement les version 12c, 18c, 19c, et 21c d'Oracle. diff --git a/openmetadata-ui/src/main/resources/ui/src/App.test.tsx b/openmetadata-ui/src/main/resources/ui/src/App.test.tsx index a4fa6260dd3a..33fe3e4cf10e 100644 --- a/openmetadata-ui/src/main/resources/ui/src/App.test.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/App.test.tsx @@ -14,8 +14,7 @@ import { render } from '@testing-library/react'; import React from 'react'; import App from './App'; -import { AuthContext } from './components/Auth/AuthProviders/AuthProvider'; -import { IAuthContext } from './components/Auth/AuthProviders/AuthProvider.interface'; +import { AuthProvider } from './components/Auth/AuthProviders/AuthProvider'; jest.mock('./components/AppRouter/AppRouter', () => { return jest.fn().mockReturnValue(

AppRouter

); @@ -34,9 +33,9 @@ jest.mock('./components/Auth/AuthProviders/AuthProvider', () => { it('renders learn react link', () => { const { getAllByTestId } = render( - + - + ); const linkElement = getAllByTestId(/content-wrapper/i); linkElement.map((elm) => expect(elm).toBeInTheDocument()); diff --git a/openmetadata-ui/src/main/resources/ui/src/App.tsx b/openmetadata-ui/src/main/resources/ui/src/App.tsx index d7abea1deac0..d7b60678454b 100644 --- a/openmetadata-ui/src/main/resources/ui/src/App.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/App.tsx @@ -11,7 +11,8 @@ * limitations under the License. */ -import React, { FC } from 'react'; +import { isEmpty } from 'lodash'; +import React, { FC, useEffect } from 'react'; import { HelmetProvider } from 'react-helmet-async'; import { I18nextProvider } from 'react-i18next'; import { Router } from 'react-router-dom'; @@ -22,18 +23,50 @@ import { AuthProvider } from './components/Auth/AuthProviders/AuthProvider'; import ErrorBoundary from './components/common/ErrorBoundary/ErrorBoundary'; import DomainProvider from './components/Domain/DomainProvider/DomainProvider'; import { EntityExportModalProvider } from './components/Entity/EntityExportModalProvider/EntityExportModalProvider.component'; +import ApplicationsProvider from './components/Settings/Applications/ApplicationsProvider/ApplicationsProvider'; import WebAnalyticsProvider from './components/WebAnalytics/WebAnalyticsProvider'; import { TOAST_OPTIONS } from './constants/Toasts.constants'; -import ApplicationConfigProvider from './context/ApplicationConfigProvider/ApplicationConfigProvider'; import DirectionProvider from './context/DirectionProvider/DirectionProvider'; import GlobalSearchProvider from './context/GlobalSearchProvider/GlobalSearchProvider'; import PermissionProvider from './context/PermissionProvider/PermissionProvider'; import TourProvider from './context/TourProvider/TourProvider'; import WebSocketProvider from './context/WebSocketProvider/WebSocketProvider'; +import { useApplicationStore } from './hooks/useApplicationStore'; +import { getCustomLogoConfig } from './rest/settingConfigAPI'; import { history } from './utils/HistoryUtils'; import i18n from './utils/i18next/LocalUtil'; const App: FC = () => { + const { applicationConfig, setApplicationConfig } = useApplicationStore(); + + const fetchApplicationConfig = async () => { + try { + const data = await getCustomLogoConfig(); + + setApplicationConfig({ + ...data, + }); + } catch (error) { + // eslint-disable-next-line no-console + console.error(error); + } + }; + + useEffect(() => { + fetchApplicationConfig(); + }, []); + + useEffect(() => { + const faviconHref = isEmpty(applicationConfig?.customFaviconUrlPath) + ? '/favicon.png' + : applicationConfig?.customFaviconUrlPath ?? '/favicon.png'; + const link = document.querySelector('link[rel~="icon"]'); + + if (link) { + link.setAttribute('href', faviconHref); + } + }, [applicationConfig]); + return (
@@ -41,27 +74,27 @@ const App: FC = () => { - - - - - - - - + + + + + + + + - - - - - - - - + + + + + + + + diff --git a/openmetadata-ui/src/main/resources/ui/src/assets/svg/MetaPilotApplication.svg b/openmetadata-ui/src/main/resources/ui/src/assets/svg/MetaPilotApplication.svg deleted file mode 100644 index ad5a5f7d5193..000000000000 --- a/openmetadata-ui/src/main/resources/ui/src/assets/svg/MetaPilotApplication.svg +++ /dev/null @@ -1,12 +0,0 @@ - - - - - - - - - - - - diff --git a/openmetadata-ui/src/main/resources/ui/src/assets/svg/external-links.svg b/openmetadata-ui/src/main/resources/ui/src/assets/svg/external-links.svg index 34d2a4870a3c..b32577bb8f7e 100644 --- a/openmetadata-ui/src/main/resources/ui/src/assets/svg/external-links.svg +++ b/openmetadata-ui/src/main/resources/ui/src/assets/svg/external-links.svg @@ -1,11 +1,16 @@ - - - - + + + + + + + + + - - + + \ No newline at end of file diff --git a/openmetadata-ui/src/main/resources/ui/src/assets/svg/ic-exit.svg b/openmetadata-ui/src/main/resources/ui/src/assets/svg/ic-exit.svg new file mode 100644 index 000000000000..058c8c553d0a --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/assets/svg/ic-exit.svg @@ -0,0 +1,11 @@ + + + + + + + + + + + \ No newline at end of file diff --git a/openmetadata-ui/src/main/resources/ui/src/assets/svg/ic-suggestions.svg b/openmetadata-ui/src/main/resources/ui/src/assets/svg/ic-suggestions.svg index 7cc52b60ab00..a018b5329cfd 100644 --- a/openmetadata-ui/src/main/resources/ui/src/assets/svg/ic-suggestions.svg +++ b/openmetadata-ui/src/main/resources/ui/src/assets/svg/ic-suggestions.svg @@ -1,24 +1,12 @@ - - - - - - - - - - - - - - - - - + + + + + - - + + diff --git a/openmetadata-ui/src/main/resources/ui/src/assets/svg/om-health-colored.svg b/openmetadata-ui/src/main/resources/ui/src/assets/svg/om-health-colored.svg new file mode 100644 index 000000000000..015acaa00bf0 --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/assets/svg/om-health-colored.svg @@ -0,0 +1,27 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/openmetadata-ui/src/main/resources/ui/src/assets/svg/om-upgrade.svg b/openmetadata-ui/src/main/resources/ui/src/assets/svg/om-upgrade.svg new file mode 100644 index 000000000000..40e3c0e3b293 --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/assets/svg/om-upgrade.svg @@ -0,0 +1,78 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/openmetadata-ui/src/main/resources/ui/src/components/ActivityFeed/ActivityFeedCard/ActivityFeedCard.tsx b/openmetadata-ui/src/main/resources/ui/src/components/ActivityFeed/ActivityFeedCard/ActivityFeedCard.tsx index 75ae38eeaeed..8ecece945433 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/ActivityFeed/ActivityFeedCard/ActivityFeedCard.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/ActivityFeed/ActivityFeedCard/ActivityFeedCard.tsx @@ -20,12 +20,12 @@ import { ReactionOperation } from '../../../enums/reactions.enum'; import { AnnouncementDetails } from '../../../generated/api/feed/createThread'; import { Post } from '../../../generated/entity/feed/thread'; import { Reaction, ReactionType } from '../../../generated/type/reaction'; +import { useApplicationStore } from '../../../hooks/useApplicationStore'; import { getEntityField, getEntityFQN, getEntityType, } from '../../../utils/FeedUtils'; -import { useAuthContext } from '../../Auth/AuthProviders/AuthProvider'; import UserPopOverCard from '../../common/PopOverCard/UserPopOverCard'; import EditAnnouncementModal from '../../Modals/AnnouncementModal/EditAnnouncementModal'; import { ActivityFeedCardProp } from './ActivityFeedCard.interface'; @@ -58,7 +58,7 @@ const ActivityFeedCard: FC = ({ const entityType = getEntityType(entityLink ?? ''); const entityFQN = getEntityFQN(entityLink ?? ''); const entityField = getEntityField(entityLink ?? ''); - const { currentUser } = useAuthContext(); + const { currentUser } = useApplicationStore(); const containerRef = useRef(null); const [feedDetail, setFeedDetail] = useState(feed); diff --git a/openmetadata-ui/src/main/resources/ui/src/components/ActivityFeed/ActivityFeedCard/PopoverContent.test.tsx b/openmetadata-ui/src/main/resources/ui/src/components/ActivityFeed/ActivityFeedCard/PopoverContent.test.tsx index 1ea2a8ac9129..4451fa188e24 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/ActivityFeed/ActivityFeedCard/PopoverContent.test.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/ActivityFeed/ActivityFeedCard/PopoverContent.test.tsx @@ -43,8 +43,8 @@ const mockUserData: User = { isAdmin: true, }; -jest.mock('../../Auth/AuthProviders/AuthProvider', () => ({ - useAuthContext: jest.fn(() => ({ +jest.mock('../../../hooks/useApplicationStore', () => ({ + useApplicationStore: jest.fn(() => ({ currentUser: mockUserData, })), })); diff --git a/openmetadata-ui/src/main/resources/ui/src/components/ActivityFeed/ActivityFeedCard/PopoverContent.tsx b/openmetadata-ui/src/main/resources/ui/src/components/ActivityFeed/ActivityFeedCard/PopoverContent.tsx index ec76c85dc581..59544e5b9adc 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/ActivityFeed/ActivityFeedCard/PopoverContent.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/ActivityFeed/ActivityFeedCard/PopoverContent.tsx @@ -23,7 +23,7 @@ import { REACTION_LIST } from '../../../constants/reactions.constant'; import { ReactionOperation } from '../../../enums/reactions.enum'; import { Post } from '../../../generated/entity/feed/thread'; import { ReactionType } from '../../../generated/type/reaction'; -import { useAuthContext } from '../../Auth/AuthProviders/AuthProvider'; +import { useApplicationStore } from '../../../hooks/useApplicationStore'; import Reaction from '../Reactions/Reaction'; import { ConfirmState } from './ActivityFeedCard.interface'; @@ -59,7 +59,7 @@ const PopoverContent: FC = ({ isAnnouncement, editAnnouncementPermission, }) => { - const { currentUser } = useAuthContext(); + const { currentUser } = useApplicationStore(); const [visible, setVisible] = useState(false); const hide = () => { diff --git a/openmetadata-ui/src/main/resources/ui/src/components/ActivityFeed/ActivityFeedProvider/ActivityFeedProvider.tsx b/openmetadata-ui/src/main/resources/ui/src/components/ActivityFeed/ActivityFeedProvider/ActivityFeedProvider.tsx index c5dd9391e186..f93cd24b0a4f 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/ActivityFeed/ActivityFeedProvider/ActivityFeedProvider.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/ActivityFeed/ActivityFeedProvider/ActivityFeedProvider.tsx @@ -42,6 +42,7 @@ import { EntityReference } from '../../../generated/entity/type'; import { TestCaseResolutionStatus } from '../../../generated/tests/testCaseResolutionStatus'; import { Paging } from '../../../generated/type/paging'; import { Reaction, ReactionType } from '../../../generated/type/reaction'; +import { useApplicationStore } from '../../../hooks/useApplicationStore'; import { deletePostById, deleteThread, @@ -59,7 +60,6 @@ import { } from '../../../utils/EntityUtils'; import { getUpdatedThread } from '../../../utils/FeedUtils'; import { showErrorToast } from '../../../utils/ToastUtils'; -import { useAuthContext } from '../../Auth/AuthProviders/AuthProvider'; import ActivityFeedDrawer from '../ActivityFeedDrawer/ActivityFeedDrawer'; import { ActivityFeedProviderContextType } from './ActivityFeedProviderContext.interface'; @@ -89,7 +89,7 @@ const ActivityFeedProvider = ({ children, user }: Props) => { const [initialAssignees, setInitialAssignees] = useState( [] ); - const { currentUser } = useAuthContext(); + const { currentUser } = useApplicationStore(); const fetchTestCaseResolution = useCallback(async (id: string) => { try { diff --git a/openmetadata-ui/src/main/resources/ui/src/components/ActivityFeed/ActivityFeedTab/ActivityFeedTab.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/ActivityFeed/ActivityFeedTab/ActivityFeedTab.component.tsx index 391c5cdd6d28..2e90e4935431 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/ActivityFeed/ActivityFeedTab/ActivityFeedTab.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/ActivityFeed/ActivityFeedTab/ActivityFeedTab.component.tsx @@ -45,6 +45,7 @@ import { ThreadType, } from '../../../generated/entity/feed/thread'; import { useAuth } from '../../../hooks/authHooks'; +import { useApplicationStore } from '../../../hooks/useApplicationStore'; import { useElementInView } from '../../../hooks/useElementInView'; import { FeedCounts } from '../../../interface/feed.interface'; import { getFeedCount } from '../../../rest/feedsAPI'; @@ -59,7 +60,6 @@ import { getEntityUserLink, } from '../../../utils/EntityUtils'; import { showErrorToast } from '../../../utils/ToastUtils'; -import { useAuthContext } from '../../Auth/AuthProviders/AuthProvider'; import Loader from '../../common/Loader/Loader'; import { TaskTab } from '../../Entity/Task/TaskTab/TaskTab.component'; import '../../MyData/Widgets/FeedsWidget/feeds-widget.less'; @@ -88,7 +88,7 @@ export const ActivityFeedTab = ({ }: ActivityFeedTabProps) => { const history = useHistory(); const { t } = useTranslation(); - const { currentUser } = useAuthContext(); + const { currentUser } = useApplicationStore(); const { isAdminUser } = useAuth(); const initialRender = useRef(true); const [elementRef, isInView] = useElementInView({ diff --git a/openmetadata-ui/src/main/resources/ui/src/components/ActivityFeed/ActivityThreadPanel/ActivityThreadPanelBody.tsx b/openmetadata-ui/src/main/resources/ui/src/components/ActivityFeed/ActivityThreadPanel/ActivityThreadPanelBody.tsx index 02bf58b4200c..3d97a0eea584 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/ActivityFeed/ActivityThreadPanel/ActivityThreadPanelBody.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/ActivityFeed/ActivityThreadPanel/ActivityThreadPanelBody.tsx @@ -31,7 +31,8 @@ import { Paging } from '../../../generated/type/paging'; import { useElementInView } from '../../../hooks/useElementInView'; import { getAllFeeds } from '../../../rest/feedsAPI'; import { showErrorToast } from '../../../utils/ToastUtils'; -import { useAuthContext } from '../../Auth/AuthProviders/AuthProvider'; + +import { useApplicationStore } from '../../../hooks/useApplicationStore'; import ErrorPlaceHolder from '../../common/ErrorWithPlaceholder/ErrorPlaceHolder'; import Loader from '../../common/Loader/Loader'; import ConfirmationModal from '../../Modals/ConfirmationModal/ConfirmationModal'; @@ -56,7 +57,7 @@ const ActivityThreadPanelBody: FC = ({ threadType, }) => { const { t } = useTranslation(); - const { currentUser } = useAuthContext(); + const { currentUser } = useApplicationStore(); const [threads, setThreads] = useState([]); const [selectedThread, setSelectedThread] = useState(); const [selectedThreadId, setSelectedThreadId] = useState(''); diff --git a/openmetadata-ui/src/main/resources/ui/src/components/ActivityFeed/FeedEditor/FeedEditor.tsx b/openmetadata-ui/src/main/resources/ui/src/components/ActivityFeed/FeedEditor/FeedEditor.tsx index 9e10661dcea2..1945312f266a 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/ActivityFeed/FeedEditor/FeedEditor.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/ActivityFeed/FeedEditor/FeedEditor.tsx @@ -35,7 +35,7 @@ import { MENTION_DENOTATION_CHARS, TOOLBAR_ITEMS, } from '../../../constants/Feeds.constants'; -import { useApplicationConfigContext } from '../../../context/ApplicationConfigProvider/ApplicationConfigProvider'; +import { useApplicationStore } from '../../../hooks/useApplicationStore'; import { getUserByName } from '../../../rest/userAPI'; import { HTMLToMarkdown, @@ -75,7 +75,8 @@ export const FeedEditor = forwardRef( const [value, setValue] = useState(defaultValue ?? ''); const [isMentionListOpen, toggleMentionList] = useState(false); const [isFocused, toggleFocus] = useState(false); - const { userProfilePics } = useApplicationConfigContext(); + + const { userProfilePics } = useApplicationStore(); const userSuggestionRenderer = async ( searchTerm: string, diff --git a/openmetadata-ui/src/main/resources/ui/src/components/ActivityFeed/Reactions/Emoji.test.tsx b/openmetadata-ui/src/main/resources/ui/src/components/ActivityFeed/Reactions/Emoji.test.tsx index c4844d71e5f0..5d1213d41ba0 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/ActivityFeed/Reactions/Emoji.test.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/ActivityFeed/Reactions/Emoji.test.tsx @@ -28,8 +28,8 @@ jest.mock('../../../hooks/useImage', () => jest.fn().mockReturnValue({ image: null }) ); -jest.mock('../../Auth/AuthProviders/AuthProvider', () => ({ - useAuthContext: jest.fn(() => ({ +jest.mock('../../../hooks/useApplicationStore', () => ({ + useApplicationStore: jest.fn(() => ({ currentUser: mockUserData, })), })); diff --git a/openmetadata-ui/src/main/resources/ui/src/components/ActivityFeed/Reactions/Emoji.tsx b/openmetadata-ui/src/main/resources/ui/src/components/ActivityFeed/Reactions/Emoji.tsx index 90c82498d27d..55baf1ffdffa 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/ActivityFeed/Reactions/Emoji.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/ActivityFeed/Reactions/Emoji.tsx @@ -19,8 +19,8 @@ import { useTranslation } from 'react-i18next'; import { REACTION_LIST } from '../../../constants/reactions.constant'; import { ReactionOperation } from '../../../enums/reactions.enum'; import { Reaction, ReactionType } from '../../../generated/type/reaction'; +import { useApplicationStore } from '../../../hooks/useApplicationStore'; import useImage from '../../../hooks/useImage'; -import { useAuthContext } from '../../Auth/AuthProviders/AuthProvider'; interface EmojiProps { reaction: ReactionType; @@ -37,7 +37,7 @@ const Emoji: FC = ({ onReactionSelect, }) => { const { t } = useTranslation(); - const { currentUser } = useAuthContext(); + const { currentUser } = useApplicationStore(); const [reactionType, setReactionType] = useState(reaction); const [isClicked, setIsClicked] = useState(false); const [visible, setVisible] = useState(false); diff --git a/openmetadata-ui/src/main/resources/ui/src/components/ActivityFeed/Reactions/Reactions.tsx b/openmetadata-ui/src/main/resources/ui/src/components/ActivityFeed/Reactions/Reactions.tsx index 7adf797a7a79..6e8501b025ac 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/ActivityFeed/Reactions/Reactions.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/ActivityFeed/Reactions/Reactions.tsx @@ -26,7 +26,7 @@ import { Reaction as ReactionProp, ReactionType, } from '../../../generated/type/reaction'; -import { useAuthContext } from '../../Auth/AuthProviders/AuthProvider'; +import { useApplicationStore } from '../../../hooks/useApplicationStore'; import Emoji from './Emoji'; import Reaction from './Reaction'; import './reactions.less'; @@ -42,7 +42,7 @@ interface ReactionsProps { const Reactions: FC = ({ reactions, onReactionSelect }) => { const { t } = useTranslation(); const [visible, setVisible] = useState(false); - const { currentUser } = useAuthContext(); + const { currentUser } = useApplicationStore(); const hide = () => { setVisible(false); diff --git a/openmetadata-ui/src/main/resources/ui/src/components/ActivityFeed/Shared/ActivityFeedActions.tsx b/openmetadata-ui/src/main/resources/ui/src/components/ActivityFeed/Shared/ActivityFeedActions.tsx index 75bc76e67031..9d54b45984d3 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/ActivityFeed/Shared/ActivityFeedActions.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/ActivityFeed/Shared/ActivityFeedActions.tsx @@ -28,7 +28,8 @@ import { Thread, ThreadType, } from '../../../generated/entity/feed/thread'; -import { useAuthContext } from '../../Auth/AuthProviders/AuthProvider'; + +import { useApplicationStore } from '../../../hooks/useApplicationStore'; import { useActivityFeedProvider } from '../ActivityFeedProvider/ActivityFeedProvider'; import Reaction from '../Reactions/Reaction'; import './activity-feed-actions.less'; @@ -47,7 +48,7 @@ const ActivityFeedActions = ({ onEditPost, }: ActivityFeedActionsProps) => { const { t } = useTranslation(); - const { currentUser } = useAuthContext(); + const { currentUser } = useApplicationStore(); const isAuthor = post.from === currentUser?.name; const [visible, setVisible] = useState(false); const [showDeleteDialog, setShowDeleteDialog] = useState(false); diff --git a/openmetadata-ui/src/main/resources/ui/src/components/AppBar/Appbar.test.tsx b/openmetadata-ui/src/main/resources/ui/src/components/AppBar/Appbar.test.tsx index 7520a605ea2e..7839e819dc6f 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/AppBar/Appbar.test.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/AppBar/Appbar.test.tsx @@ -16,16 +16,6 @@ import React from 'react'; import { MemoryRouter } from 'react-router-dom'; import Appbar from './Appbar'; -jest.mock('../../hooks/authHooks', () => ({ - useAuth: () => { - return { - isSignedIn: true, - isSignedOut: false, - isAuthenticatedRoute: true, - isAuthDisabled: true, - }; - }, -})); jest.mock('react-router-dom', () => ({ useLocation: jest.fn().mockReturnValue({ search: 'pathname' }), Link: jest @@ -35,12 +25,12 @@ jest.mock('react-router-dom', () => ({ )), useHistory: jest.fn(), })); -jest.mock('../Auth/AuthProviders/AuthProvider', () => { + +jest.mock('../../hooks/useApplicationStore', () => { return { - useAuthContext: jest.fn(() => ({ + useApplicationStore: jest.fn().mockImplementation(() => ({ isAuthenticated: true, - isProtectedRoute: jest.fn().mockReturnValue(true), - isTourRoute: jest.fn().mockReturnValue(false), + getOidcToken: jest.fn().mockReturnValue({ isExpired: false }), onLogoutHandler: jest.fn(), })), }; @@ -60,6 +50,11 @@ jest.mock('../../rest/miscAPI', () => ({ ), })); +jest.mock('../../utils/AuthProvider.util', () => ({ + ...jest.requireActual('../../utils/AuthProvider.util'), + isProtectedRoute: jest.fn().mockReturnValue(true), +})); + describe('Test Appbar Component', () => { it('Component should render', async () => { const { container } = render(, { diff --git a/openmetadata-ui/src/main/resources/ui/src/components/AppBar/Appbar.tsx b/openmetadata-ui/src/main/resources/ui/src/components/AppBar/Appbar.tsx index 00359b0a3982..4d50e98bf989 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/AppBar/Appbar.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/AppBar/Appbar.tsx @@ -11,36 +11,17 @@ * limitations under the License. */ -import Icon from '@ant-design/icons/lib/components/Icon'; -import { Col, Row } from 'antd'; -import { AxiosError } from 'axios'; import { isString } from 'lodash'; import Qs from 'qs'; import React, { useEffect, useState } from 'react'; import { useTranslation } from 'react-i18next'; -import { Link, useHistory, useLocation } from 'react-router-dom'; +import { useHistory, useLocation } from 'react-router-dom'; import { toast } from 'react-toastify'; -import { ReactComponent as IconAPI } from '../../assets/svg/api.svg'; -import { ReactComponent as IconDoc } from '../../assets/svg/doc.svg'; -import { ReactComponent as IconExternalLink } from '../../assets/svg/external-links.svg'; -import { ReactComponent as IconTour } from '../../assets/svg/icon-tour.svg'; -import { ReactComponent as IconSlackGrey } from '../../assets/svg/slack-grey.svg'; -import { ReactComponent as IconVersionBlack } from '../../assets/svg/version-black.svg'; -import { ReactComponent as IconWhatsNew } from '../../assets/svg/whats-new.svg'; -import { - getExplorePath, - ROUTES, - TOUR_SEARCH_TERM, -} from '../../constants/constants'; -import { - urlGitbookDocs, - urlGithubRepo, - urlJoinSlack, -} from '../../constants/URL.constants'; +import { getExplorePath, TOUR_SEARCH_TERM } from '../../constants/constants'; import { useGlobalSearchProvider } from '../../context/GlobalSearchProvider/GlobalSearchProvider'; import { useTourProvider } from '../../context/TourProvider/TourProvider'; import { CurrentTourPageType } from '../../enums/tour.enum'; -import { getVersion } from '../../rest/miscAPI'; +import { useApplicationStore } from '../../hooks/useApplicationStore'; import { extractDetailsFromToken, isProtectedRoute, @@ -48,8 +29,6 @@ import { } from '../../utils/AuthProvider.util'; import { addToRecentSearched } from '../../utils/CommonUtils'; import searchClassBase from '../../utils/SearchClassBase'; -import { showErrorToast } from '../../utils/ToastUtils'; -import { useAuthContext } from '../Auth/AuthProviders/AuthProvider'; import NavBar from '../NavBar/NavBar'; import './app-bar.style.less'; @@ -61,7 +40,8 @@ const Appbar: React.FC = (): JSX.Element => { const { isTourOpen, updateTourPage, updateTourSearch, tourSearchValue } = useTourProvider(); - const { isAuthenticated, onLogoutHandler } = useAuthContext(); + const { isAuthenticated, onLogoutHandler, getOidcToken } = + useApplicationStore(); const { searchCriteria } = useGlobalSearchProvider(); @@ -78,12 +58,6 @@ const Appbar: React.FC = (): JSX.Element => { const [searchValue, setSearchValue] = useState(searchQuery); const [isOpen, setIsOpen] = useState(false); - const [isFeatureModalOpen, setIsFeatureModalOpen] = useState(false); - const [version, setVersion] = useState(''); - - const handleFeatureModal = (value: boolean) => { - setIsFeatureModalOpen(value); - }; const handleSearchChange = (value: string) => { setSearchValue(value); @@ -94,163 +68,6 @@ const Appbar: React.FC = (): JSX.Element => { } }; - const supportLink = [ - { - label: ( - history.push(ROUTES.TOUR)}> - - - - - {t('label.tour')} - - - ), - key: 'tour', - }, - { - label: ( - - -
- - - - {t('label.doc-plural')} - - - - - - ), - key: 'docs', - }, - { - label: ( - - - - - - - - {t('label.api-uppercase')} - - - - - ), - key: 'api', - }, - { - label: ( - - - - - - - - {t('label.slack-support')} - - - - - - ), - key: 'slack', - }, - - { - label: ( - handleFeatureModal(true)}> - - - - - {t('label.whats-new')} - - - ), - key: 'whats-new', - }, - { - label: ( - - - - - - - {`${t('label.version')} ${ - (version ? version : '?').split('-')[0] - }`} - - - - - - ), - key: 'versions', - }, - ]; - const searchHandler = (value: string) => { if (!isTourOpen) { setIsOpen(false); @@ -293,21 +110,6 @@ const Appbar: React.FC = (): JSX.Element => { searchHandler(''); }; - const fetchOMVersion = () => { - getVersion() - .then((res) => { - setVersion(res.version); - }) - .catch((err: AxiosError) => { - showErrorToast( - err, - t('server.entity-fetch-error', { - entity: t('label.version'), - }) - ); - }); - }; - useEffect(() => { setSearchValue(searchQuery); }, [searchQuery]); @@ -317,10 +119,6 @@ const Appbar: React.FC = (): JSX.Element => { } }, [tourSearchValue, isTourOpen]); - useEffect(() => { - fetchOMVersion(); - }, []); - useEffect(() => { const handleDocumentVisibilityChange = () => { if ( @@ -329,7 +127,7 @@ const Appbar: React.FC = (): JSX.Element => { ) { return; } - const { isExpired, exp } = extractDetailsFromToken(); + const { isExpired, exp } = extractDetailsFromToken(getOidcToken()); if (!document.hidden && isExpired) { exp && toast.info(t('message.session-expired')); onLogoutHandler(); @@ -348,16 +146,13 @@ const Appbar: React.FC = (): JSX.Element => { {isProtectedRoute(location.pathname) && isAuthenticated ? ( ) : null} diff --git a/openmetadata-ui/src/main/resources/ui/src/components/AppBar/Suggestions.tsx b/openmetadata-ui/src/main/resources/ui/src/components/AppBar/Suggestions.tsx index 80cb1aedfaf1..f077b0bb930f 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/AppBar/Suggestions.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/AppBar/Suggestions.tsx @@ -79,7 +79,7 @@ const Suggestions = ({ const [containerSuggestions, setContainerSuggestions] = useState< ContainerSearchSource[] >([]); - const [glossarySuggestions, setGlossarySuggestions] = useState< + const [glossaryTermSuggestions, setGlossaryTermSuggestions] = useState< GlossarySource[] >([]); const [searchIndexSuggestions, setSearchIndexSuggestions] = useState< @@ -120,7 +120,9 @@ const Suggestions = ({ setDataModelSuggestions( filterOptionsByIndex(options, SearchIndex.DASHBOARD_DATA_MODEL) ); - setGlossarySuggestions(filterOptionsByIndex(options, SearchIndex.GLOSSARY)); + setGlossaryTermSuggestions( + filterOptionsByIndex(options, SearchIndex.GLOSSARY_TERM) + ); setTagSuggestions(filterOptionsByIndex(options, SearchIndex.TAG)); setDataProductSuggestions( filterOptionsByIndex(options, SearchIndex.DATA_PRODUCT) @@ -179,8 +181,8 @@ const Suggestions = ({ searchIndex: SearchIndex.DASHBOARD_DATA_MODEL, }, { - suggestions: glossarySuggestions, - searchIndex: SearchIndex.GLOSSARY, + suggestions: glossaryTermSuggestions, + searchIndex: SearchIndex.GLOSSARY_TERM, }, { suggestions: tagSuggestions, searchIndex: SearchIndex.TAG }, { diff --git a/openmetadata-ui/src/main/resources/ui/src/components/AppContainer/AppContainer.test.tsx b/openmetadata-ui/src/main/resources/ui/src/components/AppContainer/AppContainer.test.tsx index 4babcd0cd1a2..d557890118f8 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/AppContainer/AppContainer.test.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/AppContainer/AppContainer.test.tsx @@ -12,34 +12,17 @@ */ import { render, screen } from '@testing-library/react'; import React from 'react'; -import { MemoryRouter, Switch } from 'react-router-dom'; -import { ROUTES } from '../../constants/constants'; +import { MemoryRouter } from 'react-router-dom'; import AppContainer from './AppContainer'; -jest.mock('../Auth/AuthProviders/AuthProvider', () => { +jest.mock('../../hooks/useApplicationStore', () => { return { - useAuthContext: jest.fn(() => ({ - isAuthDisabled: false, - isAuthenticated: true, - signingIn: false, - isProtectedRoute: jest.fn().mockReturnValue(true), - isTourRoute: jest.fn().mockReturnValue(false), - onLogoutHandler: jest.fn(), + useApplicationStore: jest.fn(() => ({ + currentUser: { id: '1', email: 'user@gamil.com' }, })), }; }); -jest.mock('../../hooks/authHooks', () => ({ - useAuth: () => { - return { - isSignedIn: true, - isSignedOut: false, - isAuthenticatedRoute: true, - isAuthDisabled: true, - }; - }, -})); - jest.mock('../../components/MyData/LeftSidebar/LeftSidebar.component', () => jest.fn().mockReturnValue(

Sidebar

) ); @@ -48,28 +31,11 @@ jest.mock('../../components/AppBar/Appbar', () => jest.fn().mockReturnValue(

Appbar

) ); -jest.mock('../../pages/SignUp/SignUpPage', () => - jest.fn().mockReturnValue(

SignUpPage

) -); - jest.mock('../../components/AppRouter/AuthenticatedAppRouter', () => jest.fn().mockReturnValue(

AuthenticatedAppRouter

) ); describe('AppContainer', () => { - it('renders the SignupPage when on the signup route', () => { - render( - - - - - - ); - - expect(screen.getByText('SignUpPage')).toBeInTheDocument(); - expect(screen.queryByText('Sidebar')).not.toBeInTheDocument(); - }); - it('renders the Appbar, LeftSidebar, and AuthenticatedAppRouter components', () => { render( diff --git a/openmetadata-ui/src/main/resources/ui/src/components/AppContainer/AppContainer.tsx b/openmetadata-ui/src/main/resources/ui/src/components/AppContainer/AppContainer.tsx index 317f33f63818..56b7a63c5888 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/AppContainer/AppContainer.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/AppContainer/AppContainer.tsx @@ -17,18 +17,19 @@ import React, { useMemo } from 'react'; import { useTranslation } from 'react-i18next'; import { Redirect, Route, Switch } from 'react-router-dom'; import { ROUTES } from '../../constants/constants'; +import { useApplicationStore } from '../../hooks/useApplicationStore'; import SignUpPage from '../../pages/SignUp/SignUpPage'; import Appbar from '../AppBar/Appbar'; import AuthenticatedAppRouter from '../AppRouter/AuthenticatedAppRouter'; -import { useAuthContext } from '../Auth/AuthProviders/AuthProvider'; import LeftSidebar from '../MyData/LeftSidebar/LeftSidebar.component'; +import applicationsClassBase from '../Settings/Applications/AppDetails/ApplicationsClassBase'; import './app-container.less'; const AppContainer = () => { const { i18n } = useTranslation(); const { Header, Sider, Content } = Layout; - const { currentUser } = useAuthContext(); - + const { currentUser } = useApplicationStore(); + const ApplicationExtras = applicationsClassBase.getApplicationExtension(); const isDirectionRTL = useMemo(() => i18n.dir() === 'rtl', [i18n]); return ( @@ -52,6 +53,7 @@ const AppContainer = () => { + {ApplicationExtras && } diff --git a/openmetadata-ui/src/main/resources/ui/src/components/AppRouter/AppRouter.tsx b/openmetadata-ui/src/main/resources/ui/src/components/AppRouter/AppRouter.tsx index 7e0a6ee3dcf7..c1c20cfbfc7b 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/AppRouter/AppRouter.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/AppRouter/AppRouter.tsx @@ -13,67 +13,27 @@ import { isNil } from 'lodash'; import React, { useCallback, useEffect } from 'react'; -import { Redirect, Route, Switch, useLocation } from 'react-router-dom'; +import { Route, Switch, useLocation } from 'react-router-dom'; import { useAnalytics } from 'use-analytics'; import AppContainer from '../../components/AppContainer/AppContainer'; import { ROUTES } from '../../constants/constants'; import { CustomEventTypes } from '../../generated/analytics/webAnalyticEventData'; -import { AuthProvider } from '../../generated/settings/settings'; -import SamlCallback from '../../pages/SamlCallback'; -import AccountActivationConfirmation from '../../pages/SignUp/account-activation-confirmation.component'; -import { isProtectedRoute } from '../../utils/AuthProvider.util'; -import { useAuthContext } from '../Auth/AuthProviders/AuthProvider'; +import { useApplicationStore } from '../../hooks/useApplicationStore'; import Loader from '../common/Loader/Loader'; +import { UnAuthenticatedAppRouter } from './UnAuthenticatedAppRouter'; import withSuspenseFallback from './withSuspenseFallback'; -const SigninPage = withSuspenseFallback( - React.lazy(() => import('../../pages/LoginPage/SignInPage')) -); const PageNotFound = withSuspenseFallback( React.lazy(() => import('../../pages/PageNotFound/PageNotFound')) ); -const ForgotPassword = withSuspenseFallback( - React.lazy( - () => import('../../pages/ForgotPassword/ForgotPassword.component') - ) -); - -const ResetPassword = withSuspenseFallback( - React.lazy(() => import('../../pages/ResetPassword/ResetPassword.component')) -); - -const BasicSignupPage = withSuspenseFallback( - React.lazy(() => import('../../pages/SignUp/BasicSignup.component')) -); - const AppRouter = () => { const location = useLocation(); // web analytics instance const analytics = useAnalytics(); - const { - authConfig, - isAuthenticated, - loading, - isSigningIn, - getCallBackComponent, - } = useAuthContext(); - - const callbackComponent = getCallBackComponent(); - const oidcProviders = [ - AuthProvider.Google, - AuthProvider.AwsCognito, - AuthProvider.CustomOidc, - ]; - const isOidcProvider = - authConfig?.provider && oidcProviders.includes(authConfig.provider); - - const isBasicAuthProvider = - authConfig && - (authConfig.provider === AuthProvider.Basic || - authConfig.provider === AuthProvider.LDAP); + const { isAuthenticated, loading } = useApplicationStore(); useEffect(() => { const { pathname } = location; @@ -115,56 +75,11 @@ const AppRouter = () => { return ; } - if (!isAuthenticated && isProtectedRoute(location.pathname)) { - return ; - } - - if (isOidcProvider || isAuthenticated) { - return ; - } - return ( - <> - - - {callbackComponent ? ( - - ) : null} - - - {!isAuthenticated && !isSigningIn ? ( - <> - - - ) : ( - - )} - - - {isBasicAuthProvider && ( - <> - - - - - - )} - {isAuthenticated && } - - - + + {isAuthenticated ? : } + + ); }; diff --git a/openmetadata-ui/src/main/resources/ui/src/components/AppRouter/AuthenticatedAppRouter.tsx b/openmetadata-ui/src/main/resources/ui/src/components/AppRouter/AuthenticatedAppRouter.tsx index b901da422247..03fcc9305b6b 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/AppRouter/AuthenticatedAppRouter.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/AppRouter/AuthenticatedAppRouter.tsx @@ -28,6 +28,7 @@ import AddCustomMetricPage from '../../pages/AddCustomMetricPage/AddCustomMetric import { CustomizablePage } from '../../pages/CustomizablePage/CustomizablePage'; import { CustomPageSettings } from '../../pages/CustomPageSettings/CustomPageSettings'; import DataQualityPage from '../../pages/DataQuality/DataQualityPage'; +import OmHealthPage from '../../pages/OmHealth/OmHealthPage'; import { PersonaDetailsPage } from '../../pages/Persona/PersonaDetailsPage/PersonaDetailsPage'; import { PersonaPage } from '../../pages/Persona/PersonaListPage/PersonaPage'; import applicationRoutesClass from '../../utils/ApplicationRoutesClassBase'; @@ -1269,7 +1270,7 @@ const AuthenticatedAppRouter: FunctionComponent = () => { component={EmailConfigSettingsPage} hasPermission={false} path={getSettingPath( - GlobalSettingsMenuCategory.OPEN_METADATA, + GlobalSettingsMenuCategory.PREFERENCES, GlobalSettingOptions.EMAIL )} /> @@ -1278,7 +1279,7 @@ const AuthenticatedAppRouter: FunctionComponent = () => { component={CustomLogoConfigSettingsPage} hasPermission={false} path={getSettingPath( - GlobalSettingsMenuCategory.OPEN_METADATA, + GlobalSettingsMenuCategory.PREFERENCES, GlobalSettingOptions.CUSTOM_LOGO )} /> @@ -1287,7 +1288,7 @@ const AuthenticatedAppRouter: FunctionComponent = () => { component={LoginConfigurationPage} hasPermission={false} path={getSettingPath( - GlobalSettingsMenuCategory.OPEN_METADATA, + GlobalSettingsMenuCategory.PREFERENCES, GlobalSettingOptions.LOGIN_CONFIGURATION )} /> @@ -1295,11 +1296,20 @@ const AuthenticatedAppRouter: FunctionComponent = () => { exact component={CustomPageSettings} path={getSettingPath( - GlobalSettingsMenuCategory.OPEN_METADATA, + GlobalSettingsMenuCategory.PREFERENCES, GlobalSettingOptions.CUSTOMIZE_LANDING_PAGE )} /> + + { GlobalSettingsMenuCategory.CUSTOM_PROPERTIES )} /> + {RouteElements && } import('../../pages/LoginPage/SignInPage')) +); + +const ForgotPassword = withSuspenseFallback( + React.lazy( + () => import('../../pages/ForgotPassword/ForgotPassword.component') + ) +); + +const ResetPassword = withSuspenseFallback( + React.lazy(() => import('../../pages/ResetPassword/ResetPassword.component')) +); + +const BasicSignupPage = withSuspenseFallback( + React.lazy(() => import('../../pages/SignUp/BasicSignup.component')) +); + +export const UnAuthenticatedAppRouter = () => { + const { authConfig, isSigningIn } = useApplicationStore(); + + const isBasicAuthProvider = + authConfig && + (authConfig.provider === AuthProvider.Basic || + authConfig.provider === AuthProvider.LDAP); + + const callbackComponent = useMemo(() => { + switch (authConfig?.provider) { + case AuthProvider.Okta: { + return LoginCallback; + } + case AuthProvider.Auth0: { + return Auth0Callback; + } + default: { + return null; + } + } + }, [authConfig?.provider]); + + if (isProtectedRoute(location.pathname)) { + return ; + } + + return ( + + + + {callbackComponent && ( + + )} + + {!isSigningIn && ( + + + + )} + + {isBasicAuthProvider && ( + <> + + + + + + )} + + ); +}; diff --git a/openmetadata-ui/src/main/resources/ui/src/components/AppRouter/withSuggestions.tsx b/openmetadata-ui/src/main/resources/ui/src/components/AppRouter/withSuggestions.tsx new file mode 100644 index 000000000000..e0bb19062ab0 --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/components/AppRouter/withSuggestions.tsx @@ -0,0 +1,24 @@ +/* + * Copyright 2024 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import React, { FC } from 'react'; +import SuggestionsProvider from '../Suggestions/SuggestionsProvider/SuggestionsProvider'; + +export const withSuggestions = + (Component: FC) => + (props: JSX.IntrinsicAttributes & { children?: React.ReactNode }) => { + return ( + + + + ); + }; diff --git a/openmetadata-ui/src/main/resources/ui/src/components/AppRouter/withSuspenseFallback.js b/openmetadata-ui/src/main/resources/ui/src/components/AppRouter/withSuspenseFallback.tsx similarity index 77% rename from openmetadata-ui/src/main/resources/ui/src/components/AppRouter/withSuspenseFallback.js rename to openmetadata-ui/src/main/resources/ui/src/components/AppRouter/withSuspenseFallback.tsx index 5c69f2be4081..4fdd5d3b1e54 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/AppRouter/withSuspenseFallback.js +++ b/openmetadata-ui/src/main/resources/ui/src/components/AppRouter/withSuspenseFallback.tsx @@ -11,11 +11,15 @@ * limitations under the License. */ -import React, { Suspense } from 'react'; +import React, { FC, Suspense } from 'react'; import Loader from '../common/Loader/Loader'; -export default function withSuspenseFallback(Component) { - return function DefaultFallback(props) { +export default function withSuspenseFallback( + Component: FC +) { + return function DefaultFallback( + props: JSX.IntrinsicAttributes & { children?: React.ReactNode } & T + ) { return ( { +jest.mock('../../../hooks/useApplicationStore', () => { return { - useAuthContext: jest.fn(() => ({ + useApplicationStore: jest.fn(() => ({ authConfig: {}, setIsAuthenticated: jest.fn(), onLogoutHandler: jest.fn(), diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Auth/AppAuthenticators/Auth0Authenticator.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Auth/AppAuthenticators/Auth0Authenticator.tsx index a953a0a4e1f1..2666e3cd9e12 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Auth/AppAuthenticators/Auth0Authenticator.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Auth/AppAuthenticators/Auth0Authenticator.tsx @@ -20,8 +20,8 @@ import React, { } from 'react'; import { useTranslation } from 'react-i18next'; import { AuthProvider } from '../../../generated/settings/settings'; -import localState from '../../../utils/LocalStorageUtils'; -import { useAuthContext } from '../AuthProviders/AuthProvider'; + +import { useApplicationStore } from '../../../hooks/useApplicationStore'; import { AuthenticatorRef } from '../AuthProviders/AuthProvider.interface'; interface Props { @@ -31,7 +31,8 @@ interface Props { const Auth0Authenticator = forwardRef( ({ children, onLogoutSuccess }: Props, ref) => { - const { setIsAuthenticated, authConfig } = useAuthContext(); + const { setIsAuthenticated, authConfig, setOidcToken } = + useApplicationStore(); const { t } = useTranslation(); const { loginWithRedirect, getAccessTokenSilently, getIdTokenClaims } = useAuth0(); @@ -59,7 +60,7 @@ const Auth0Authenticator = forwardRef( .then((token) => { if (token !== undefined) { idToken = token.__raw; - localState.setOidcToken(idToken); + setOidcToken(idToken); resolve(idToken); } }) diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Auth/AppAuthenticators/BasicAuthAuthenticator.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Auth/AppAuthenticators/BasicAuthAuthenticator.tsx index bab42adf34e6..09cf9ca37400 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Auth/AppAuthenticators/BasicAuthAuthenticator.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Auth/AppAuthenticators/BasicAuthAuthenticator.tsx @@ -23,8 +23,8 @@ import { AccessTokenResponse, getAccessTokenOnExpiry, } from '../../../rest/auth-API'; -import localState from '../../../utils/LocalStorageUtils'; -import { useAuthContext } from '../AuthProviders/AuthProvider'; + +import { useApplicationStore } from '../../../hooks/useApplicationStore'; import { useBasicAuth } from '../AuthProviders/BasicAuthProvider'; interface BasicAuthenticatorInterface { @@ -35,10 +35,16 @@ const BasicAuthenticator = forwardRef( ({ children }: BasicAuthenticatorInterface, ref) => { const { handleLogout } = useBasicAuth(); const { t } = useTranslation(); - const { setIsAuthenticated, authConfig } = useAuthContext(); + const { + setIsAuthenticated, + authConfig, + getRefreshToken, + setRefreshToken, + setOidcToken, + } = useApplicationStore(); const handleSilentSignIn = async (): Promise => { - const refreshToken = localState.getRefreshToken(); + const refreshToken = getRefreshToken(); if ( authConfig?.provider !== AuthProvider.Basic && @@ -51,8 +57,8 @@ const BasicAuthenticator = forwardRef( refreshToken: refreshToken as string, }); - localState.setRefreshToken(response.refreshToken); - localState.setOidcToken(response.accessToken); + setRefreshToken(response.refreshToken); + setOidcToken(response.accessToken); return Promise.resolve(response); }; diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Auth/AppAuthenticators/GenericAuthenticator.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Auth/AppAuthenticators/GenericAuthenticator.tsx new file mode 100644 index 000000000000..10a77b5ade88 --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/components/Auth/AppAuthenticators/GenericAuthenticator.tsx @@ -0,0 +1,63 @@ +/* + * Copyright 2024 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import React, { + forwardRef, + Fragment, + ReactNode, + useImperativeHandle, +} from 'react'; +import { useHistory } from 'react-router-dom'; +import { ROUTES } from '../../../constants/constants'; +import { useApplicationStore } from '../../../hooks/useApplicationStore'; +import { logoutUser, renewToken } from '../../../rest/LoginAPI'; + +export const GenericAuthenticator = forwardRef( + ({ children }: { children: ReactNode }, ref) => { + const { + setIsAuthenticated, + setIsSigningIn, + removeOidcToken, + setOidcToken, + } = useApplicationStore(); + const history = useHistory(); + + const handleLogin = () => { + setIsAuthenticated(false); + setIsSigningIn(true); + window.location.assign('api/v1/auth/login'); + }; + + const handleLogout = async () => { + await logoutUser(); + + history.push(ROUTES.SIGNIN); + removeOidcToken(); + setIsAuthenticated(false); + }; + + const handleSilentSignIn = async () => { + const resp = await renewToken(); + setOidcToken(resp.accessToken); + + return Promise.resolve(resp); + }; + + useImperativeHandle(ref, () => ({ + invokeLogout: handleLogout, + renewIdToken: handleSilentSignIn, + invokeLogin: handleLogin, + })); + + return {children}; + } +); diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Auth/AppAuthenticators/MsalAuthenticator.test.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Auth/AppAuthenticators/MsalAuthenticator.test.tsx index a551e866c238..ac2036d510e3 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Auth/AppAuthenticators/MsalAuthenticator.test.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Auth/AppAuthenticators/MsalAuthenticator.test.tsx @@ -16,12 +16,12 @@ import React from 'react'; import { MemoryRouter } from 'react-router-dom'; import MsalAuthenticator from './MsalAuthenticator'; -jest.mock('../AuthProviders/AuthProvider', () => { +jest.mock('../../../hooks/useApplicationStore', () => { return { - useAuthContext: jest.fn(() => ({ + useApplicationStore: jest.fn(() => ({ authConfig: {}, - setIsAuthenticated: jest.fn(), - onLogoutHandler: jest.fn(), + getOidcToken: jest.fn(), + setOidcToken: jest.fn(), })), }; }); diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Auth/AppAuthenticators/MsalAuthenticator.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Auth/AppAuthenticators/MsalAuthenticator.tsx index f3802e30478b..6ec4eba160b0 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Auth/AppAuthenticators/MsalAuthenticator.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Auth/AppAuthenticators/MsalAuthenticator.tsx @@ -26,8 +26,8 @@ import React, { useImperativeHandle, } from 'react'; import { useMutex } from 'react-context-mutex'; +import { useApplicationStore } from '../../../hooks/useApplicationStore'; import { msalLoginRequest } from '../../../utils/AuthProvider.util'; -import localState from '../../../utils/LocalStorageUtils'; import { AuthenticatorRef, OidcUser, @@ -45,6 +45,7 @@ const MsalAuthenticator = forwardRef( { children, onLoginSuccess, onLogoutSuccess, onLoginFailure }: Props, ref ) => { + const { setOidcToken, getOidcToken } = useApplicationStore(); const { instance, accounts, inProgress } = useMsal(); const account = useAccount(accounts[0] || {}); const MutexRunner = useMutex(); @@ -87,7 +88,7 @@ const MsalAuthenticator = forwardRef( }, }; - localState.setOidcToken(idToken); + setOidcToken(idToken); return user; }; @@ -128,7 +129,7 @@ const MsalAuthenticator = forwardRef( }; useEffect(() => { - const oidcUserToken = localState.getOidcToken(); + const oidcUserToken = getOidcToken(); if ( !oidcUserToken && inProgress === InteractionStatus.None && diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Auth/AppAuthenticators/OidcAuthenticator.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Auth/AppAuthenticators/OidcAuthenticator.tsx index 574cd18097e6..5f312a8ff4f5 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Auth/AppAuthenticators/OidcAuthenticator.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Auth/AppAuthenticators/OidcAuthenticator.tsx @@ -24,12 +24,11 @@ import React, { import { Callback, makeAuthenticator, makeUserManager } from 'react-oidc'; import { Redirect, Route, Switch, useHistory } from 'react-router-dom'; import { ROUTES } from '../../../constants/constants'; +import { useApplicationStore } from '../../../hooks/useApplicationStore'; import SignInPage from '../../../pages/LoginPage/SignInPage'; import PageNotFound from '../../../pages/PageNotFound/PageNotFound'; -import localState from '../../../utils/LocalStorageUtils'; import { showErrorToast } from '../../../utils/ToastUtils'; import Loader from '../../common/Loader/Loader'; -import { useAuthContext } from '../AuthProviders/AuthProvider'; import { AuthenticatorRef, OidcUser, @@ -73,9 +72,11 @@ const OidcAuthenticator = forwardRef( setIsSigningIn, setLoadingIndicator, updateAxiosInterceptors, - } = useAuthContext(); + currentUser, + newUser, + setOidcToken, + } = useApplicationStore(); const history = useHistory(); - const { currentUser, newUser } = useAuthContext(); const userManager = useMemo( () => makeUserManager(userConfig), [userConfig] @@ -94,7 +95,7 @@ const OidcAuthenticator = forwardRef( // Performs silent signIn and returns with IDToken const signInSilently = async () => { const user = await userManager.signinSilent(); - localState.setOidcToken(user.id_token); + setOidcToken(user.id_token); return user.id_token; }; @@ -138,7 +139,7 @@ const OidcAuthenticator = forwardRef( onLoginFailure(); }} onSuccess={(user) => { - localState.setOidcToken(user.id_token); + setOidcToken(user.id_token); setIsAuthenticated(true); onLoginSuccess(user as OidcUser); }} @@ -146,6 +147,7 @@ const OidcAuthenticator = forwardRef( )} /> + ( @@ -159,7 +161,7 @@ const OidcAuthenticator = forwardRef( history.push(ROUTES.SIGNIN); }} onSuccess={(user) => { - localState.setOidcToken(user.id_token); + setOidcToken(user.id_token); updateAxiosInterceptors(); }} /> diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Auth/AppAuthenticators/OktaAuthenticator.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Auth/AppAuthenticators/OktaAuthenticator.tsx index d82cfab153ad..dce1fd375779 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Auth/AppAuthenticators/OktaAuthenticator.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Auth/AppAuthenticators/OktaAuthenticator.tsx @@ -18,8 +18,8 @@ import React, { ReactNode, useImperativeHandle, } from 'react'; -import localState from '../../../utils/LocalStorageUtils'; -import { useAuthContext } from '../AuthProviders/AuthProvider'; + +import { useApplicationStore } from '../../../hooks/useApplicationStore'; import { AuthenticatorRef } from '../AuthProviders/AuthProvider.interface'; interface Props { @@ -30,7 +30,7 @@ interface Props { const OktaAuthenticator = forwardRef( ({ children, onLogoutSuccess }: Props, ref) => { const { oktaAuth } = useOktaAuth(); - const { setIsAuthenticated } = useAuthContext(); + const { setIsAuthenticated, setOidcToken } = useApplicationStore(); const login = async () => { oktaAuth.signInWithRedirect(); @@ -54,7 +54,7 @@ const OktaAuthenticator = forwardRef( oktaAuth.tokenManager.setTokens(renewToken); const newToken = renewToken?.idToken?.idToken ?? oktaAuth.getIdToken() ?? ''; - localState.setOidcToken(newToken); + setOidcToken(newToken); return Promise.resolve(newToken); }, diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Auth/AppAuthenticators/SamlAuthenticator.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Auth/AppAuthenticators/SamlAuthenticator.tsx index 009989dfeabc..c53e87358066 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Auth/AppAuthenticators/SamlAuthenticator.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Auth/AppAuthenticators/SamlAuthenticator.tsx @@ -29,11 +29,11 @@ import React, { ReactNode, useImperativeHandle, } from 'react'; -import { oidcTokenKey } from '../../../constants/constants'; import { SamlSSOClientConfig } from '../../../generated/configuration/authenticationConfiguration'; import { postSamlLogout } from '../../../rest/miscAPI'; import { showErrorToast } from '../../../utils/ToastUtils'; -import { useAuthContext } from '../AuthProviders/AuthProvider'; + +import { useApplicationStore } from '../../../hooks/useApplicationStore'; import { AuthenticatorRef } from '../AuthProviders/AuthProvider.interface'; interface Props { @@ -43,7 +43,8 @@ interface Props { const SamlAuthenticator = forwardRef( ({ children, onLogoutSuccess }: Props, ref) => { - const { setIsAuthenticated, authConfig } = useAuthContext(); + const { setIsAuthenticated, authConfig, getOidcToken } = + useApplicationStore(); const config = authConfig?.samlConfiguration as SamlSSOClientConfig; const login = async () => { @@ -55,7 +56,7 @@ const SamlAuthenticator = forwardRef( }; const logout = () => { - const token = localStorage.getItem(oidcTokenKey); + const token = getOidcToken(); if (token) { postSamlLogout({ token }) .then(() => { diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Auth/AppCallbacks/Auth0Callback/Auth0Callback.test.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Auth/AppCallbacks/Auth0Callback/Auth0Callback.test.tsx index af9749e9ca1f..f78735f194c4 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Auth/AppCallbacks/Auth0Callback/Auth0Callback.test.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Auth/AppCallbacks/Auth0Callback/Auth0Callback.test.tsx @@ -16,7 +16,6 @@ import { render, screen } from '@testing-library/react'; import { t } from 'i18next'; import React from 'react'; import { MemoryRouter } from 'react-router-dom'; -import { oidcTokenKey } from '../../../../constants/constants'; import Auth0Callback from './Auth0Callback'; const localStorageMock = (() => { @@ -50,12 +49,13 @@ jest.mock('@auth0/auth0-react', () => ({ useAuth0: jest.fn(), })); -jest.mock('../../../Auth/AuthProviders/AuthProvider', () => { +jest.mock('../../../../hooks/useApplicationStore', () => { return { - useAuthContext: jest.fn(() => ({ + useApplicationStore: jest.fn(() => ({ authConfig: {}, setIsAuthenticated: mockSetIsAuthenticated, handleSuccessfulLogin: mockHandleSuccessfulLogin, + setOidcToken: jest.fn(), })), }; }); @@ -108,7 +108,6 @@ describe('Test Auth0Callback component', () => { // eslint-disable-next-line no-undef await new Promise(process.nextTick); - expect(localStorageMock.getItem(oidcTokenKey)).toBe('raw_id_token'); expect(mockSetIsAuthenticated).toHaveBeenCalledTimes(1); expect(mockSetIsAuthenticated).toHaveBeenCalledWith(true); expect(mockHandleSuccessfulLogin).toHaveBeenCalledTimes(1); diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Auth/AppCallbacks/Auth0Callback/Auth0Callback.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Auth/AppCallbacks/Auth0Callback/Auth0Callback.tsx index d085ef3ffef0..a7560ca12bd3 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Auth/AppCallbacks/Auth0Callback/Auth0Callback.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Auth/AppCallbacks/Auth0Callback/Auth0Callback.tsx @@ -14,17 +14,18 @@ import { useAuth0 } from '@auth0/auth0-react'; import { t } from 'i18next'; import React, { VFC } from 'react'; -import localState from '../../../../utils/LocalStorageUtils'; -import { useAuthContext } from '../../AuthProviders/AuthProvider'; + +import { useApplicationStore } from '../../../../hooks/useApplicationStore'; import { OidcUser } from '../../AuthProviders/AuthProvider.interface'; const Auth0Callback: VFC = () => { const { isAuthenticated, user, getIdTokenClaims, error } = useAuth0(); - const { setIsAuthenticated, handleSuccessfulLogin } = useAuthContext(); + const { setIsAuthenticated, handleSuccessfulLogin, setOidcToken } = + useApplicationStore(); if (isAuthenticated) { getIdTokenClaims() .then((token) => { - localState.setOidcToken(token?.__raw || ''); + setOidcToken(token?.__raw || ''); setIsAuthenticated(true); const oidcUser: OidcUser = { id_token: token?.__raw || '', diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Auth/AuthProviders/AuthProvider.interface.ts b/openmetadata-ui/src/main/resources/ui/src/components/Auth/AuthProviders/AuthProvider.interface.ts index 80e1dbe81fe3..a967f22f68e9 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Auth/AuthProviders/AuthProvider.interface.ts +++ b/openmetadata-ui/src/main/resources/ui/src/components/Auth/AuthProviders/AuthProvider.interface.ts @@ -12,7 +12,7 @@ */ import { Profile } from 'oidc-client'; -import { ComponentType, FC, ReactNode } from 'react'; +import { ComponentType, ReactNode } from 'react'; import { AuthenticationConfiguration } from '../../../generated/configuration/authenticationConfiguration'; import { AuthorizerConfiguration } from '../../../generated/configuration/authorizerConfiguration'; import { User } from '../../../generated/entity/teams/user'; @@ -56,13 +56,13 @@ export interface IAuthContext { setIsSigningIn: (authenticated: boolean) => void; onLoginHandler: () => void; onLogoutHandler: () => void; - getCallBackComponent: () => FC | null; loading: boolean; currentUser?: User; newUser?: UserProfile; updateNewUser: (user: UserProfile) => void; setLoadingIndicator: (authenticated: boolean) => void; handleSuccessfulLogin: (user: OidcUser) => void; + handleFailedLogin: () => void; updateAxiosInterceptors: () => void; updateCurrentUser: (user: User) => void; jwtPrincipalClaims: string[]; diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Auth/AuthProviders/AuthProvider.test.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Auth/AuthProviders/AuthProvider.test.tsx index c4c961aabe5a..24c882447c80 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Auth/AuthProviders/AuthProvider.test.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Auth/AuthProviders/AuthProvider.test.tsx @@ -18,9 +18,9 @@ import { } from '@testing-library/react'; import userEvent from '@testing-library/user-event'; import React from 'react'; -import { refreshTokenKey } from '../../../constants/constants'; import { AuthProvider as AuthProviderProps } from '../../../generated/configuration/authenticationConfiguration'; -import AuthProvider, { useAuthContext } from './AuthProvider'; +import { useApplicationStore } from '../../../hooks/useApplicationStore'; +import AuthProvider from './AuthProvider'; const localStorageMock = { getItem: jest.fn(), @@ -33,6 +33,8 @@ Object.defineProperty(window, 'localStorage', { value: localStorageMock, }); +const mockOnLogoutHandler = jest.fn(); + jest.mock('react-router-dom', () => ({ useHistory: jest.fn().mockReturnValue({ push: jest.fn(), listen: jest.fn() }), useLocation: jest.fn().mockReturnValue({ pathname: 'pathname' }), @@ -55,7 +57,7 @@ jest.mock('../../../rest/userAPI', () => ({ describe('Test auth provider', () => { it('Logout handler should call the "updateUserDetails" method', async () => { const ConsumerComponent = () => { - const { onLogoutHandler } = useAuthContext(); + const { onLogoutHandler } = useApplicationStore(); return ( ); @@ -94,8 +94,6 @@ describe('Test auth provider', () => { ); - await waitForElementToBeRemoved(() => screen.getByTestId('loader')); - const logoutButton = screen.getByTestId('logout-button'); expect(logoutButton).toBeInTheDocument(); @@ -104,6 +102,6 @@ describe('Test auth provider', () => { userEvent.click(logoutButton); }); - expect(localStorageMock.removeItem).toHaveBeenCalledWith(refreshTokenKey); + expect(mockOnLogoutHandler).toHaveBeenCalled(); }); }); diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Auth/AuthProviders/AuthProvider.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Auth/AuthProviders/AuthProvider.tsx index d4f9527a5a62..bb4e741ea2f7 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Auth/AuthProviders/AuthProvider.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Auth/AuthProviders/AuthProvider.tsx @@ -15,7 +15,6 @@ import { removeSession } from '@analytics/session-utils'; import { Auth0Provider } from '@auth0/auth0-react'; import { Configuration } from '@azure/msal-browser'; import { MsalProvider } from '@azure/msal-react'; -import { LoginCallback } from '@okta/okta-react'; import { AxiosError, AxiosRequestHeaders, @@ -27,10 +26,8 @@ import { isEmpty, isNil, isNumber } from 'lodash'; import Qs from 'qs'; import React, { ComponentType, - createContext, ReactNode, useCallback, - useContext, useEffect, useMemo, useRef, @@ -44,13 +41,15 @@ import { REDIRECT_PATHNAME, ROUTES, } from '../../../constants/constants'; -import { useApplicationConfigContext } from '../../../context/ApplicationConfigProvider/ApplicationConfigProvider'; import { ClientErrors } from '../../../enums/Axios.enum'; import { SearchIndex } from '../../../enums/search.enum'; -import { AuthenticationConfiguration } from '../../../generated/configuration/authenticationConfiguration'; -import { AuthorizerConfiguration } from '../../../generated/configuration/authorizerConfiguration'; +import { + AuthenticationConfiguration, + ClientType, +} from '../../../generated/configuration/authenticationConfiguration'; import { User } from '../../../generated/entity/teams/user'; import { AuthProvider as AuthProviderEnum } from '../../../generated/settings/settings'; +import { useApplicationStore } from '../../../hooks/useApplicationStore'; import axiosClient from '../../../rest'; import { fetchAuthenticationConfig, @@ -66,7 +65,6 @@ import { msalInstance, setMsalInstance, } from '../../../utils/AuthProvider.util'; -import localState from '../../../utils/LocalStorageUtils'; import { escapeESReservedCharacters } from '../../../utils/StringsUtils'; import { showErrorToast } from '../../../utils/ToastUtils'; import { @@ -77,18 +75,12 @@ import { resetWebAnalyticSession } from '../../../utils/WebAnalyticsUtils'; import Loader from '../../common/Loader/Loader'; import Auth0Authenticator from '../AppAuthenticators/Auth0Authenticator'; import BasicAuthAuthenticator from '../AppAuthenticators/BasicAuthAuthenticator'; +import { GenericAuthenticator } from '../AppAuthenticators/GenericAuthenticator'; import MsalAuthenticator from '../AppAuthenticators/MsalAuthenticator'; import OidcAuthenticator from '../AppAuthenticators/OidcAuthenticator'; import OktaAuthenticator from '../AppAuthenticators/OktaAuthenticator'; import SamlAuthenticator from '../AppAuthenticators/SamlAuthenticator'; -import Auth0Callback from '../AppCallbacks/Auth0Callback/Auth0Callback'; -import { - AuthenticationConfigurationWithScope, - AuthenticatorRef, - IAuthContext, - OidcUser, - UserProfile, -} from './AuthProvider.interface'; +import { AuthenticatorRef, OidcUser } from './AuthProvider.interface'; import BasicAuthProvider from './BasicAuthProvider'; import OktaAuthProvider from './OktaAuthProvider'; @@ -97,8 +89,6 @@ interface AuthProviderProps { children: ReactNode; } -export const AuthContext = createContext({} as IAuthContext); - const cookieStorage = new CookieStorage(); const userAPIQueryFields = 'profile,teams,roles,personas,defaultPersona'; @@ -112,31 +102,30 @@ export const AuthProvider = ({ childComponentType, children, }: AuthProviderProps) => { + const { + setHelperFunctionsRef, + setCurrentUser, + updateNewUser: setNewUserProfile, + setIsAuthenticated: setIsUserAuthenticated, + setLoadingIndicator: setLoading, + authConfig, + setAuthConfig, + setAuthorizerConfig, + setIsSigningIn, + setJwtPrincipalClaims, + removeRefreshToken, + removeOidcToken, + getOidcToken, + getRefreshToken, + urlPathName, + setUrlPathName, + } = useApplicationStore(); + const location = useLocation(); const history = useHistory(); const { t } = useTranslation(); const [timeoutId, setTimeoutId] = useState(); const authenticatorRef = useRef(null); - const [currentUser, setCurrentUser] = useState(); - const { urlPathName } = useApplicationConfigContext(); - - const oidcUserToken = localState.getOidcToken(); - const [newUserProfile, setNewUserProfile] = useState(); - const [isUserAuthenticated, setIsUserAuthenticated] = useState( - Boolean(oidcUserToken) - ); - - const [loading, setLoading] = useState(true); - const [authConfig, setAuthConfig] = - useState(); - - const [authorizerConfig, setAuthorizerConfig] = - useState(); - const [isSigningIn, setIsSigningIn] = useState(false); - - const [jwtPrincipalClaims, setJwtPrincipalClaims] = useState< - AuthenticationConfiguration['jwtPrincipalClaims'] - >([]); let silentSignInRetries = 0; @@ -145,8 +134,11 @@ export const AuthProvider = ({ [authConfig] ); + const clientType = authConfig?.clientType ?? ClientType.Public; + const onLoginHandler = () => { setLoading(true); + authenticatorRef.current?.invokeLogin(); resetWebAnalyticSession(); @@ -154,7 +146,9 @@ export const AuthProvider = ({ const onLogoutHandler = useCallback(() => { clearTimeout(timeoutId); + authenticatorRef.current?.invokeLogout(); + setIsUserAuthenticated(false); // reset the user details on logout setCurrentUser({} as User); @@ -163,7 +157,7 @@ export const AuthProvider = ({ removeSession(); // remove the refresh token on logout - localState.removeRefreshToken(); + removeRefreshToken(); setLoading(false); }, [timeoutId]); @@ -197,7 +191,7 @@ export const AuthProvider = ({ const resetUserDetails = (forceLogout = false) => { setCurrentUser({} as User); - localState.removeOidcToken(); + removeOidcToken(); setIsUserAuthenticated(false); setLoadingIndicator(false); clearTimeout(timeoutId); @@ -214,6 +208,7 @@ export const AuthProvider = ({ .then((res) => { if (res) { setCurrentUser(res); + setIsUserAuthenticated(true); } else { resetUserDetails(); } @@ -276,7 +271,7 @@ export const AuthProvider = ({ throw error; } - return localState.getOidcToken(); + return getOidcToken(); }; /** @@ -320,8 +315,11 @@ export const AuthProvider = ({ */ const startTokenExpiryTimer = () => { // Extract expiry - const { isExpired, timeoutExpiry } = extractDetailsFromToken(); - const refreshToken = localState.getRefreshToken(); + const { isExpired, timeoutExpiry } = extractDetailsFromToken( + getOidcToken(), + clientType + ); + const refreshToken = getRefreshToken(); // Basic & LDAP renewToken depends on RefreshToken hence adding a check here for the same const shouldStartExpiry = @@ -368,11 +366,12 @@ export const AuthProvider = ({ .then((res) => { if (res) { const updatedUserData = getUserDataFromOidc(res, user); - if (!matchUserDetails(res, updatedUserData, ['profile', 'email'])) { + if (!matchUserDetails(res, updatedUserData, ['email'])) { getUpdatedUser(updatedUserData, res); } else { setCurrentUser(res); } + handledVerifiedUser(); // Start expiry timer on successful login startTokenExpiryTimer(); @@ -419,8 +418,10 @@ export const AuthProvider = ({ const hasActiveDomain = activeDomain !== DEFAULT_DOMAIN_VALUE; const currentPath = window.location.pathname; - // Do not intercept requests from domains page - if (currentPath.includes('/domain')) { + // Do not intercept requests from domains page or /auth endpoints + if ( + ['/domain', '/auth/logout', '/auth/refresh'].indexOf(currentPath) > -1 + ) { return config; } @@ -473,7 +474,7 @@ export const AuthProvider = ({ // eslint-disable-next-line @typescript-eslint/no-explicit-any config: InternalAxiosRequestConfig ) { - const token: string = localState.getOidcToken() || ''; + const token: string = getOidcToken() || ''; if (token) { if (config.headers) { config.headers['Authorization'] = `Bearer ${token}`; @@ -523,13 +524,15 @@ export const AuthProvider = ({ setAuthConfig(configJson); setAuthorizerConfig(authorizerConfig); updateAuthInstance(configJson); - if (!oidcUserToken) { + if (!getOidcToken()) { if (isProtectedRoute(location.pathname)) { storeRedirectPath(location.pathname); } setLoading(false); } else { - getLoggedInUserDetails(); + if (location.pathname !== ROUTES.AUTH_CALLBACK) { + getLoggedInUserDetails(); + } } } else { // provider is either null or not supported @@ -555,21 +558,14 @@ export const AuthProvider = ({ } }; - const getCallBackComponent = () => { - switch (authConfig?.provider) { - case AuthProviderEnum.Okta: { - return LoginCallback; - } - case AuthProviderEnum.Auth0: { - return Auth0Callback; - } - default: { - return null; - } - } - }; - const getProtectedApp = () => { + if (clientType === ClientType.Confidential) { + return ( + + {children} + + ); + } switch (authConfig?.provider) { case AuthProviderEnum.LDAP: case AuthProviderEnum.Basic: { @@ -662,41 +658,28 @@ export const AuthProvider = ({ startTokenExpiryTimer(); initializeAxiosInterceptors(); + setHelperFunctionsRef({ + onLoginHandler, + onLogoutHandler, + handleSuccessfulLogin, + handleFailedLogin, + updateAxiosInterceptors: initializeAxiosInterceptors, + }); + return cleanup; }, []); + useEffect(() => { + if (isProtectedRoute(location.pathname)) { + setUrlPathName(location.pathname); + } + }, [location.pathname]); + const isLoading = !authConfig || (authConfig.provider === AuthProviderEnum.Azure && !msalInstance); - const authContext: IAuthContext = { - currentUser: currentUser, - isAuthenticated: isUserAuthenticated, - setIsAuthenticated: setIsUserAuthenticated, - newUser: newUserProfile, - updateNewUser: setNewUserProfile, - authConfig, - authorizerConfig, - isSigningIn, - setIsSigningIn, - onLoginHandler, - onLogoutHandler, - getCallBackComponent, - loading, - setLoadingIndicator, - handleSuccessfulLogin, - updateAxiosInterceptors: initializeAxiosInterceptors, - jwtPrincipalClaims, - updateCurrentUser: setCurrentUser, - }; - - return ( - - {isLoading ? : getProtectedApp()} - - ); + return <>{isLoading ? : getProtectedApp()}; }; -export const useAuthContext = () => useContext(AuthContext); - export default AuthProvider; diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Auth/AuthProviders/BasicAuthProvider.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Auth/AuthProviders/BasicAuthProvider.tsx index de92b773fce8..48a65f5c5da7 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Auth/AuthProviders/BasicAuthProvider.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Auth/AuthProviders/BasicAuthProvider.tsx @@ -32,14 +32,14 @@ import { resetPassword, } from '../../../rest/auth-API'; import { getBase64EncodedString } from '../../../utils/CommonUtils'; -import localState from '../../../utils/LocalStorageUtils'; import { showErrorToast, showInfoToast, showSuccessToast, } from '../../../utils/ToastUtils'; import { resetWebAnalyticSession } from '../../../utils/WebAnalyticsUtils'; -import { useAuthContext } from './AuthProvider'; + +import { useApplicationStore } from '../../../hooks/useApplicationStore'; import { OidcUser } from './AuthProvider.interface'; export interface BasicAuthJWTPayload extends JwtPayload { @@ -89,7 +89,14 @@ const BasicAuthProvider = ({ onLoginFailure, }: BasicAuthProps) => { const { t } = useTranslation(); - const { setLoadingIndicator } = useAuthContext(); + const { + setLoadingIndicator, + setRefreshToken, + setOidcToken, + getOidcToken, + removeOidcToken, + getRefreshToken, + } = useApplicationStore(); const [loginError, setLoginError] = useState(null); const history = useHistory(); @@ -103,8 +110,8 @@ const BasicAuthProvider = ({ }); if (response.accessToken) { - localState.setRefreshToken(response.refreshToken); - localState.setOidcToken(response.accessToken); + setRefreshToken(response.refreshToken); + setOidcToken(response.accessToken); onLoginSuccess({ id_token: response.accessToken, @@ -191,12 +198,12 @@ const BasicAuthProvider = ({ }; const handleLogout = async () => { - const token = localState.getOidcToken(); - const refreshToken = localState.getRefreshToken(); + const token = getOidcToken(); + const refreshToken = getRefreshToken(); if (token) { try { await logoutUser({ token, refreshToken }); - localState.removeOidcToken(); + removeOidcToken(); history.push(ROUTES.SIGNIN); } catch (error) { showErrorToast(error as AxiosError); diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Auth/AuthProviders/OktaAuthProvider.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Auth/AuthProviders/OktaAuthProvider.tsx index 57b5d00a73dc..af6f740d46b4 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Auth/AuthProviders/OktaAuthProvider.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Auth/AuthProviders/OktaAuthProvider.tsx @@ -19,8 +19,7 @@ import React, { useCallback, useMemo, } from 'react'; -import localState from '../../../utils/LocalStorageUtils'; -import { useAuthContext } from './AuthProvider'; +import { useApplicationStore } from '../../../hooks/useApplicationStore'; import { OidcUser } from './AuthProvider.interface'; interface Props { @@ -32,7 +31,8 @@ export const OktaAuthProvider: FunctionComponent = ({ children, onLoginSuccess, }: Props) => { - const { authConfig, setIsAuthenticated } = useAuthContext(); + const { authConfig, setIsAuthenticated, setOidcToken } = + useApplicationStore(); const { clientId, issuer, redirectUri, scopes, pkce } = authConfig as OktaAuthOptions; @@ -58,8 +58,8 @@ export const OktaAuthProvider: FunctionComponent = ({ const restoreOriginalUri = useCallback(async (_oktaAuth: OktaAuth) => { const idToken = _oktaAuth.getIdToken() ?? ''; const scopes = - _oktaAuth.authStateManager.getAuthState()?.idToken?.scopes.join() ?? ''; - localState.setOidcToken(idToken); + _oktaAuth.authStateManager.getAuthState()?.idToken?.scopes.join() || ''; + setOidcToken(idToken); _oktaAuth .getUser() .then((info) => { diff --git a/openmetadata-ui/src/main/resources/ui/src/components/BlockEditor/Extensions/index.ts b/openmetadata-ui/src/main/resources/ui/src/components/BlockEditor/Extensions/index.ts index c20b086768ad..da6f7d8adb34 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/BlockEditor/Extensions/index.ts +++ b/openmetadata-ui/src/main/resources/ui/src/components/BlockEditor/Extensions/index.ts @@ -124,6 +124,7 @@ export const extensions = [ class: 'om-table', 'data-om-table': 'om-table', }, + resizable: true, }), TableRow.configure({ HTMLAttributes: { diff --git a/openmetadata-ui/src/main/resources/ui/src/components/BlockEditor/TableMenu/TableMenu.tsx b/openmetadata-ui/src/main/resources/ui/src/components/BlockEditor/TableMenu/TableMenu.tsx index ed2fdb5e2841..acee3fd46763 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/BlockEditor/TableMenu/TableMenu.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/BlockEditor/TableMenu/TableMenu.tsx @@ -32,7 +32,7 @@ const TableMenu = (props: TableMenuProps) => { const handleMouseDown = useCallback((event: MouseEvent) => { const target = event.target as HTMLElement; - const table = target?.closest('[data-om-table]'); + const table = target?.closest('.tableWrapper'); if (table?.contains(target)) { tableMenuPopup.current?.setProps({ diff --git a/openmetadata-ui/src/main/resources/ui/src/components/BlockEditor/block-editor.less b/openmetadata-ui/src/main/resources/ui/src/components/BlockEditor/block-editor.less index ca6628b9b44f..1dd3acd64910 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/BlockEditor/block-editor.less +++ b/openmetadata-ui/src/main/resources/ui/src/components/BlockEditor/block-editor.less @@ -27,6 +27,10 @@ // this is to have enough space after last node, referred from the reference editor padding-bottom: 30vh; } + .om-block-editor > .tableWrapper { + width: 100%; + overflow-x: auto; + } // show placeholder when editor is in focused mode .tiptap.ProseMirror-focused .is-node-empty.has-focus::before { color: @grey-3; @@ -323,11 +327,11 @@ .om-list-decimal { list-style-type: decimal !important; - padding-left: 16px; + padding-left: 24px; } .om-list-disc { list-style-type: disc !important; - padding-left: 16px; + padding-left: 24px; } .om-leading-normal { line-height: 1.5; diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Classifications/ClassificationDetails/ClassificationDetails.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Classifications/ClassificationDetails/ClassificationDetails.tsx index f08b363fa2da..78ed69148b97 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Classifications/ClassificationDetails/ClassificationDetails.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Classifications/ClassificationDetails/ClassificationDetails.tsx @@ -30,7 +30,11 @@ import { useHistory } from 'react-router-dom'; import { ReactComponent as IconTag } from '../../../assets/svg/classification.svg'; import { ReactComponent as LockIcon } from '../../../assets/svg/closed-lock.svg'; import { ReactComponent as VersionIcon } from '../../../assets/svg/ic-version.svg'; -import { DE_ACTIVE_COLOR, PRIMERY_COLOR } from '../../../constants/constants'; +import { + DATA_ASSET_ICON_DIMENSION, + DE_ACTIVE_COLOR, + PRIMERY_COLOR, +} from '../../../constants/constants'; import { EntityField } from '../../../constants/Feeds.constants'; import { usePermissionProvider } from '../../../context/PermissionProvider/PermissionProvider'; import { ResourceEntity } from '../../../context/PermissionProvider/PermissionProvider.interface'; @@ -57,7 +61,7 @@ import { import { getErrorText } from '../../../utils/StringsUtils'; import { showErrorToast } from '../../../utils/ToastUtils'; import AppBadge from '../../common/Badge/Badge.component'; -import Description from '../../common/EntityDescription/Description'; +import DescriptionV1 from '../../common/EntityDescription/DescriptionV1'; import ManageButton from '../../common/EntityPageInfos/ManageButton/ManageButton'; import ErrorPlaceHolder from '../../common/ErrorWithPlaceholder/ErrorPlaceHolder'; import NextPrevious from '../../common/NextPrevious/NextPrevious'; @@ -446,7 +450,13 @@ const ClassificationDetails = forwardRef( @@ -480,14 +490,16 @@ const ClassificationDetails = forwardRef( )}
- { const { t } = useTranslation(); - const { currentUser } = useAuthContext(); + const { currentUser } = useApplicationStore(); const history = useHistory(); const { tab: activeTab = EntityTabs.DETAILS } = useParams<{ tab: EntityTabs }>(); @@ -437,10 +441,10 @@ const DashboardDetails = ({ {chartName} -
diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Dashboard/DashboardDetails/DashboardDetails.test.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Dashboard/DashboardDetails/DashboardDetails.test.tsx index 1ddeebef447f..2c3080a47e0e 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Dashboard/DashboardDetails/DashboardDetails.test.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Dashboard/DashboardDetails/DashboardDetails.test.tsx @@ -97,7 +97,7 @@ jest.mock('../../common/TabsLabel/TabsLabel.component', () => { return jest.fn().mockImplementation(({ name }) =>

{name}

); }); -jest.mock('../../common/EntityDescription/Description', () => { +jest.mock('../../common/EntityDescription/DescriptionV1', () => { return jest.fn().mockReturnValue(

Description Component

); }); jest.mock('../../common/RichTextEditor/RichTextEditorPreviewer', () => { diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Dashboard/DashboardVersion/DashboardVersion.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Dashboard/DashboardVersion/DashboardVersion.component.tsx index 2390ec6ca2c1..dc67fb101202 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Dashboard/DashboardVersion/DashboardVersion.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Dashboard/DashboardVersion/DashboardVersion.component.tsx @@ -11,6 +11,7 @@ * limitations under the License. */ +import Icon from '@ant-design/icons/lib/components/Icon'; import { Col, Row, Space, Table, Tabs, TabsProps } from 'antd'; import { ColumnsType } from 'antd/lib/table'; import classNames from 'classnames'; @@ -18,7 +19,10 @@ import React, { FC, useEffect, useMemo, useState } from 'react'; import { useTranslation } from 'react-i18next'; import { Link, useHistory, useParams } from 'react-router-dom'; import { ReactComponent as IconExternalLink } from '../../../assets/svg/external-links.svg'; -import { getVersionPathWithTab } from '../../../constants/constants'; +import { + DATA_ASSET_ICON_DIMENSION, + getVersionPathWithTab, +} from '../../../constants/constants'; import { EntityField } from '../../../constants/Feeds.constants'; import { EntityTabs, EntityType } from '../../../enums/entity.enum'; import { @@ -107,7 +111,11 @@ const DashboardVersion: FC = ({ {getEntityName(record)} - + ), diff --git a/openmetadata-ui/src/main/resources/ui/src/components/DataAssets/DataAssetsHeader/DataAssetsHeader.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/DataAssets/DataAssetsHeader/DataAssetsHeader.component.tsx index 40380d667dbf..6124ab9c9b35 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/DataAssets/DataAssetsHeader/DataAssetsHeader.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/DataAssets/DataAssetsHeader/DataAssetsHeader.component.tsx @@ -30,7 +30,10 @@ import { DomainLabel } from '../../../components/common/DomainLabel/DomainLabel. import { OwnerLabel } from '../../../components/common/OwnerLabel/OwnerLabel.component'; import TierCard from '../../../components/common/TierCard/TierCard'; import EntityHeaderTitle from '../../../components/Entity/EntityHeaderTitle/EntityHeaderTitle.component'; -import { DE_ACTIVE_COLOR } from '../../../constants/constants'; +import { + DATA_ASSET_ICON_DIMENSION, + DE_ACTIVE_COLOR, +} from '../../../constants/constants'; import { SERVICE_TYPES } from '../../../constants/Services.constant'; import { useTourProvider } from '../../../context/TourProvider/TourProvider'; import { EntityTabs, EntityType } from '../../../enums/entity.enum'; @@ -51,7 +54,8 @@ import { import serviceUtilClassBase from '../../../utils/ServiceUtilClassBase'; import { getTierTags } from '../../../utils/TableUtils'; import { showErrorToast } from '../../../utils/ToastUtils'; -import { useAuthContext } from '../../Auth/AuthProviders/AuthProvider'; + +import { useApplicationStore } from '../../../hooks/useApplicationStore'; import AnnouncementCard from '../../common/EntityPageInfos/AnnouncementCard/AnnouncementCard'; import AnnouncementDrawer from '../../common/EntityPageInfos/AnnouncementDrawer/AnnouncementDrawer'; import ManageButton from '../../common/EntityPageInfos/ManageButton/ManageButton'; @@ -76,7 +80,7 @@ export const ExtraInfoLabel = ({ value: string | number; }) => ( <> - + {!isEmpty(label) && ( {`${label}: `} @@ -96,7 +100,7 @@ export const ExtraInfoLink = ({ href: string; }) => ( <> - +
{!isEmpty(label) && ( {`${label}: `} @@ -104,7 +108,11 @@ export const ExtraInfoLink = ({ {value}{' '} - {' '} +
); @@ -129,7 +137,7 @@ export const DataAssetsHeader = ({ onProfilerSettingUpdate, onUpdateRetentionPeriod, }: DataAssetsHeaderProps) => { - const { currentUser } = useAuthContext(); + const { currentUser } = useApplicationStore(); const USER_ID = currentUser?.id ?? ''; const { t } = useTranslation(); const { isTourPage } = useTourProvider(); @@ -346,7 +354,7 @@ export const DataAssetsHeader = ({ /> -
+
{showDomain && ( <> - + )} - + {tier ? ( @@ -381,13 +389,20 @@ export const DataAssetsHeader = ({ )} {editTierPermission && ( - @@ -433,7 +454,13 @@ export const DataAssetsHeader = ({ @@ -452,6 +479,7 @@ export const DataAssetsHeader = ({ icon={ } loading={isFollowingLoading} @@ -462,11 +490,17 @@ export const DataAssetsHeader = ({ )} + placement="topRight" + title={copyTooltip ?? t('message.copy-to-clipboard')}> diff --git a/openmetadata-ui/src/main/resources/ui/src/components/DataProducts/DataProductsDetailsPage/DataProductsDetailsPage.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/DataProducts/DataProductsDetailsPage/DataProductsDetailsPage.component.tsx index 004b50ad8ce1..3a4382dbc7ec 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/DataProducts/DataProductsDetailsPage/DataProductsDetailsPage.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/DataProducts/DataProductsDetailsPage/DataProductsDetailsPage.component.tsx @@ -32,7 +32,10 @@ import { ReactComponent as DeleteIcon } from '../../../assets/svg/ic-delete.svg' import { ReactComponent as VersionIcon } from '../../../assets/svg/ic-version.svg'; import { ReactComponent as IconDropdown } from '../../../assets/svg/menu.svg'; import { ReactComponent as StyleIcon } from '../../../assets/svg/style.svg'; -import { DE_ACTIVE_COLOR } from '../../../constants/constants'; +import { + DATA_ASSET_ICON_DIMENSION, + DE_ACTIVE_COLOR, +} from '../../../constants/constants'; import { EntityField } from '../../../constants/Feeds.constants'; import { usePermissionProvider } from '../../../context/PermissionProvider/PermissionProvider'; import { @@ -512,7 +515,13 @@ const DataProductsDetailsPage = ({ 'text-primary border-primary': version, })} data-testid="version-button" - icon={} + icon={ + + } onClick={handleVersionClick}> - +
axisTickFormatter(props)} - type="category" + type="number" /> diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Database/Profiler/TableProfiler/QualityTab/QualityTab.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Database/Profiler/TableProfiler/QualityTab/QualityTab.component.tsx index 9bf06e9d623b..c09974806b12 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Database/Profiler/TableProfiler/QualityTab/QualityTab.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Database/Profiler/TableProfiler/QualityTab/QualityTab.component.tsx @@ -13,7 +13,7 @@ import { DownOutlined } from '@ant-design/icons'; import { Button, Col, Dropdown, Form, Row, Select, Space, Tabs } from 'antd'; import { AxiosError } from 'axios'; -import { isUndefined } from 'lodash'; +import { isEmpty, isUndefined } from 'lodash'; import React, { useEffect, useMemo, useState } from 'react'; import { useTranslation } from 'react-i18next'; import { useHistory } from 'react-router-dom'; @@ -27,16 +27,18 @@ import { INITIAL_TEST_SUMMARY } from '../../../../../constants/TestSuite.constan import { EntityTabs, TabSpecificField } from '../../../../../enums/entity.enum'; import { ProfilerDashboardType } from '../../../../../enums/table.enum'; import { Table } from '../../../../../generated/entity/data/table'; -import { TestCase } from '../../../../../generated/tests/testCase'; -import { EntityType as TestType } from '../../../../../generated/tests/testDefinition'; +import { TestCaseStatus } from '../../../../../generated/tests/testCase'; import { useFqn } from '../../../../../hooks/useFqn'; import { getTableDetailsByFQN } from '../../../../../rest/tableAPI'; +import { TestCaseType } from '../../../../../rest/testAPI'; import { getBreadcrumbForTable, getEntityName, } from '../../../../../utils/EntityUtils'; import { getAddDataQualityTableTestPath } from '../../../../../utils/RouterUtils'; import { showErrorToast } from '../../../../../utils/ToastUtils'; +import NextPrevious from '../../../../common/NextPrevious/NextPrevious'; +import { NextPreviousProps } from '../../../../common/NextPrevious/NextPrevious.interface'; import TabsLabel from '../../../../common/TabsLabel/TabsLabel.component'; import { SummaryPanel } from '../../../../DataQuality/SummaryPannel/SummaryPanel.component'; import TestSuitePipelineTab from '../../../../DataQuality/TestSuite/TestSuitePipelineTab/TestSuitePipelineTab.component'; @@ -51,23 +53,48 @@ export const QualityTab = () => { fetchAllTests, onTestCaseUpdate, allTestCases, - splitTestCases, isTestsLoading, isTableDeleted, + testCasePaging, } = useTableProfiler(); + const { + currentPage, + pageSize, + paging, + handlePageChange, + handlePageSizeChange, + showPagination, + } = testCasePaging; + const editTest = permissions.EditAll || permissions.EditTests; const { fqn: datasetFQN } = useFqn(); const history = useHistory(); const { t } = useTranslation(); const [selectedTestCaseStatus, setSelectedTestCaseStatus] = - useState(''); - const [selectedTestType, setSelectedTestType] = useState(''); + useState('' as TestCaseStatus); + const [selectedTestType, setSelectedTestType] = useState(TestCaseType.all); const [table, setTable] = useState
(); const [isTestSuiteLoading, setIsTestSuiteLoading] = useState(true); const testSuite = useMemo(() => table?.testSuite, [table]); + const handleTestCasePageChange: NextPreviousProps['pagingHandler'] = ({ + cursorType, + currentPage, + }) => { + if (cursorType) { + fetchAllTests({ + [cursorType]: paging[cursorType], + testCaseType: selectedTestType, + testCaseStatus: isEmpty(selectedTestCaseStatus) + ? undefined + : selectedTestCaseStatus, + }); + } + handlePageChange(currentPage); + }; + const tableBreadcrumb = useMemo(() => { return table ? [ @@ -84,37 +111,36 @@ export const QualityTab = () => { : undefined; }, [table]); - const filteredTestCase = useMemo(() => { - let tests: TestCase[] = allTestCases ?? []; - if (selectedTestType === TestType.Table) { - tests = splitTestCases.table; - } else if (selectedTestType === TestType.Column) { - tests = splitTestCases.column; - } - - return tests.filter( - (data) => - selectedTestCaseStatus === '' || - data.testCaseResult?.testCaseStatus === selectedTestCaseStatus - ); - }, [selectedTestCaseStatus, selectedTestType, allTestCases, splitTestCases]); const tabs = useMemo( () => [ { label: t('label.test-case-plural'), key: EntityTabs.TEST_CASES, children: ( -
- -
+ + + + + + {showPagination && ( + + )} + + ), }, { @@ -125,23 +151,34 @@ export const QualityTab = () => { ], [ isTestsLoading, - filteredTestCase, + allTestCases, onTestCaseUpdate, testSuite, fetchAllTests, tableBreadcrumb, + testCasePaging, ] ); - const handleTestCaseStatusChange = (value: string) => { + const handleTestCaseStatusChange = (value: TestCaseStatus) => { if (value !== selectedTestCaseStatus) { setSelectedTestCaseStatus(value); + fetchAllTests({ + testCaseType: selectedTestType, + testCaseStatus: isEmpty(value) ? undefined : value, + }); } }; - const handleTestCaseTypeChange = (value: string) => { + const handleTestCaseTypeChange = (value: TestCaseType) => { if (value !== selectedTestType) { setSelectedTestType(value); + fetchAllTests({ + testCaseType: value, + testCaseStatus: isEmpty(selectedTestCaseStatus) + ? undefined + : selectedTestCaseStatus, + }); } }; diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Database/Profiler/TableProfiler/QualityTab/QualityTab.test.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Database/Profiler/TableProfiler/QualityTab/QualityTab.test.tsx index ae8a5755a4aa..1e81b386f81e 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Database/Profiler/TableProfiler/QualityTab/QualityTab.test.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Database/Profiler/TableProfiler/QualityTab/QualityTab.test.tsx @@ -10,7 +10,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import { act, render, screen } from '@testing-library/react'; +import { act, fireEvent, render, screen } from '@testing-library/react'; import React from 'react'; import { MOCK_TABLE } from '../../../../../mocks/TableData.mock'; import { useTableProfiler } from '../TableProfilerProvider'; @@ -32,15 +32,30 @@ const mockTable = { }; const mockPush = jest.fn(); +const mockUseTableProfiler = { + tableProfiler: MOCK_TABLE, + permissions: { + EditAll: true, + EditDataProfile: true, + EditTests: true, + }, + fetchAllTests: jest.fn(), + onTestCaseUpdate: jest.fn(), + allTestCases: [], + isTestsLoading: false, + isTableDeleted: false, + testCasePaging: { + currentPage: 1, + pageSize: 10, + paging: { total: 16, after: 'after' }, + handlePageChange: jest.fn(), + handlePageSizeChange: jest.fn(), + showPagination: true, + }, +}; jest.mock('../TableProfilerProvider', () => ({ - useTableProfiler: jest.fn().mockImplementation(() => ({ - tableProfiler: MOCK_TABLE, - permissions: { - EditAll: true, - EditDataProfile: true, - }, - })), + useTableProfiler: jest.fn().mockImplementation(() => mockUseTableProfiler), })); jest.mock( @@ -70,6 +85,23 @@ jest.mock('../../../../../rest/tableAPI', () => ({ .fn() .mockImplementation(() => Promise.resolve(mockTable)), })); +jest.mock('../../../../common/NextPrevious/NextPrevious', () => { + return jest.fn().mockImplementation(({ pagingHandler }) => ( +
+

NextPrevious.component

+ +
+ )); +}); +jest.mock('../../DataQualityTab/DataQualityTab', () => { + return jest + .fn() + .mockImplementation(() =>
DataQualityTab.component
); +}); describe('QualityTab', () => { it('should render QualityTab', async () => { @@ -89,18 +121,36 @@ describe('QualityTab', () => { expect( await screen.findByText('label.test-case-plural') ).toBeInTheDocument(); + expect( + await screen.findByText('NextPrevious.component') + ).toBeInTheDocument(); + expect( + await screen.findByText('DataQualityTab.component') + ).toBeInTheDocument(); expect(await screen.findByText('label.pipeline')).toBeInTheDocument(); }); - it('should render the Add button if editTest is true and isTableDeleted is false', async () => { - (useTableProfiler as jest.Mock).mockReturnValue({ - permissions: { - EditAll: true, - EditTests: true, - }, - isTableDeleted: false, + it("Pagination should be called with 'handlePageChange'", async () => { + await act(async () => { + render(); + }); + const nextBtn = await screen.findByTestId('next-btn'); + + await act(async () => { + fireEvent.click(nextBtn); }); + expect( + mockUseTableProfiler.testCasePaging.handlePageChange + ).toHaveBeenCalledWith(2); + expect(mockUseTableProfiler.fetchAllTests).toHaveBeenCalledWith({ + after: 'after', + testCaseStatus: undefined, + testCaseType: 'all', + }); + }); + + it('should render the Add button if editTest is true and isTableDeleted is false', async () => { await act(async () => { render(); }); @@ -112,6 +162,7 @@ describe('QualityTab', () => { it('should not render the Add button if editTest is false', async () => { (useTableProfiler as jest.Mock).mockReturnValue({ + ...mockUseTableProfiler, permissions: { EditAll: false, EditTests: false, @@ -130,6 +181,7 @@ describe('QualityTab', () => { it('should not render the Add button if isTableDeleted is true', async () => { (useTableProfiler as jest.Mock).mockReturnValue({ + ...mockUseTableProfiler, permissions: { EditAll: true, EditTests: true, @@ -159,32 +211,15 @@ describe('QualityTab', () => { ).toHaveAttribute('aria-selected', 'false'); }); - it('should show skeleton loader when data is loading', async () => { - (useTableProfiler as jest.Mock).mockReturnValue({ - permissions: { - EditAll: true, - EditTests: true, - }, - isTableDeleted: true, - isTestsLoading: true, - isProfilerDataLoading: true, - }); - - await act(async () => { - render(); - }); - - expect(await screen.findByTestId('skeleton-table')).toBeInTheDocument(); - }); - it('should display the initial summary data', async () => { - (useTableProfiler as jest.Mock).mockReturnValue({ + (useTableProfiler as jest.Mock).mockImplementationOnce(() => ({ + ...mockUseTableProfiler, permissions: { EditAll: true, EditTests: true, }, isTableDeleted: false, - }); + })); await act(async () => { render(); diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Database/Profiler/TableProfiler/TableProfiler.interface.ts b/openmetadata-ui/src/main/resources/ui/src/components/Database/Profiler/TableProfiler/TableProfiler.interface.ts index e21cb97ba337..48576d9746fc 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Database/Profiler/TableProfiler/TableProfiler.interface.ts +++ b/openmetadata-ui/src/main/resources/ui/src/components/Database/Profiler/TableProfiler/TableProfiler.interface.ts @@ -25,6 +25,7 @@ import { TableProfilerConfig, } from '../../../../generated/entity/data/table'; import { TestCase } from '../../../../generated/tests/testCase'; +import { UsePagingInterface } from '../../../../hooks/paging/usePaging'; import { ListTestCaseParams } from '../../../../rest/testAPI'; export interface TableProfilerProps { @@ -50,15 +51,11 @@ export interface TableProfilerContextInterface { fetchAllTests: (params?: ListTestCaseParams) => Promise; onCustomMetricUpdate: (table: Table) => void; isProfilingEnabled: boolean; - splitTestCases: SplitTestCasesType; dateRangeObject: DateRangeObject; onDateRangeChange: (dateRange: DateRangeObject) => void; + testCasePaging: UsePagingInterface; } -export type SplitTestCasesType = { - column: TestCase[]; - table: TestCase[]; -}; export type TableTestsType = { tests: TestCase[]; results: { diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Database/Profiler/TableProfiler/TableProfilerProvider.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Database/Profiler/TableProfiler/TableProfilerProvider.tsx index 94680cd4779c..d00463996071 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Database/Profiler/TableProfiler/TableProfilerProvider.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Database/Profiler/TableProfiler/TableProfilerProvider.tsx @@ -11,7 +11,7 @@ * limitations under the License. */ import { AxiosError } from 'axios'; -import { isEmpty, isUndefined } from 'lodash'; +import { isUndefined } from 'lodash'; import { DateTime } from 'luxon'; import { DateRangeObject } from 'Models'; import Qs from 'qs'; @@ -25,13 +25,14 @@ import React, { } from 'react'; import { useTranslation } from 'react-i18next'; import { useLocation } from 'react-router-dom'; -import { API_RES_MAX_SIZE } from '../../../../constants/constants'; +import { PAGE_SIZE } from '../../../../constants/constants'; import { mockDatasetData } from '../../../../constants/mockTourData.constants'; import { DEFAULT_RANGE_DATA } from '../../../../constants/profiler.constant'; import { useTourProvider } from '../../../../context/TourProvider/TourProvider'; import { Table } from '../../../../generated/entity/data/table'; import { ProfileSampleType } from '../../../../generated/metadataIngestion/databaseServiceProfilerPipeline'; import { TestCase } from '../../../../generated/tests/testCase'; +import { usePaging } from '../../../../hooks/paging/usePaging'; import { useFqn } from '../../../../hooks/useFqn'; import { getLatestTableProfileByFqn, @@ -45,7 +46,6 @@ import { TableProfilerTab } from '../ProfilerDashboard/profilerDashboard.interfa import ProfilerSettingsModal from './ProfilerSettingsModal/ProfilerSettingsModal'; import { OverallTableSummaryType, - SplitTestCasesType, TableProfilerContextInterface, TableProfilerProviderProps, } from './TableProfiler.interface'; @@ -63,6 +63,7 @@ export const TableProfilerProvider = ({ const { t } = useTranslation(); const { fqn: datasetFQN } = useFqn(); const { isTourOpen } = useTourProvider(); + const testCasePaging = usePaging(PAGE_SIZE); const location = useLocation(); // profiler has its own api but sent's the data in Table type const [tableProfiler, setTableProfiler] = useState
(); @@ -72,10 +73,6 @@ export const TableProfilerProvider = ({ const [isProfilerDataLoading, setIsProfilerDataLoading] = useState(true); const [allTestCases, setAllTestCases] = useState([]); const [settingModalVisible, setSettingModalVisible] = useState(false); - const [splitTestCases, setSplitTestCases] = useState({ - column: [], - table: [], - }); const [dateRangeObject, setDateRangeObject] = useState(DEFAULT_RANGE_DATA); @@ -158,19 +155,6 @@ export const TableProfilerProvider = ({ setDateRangeObject(data); }; - const splitTableAndColumnTest = (data: TestCase[]) => { - const columnTestsCase: TestCase[] = []; - const tableTests: TestCase[] = []; - data.forEach((test) => { - if (test.entityFQN === datasetFQN) { - tableTests.push(test); - } else { - columnTestsCase.push(test); - } - }); - setSplitTestCases({ column: columnTestsCase, table: tableTests }); - }; - const onTestCaseUpdate = useCallback( (testCase?: TestCase) => { if (isUndefined(testCase)) { @@ -180,7 +164,6 @@ export const TableProfilerProvider = ({ const updatedTests = prevTestCases.map((test) => { return testCase.id === test.id ? { ...test, ...testCase } : test; }); - splitTableAndColumnTest(updatedTests); return updatedTests; }); @@ -218,15 +201,16 @@ export const TableProfilerProvider = ({ const fetchAllTests = async (params?: ListTestCaseParams) => { setIsTestsLoading(true); try { - const { data } = await getListTestCase({ + const { data, paging } = await getListTestCase({ ...params, - fields: 'testCaseResult, testDefinition, incidentId', + fields: 'testCaseResult, incidentId', entityLink: generateEntityLink(datasetFQN ?? ''), includeAllTests: true, - limit: API_RES_MAX_SIZE, + limit: testCasePaging.pageSize, }); - splitTableAndColumnTest(data); + setAllTestCases(data); + testCasePaging.handlePagingChange(paging); } catch (error) { showErrorToast(error as AxiosError); } finally { @@ -257,19 +241,14 @@ export const TableProfilerProvider = ({ useEffect(() => { const fetchTest = - viewTest && - !isTourOpen && - [TableProfilerTab.DATA_QUALITY, TableProfilerTab.COLUMN_PROFILE].includes( - activeTab - ) && - isEmpty(allTestCases); + !isTourOpen && activeTab === TableProfilerTab.DATA_QUALITY && viewTest; if (fetchTest) { fetchAllTests(); } else { setIsTestsLoading(false); } - }, [viewTest, isTourOpen, activeTab]); + }, [viewTest, isTourOpen, activeTab, testCasePaging.pageSize]); const tableProfilerPropsData: TableProfilerContextInterface = useMemo(() => { return { @@ -284,11 +263,11 @@ export const TableProfilerProvider = ({ onSettingButtonClick: () => setSettingModalVisible(true), fetchAllTests, isProfilingEnabled: !isUndefined(tableProfiler?.profile), - splitTestCases, customMetric, onCustomMetricUpdate: handleUpdateCustomMetrics, onDateRangeChange: handleDateRangeChange, dateRangeObject, + testCasePaging, }; }, [ isTestsLoading, @@ -299,9 +278,9 @@ export const TableProfilerProvider = ({ isTableDeleted, overallSummary, onTestCaseUpdate, - splitTestCases, customMetric, dateRangeObject, + testCasePaging, ]); return ( diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Database/Profiler/TestSummary/TestSummary.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Database/Profiler/TestSummary/TestSummary.tsx index b7ab34aa8083..6c942ffb89c1 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Database/Profiler/TestSummary/TestSummary.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Database/Profiler/TestSummary/TestSummary.tsx @@ -22,7 +22,6 @@ import { omitBy, pick, round, - uniqueId, } from 'lodash'; import { DateRangeObject } from 'Models'; import React, { @@ -327,6 +326,7 @@ const TestSummary: React.FC = ({ data }) => { /> axisTickFormatter(value)} /> @@ -342,7 +342,7 @@ const TestSummary: React.FC = ({ data }) => { diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Database/Profiler/TestSummaryCustomTooltip/TestSummaryCustomTooltip.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Database/Profiler/TestSummaryCustomTooltip/TestSummaryCustomTooltip.component.tsx index b352017a824f..251dc15cf00c 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Database/Profiler/TestSummaryCustomTooltip/TestSummaryCustomTooltip.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Database/Profiler/TestSummaryCustomTooltip/TestSummaryCustomTooltip.component.tsx @@ -12,7 +12,7 @@ */ import { Card, Typography } from 'antd'; import { entries, isNumber, isString, omit, startCase } from 'lodash'; -import React from 'react'; +import React, { Fragment } from 'react'; import { useTranslation } from 'react-i18next'; import { Link } from 'react-router-dom'; import { TooltipProps } from 'recharts'; @@ -40,7 +40,7 @@ const TestSummaryCustomTooltip = ( ]) => { if (key === 'task' && !isString(value) && !isNumber(value)) { return value?.task ? ( - <> +
  • @@ -66,7 +66,7 @@ const TestSummaryCustomTooltip = (
    • - +     ) : null; } diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Database/RetentionPeriod/RetentionPeriod.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Database/RetentionPeriod/RetentionPeriod.component.tsx index 86dd2bab7614..d23808f82a0f 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Database/RetentionPeriod/RetentionPeriod.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Database/RetentionPeriod/RetentionPeriod.component.tsx @@ -10,7 +10,16 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import { Alert, Button, Form, FormProps, Input, Modal, Space } from 'antd'; +import { + Alert, + Button, + Form, + FormProps, + Input, + Modal, + Space, + Tooltip, +} from 'antd'; import { useForm } from 'antd/lib/form/Form'; import { AxiosError } from 'axios'; import React, { useCallback, useEffect, useState } from 'react'; @@ -62,14 +71,19 @@ const RetentionPeriod = ({ /> {permissions?.EditAll && ( - + + + )} ); diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Database/TableDescription/TableDescription.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Database/TableDescription/TableDescription.component.tsx index a910efc7e805..d2797c332b3f 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Database/TableDescription/TableDescription.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Database/TableDescription/TableDescription.component.tsx @@ -11,14 +11,19 @@ * limitations under the License. */ -import { Button, Space } from 'antd'; -import React from 'react'; +import { Button, Space, Tooltip } from 'antd'; +import React, { useMemo } from 'react'; import { useTranslation } from 'react-i18next'; import { ReactComponent as EditIcon } from '../../../assets/svg/edit-new.svg'; import { DE_ACTIVE_COLOR, ICON_DIMENSION } from '../../../constants/constants'; import { EntityField } from '../../../constants/Feeds.constants'; +import { EntityType } from '../../../enums/entity.enum'; import EntityTasks from '../../../pages/TasksPage/EntityTasks/EntityTasks.component'; +import EntityLink from '../../../utils/EntityLink'; +import { getEntityFeedLink } from '../../../utils/EntityUtils'; import RichTextEditorPreviewer from '../../common/RichTextEditor/RichTextEditorPreviewer'; +import SuggestionsAlert from '../../Suggestions/SuggestionsAlert/SuggestionsAlert'; +import { useSuggestionsContext } from '../../Suggestions/SuggestionsProvider/SuggestionsProvider'; import { TableDescriptionProps } from './TableDescription.interface'; const TableDescription = ({ @@ -32,37 +37,84 @@ const TableDescription = ({ onThreadLinkSelect, }: TableDescriptionProps) => { const { t } = useTranslation(); + const { selectedUserSuggestions = [] } = useSuggestionsContext(); - return ( - - {columnData.field ? ( - - ) : ( + const entityLink = useMemo( + () => + entityType === EntityType.TABLE + ? EntityLink.getTableEntityLink( + entityFqn, + columnData.record?.name ?? '' + ) + : getEntityFeedLink(entityType, columnData.fqn), + [entityType, entityFqn] + ); + + const suggestionData = useMemo(() => { + const activeSuggestion = selectedUserSuggestions.find( + (suggestion) => suggestion.entityLink === entityLink + ); + + if (activeSuggestion?.entityLink === entityLink) { + return ( + + ); + } + + return null; + }, [hasEditPermission, entityLink, selectedUserSuggestions]); + + const descriptionContent = useMemo(() => { + if (suggestionData) { + return suggestionData; + } else if (columnData.field) { + return ; + } else { + return ( {t('label.no-entity', { entity: t('label.description'), })} - )} - {!isReadOnly ? ( + ); + } + }, [columnData, suggestionData]); + + return ( + + {descriptionContent} + + {!suggestionData && !isReadOnly ? ( {hasEditPermission && ( - + + + )} = ({ afterDeleteAction, }: QueryCardProp) => { const { t } = useTranslation(); + const QueryExtras = queryClassBase.getQueryExtras(); const { fqn: datasetFQN } = useFqn(); const location = useLocation(); const history = useHistory(); @@ -165,7 +167,7 @@ const QueryCard: FC = ({ return ( -     + = ({ onChange={handleQueryChange} /> - - + + = ({ + {isExpanded && QueryExtras && } ); }; diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Database/TableQueries/QueryCardExtraOption/QueryCardExtraOption.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Database/TableQueries/QueryCardExtraOption/QueryCardExtraOption.component.tsx index e8e8b95c2925..44101b5af343 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Database/TableQueries/QueryCardExtraOption/QueryCardExtraOption.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Database/TableQueries/QueryCardExtraOption/QueryCardExtraOption.component.tsx @@ -12,7 +12,7 @@ */ import { Button, Dropdown, MenuProps, Space, Tag, Tooltip } from 'antd'; import { isUndefined, split } from 'lodash'; -import React, { useMemo, useState } from 'react'; +import React, { useCallback, useMemo, useState } from 'react'; import { useTranslation } from 'react-i18next'; import { ReactComponent as EditIcon } from '../../../../assets/svg/edit-new.svg'; import { ReactComponent as DeleteIcon } from '../../../../assets/svg/ic-delete.svg'; @@ -25,9 +25,14 @@ import { QueryVoteType } from '../TableQueries.interface'; import { QueryCardExtraOptionProps } from './QueryCardExtraOption.interface'; import { AxiosError } from 'axios'; +import Qs from 'qs'; +import { useHistory } from 'react-router-dom'; +import { useApplicationStore } from '../../../../hooks/useApplicationStore'; +import { useFqn } from '../../../../hooks/useFqn'; import { deleteQuery } from '../../../../rest/queryAPI'; +import queryClassBase from '../../../../utils/QueryClassBase'; +import { getQueryPath } from '../../../../utils/RouterUtils'; import { showErrorToast } from '../../../../utils/ToastUtils'; -import { useAuthContext } from '../../../Auth/AuthProviders/AuthProvider'; import ConfirmationModal from '../../../Modals/ConfirmationModal/ConfirmationModal'; import './query-card-extra-option.style.less'; @@ -39,7 +44,10 @@ const QueryCardExtraOption = ({ afterDeleteAction, }: QueryCardExtraOptionProps) => { const { EditAll, EditQueries, Delete } = permission; - const { currentUser } = useAuthContext(); + const { fqn: datasetFQN } = useFqn(); + const history = useHistory(); + const QueryHeaderButton = queryClassBase.getQueryHeaderActionsButtons(); + const { currentUser } = useApplicationStore(); const { t } = useTranslation(); const [showDeleteModal, setShowDeleteModal] = useState(false); const [loading, setLoading] = useState(null); @@ -57,6 +65,13 @@ const QueryCardExtraOption = ({ } }; + const onExpandClick = useCallback(() => { + history.push({ + search: Qs.stringify({ query: query.id }), + pathname: getQueryPath(datasetFQN, query.id ?? ''), + }); + }, [query]); + const dropdownItems = useMemo(() => { const items: MenuProps['items'] = [ { @@ -130,9 +145,14 @@ const QueryCardExtraOption = ({ className="query-card-extra-option" data-testid="extra-option-container" size={8}> + {QueryHeaderButton && ( + + )} + {queryLine} + - - - {t('label.description')} - - - {(EditDescription || EditAll) && ( - + + )), }) ); -jest.mock('../../../common/EntityDescription/Description', () => { - return jest.fn().mockImplementation(() =>
    Description.component
    ); +jest.mock('../../../common/EntityDescription/DescriptionV1', () => { + return jest.fn().mockImplementation(({ onDescriptionUpdate }) => ( +
    + Description.component +
    + )); }); jest.mock('../../../TagsInput/TagsInput.component', () => { - return jest.fn().mockImplementation(() =>
    TagsInput.component
    ); + return jest.fn().mockImplementation(({ onTagsUpdate }) => ( +
    + TagsInput.component + +
    + )); }); jest.mock('../../../common/Loader/Loader', () => { return jest.fn().mockImplementation(() =>
    Loader
    ); }); -jest.mock('../../../../utils/TagsUtils', () => ({ - fetchTagsAndGlossaryTerms: jest - .fn() - .mockImplementation(() => Promise.resolve({ data: [] })), -})); - jest.mock('../../../common/ProfilePicture/ProfilePicture', () => { return jest.fn().mockImplementation(() => <>testProfilePicture); }); @@ -64,9 +110,6 @@ describe('TableQueryRightPanel component test', () => { expect(owner).toBeInTheDocument(); expect(owner.textContent).toEqual(MOCK_QUERIES[0].owner?.displayName); - expect( - await screen.findByTestId('edit-description-btn') - ).toBeInTheDocument(); expect( await screen.findByText('Description.component') ).toBeInTheDocument(); @@ -89,23 +132,67 @@ describe('TableQueryRightPanel component test', () => { expect(editDescriptionBtn).not.toBeInTheDocument(); }); - it('If Edit All permission is granted, editing of the Owner, Description, and tags should not be disabled.', async () => { + it('Loader should visible', async () => { + render(, { + wrapper: MemoryRouter, + }); + + expect(await screen.findByText('Loader')).toBeInTheDocument(); + }); + + it('call onupdate owner', async () => { render(, { wrapper: MemoryRouter, }); + const updateButton = await screen.findByTestId('update-button'); + await act(async () => { + fireEvent.click(updateButton); + }); - const editDescriptionBtn = await screen.findByTestId( - 'edit-description-btn' + expect(mockQueryUpdate).toHaveBeenCalledWith( + { + ...MOCK_QUERIES[0], + owner: mockNewOwner, + }, + 'owner' ); + }); - expect(editDescriptionBtn).not.toBeDisabled(); + it('call description update', async () => { + render(, { + wrapper: MemoryRouter, + }); + const updateDescriptionButton = await screen.findByTestId( + 'update-description-button' + ); + await act(async () => { + fireEvent.click(updateDescriptionButton); + }); + + expect(mockQueryUpdate).toHaveBeenCalledWith( + { + ...MOCK_QUERIES[0], + description: 'new description', + }, + 'description' + ); }); - it('Loader should visible', async () => { - render(, { + it('call tags update', async () => { + render(, { wrapper: MemoryRouter, }); + const updateTagsButton = await screen.findByTestId('update-tags-button'); + await act(async () => { + fireEvent.click(updateTagsButton); + }); - expect(await screen.findByText('Loader')).toBeInTheDocument(); + expect(mockQueryUpdate).toHaveBeenCalledWith( + { + ...MOCK_QUERIES[0], + tags: mockNewTag, + }, + 'tags' + ); }); }); diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Domain/DomainDetailsPage/DomainDetailsPage.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Domain/DomainDetailsPage/DomainDetailsPage.component.tsx index f6c1911f46c8..a24f1ef42ee0 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Domain/DomainDetailsPage/DomainDetailsPage.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Domain/DomainDetailsPage/DomainDetailsPage.component.tsx @@ -51,7 +51,11 @@ import { AssetsOfEntity } from '../../../components/Glossary/GlossaryTerms/tabs/ import EntityNameModal from '../../../components/Modals/EntityNameModal/EntityNameModal.component'; import PageLayoutV1 from '../../../components/PageLayoutV1/PageLayoutV1'; import { FQN_SEPARATOR_CHAR } from '../../../constants/char.constants'; -import { DE_ACTIVE_COLOR, ERROR_MESSAGE } from '../../../constants/constants'; +import { + DATA_ASSET_ICON_DIMENSION, + DE_ACTIVE_COLOR, + ERROR_MESSAGE, +} from '../../../constants/constants'; import { EntityField } from '../../../constants/Feeds.constants'; import { usePermissionProvider } from '../../../context/PermissionProvider/PermissionProvider'; import { @@ -573,7 +577,13 @@ const DomainDetailsPage = ({ 'text-primary border-primary': version, })} data-testid="version-button" - icon={} + icon={ + + } onClick={handleVersionClick}> - + - {node.entityType === EntityType.TABLE && testSuite && ( -
    -
    -
    - {formTwoDigitNumber(testSuite?.summary?.success ?? 0)} -
    -
    -
    -
    - {formTwoDigitNumber(testSuite?.summary?.aborted ?? 0)} -
    -
    -
    -
    - {formTwoDigitNumber(testSuite?.summary?.failed ?? 0)} -
    -
    -
    - )} - - - {isExpanded && ( -
    -
    - } - value={searchValue} - onChange={handleSearchChange} - /> -
    - -
    -
    - {filteredColumns.map((column) => { - const isColumnTraced = tracedColumns.includes( - column.fullyQualifiedName ?? '' - ); - - return ( -
    { - e.stopPropagation(); - onColumnClick(column.fullyQualifiedName ?? ''); - }}> - {getColumnHandle( - column.type, - isConnectable, - 'lineage-column-node-handle', - column.fullyQualifiedName - )} - {getConstraintIcon({ constraint: column.constraint })} -

    {getEntityName(column)}

    -
    - ); - })} -
    -
    - - {!showAllColumns && ( - - )} -
    - )} - - )} + ); diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/NodeChildren/NodeChildren.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/NodeChildren/NodeChildren.component.tsx new file mode 100644 index 000000000000..b818c1ca9e66 --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/NodeChildren/NodeChildren.component.tsx @@ -0,0 +1,263 @@ +/* + * Copyright 2024 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { DownOutlined, SearchOutlined, UpOutlined } from '@ant-design/icons'; +import { Button, Collapse, Input } from 'antd'; +import { isEmpty } from 'lodash'; +import React, { useCallback, useEffect, useMemo, useState } from 'react'; +import { useTranslation } from 'react-i18next'; +import { BORDER_COLOR } from '../../../../constants/constants'; +import { LINEAGE_COLUMN_NODE_SUPPORTED } from '../../../../constants/Lineage.constants'; +import { useLineageProvider } from '../../../../context/LineageProvider/LineageProvider'; +import { EntityType } from '../../../../enums/entity.enum'; +import { Container } from '../../../../generated/entity/data/container'; +import { Dashboard } from '../../../../generated/entity/data/dashboard'; +import { Mlmodel } from '../../../../generated/entity/data/mlmodel'; +import { Column, Table } from '../../../../generated/entity/data/table'; +import { Topic } from '../../../../generated/entity/data/topic'; +import { getEntityName } from '../../../../utils/EntityUtils'; +import { getEntityIcon } from '../../../../utils/TableUtils'; +import { getColumnContent, getTestSuiteSummary } from '../CustomNode.utils'; +import { EntityChildren, NodeChildrenProps } from './NodeChildren.interface'; + +const NodeChildren = ({ node, isConnectable }: NodeChildrenProps) => { + const { t } = useTranslation(); + const { Panel } = Collapse; + const { isEditMode, tracedColumns, expandedNodes, onColumnClick } = + useLineageProvider(); + const { entityType, id } = node; + const [searchValue, setSearchValue] = useState(''); + const [filteredColumns, setFilteredColumns] = useState([]); + const [showAllColumns, setShowAllColumns] = useState(false); + const [isExpanded, setIsExpanded] = useState(false); + + const supportsColumns = useMemo(() => { + return ( + node && + LINEAGE_COLUMN_NODE_SUPPORTED.includes(node.entityType as EntityType) + ); + }, [node.id]); + + const { children, childrenHeading } = useMemo(() => { + const entityMappings: Record< + string, + { data: EntityChildren; label: string } + > = { + [EntityType.TABLE]: { + data: (node as Table).columns ?? [], + label: t('label.column-plural'), + }, + [EntityType.DASHBOARD]: { + data: (node as Dashboard).charts ?? [], + label: t('label.chart-plural'), + }, + [EntityType.MLMODEL]: { + data: (node as Mlmodel).mlFeatures ?? [], + label: t('label.feature-plural'), + }, + [EntityType.DASHBOARD_DATA_MODEL]: { + data: (node as Table).columns ?? [], + label: t('label.column-plural'), + }, + [EntityType.CONTAINER]: { + data: (node as Container).dataModel?.columns ?? [], + label: t('label.column-plural'), + }, + [EntityType.TOPIC]: { + data: (node as Topic).messageSchema?.schemaFields ?? [], + label: t('label.field-plural'), + }, + }; + + const { data, label } = entityMappings[node.entityType as EntityType] || { + data: [], + label: '', + }; + + return { + children: data, + childrenHeading: label, + }; + }, [node.id]); + + const handleSearchChange = useCallback( + (e: React.ChangeEvent) => { + e.stopPropagation(); + const value = e.target.value; + setSearchValue(value); + + if (value.trim() === '') { + // If search value is empty, show all columns or the default number of columns + const filterColumns = Object.values(children ?? {}); + setFilteredColumns( + showAllColumns ? filterColumns : filterColumns.slice(0, 5) + ); + } else { + // Filter columns based on search value + const filtered = Object.values(children ?? {}).filter((column) => + getEntityName(column).toLowerCase().includes(value.toLowerCase()) + ); + setFilteredColumns(filtered); + } + }, + [children] + ); + + useEffect(() => { + setIsExpanded(expandedNodes.includes(id ?? '')); + }, [expandedNodes, id]); + + useEffect(() => { + if (!isEmpty(children)) { + setFilteredColumns(children.slice(0, 5)); + } + }, [children]); + + useEffect(() => { + if (!isExpanded) { + setShowAllColumns(false); + } else if (!isEmpty(children) && Object.values(children).length < 5) { + setShowAllColumns(true); + } + }, [isEditMode, isExpanded, children]); + + const handleShowMoreClick = useCallback( + (e: React.MouseEvent) => { + e.stopPropagation(); + setShowAllColumns(true); + setFilteredColumns(children ?? []); + }, + [children] + ); + + const renderRecord = useCallback( + (record: Column) => { + return ( + + + {record?.children?.map((child) => { + const { fullyQualifiedName, dataType } = child; + if (['RECORD', 'STRUCT'].includes(dataType)) { + return renderRecord(child); + } else { + const isColumnTraced = tracedColumns.includes( + fullyQualifiedName ?? '' + ); + + return getColumnContent( + child, + isColumnTraced, + isConnectable, + onColumnClick + ); + } + })} + + + ); + }, + [isConnectable, tracedColumns] + ); + + const renderColumnsData = useCallback( + (column: Column) => { + const { fullyQualifiedName, dataType } = column; + if (['RECORD', 'STRUCT'].includes(dataType)) { + return renderRecord(column); + } else { + const isColumnTraced = tracedColumns.includes(fullyQualifiedName ?? ''); + + return getColumnContent( + column, + isColumnTraced, + isConnectable, + onColumnClick + ); + } + }, + [isConnectable, tracedColumns] + ); + + if (supportsColumns) { + return ( +
    +
    +
    + } + type="text" + onClick={(e) => { + e.stopPropagation(); + setIsExpanded((prevIsExpanded: boolean) => !prevIsExpanded); + }}> + {childrenHeading} + {isExpanded ? ( + + ) : ( + + )} + + {entityType === EntityType.TABLE && + getTestSuiteSummary((node as Table).testSuite)} +
    + + {isExpanded && ( +
    +
    + } + value={searchValue} + onChange={handleSearchChange} + /> +
    + +
    +
    + {filteredColumns.map((column) => + renderColumnsData(column as Column) + )} +
    +
    + + {!showAllColumns && ( + + )} +
    + )} + + ); + } else { + return null; + } +}; + +export default NodeChildren; diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/NodeChildren/NodeChildren.interface.ts b/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/NodeChildren/NodeChildren.interface.ts new file mode 100644 index 000000000000..db2aa15ee817 --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/NodeChildren/NodeChildren.interface.ts @@ -0,0 +1,28 @@ +/* + * Copyright 2024 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { MlFeature } from '../../../../generated/entity/data/mlmodel'; +import { Column } from '../../../../generated/entity/data/table'; +import { Field } from '../../../../generated/entity/data/topic'; +import { EntityReference } from '../../../../generated/entity/type'; +import { SearchedDataProps } from '../../../SearchedData/SearchedData.interface'; + +export interface NodeChildrenProps { + node: SearchedDataProps['data'][number]['_source']; + isConnectable: boolean; +} + +export type EntityChildren = + | Column[] + | EntityReference[] + | MlFeature[] + | Field[]; diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/NodeSuggestions.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/NodeSuggestions.component.tsx index 99e7bdcbe4f9..1615f8056161 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/NodeSuggestions.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/NodeSuggestions.component.tsx @@ -14,7 +14,6 @@ import { Select } from 'antd'; import { AxiosError } from 'axios'; import { capitalize, debounce } from 'lodash'; -import { FormattedTableData } from 'Models'; import React, { FC, HTMLAttributes, @@ -28,15 +27,14 @@ import { PAGE_SIZE } from '../../../constants/constants'; import { EntityType, FqnPart } from '../../../enums/entity.enum'; import { SearchIndex } from '../../../enums/search.enum'; import { EntityReference } from '../../../generated/entity/type'; -import { SearchSourceAlias } from '../../../interface/search.interface'; import { searchData } from '../../../rest/miscAPI'; -import { formatDataResponse } from '../../../utils/APIUtils'; import { getPartialNameFromTableFQN } from '../../../utils/CommonUtils'; import { getEntityNodeIcon } from '../../../utils/EntityLineageUtils'; import { getEntityName } from '../../../utils/EntityUtils'; import serviceUtilClassBase from '../../../utils/ServiceUtilClassBase'; import { showErrorToast } from '../../../utils/ToastUtils'; import { ExploreSearchIndex } from '../../Explore/ExplorePage.interface'; +import { SourceType } from '../../SearchedData/SearchedData.interface'; import './node-suggestion.less'; interface EntitySuggestionProps extends HTMLAttributes { @@ -50,7 +48,7 @@ const NodeSuggestions: FC = ({ }) => { const { t } = useTranslation(); - const [data, setData] = useState>([]); + const [data, setData] = useState>([]); const [searchValue, setSearchValue] = useState(''); @@ -78,7 +76,8 @@ const NodeSuggestions: FC = ({ '', (entityType as ExploreSearchIndex) ?? SearchIndex.TABLE ); - setData(formatDataResponse(data.data.hits.hits)); + const sources = data.data.hits.hits.map((hit) => hit._source); + setData(sources); } catch (error) { showErrorToast( error as AxiosError, @@ -133,16 +132,14 @@ const NodeSuggestions: FC = ({ alt={entity.serviceType} className="m-r-xs" height="16px" - src={serviceUtilClassBase.getServiceTypeLogo( - entity as SearchSourceAlias - )} + src={serviceUtilClassBase.getServiceTypeLogo(entity)} width="16px" />
    {entity.entityType === EntityType.TABLE && (

    {getSuggestionLabelHeading( - entity.fullyQualifiedName, + entity.fullyQualifiedName ?? '', entity.entityType as string )}

    diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/custom-node.less b/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/custom-node.less index 662251410390..633b3fdabc90 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/custom-node.less +++ b/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/custom-node.less @@ -59,6 +59,25 @@ border: 1px solid @red-5; } } + .column-container { + .ant-collapse-arrow { + margin-right: 0 !important; + } + .ant-collapse { + border-radius: 0; + } + .ant-collapse-header { + padding: 4px 12px !important; + font-size: 12px; + } + .ant-collapse-content-box { + padding: 4px !important; + } + .ant-collapse-item { + border: none !important; + border-radius: 0; + } + } } .react-flow__node-default, @@ -151,13 +170,13 @@ } .react-flow .lineage-node-handle { - width: 35px; - min-width: 35px; - height: 35px; - border-radius: 50%; - border-color: @lineage-border; - background: @white; - top: 43px; // Need to show handles on top half + width: 35px !important; + min-width: 35px !important; + height: 35px !important; + border-radius: 50% !important; + border-color: @lineage-border !important; + background: @white !important; + top: 43px !important; // Need to show handles on top half svg { color: @text-grey-muted; } diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/entity-lineage-sidebar.less b/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/entity-lineage-sidebar.less index 769bebb5e31f..bb123b55f56c 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/entity-lineage-sidebar.less +++ b/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/entity-lineage-sidebar.less @@ -20,7 +20,7 @@ } width: 46px; - box-sizing: content-box; + box-sizing: content-box !important; border: @global-border; display: flex; align-items: center; diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/entity-lineage.style.less b/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/entity-lineage.style.less index a4f5a01c736b..17e0c3f70645 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/entity-lineage.style.less +++ b/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/entity-lineage.style.less @@ -68,11 +68,15 @@ } } .custom-node-column-lineage-normal { - border-bottom: 1px solid @border-color; + border-top: 1px solid @border-color; display: flex; justify-content: center; align-items: center; position: relative; + font-size: 12px; + &:first-child { + border-top: none; + } } .custom-node { @@ -134,6 +138,12 @@ &.active { background-color: @primary-color; } + &.active:hover { + background-color: @primary-color; + } + &.active:focus { + background-color: @primary-color; + } } .custom-lineage-heading { @@ -185,12 +195,6 @@ } // lineage -.lineage-card { - > .ant-card-body { - padding: 0; - } -} - .lineage-card { height: calc(100vh - 240px); @@ -223,7 +227,7 @@ } .lineage-node-remove-btn { - position: absolute; + position: absolute !important; top: -20px; right: -20px; cursor: pointer; diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Entity/Task/TaskTab/TaskTab.component.test.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Entity/Task/TaskTab/TaskTab.component.test.tsx index e0c85acd3075..b8365b83881d 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Entity/Task/TaskTab/TaskTab.component.test.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Entity/Task/TaskTab/TaskTab.component.test.tsx @@ -96,8 +96,8 @@ jest.mock('../../../../utils/ToastUtils', () => ({ showSuccessToast: jest.fn(), })); -jest.mock('../../../Auth/AuthProviders/AuthProvider', () => ({ - useAuthContext: jest.fn(() => ({ +jest.mock('../../../../hooks/useApplicationStore', () => ({ + useApplicationStore: jest.fn(() => ({ currentUser: mockUserData, })), })); diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Entity/Task/TaskTab/TaskTab.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Entity/Task/TaskTab/TaskTab.component.tsx index 110351b39c11..961b0fc5b9f8 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Entity/Task/TaskTab/TaskTab.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Entity/Task/TaskTab/TaskTab.component.tsx @@ -68,6 +68,7 @@ import { } from '../../../../generated/tests/testCaseResolutionStatus'; import { TagLabel } from '../../../../generated/type/tagLabel'; import { useAuth } from '../../../../hooks/authHooks'; +import { useApplicationStore } from '../../../../hooks/useApplicationStore'; import Assignees from '../../../../pages/TasksPage/shared/Assignees'; import DescriptionTask from '../../../../pages/TasksPage/shared/DescriptionTask'; import TagsTask from '../../../../pages/TasksPage/shared/TagsTask'; @@ -82,6 +83,7 @@ import { getNameFromFQN } from '../../../../utils/CommonUtils'; import EntityLink from '../../../../utils/EntityLink'; import { getEntityFQN } from '../../../../utils/FeedUtils'; import { checkPermission } from '../../../../utils/PermissionsUtils'; +import { getErrorText } from '../../../../utils/StringsUtils'; import { fetchOptions, generateOptions, @@ -95,7 +97,6 @@ import { showErrorToast, showSuccessToast } from '../../../../utils/ToastUtils'; import ActivityFeedCardV1 from '../../../ActivityFeed/ActivityFeedCard/ActivityFeedCardV1'; import ActivityFeedEditor from '../../../ActivityFeed/ActivityFeedEditor/ActivityFeedEditor'; import { useActivityFeedProvider } from '../../../ActivityFeed/ActivityFeedProvider/ActivityFeedProvider'; -import { useAuthContext } from '../../../Auth/AuthProviders/AuthProvider'; import AssigneeList from '../../../common/AssigneeList/AssigneeList'; import InlineEdit from '../../../common/InlineEdit/InlineEdit.component'; import { OwnerLabel } from '../../../common/OwnerLabel/OwnerLabel.component'; @@ -114,7 +115,7 @@ export const TaskTab = ({ }: TaskTabProps) => { const history = useHistory(); const [assigneesForm] = useForm(); - const { currentUser } = useAuthContext(); + const { currentUser } = useApplicationStore(); const markdownRef = useRef(); const updatedAssignees = Form.useWatch('assignees', assigneesForm); const { permissions } = usePermissionProvider(); @@ -266,7 +267,9 @@ export const TaskTab = ({ rest.onAfterClose?.(); rest.onUpdateEntityDetails?.(); }) - .catch((err: AxiosError) => showErrorToast(err)); + .catch((err: AxiosError) => + showErrorToast(getErrorText(err, t('server.unexpected-error'))) + ); }; const onTaskResolve = () => { @@ -429,7 +432,9 @@ export const TaskTab = ({ rest.onAfterClose?.(); setShowEditTaskModel(false); } catch (error) { - showErrorToast(error as AxiosError); + showErrorToast( + getErrorText(error as AxiosError, t('server.unexpected-error')) + ); } finally { setIsActionLoading(false); } diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Explore/EntitySummaryPanel/CommonEntitySummaryInfo/CommonEntitySummaryInfo.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Explore/EntitySummaryPanel/CommonEntitySummaryInfo/CommonEntitySummaryInfo.tsx index d17aed9742e3..df67bc318a81 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Explore/EntitySummaryPanel/CommonEntitySummaryInfo/CommonEntitySummaryInfo.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Explore/EntitySummaryPanel/CommonEntitySummaryInfo/CommonEntitySummaryInfo.tsx @@ -11,12 +11,14 @@ * limitations under the License. */ +import Icon from '@ant-design/icons/lib/components/Icon'; import { Col, Row, Typography } from 'antd'; import classNames from 'classnames'; import React from 'react'; import { useTranslation } from 'react-i18next'; import { Link } from 'react-router-dom'; import { ReactComponent as IconExternalLink } from '../../../../assets/svg/external-links.svg'; +import { ICON_DIMENSION } from '../../../../constants/constants'; import { CommonEntitySummaryInfoProps } from './CommonEntitySummaryInfo.interface'; function CommonEntitySummaryInfo({ @@ -55,10 +57,11 @@ function CommonEntitySummaryInfo({ to={{ pathname: info.url }}> {info.value} {info.isExternal ? ( - ) : null} diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Explore/EntitySummaryPanel/TagsSummary/TagsSummary.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Explore/EntitySummaryPanel/TagsSummary/TagsSummary.component.tsx index 0acd0d2fc848..bba441d3d673 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Explore/EntitySummaryPanel/TagsSummary/TagsSummary.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Explore/EntitySummaryPanel/TagsSummary/TagsSummary.component.tsx @@ -34,7 +34,7 @@ function TagsSummary({ entityDetails, isLoading }: TagsSummaryProps) { SearchIndex.TOPIC, SearchIndex.DASHBOARD, SearchIndex.CONTAINER, - SearchIndex.GLOSSARY, + SearchIndex.GLOSSARY_TERM, SearchIndex.MLMODEL, SearchIndex.PIPELINE, SearchIndex.STORED_PROCEDURE, diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Explore/ExplorePage.interface.ts b/openmetadata-ui/src/main/resources/ui/src/components/Explore/ExplorePage.interface.ts index aa803f4e93ce..b625b7578b6b 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Explore/ExplorePage.interface.ts +++ b/openmetadata-ui/src/main/resources/ui/src/components/Explore/ExplorePage.interface.ts @@ -56,7 +56,7 @@ export type ExploreSearchIndex = | SearchIndex.MLMODEL | SearchIndex.TOPIC | SearchIndex.CONTAINER - | SearchIndex.GLOSSARY + | SearchIndex.GLOSSARY_TERM | SearchIndex.TAG | SearchIndex.SEARCH_INDEX | SearchIndex.STORED_PROCEDURE diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Explore/ExploreQuickFilters.test.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Explore/ExploreQuickFilters.test.tsx index abc077d7b6f9..108412d2a845 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Explore/ExploreQuickFilters.test.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Explore/ExploreQuickFilters.test.tsx @@ -20,10 +20,7 @@ import { ExploreQuickFilterField } from './ExplorePage.interface'; import ExploreQuickFilters from './ExploreQuickFilters'; import { mockAdvancedFieldDefaultOptions, - mockAdvancedFieldOptions, mockAggregations, - mockTagSuggestions, - mockUserSuggestions, } from './mocks/ExploreQuickFilters.mock'; jest.mock('react-router-dom', () => ({ @@ -79,15 +76,6 @@ jest.mock('../../rest/miscAPI', () => ({ getAggregateFieldOptions: jest .fn() .mockImplementation(() => Promise.resolve(mockAdvancedFieldDefaultOptions)), - getAdvancedFieldOptions: jest - .fn() - .mockImplementation(() => Promise.resolve(mockAdvancedFieldOptions)), - getTagSuggestions: jest - .fn() - .mockImplementation(() => Promise.resolve(mockTagSuggestions)), - getUserSuggestions: jest - .fn() - .mockImplementation(() => Promise.resolve(mockUserSuggestions)), })); const index = SearchIndex.TABLE; diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Glossary/AddGlossary/AddGlossary.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Glossary/AddGlossary/AddGlossary.component.tsx index 5dce27be5a59..7675075c97d5 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Glossary/AddGlossary/AddGlossary.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Glossary/AddGlossary/AddGlossary.component.tsx @@ -29,7 +29,8 @@ import { } from '../../../interface/FormUtils.interface'; import { getEntityName } from '../../../utils/EntityUtils'; import { generateFormFields, getField } from '../../../utils/formUtils'; -import { useAuthContext } from '../../Auth/AuthProviders/AuthProvider'; + +import { useApplicationStore } from '../../../hooks/useApplicationStore'; import { UserTeam } from '../../common/AssigneeList/AssigneeList.interface'; import ResizablePanels from '../../common/ResizablePanels/ResizablePanels'; import TitleBreadcrumb from '../../common/TitleBreadcrumb/TitleBreadcrumb.component'; @@ -47,7 +48,7 @@ const AddGlossary = ({ }: AddGlossaryProps) => { const { t } = useTranslation(); const [form] = useForm(); - const { currentUser } = useAuthContext(); + const { currentUser } = useApplicationStore(); const selectedOwner = Form.useWatch( 'owner', diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Glossary/AddGlossaryTermForm/AddGlossaryTermForm.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Glossary/AddGlossaryTermForm/AddGlossaryTermForm.component.tsx index 73e10d746792..fa9bb17db5f8 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Glossary/AddGlossaryTermForm/AddGlossaryTermForm.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Glossary/AddGlossaryTermForm/AddGlossaryTermForm.component.tsx @@ -30,7 +30,8 @@ import { import { getEntityName } from '../../../utils/EntityUtils'; import { generateFormFields, getField } from '../../../utils/formUtils'; import { fetchGlossaryList } from '../../../utils/TagsUtils'; -import { useAuthContext } from '../../Auth/AuthProviders/AuthProvider'; + +import { useApplicationStore } from '../../../hooks/useApplicationStore'; import { UserTeam } from '../../common/AssigneeList/AssigneeList.interface'; import { UserTag } from '../../common/UserTag/UserTag.component'; import { UserTagSize } from '../../common/UserTag/UserTag.interface'; @@ -42,7 +43,7 @@ const AddGlossaryTermForm = ({ glossaryTerm, formRef: form, }: AddGlossaryTermFormProps) => { - const { currentUser } = useAuthContext(); + const { currentUser } = useApplicationStore(); const owner = Form.useWatch('owner', form); const reviewersList = Form.useWatch('reviewers', form) ?? []; diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryDetailsRightPanel/GlossaryDetailsRightPanel.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryDetailsRightPanel/GlossaryDetailsRightPanel.component.tsx index 6b018f48981c..e1721245f888 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryDetailsRightPanel/GlossaryDetailsRightPanel.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryDetailsRightPanel/GlossaryDetailsRightPanel.component.tsx @@ -10,7 +10,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import { Button, Col, Row, Space, Typography } from 'antd'; +import { Button, Col, Row, Space, Tooltip, Typography } from 'antd'; import { t } from 'i18next'; import { cloneDeep, includes, isEmpty, isEqual } from 'lodash'; import React, { ReactNode, useCallback, useMemo } from 'react'; @@ -224,13 +224,18 @@ const GlossaryDetailsRightPanel = ({ hasPermission={permissions.EditOwner || permissions.EditAll} owner={selectedData.owner} onUpdate={handleUpdatedOwner}> -
    @@ -272,13 +277,18 @@ const GlossaryDetailsRightPanel = ({ popoverProps={{ placement: 'topLeft' }} selectedUsers={selectedData.reviewers ?? []} onUpdate={handleReviewerSave}> - + + + ) : null } @@ -655,7 +640,6 @@ const AssetsTabs = forwardRef( openKeys, visible, currentPage, - tabs, itemCount, onOpenChange, handleAssetButtonVisibleChange, diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryTerms/tabs/AssetsTabs.interface.ts b/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryTerms/tabs/AssetsTabs.interface.ts index c69acd09c503..aed1d532ae3e 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryTerms/tabs/AssetsTabs.interface.ts +++ b/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryTerms/tabs/AssetsTabs.interface.ts @@ -36,7 +36,7 @@ export interface AssetsTabsProps { isEntityDeleted?: boolean; type?: AssetsOfEntity; queryFilter?: string; - noDataPlaceholder?: boolean | AssetNoDataPlaceholderProps; + noDataPlaceholder?: string | AssetNoDataPlaceholderProps; } export interface AssetNoDataPlaceholderProps { diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryTerms/tabs/GlossaryTermReferences.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryTerms/tabs/GlossaryTermReferences.tsx index f6ba148cae01..8f0dff3c8b71 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryTerms/tabs/GlossaryTermReferences.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryTerms/tabs/GlossaryTermReferences.tsx @@ -11,6 +11,7 @@ * limitations under the License. */ +import Icon from '@ant-design/icons/lib/components/Icon'; import { Button, Space, Tag, Tooltip, Typography } from 'antd'; import classNames from 'classnames'; import { t } from 'i18next'; @@ -21,6 +22,7 @@ import { ReactComponent as ExternalLinkIcon } from '../../../../assets/svg/exter import { ReactComponent as PlusIcon } from '../../../../assets/svg/plus-primary.svg'; import { DE_ACTIVE_COLOR, + ICON_DIMENSION, NO_DATA_PLACEHOLDER, SUCCESS_COLOR, TEXT_BODY_COLOR, @@ -119,10 +121,11 @@ const GlossaryTermReferences = ({ rel="noopener noreferrer" target="_blank">
    - {ref.name}
    @@ -194,7 +197,9 @@ const GlossaryTermReferences = ({ ); }; diff --git a/openmetadata-ui/src/main/resources/ui/src/components/MyData/RightSidebar/FollowingWidget.test.tsx b/openmetadata-ui/src/main/resources/ui/src/components/MyData/RightSidebar/FollowingWidget.test.tsx index 2983fa5304b7..96bff3c6ee4f 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/MyData/RightSidebar/FollowingWidget.test.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/MyData/RightSidebar/FollowingWidget.test.tsx @@ -14,12 +14,6 @@ import { act, fireEvent, render, screen } from '@testing-library/react'; import React from 'react'; import FollowingWidget from './FollowingWidget'; -jest.mock('../../Auth/AuthProviders/AuthProvider', () => ({ - useAuthContext: jest.fn().mockImplementation(() => ({ - currentUser: { name: 'testUser' }, - })), -})); - jest.mock('react-router-dom', () => ({ Link: jest .fn() diff --git a/openmetadata-ui/src/main/resources/ui/src/components/MyData/RightSidebar/FollowingWidget.tsx b/openmetadata-ui/src/main/resources/ui/src/components/MyData/RightSidebar/FollowingWidget.tsx index a0ae16fc0b90..562c377db3e0 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/MyData/RightSidebar/FollowingWidget.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/MyData/RightSidebar/FollowingWidget.tsx @@ -22,7 +22,8 @@ import { FOLLOW_DATA_ASSET } from '../../../constants/docs.constants'; import { ERROR_PLACEHOLDER_TYPE, SIZE } from '../../../enums/common.enum'; import { EntityReference } from '../../../generated/entity/type'; import { WidgetCommonProps } from '../../../pages/CustomizablePage/CustomizablePage.interface'; -import { useAuthContext } from '../../Auth/AuthProviders/AuthProvider'; + +import { useApplicationStore } from '../../../hooks/useApplicationStore'; import ErrorPlaceHolder from '../../common/ErrorWithPlaceholder/ErrorPlaceHolder'; import { EntityListWithV1 } from '../../Entity/EntityList/EntityList'; import './following-widget.less'; @@ -42,7 +43,7 @@ function FollowingWidget({ widgetKey, }: Readonly) { const { t } = useTranslation(); - const { currentUser } = useAuthContext(); + const { currentUser } = useApplicationStore(); const handleCloseClick = useCallback(() => { !isUndefined(handleRemoveWidget) && handleRemoveWidget(widgetKey); diff --git a/openmetadata-ui/src/main/resources/ui/src/components/MyData/WelcomeScreen/WelcomeScreen.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/MyData/WelcomeScreen/WelcomeScreen.component.tsx index 720d365ef548..f7e7d97c7f9a 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/MyData/WelcomeScreen/WelcomeScreen.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/MyData/WelcomeScreen/WelcomeScreen.component.tsx @@ -19,8 +19,8 @@ import WelcomeScreenImg from '../../../assets/img/welcome-screen.png'; import { ReactComponent as CloseIcon } from '../../../assets/svg/close.svg'; import { ReactComponent as LineArrowRight } from '../../../assets/svg/line-arrow-right.svg'; import { ROUTES } from '../../../constants/constants'; +import { useApplicationStore } from '../../../hooks/useApplicationStore'; import { getEntityName } from '../../../utils/EntityUtils'; -import { useAuthContext } from '../../Auth/AuthProviders/AuthProvider'; import './welcome-screen.style.less'; const { Paragraph, Text } = Typography; @@ -31,7 +31,7 @@ interface WelcomeScreenProps { const WelcomeScreen = ({ onClose }: WelcomeScreenProps) => { const { t } = useTranslation(); - const { currentUser } = useAuthContext(); + const { currentUser } = useApplicationStore(); const userName = useMemo(() => { return split(getEntityName(currentUser), ' ')[0]; diff --git a/openmetadata-ui/src/main/resources/ui/src/components/MyData/Widgets/FeedsWidget/FeedsWidget.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/MyData/Widgets/FeedsWidget/FeedsWidget.component.tsx index 2114c05bafc7..18d50ef27952 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/MyData/Widgets/FeedsWidget/FeedsWidget.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/MyData/Widgets/FeedsWidget/FeedsWidget.component.tsx @@ -27,6 +27,7 @@ import { ThreadTaskStatus, ThreadType, } from '../../../../generated/entity/feed/thread'; +import { useApplicationStore } from '../../../../hooks/useApplicationStore'; import { FeedCounts } from '../../../../interface/feed.interface'; import { WidgetCommonProps } from '../../../../pages/CustomizablePage/CustomizablePage.interface'; import { getFeedCount } from '../../../../rest/feedsAPI'; @@ -40,7 +41,6 @@ import { showErrorToast } from '../../../../utils/ToastUtils'; import ActivityFeedListV1 from '../../../ActivityFeed/ActivityFeedList/ActivityFeedListV1.component'; import { useActivityFeedProvider } from '../../../ActivityFeed/ActivityFeedProvider/ActivityFeedProvider'; import { ActivityFeedTabs } from '../../../ActivityFeed/ActivityFeedTab/ActivityFeedTab.interface'; -import { useAuthContext } from '../../../Auth/AuthProviders/AuthProvider'; import FeedsFilterPopover from '../../../common/FeedsFilterPopover/FeedsFilterPopover.component'; import './feeds-widget.less'; @@ -52,7 +52,7 @@ const FeedsWidget = ({ const { t } = useTranslation(); const history = useHistory(); const { isTourOpen } = useTourProvider(); - const { currentUser } = useAuthContext(); + const { currentUser } = useApplicationStore(); const [activeTab, setActiveTab] = useState( ActivityFeedTabs.ALL ); diff --git a/openmetadata-ui/src/main/resources/ui/src/components/NavBar/NavBar.interface.ts b/openmetadata-ui/src/main/resources/ui/src/components/NavBar/NavBar.interface.ts index f3888b46793c..8d82451ddd6b 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/NavBar/NavBar.interface.ts +++ b/openmetadata-ui/src/main/resources/ui/src/components/NavBar/NavBar.interface.ts @@ -11,7 +11,6 @@ * limitations under the License. */ -import { MenuProps } from 'antd'; import { ReactNode } from 'react'; export interface DropdownOption { @@ -24,14 +23,11 @@ export interface DropdownOption { } export interface NavBarProps { - supportDropdown: MenuProps['items']; searchValue: string; isTourRoute?: boolean; - isFeatureModalOpen: boolean; pathname: string; isSearchBoxOpen: boolean; handleSearchBoxOpen: (value: boolean) => void; - handleFeatureModal: (value: boolean) => void; handleSearchChange: (value: string) => void; handleOnClick: () => void; handleClear: () => void; diff --git a/openmetadata-ui/src/main/resources/ui/src/components/NavBar/NavBar.test.tsx b/openmetadata-ui/src/main/resources/ui/src/components/NavBar/NavBar.test.tsx new file mode 100644 index 000000000000..3f335b2e4ad9 --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/components/NavBar/NavBar.test.tsx @@ -0,0 +1,229 @@ +/* + * Copyright 2024 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { act, fireEvent, render, screen } from '@testing-library/react'; +import React from 'react'; +import { HELP_ITEMS_ENUM } from '../../constants/Navbar.constants'; +import { getVersion } from '../../rest/miscAPI'; +import { getHelpDropdownItems } from '../../utils/NavbarUtils'; +import { mockUserData } from '../Settings/Users/mocks/User.mocks'; +import NavBar from './NavBar'; + +const mockHandleSearchBoxOpen = jest.fn(); +const mockHandleSearchChange = jest.fn(); +const mockHandleOnClick = jest.fn(); +const mockHandleKeyDown = jest.fn(); +const mockHandleClear = jest.fn(); +const mockProps = { + searchValue: 'searchValue', + isTourRoute: false, + pathname: '', + isSearchBoxOpen: false, + handleSearchBoxOpen: mockHandleSearchBoxOpen, + handleSearchChange: mockHandleSearchChange, + handleOnClick: mockHandleOnClick, + handleClear: mockHandleClear, + handleKeyDown: mockHandleKeyDown, +}; +jest.mock('../../context/GlobalSearchProvider/GlobalSearchProvider', () => ({ + useGlobalSearchProvider: jest.fn().mockImplementation(() => ({ + searchCriteria: '', + updateSearchCriteria: jest.fn(), + })), +})); +jest.mock('../../context/WebSocketProvider/WebSocketProvider', () => ({ + useWebSocketConnector: jest.fn().mockImplementation(() => ({ + socket: { + on: jest.fn(), + off: jest.fn(), + }, + })), +})); +jest.mock('../../utils/BrowserNotificationUtils', () => ({ + hasNotificationPermission: jest.fn(), + shouldRequestPermission: jest.fn(), +})); +jest.mock('../../utils/CommonUtils', () => ({ + refreshPage: jest.fn(), + getEntityDetailLink: jest.fn(), +})); +jest.mock('../../utils/FeedUtils', () => ({ + getEntityFQN: jest.fn().mockReturnValue('entityFQN'), + getEntityType: jest.fn().mockReturnValue('entityType'), + prepareFeedLink: jest.fn().mockReturnValue('entity-link'), +})); +jest.mock('../Domain/DomainProvider/DomainProvider', () => ({ + useDomainProvider: jest.fn().mockImplementation(() => ({ + domainOptions: jest.fn().mockReturnValue('domainOptions'), + activeDomain: jest.fn().mockReturnValue('activeDomain'), + updateActiveDomain: jest.fn(), + })), +})); +jest.mock('../Modals/WhatsNewModal/WhatsNewModal', () => { + return jest + .fn() + .mockImplementation(() => ( +

    WhatsNewModal

    + )); +}); + +jest.mock('../NotificationBox/NotificationBox.component', () => { + return jest.fn().mockImplementation(({ onTabChange }) => ( +
    + tab change +
    + )); +}); + +jest.mock( + '../Settings/Users/UserProfileIcon/UserProfileIcon.component', + () => ({ + UserProfileIcon: jest + .fn() + .mockReturnValue( +
    UserProfileIcon
    + ), + }) +); +jest.mock('../Auth/AuthProviders/AuthProvider', () => ({ + useAuthContext: jest.fn(() => ({ + currentUser: mockUserData, + })), +})); +jest.mock('react-router-dom', () => ({ + useLocation: jest + .fn() + .mockReturnValue({ search: 'search', pathname: '/my-data' }), + useHistory: jest.fn(), +})); + +jest.mock('../common/CmdKIcon/CmdKIcon.component', () => { + return jest.fn().mockReturnValue(
    CmdKIcon
    ); +}); +jest.mock('../AppBar/SearchOptions', () => { + return jest.fn().mockReturnValue(
    SearchOptions
    ); +}); +jest.mock('../AppBar/Suggestions', () => { + return jest.fn().mockReturnValue(
    Suggestions
    ); +}); +jest.mock('antd', () => ({ + ...jest.requireActual('antd'), + + Dropdown: jest.fn().mockImplementation(({ dropdownRender }) => { + return ( +
    +
    {dropdownRender}
    +
    + ); + }), +})); + +jest.mock('../../rest/miscAPI', () => ({ + getVersion: jest.fn().mockImplementation(() => + Promise.resolve({ + data: { + version: '0.5.0-SNAPSHOT', + }, + }) + ), +})); + +jest.mock('../../utils/NavbarUtils', () => ({ + getHelpDropdownItems: jest.fn().mockReturnValue([ + { + label:

    Whats New

    , + key: HELP_ITEMS_ENUM.WHATS_NEW, + }, + ]), +})); + +describe('Test NavBar Component', () => { + it('Should render NavBar component', async () => { + render(); + + expect( + await screen.findByTestId('navbar-search-container') + ).toBeInTheDocument(); + expect( + await screen.findByTestId('global-search-selector') + ).toBeInTheDocument(); + expect(await screen.findByTestId('searchBox')).toBeInTheDocument(); + expect(await screen.findByTestId('cmd')).toBeInTheDocument(); + expect(await screen.findByTestId('user-profile-icon')).toBeInTheDocument(); + expect( + await screen.findByTestId('whats-new-alert-card') + ).toBeInTheDocument(); + expect( + await screen.findByTestId('whats-new-alert-header') + ).toBeInTheDocument(); + expect( + await screen.findByTestId('close-whats-new-alert') + ).toBeInTheDocument(); + expect( + await screen.findByText('label.whats-new-version') + ).toBeInTheDocument(); + }); + + it('should call getVersion onMount', () => { + render(); + + expect(getVersion).toHaveBeenCalled(); + }); + + it('should handle search box open', () => { + render(); + const searchBox = screen.getByTestId('searchBox'); + fireEvent.click(searchBox); + + expect(mockHandleSearchBoxOpen).toHaveBeenCalled(); + }); + + it('should handle search change', () => { + render(); + const searchBox = screen.getByTestId('searchBox'); + fireEvent.change(searchBox, { target: { value: 'test' } }); + + expect(mockHandleSearchChange).toHaveBeenCalledWith('test'); + }); + + it('should handle key down', () => { + render(); + const searchBox = screen.getByTestId('searchBox'); + fireEvent.keyDown(searchBox, { key: 'Enter', code: 'Enter' }); + + expect(mockHandleKeyDown).toHaveBeenCalled(); + }); + + it('should render cancel icon', () => { + render(); + const searchBox = screen.getByTestId('searchBox'); + fireEvent.keyDown(searchBox, { key: 'Enter', code: 'Enter' }); + + expect(mockHandleKeyDown).toHaveBeenCalled(); + }); + + it('should call function on icon search', async () => { + render(); + const searchBox = await screen.findByTestId('search-icon'); + await act(async () => { + fireEvent.click(searchBox); + }); + + expect(mockHandleOnClick).toHaveBeenCalled(); + }); + + it('should call getHelpDropdownItems function', async () => { + render(); + + expect(getHelpDropdownItems).toHaveBeenCalled(); + }); +}); diff --git a/openmetadata-ui/src/main/resources/ui/src/components/NavBar/NavBar.tsx b/openmetadata-ui/src/main/resources/ui/src/components/NavBar/NavBar.tsx index 9edeeadabd72..c2a02d81a048 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/NavBar/NavBar.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/NavBar/NavBar.tsx @@ -22,10 +22,14 @@ import { Row, Select, Space, + Tooltip, } from 'antd'; +import { AxiosError } from 'axios'; +import classNames from 'classnames'; import { CookieStorage } from 'cookie-storage'; import i18next from 'i18next'; import { debounce, upperCase } from 'lodash'; +import { MenuInfo } from 'rc-menu/lib/interface'; import React, { useCallback, useEffect, @@ -42,14 +46,15 @@ import { ReactComponent as DomainIcon } from '../../assets/svg/ic-domain.svg'; import { ReactComponent as Help } from '../../assets/svg/ic-help.svg'; import { ReactComponent as IconSearch } from '../../assets/svg/search.svg'; -import classNames from 'classnames'; import { NOTIFICATION_READ_TIMER, SOCKET_EVENTS, } from '../../constants/constants'; +import { HELP_ITEMS_ENUM } from '../../constants/Navbar.constants'; import { useGlobalSearchProvider } from '../../context/GlobalSearchProvider/GlobalSearchProvider'; import { useWebSocketConnector } from '../../context/WebSocketProvider/WebSocketProvider'; import { EntityTabs, EntityType } from '../../enums/entity.enum'; +import { getVersion } from '../../rest/miscAPI'; import brandImageClassBase from '../../utils/BrandImage/BrandImageClassBase'; import { hasNotificationPermission, @@ -66,11 +71,13 @@ import { SupportedLocales, } from '../../utils/i18next/i18nextUtil'; import { isCommandKeyPress, Keys } from '../../utils/KeyboardUtil'; +import { getHelpDropdownItems } from '../../utils/NavbarUtils'; import { inPageSearchOptions, isInPageSearchAllowed, } from '../../utils/RouterUtils'; import searchClassBase from '../../utils/SearchClassBase'; +import { showErrorToast } from '../../utils/ToastUtils'; import { ActivityFeedTabs } from '../ActivityFeed/ActivityFeedTab/ActivityFeedTab.interface'; import SearchOptions from '../AppBar/SearchOptions'; import Suggestions from '../AppBar/Suggestions'; @@ -86,14 +93,11 @@ import popupAlertsCardsClassBase from './PopupAlertClassBase'; const cookieStorage = new CookieStorage(); const NavBar = ({ - supportDropdown, searchValue, - isFeatureModalOpen, isTourRoute = false, pathname, isSearchBoxOpen, handleSearchBoxOpen, - handleFeatureModal, handleSearchChange, handleKeyDown, handleOnClick, @@ -116,6 +120,22 @@ const NavBar = ({ const [hasMentionNotification, setHasMentionNotification] = useState(false); const [activeTab, setActiveTab] = useState('Task'); + const [isFeatureModalOpen, setIsFeatureModalOpen] = useState(false); + const [version, setVersion] = useState(); + + const fetchOMVersion = async () => { + try { + const res = await getVersion(); + setVersion(res.version); + } catch (err) { + showErrorToast( + err as AxiosError, + t('server.entity-fetch-error', { + entity: t('label.version'), + }) + ); + } + }; const renderAlertCards = useMemo(() => { const cardList = popupAlertsCardsClassBase.alertsCards(); @@ -127,6 +147,12 @@ const NavBar = ({ }); }, []); + const handleSupportClick = ({ key }: MenuInfo): void => { + if (key === HELP_ITEMS_ENUM.WHATS_NEW) { + setIsFeatureModalOpen(true); + } + }; + const entitiesSelect = useMemo( () => (
    + ); + }, + [tableColumn] + ); + return ( <> {successContext?.stats && statsRender(successContext?.stats.jobStats)} {failureContext?.stats && statsRender(failureContext?.stats.jobStats)} + + {successContext?.stats?.entityStats && + entityStatsRenderer(successContext.stats.entityStats)} + {failureContext?.stats?.entityStats && + entityStatsRenderer(failureContext.stats.entityStats)} + {logsRender( formatJsonString( JSON.stringify( diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Settings/Applications/AppLogsViewer/AppLogsViewer.interface.ts b/openmetadata-ui/src/main/resources/ui/src/components/Settings/Applications/AppLogsViewer/AppLogsViewer.interface.ts index 715262b6a0f0..132933f1cdce 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Settings/Applications/AppLogsViewer/AppLogsViewer.interface.ts +++ b/openmetadata-ui/src/main/resources/ui/src/components/Settings/Applications/AppLogsViewer/AppLogsViewer.interface.ts @@ -11,15 +11,52 @@ * limitations under the License. */ +import { EntityType } from '../../../../enums/entity.enum'; import { AppRunRecord } from '../../../../generated/entity/applications/appRunRecord'; export interface AppLogsViewerProps { data: AppRunRecord; } -export interface JobStats { - totalRecords: string; - successRecords: string; - failedRecords: string; +export interface TotalRecords { + totalRecords: number; + successRecords: number; + failedRecords: number; +} +export interface JobStats extends TotalRecords { processedRecords: string; } + +export type EntityTypeSearchIndex = Exclude< + EntityType, + | EntityType.BOT + | EntityType.ALERT + | EntityType.knowledgePanels + | EntityType.WEBHOOK + | EntityType.USER + | EntityType.ROLE + | EntityType.TEAM + | EntityType.SUBSCRIPTION + | EntityType.POLICY + | EntityType.DATA_INSIGHT_CHART + | EntityType.KPI + | EntityType.TYPE + | EntityType.APP_MARKET_PLACE_DEFINITION + | EntityType.APPLICATION + | EntityType.PERSONA + | EntityType.DOC_STORE + | EntityType.PAGE + | EntityType.SAMPLE_DATA + | EntityType.GOVERN + | EntityType.CUSTOM_METRIC + | EntityType.ALL +>; + +export type EntityStats = Record; + +export interface EntityStatsData { + name: string; + totalRecords: number; + successRecords: number; + failedRecords: number; +} diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Settings/Applications/AppLogsViewer/AppLogsViewer.test.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Settings/Applications/AppLogsViewer/AppLogsViewer.test.tsx index 74619d8da2b1..fec8d841099d 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Settings/Applications/AppLogsViewer/AppLogsViewer.test.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Settings/Applications/AppLogsViewer/AppLogsViewer.test.tsx @@ -14,7 +14,6 @@ import { render, screen } from '@testing-library/react'; import userEvent from '@testing-library/user-event'; import React from 'react'; import { - RunType, ScheduleTimeline, Status, } from '../../../../generated/entity/applications/appRunRecord'; @@ -39,11 +38,28 @@ jest.mock('antd', () => ({ Badge: jest.fn().mockReturnValue(
    Badge
    ), })); +jest.mock('../../../../utils/ApplicationUtils', () => ({ + getEntityStatsData: jest.fn().mockReturnValue([ + { + name: 'chart', + totalRecords: 100, + failedRecords: 10, + successRecords: 90, + }, + ]), +})); + +jest.mock('../../../common/Badge/Badge.component', () => + jest.fn().mockImplementation(({ label }) => { + return
    {`${label}-AppBadge`}
    ; + }) +); + const mockProps1 = { data: { appId: '6e4d3dcf-238d-4874-b4e4-dd863ede6544', status: Status.Success, - runType: RunType.OnDemand, + runType: 'OnDemand', startTime: 1706871884587, endTime: 1706871891251, timestamp: 1706871884587, @@ -54,11 +70,10 @@ const mockProps1 = { failedRecords: 0, successRecords: 274, }, - entityStats: {}, }, }, scheduleInfo: { - scheduleType: ScheduleTimeline.Custom, + scheduleTimeline: ScheduleTimeline.Custom, cronExpression: '0 0 0 1/1 * ? *', }, id: '6e4d3dcf-238d-4874-b4e4-dd863ede6544-OnDemand-1706871884587', @@ -77,12 +92,68 @@ const mockProps2 = { failedRecords: 0, successRecords: 274, }, - entityStats: {}, }, }, }, }; +const mockProps3 = { + data: { + ...mockProps1.data, + successContext: { + stats: { + jobStats: { + totalRecords: 274, + failedRecords: 4, + successRecords: 270, + }, + entityStats: { + chart: { + totalRecords: 100, + failedRecords: 10, + successRecords: 90, + }, + }, + }, + }, + }, +}; + +const mockProps4 = { + data: { + ...mockProps1.data, + successContext: undefined, + failureContext: { + stats: { + jobStats: { + totalRecords: 274, + failedRecords: 4, + successRecords: 270, + }, + entityStats: { + chart: { + totalRecords: 100, + failedRecords: 10, + successRecords: 90, + }, + }, + }, + }, + }, +}; + +const mockProps5 = { + data: { + ...mockProps1.data, + successContext: { + stats: null, + }, + failureContext: { + stats: null, + }, + }, +}; + describe('AppLogsViewer component', () => { it('should contain all necessary elements', () => { render(); @@ -105,4 +176,60 @@ describe('AppLogsViewer component', () => { expect(screen.getByText('--')).toBeInTheDocument(); // Note: not asserting other elements as for failure also same elements will render }); + + it("should not render entity stats table based if successContext doesn't have data", () => { + render(); + + expect( + screen.queryByTestId('app-entity-stats-history-table') + ).not.toBeInTheDocument(); + }); + + it('should render entity stats table based if SuccessContext has data', () => { + render(); + + expect( + screen.getByTestId('app-entity-stats-history-table') + ).toBeInTheDocument(); + + expect(screen.getByText('label.name')).toBeInTheDocument(); + + expect(screen.getAllByTestId('app-badge')).toHaveLength(3); + expect(screen.getByText('274-AppBadge')).toBeInTheDocument(); + expect(screen.getByText('270-AppBadge')).toBeInTheDocument(); + expect(screen.getByText('4-AppBadge')).toBeInTheDocument(); + }); + + it("should not render entity stats table based if failedContext doesn't have data", () => { + render(); + + expect( + screen.queryByTestId('app-entity-stats-history-table') + ).not.toBeInTheDocument(); + }); + + it('should render entity stats table based if failedContext has data', () => { + render(); + + expect( + screen.getByTestId('app-entity-stats-history-table') + ).toBeInTheDocument(); + + expect(screen.getByText('label.name')).toBeInTheDocument(); + + expect(screen.getAllByTestId('app-badge')).toHaveLength(3); + expect(screen.getByText('274-AppBadge')).toBeInTheDocument(); + expect(screen.getByText('270-AppBadge')).toBeInTheDocument(); + expect(screen.getByText('4-AppBadge')).toBeInTheDocument(); + }); + + it('should not render stats and entityStats component if successContext and failureContext stats is empty', () => { + render(); + + expect(screen.queryByTestId('stats-component')).not.toBeInTheDocument(); + + expect( + screen.queryByTestId('app-entity-stats-history-table') + ).not.toBeInTheDocument(); + }); }); diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Settings/Applications/AppLogsViewer/app-logs-viewer.less b/openmetadata-ui/src/main/resources/ui/src/components/Settings/Applications/AppLogsViewer/app-logs-viewer.less new file mode 100644 index 000000000000..70137ec19a99 --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/components/Settings/Applications/AppLogsViewer/app-logs-viewer.less @@ -0,0 +1,44 @@ +/* + * Copyright 2024 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +@import (reference) url('../../../../styles/variables.less'); + +.entity-stats { + width: max-content; + + .ant-typography { + font-weight: 500; + } + + &.total { + background-color: @primary-color-hover; + border: 1px solid @primary-color; + .ant-typography { + color: @primary-color; + } + } + &.success { + background-color: @green-2; + border: 1px solid @success-color; + .ant-typography { + color: @success-color; + } + } + &.failure { + background-color: @error-light-color; + border: 1px solid @failed-color; + .ant-typography { + color: @failed-color; + } + } +} diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Settings/Applications/AppRunsHistory/AppRunsHistory.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Settings/Applications/AppRunsHistory/AppRunsHistory.component.tsx index 0b9a07bad5a7..a1bdd3bcb2f2 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Settings/Applications/AppRunsHistory/AppRunsHistory.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Settings/Applications/AppRunsHistory/AppRunsHistory.component.tsx @@ -152,7 +152,7 @@ const AppRunsHistory = forwardRef( return NO_DATA_PLACEHOLDER; } }, - [showLogAction, appData, isExternalApp] + [showLogAction, appData, isExternalApp, handleRowExpandable] ); const tableColumn: ColumnsType = useMemo( diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Settings/Applications/AppSchedule/AppSchedule.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Settings/Applications/AppSchedule/AppSchedule.component.tsx index b8402f76671d..ac7a7270c35c 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Settings/Applications/AppSchedule/AppSchedule.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Settings/Applications/AppSchedule/AppSchedule.component.tsx @@ -12,6 +12,7 @@ */ import { Button, Col, Divider, Modal, Row, Space, Typography } from 'antd'; import cronstrue from 'cronstrue'; +import { isEmpty } from 'lodash'; import React, { useCallback, useEffect, @@ -68,11 +69,10 @@ const AppSchedule = ({ }, [appData]); const cronString = useMemo(() => { - if (appData.appSchedule) { - const cronExp = - (appData.appSchedule as AppScheduleClass).cronExpression ?? ''; - - return cronstrue.toString(cronExp, { + const cronExpression = (appData.appSchedule as AppScheduleClass) + ?.cronExpression; + if (cronExpression) { + return cronstrue.toString(cronExpression, { throwExceptionOnParseError: false, }); } @@ -150,19 +150,18 @@ const AppSchedule = ({     {appData.appSchedule && ( <> -
    - - - {t('label.schedule-type')} - - - {(appData.appSchedule as AppScheduleClass).scheduleType ?? - ''} - - +
    + + {t('label.schedule-type')} + + + {(appData.appSchedule as AppScheduleClass).scheduleTimeline ?? + ''} +
    -
    - + + {!isEmpty(cronString) && ( +
    {t('label.schedule-interval')} @@ -171,8 +170,8 @@ const AppSchedule = ({ data-testid="cron-string"> {cronString} - -
    +
    + )} )} @@ -231,7 +230,7 @@ const AppSchedule = ({ }} includePeriodOptions={initialOptions} initialData={ - (appData.appSchedule as AppScheduleClass)?.cronExpression ?? '' + (appData.appSchedule as AppScheduleClass)?.cronExpression } isLoading={isSaveLoading} onCancel={onDialogCancel} diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Settings/Applications/ApplicationsProvider/ApplicationsProvider.interface.ts b/openmetadata-ui/src/main/resources/ui/src/components/Settings/Applications/ApplicationsProvider/ApplicationsProvider.interface.ts new file mode 100644 index 000000000000..2e21220e2b27 --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/components/Settings/Applications/ApplicationsProvider/ApplicationsProvider.interface.ts @@ -0,0 +1,18 @@ +/* + * Copyright 2024 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { App } from '../../../../generated/entity/applications/app'; + +export type ApplicationsContextType = { + applications: App[]; + loading: boolean; +}; diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Settings/Applications/ApplicationsProvider/ApplicationsProvider.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Settings/Applications/ApplicationsProvider/ApplicationsProvider.tsx new file mode 100644 index 000000000000..16ca60af4cca --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/components/Settings/Applications/ApplicationsProvider/ApplicationsProvider.tsx @@ -0,0 +1,69 @@ +/* + * Copyright 2024 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { isEmpty } from 'lodash'; +import React, { + createContext, + ReactNode, + useCallback, + useContext, + useEffect, + useMemo, + useState, +} from 'react'; +import { usePermissionProvider } from '../../../../context/PermissionProvider/PermissionProvider'; +import { App } from '../../../../generated/entity/applications/app'; +import { getApplicationList } from '../../../../rest/applicationAPI'; +import { ApplicationsContextType } from './ApplicationsProvider.interface'; + +export const ApplicationsContext = createContext({} as ApplicationsContextType); + +export const ApplicationsProvider = ({ children }: { children: ReactNode }) => { + const [applications, setApplications] = useState([]); + const [loading, setLoading] = useState(false); + const { permissions } = usePermissionProvider(); + + const fetchApplicationList = useCallback(async () => { + try { + setLoading(true); + const { data } = await getApplicationList({ + limit: 100, + }); + + setApplications(data); + } catch (err) { + // do not handle error + } finally { + setLoading(false); + } + }, []); + + useEffect(() => { + if (!isEmpty(permissions)) { + fetchApplicationList(); + } + }, [permissions]); + + const appContext = useMemo(() => { + return { applications, loading }; + }, [applications, loading]); + + return ( + + {children} + + ); +}; + +export const useApplicationsProvider = () => useContext(ApplicationsContext); + +export default ApplicationsProvider; diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Settings/Applications/MarketPlaceAppDetails/MarketPlaceAppDetails.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Settings/Applications/MarketPlaceAppDetails/MarketPlaceAppDetails.component.tsx index 804f3ccb0c9d..d916d1d450b3 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Settings/Applications/MarketPlaceAppDetails/MarketPlaceAppDetails.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Settings/Applications/MarketPlaceAppDetails/MarketPlaceAppDetails.component.tsx @@ -11,7 +11,16 @@ * limitations under the License. */ import { LeftOutlined } from '@ant-design/icons'; -import { Button, Carousel, Col, Row, Space, Tooltip, Typography } from 'antd'; +import { + Alert, + Button, + Carousel, + Col, + Row, + Space, + Tooltip, + Typography, +} from 'antd'; import { AxiosError } from 'axios'; import { uniqueId } from 'lodash'; import React, { useCallback, useEffect, useMemo, useState } from 'react'; @@ -24,12 +33,14 @@ import { Include } from '../../../../generated/type/include'; import { useFqn } from '../../../../hooks/useFqn'; import { getApplicationByName } from '../../../../rest/applicationAPI'; import { getMarketPlaceApplicationByFqn } from '../../../../rest/applicationMarketPlaceAPI'; +import { Transi18next } from '../../../../utils/CommonUtils'; import { getEntityName } from '../../../../utils/EntityUtils'; import { getAppInstallPath } from '../../../../utils/RouterUtils'; import { showErrorToast } from '../../../../utils/ToastUtils'; import Loader from '../../../common/Loader/Loader'; import RichTextEditorPreviewer from '../../../common/RichTextEditor/RichTextEditorPreviewer'; import PageLayoutV1 from '../../../PageLayoutV1/PageLayoutV1'; +import applicationsClassBase from '../AppDetails/ApplicationsClassBase'; import AppLogo from '../AppLogo/AppLogo.component'; import './market-place-app-details.less'; @@ -42,10 +53,12 @@ const MarketPlaceAppDetails = () => { const [isInstalled, setIsInstalled] = useState(false); const [appScreenshots, setAppScreenshots] = useState([]); + const isPreviewApp = useMemo(() => !!appData?.preview, [appData]); + const loadScreenshot = async (screenshotName: string) => { try { - const imageModule = await import( - `../../../../assets/img/appScreenshots/${screenshotName}` + const imageModule = await applicationsClassBase.importAppScreenshot( + screenshotName ); const imageSrc = imageModule.default; @@ -106,6 +119,27 @@ const MarketPlaceAppDetails = () => { history.push(ROUTES.MARKETPLACE); }; + const tooltipTitle = useMemo(() => { + if (isInstalled) { + return t('message.app-already-installed'); + } + if (isPreviewApp) { + return ( + + } + values={{ + app: appData?.displayName, + }} + /> + ); + } + + return ''; + }, [isInstalled, isPreviewApp, appData?.displayName]); + const leftPanel = useMemo(() => { return (
    @@ -119,32 +153,52 @@ const MarketPlaceAppDetails = () => { {t('label.browse-app-plural')} -
    - + + {isPreviewApp && ( + + + + } + values={{ + app: appData?.displayName, + }} + /> + + + + {t('message.please-contact-us')} + + + } + type="info" + /> + )}
    {t('message.marketplace-verify-msg')}
    - {appData?.supportEmail && ( @@ -166,7 +220,7 @@ const MarketPlaceAppDetails = () => {
    ); - }, [appData, isInstalled]); + }, [appData, isInstalled, tooltipTitle]); useEffect(() => { fetchAppDetails(); diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Settings/Applications/MarketPlaceAppDetails/MarketPlaceAppDetails.test.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Settings/Applications/MarketPlaceAppDetails/MarketPlaceAppDetails.test.tsx index 516e4846cca9..d3a4bdb6fc3e 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Settings/Applications/MarketPlaceAppDetails/MarketPlaceAppDetails.test.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Settings/Applications/MarketPlaceAppDetails/MarketPlaceAppDetails.test.tsx @@ -11,27 +11,28 @@ * limitations under the License. */ import { + queryByTestId, render, screen, waitForElementToBeRemoved, } from '@testing-library/react'; import userEvent from '@testing-library/user-event'; import React from 'react'; -// import { ROUTES } from '../../../constants/constants'; import { ROUTES } from '../../../../constants/constants'; import { mockApplicationData } from '../../../../mocks/rests/applicationAPI.mock'; import MarketPlaceAppDetails from './MarketPlaceAppDetails.component'; const mockPush = jest.fn(); const mockShowErrorToast = jest.fn(); -const mockGetApplicationByName = jest.fn().mockReturnValue(mockApplicationData); -const mockGetMarketPlaceApplicationByFqn = jest.fn().mockReturnValue({ +let mockGetApplicationByName = jest.fn().mockReturnValue(mockApplicationData); +let mockGetMarketPlaceApplicationByFqn = jest.fn().mockReturnValue({ description: 'marketplace description', fullyQualifiedName: 'marketplace fqn', supportEmail: 'support@email.com', developerUrl: 'https://xyz.com', privacyPolicyUrl: 'https://xyz.com', appScreenshots: ['screenshot1', 'screenshot2'], + preview: false, }); jest.mock('react-router-dom', () => ({ @@ -95,7 +96,7 @@ jest.mock('../AppLogo/AppLogo.component', () => describe('MarketPlaceAppDetails component', () => { it('should render all necessary elements if app details fetch successfully', async () => { - render(); + const { container } = render(); await waitForElementToBeRemoved(() => screen.getByText('Loader')); @@ -116,6 +117,10 @@ describe('MarketPlaceAppDetails component', () => { expect(screen.getByText('label.privacy-policy')).toBeInTheDocument(); expect(screen.getByText('label.get-app-support')).toBeInTheDocument(); + const appName = queryByTestId(container, 'appName'); + + expect(appName).not.toBeInTheDocument(); + // actions check userEvent.click( screen.getByRole('button', { name: 'left label.browse-app-plural' }) @@ -124,6 +129,31 @@ describe('MarketPlaceAppDetails component', () => { expect(mockPush).toHaveBeenCalledWith(ROUTES.MARKETPLACE); }); + it('should show install button disabled', async () => { + mockGetApplicationByName = jest.fn().mockReturnValue([]); + mockGetMarketPlaceApplicationByFqn = jest.fn().mockReturnValue({ + description: 'marketplace description', + fullyQualifiedName: 'marketplace fqn', + supportEmail: 'support@email.com', + developerUrl: 'https://xyz.com', + privacyPolicyUrl: 'https://xyz.com', + appScreenshots: ['screenshot1', 'screenshot2'], + preview: true, + }); + + render(); + + await waitForElementToBeRemoved(() => screen.getByText('Loader')); + + expect(mockGetMarketPlaceApplicationByFqn).toHaveBeenCalledWith('mockFQN', { + fields: expect.anything(), + }); + expect(mockGetApplicationByName).toHaveBeenCalled(); + + expect(screen.getByTestId('install-application')).toBeDisabled(); + expect(screen.getByTestId('appName')).toBeInTheDocument(); + }); + it('should show toast error, if failed to fetch app details', async () => { const MARKETPLACE_APP_DETAILS_ERROR = 'marketplace app data fetch failed.'; const APP_DETAILS_ERROR = 'app data fetch failed.'; diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Settings/Bot/BotDetails/BotDetails.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Settings/Bot/BotDetails/BotDetails.component.tsx index 048ebb6af3ce..2f0a9d1e69e0 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Settings/Bot/BotDetails/BotDetails.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Settings/Bot/BotDetails/BotDetails.component.tsx @@ -12,7 +12,16 @@ */ import { CheckOutlined, CloseOutlined } from '@ant-design/icons'; -import { Button, Card, Col, Input, Row, Space, Typography } from 'antd'; +import { + Button, + Card, + Col, + Input, + Row, + Space, + Tooltip, + Typography, +} from 'antd'; import { AxiosError } from 'axios'; import { toLower } from 'lodash'; import React, { FC, useEffect, useMemo, useState } from 'react'; @@ -25,7 +34,6 @@ import { getRoles } from '../../../../rest/userAPI'; import { getEntityName } from '../../../../utils/EntityUtils'; import { getSettingPath } from '../../../../utils/RouterUtils'; import { showErrorToast } from '../../../../utils/ToastUtils'; -import Description from '../../../common/EntityDescription/Description'; import InheritedRolesCard from '../../../common/InheritedRolesCard/InheritedRolesCard.component'; import RolesCard from '../../../common/RolesCard/RolesCard.component'; import TitleBreadcrumb from '../../../common/TitleBreadcrumb/TitleBreadcrumb.component'; @@ -34,6 +42,8 @@ import './bot-details.less'; import { BotsDetailProps } from './BotDetails.interfaces'; import { ReactComponent as IconBotProfile } from '../../../../assets/svg/bot-profile.svg'; +import { EntityType } from '../../../../enums/entity.enum'; +import DescriptionV1 from '../../../common/EntityDescription/DescriptionV1'; import AccessTokenCard from '../../Users/AccessTokenCard/AccessTokenCard.component'; const BotDetails: FC = ({ @@ -157,22 +167,29 @@ const BotDetails: FC = ({ )} {(displayNamePermission || editAllPermission) && ( -
    - setIsDescriptionEdit(false)} onDescriptionEdit={() => setIsDescriptionEdit(true)} onDescriptionUpdate={handleDescriptionChange} diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Settings/Bot/BotDetails/BotDetails.test.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Settings/Bot/BotDetails/BotDetails.test.tsx index 45d65b700c38..a0796267a79b 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Settings/Bot/BotDetails/BotDetails.test.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Settings/Bot/BotDetails/BotDetails.test.tsx @@ -111,7 +111,7 @@ jest.mock('../../../../rest/userAPI', () => { }; }); -jest.mock('../../../common/EntityDescription/Description', () => { +jest.mock('../../../common/EntityDescription/DescriptionV1', () => { return jest.fn().mockReturnValue(

    Description Component

    ); }); diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Settings/CustomProperty/AddCustomProperty/AddCustomProperty.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Settings/CustomProperty/AddCustomProperty/AddCustomProperty.tsx index ef7f9f7bba6d..88c9d8ee403f 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Settings/CustomProperty/AddCustomProperty/AddCustomProperty.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Settings/CustomProperty/AddCustomProperty/AddCustomProperty.tsx @@ -14,7 +14,7 @@ import { Button, Col, Form, Row } from 'antd'; import { AxiosError } from 'axios'; import { t } from 'i18next'; -import { isUndefined, map, startCase } from 'lodash'; +import { isUndefined, map, omit, startCase } from 'lodash'; import React, { FocusEvent, useCallback, @@ -40,6 +40,7 @@ import { import { FieldProp, FieldTypes, + FormItemLayout, } from '../../../../interface/FormUtils.interface'; import { addPropertyToEntity, @@ -55,6 +56,7 @@ import ServiceDocPanel from '../../../common/ServiceDocPanel/ServiceDocPanel'; import TitleBreadcrumb from '../../../common/TitleBreadcrumb/TitleBreadcrumb.component'; const AddCustomProperty = () => { + const [form] = Form.useForm(); const { entityType } = useParams<{ entityType: EntityType }>(); const history = useHistory(); @@ -64,6 +66,8 @@ const AddCustomProperty = () => { const [activeField, setActiveField] = useState(''); const [isCreating, setIsCreating] = useState(false); + const watchedPropertyType = Form.useWatch('propertyType', form); + const slashedBreadcrumb = useMemo( () => [ { @@ -99,6 +103,10 @@ const AddCustomProperty = () => { })); }, [propertyTypes]); + const isEnumType = + propertyTypeOptions.find((option) => option.value === watchedPropertyType) + ?.key === 'enum'; + const fetchPropertyType = async () => { try { const response = await getTypeListByCategory(Category.Field); @@ -130,7 +138,15 @@ const AddCustomProperty = () => { * In CustomProperty the propertyType is type of entity reference, however from the form we * get propertyType as string */ - data: Exclude & { propertyType: string } + /** + * In CustomProperty the customPropertyConfig is type of CustomPropertyConfig, however from the + * form we get customPropertyConfig as string[] + */ + data: Exclude & { + propertyType: string; + customPropertyConfig: string[]; + multiSelect?: boolean; + } ) => { if (isUndefined(typeDetail)) { return; @@ -139,11 +155,22 @@ const AddCustomProperty = () => { try { setIsCreating(true); await addPropertyToEntity(typeDetail?.id ?? '', { - ...data, + ...omit(data, 'multiSelect'), propertyType: { id: data.propertyType, type: 'type', }, + // Only add customPropertyConfig if it is an enum type + ...(isEnumType + ? { + customPropertyConfig: { + config: { + multiSelect: Boolean(data?.multiSelect), + values: data.customPropertyConfig, + }, + }, + } + : {}), }); history.goBack(); } catch (error) { @@ -194,18 +221,52 @@ const AddCustomProperty = () => { })}`, }, }, - { - name: 'description', - required: true, - label: t('label.description'), - id: 'root/description', - type: FieldTypes.DESCRIPTION, - props: { - 'data-testid': 'description', - initialValue: '', + ]; + + const descriptionField: FieldProp = { + name: 'description', + required: true, + label: t('label.description'), + id: 'root/description', + type: FieldTypes.DESCRIPTION, + props: { + 'data-testid': 'description', + initialValue: '', + }, + }; + + const customPropertyConfigTypeValueField: FieldProp = { + name: 'customPropertyConfig', + required: false, + label: t('label.enum-value-plural'), + id: 'root/customPropertyConfig', + type: FieldTypes.SELECT, + props: { + 'data-testid': 'customPropertyConfig', + mode: 'tags', + placeholder: t('label.enum-value-plural'), + }, + rules: [ + { + required: true, + message: t('label.field-required', { + field: t('label.enum-value-plural'), + }), }, + ], + }; + + const multiSelectField: FieldProp = { + name: 'multiSelect', + label: t('label.multi-select'), + type: FieldTypes.SWITCH, + required: false, + props: { + 'data-testid': 'multiSelect', }, - ]; + id: 'root/multiSelect', + formItemLayout: FormItemLayout.HORIZONTAL, + }; const firstPanelChildren = (
    @@ -213,10 +274,20 @@ const AddCustomProperty = () => {
    {generateFormFields(formFields)} + {isEnumType && ( + <> + {generateFormFields([ + customPropertyConfigTypeValueField, + multiSelectField, + ])} + + )} + {generateFormFields([descriptionField])}
    - + ) : ( @@ -1119,11 +1119,9 @@ const TeamDetailsV1 = ({ + className="header-collapse-custom-collapse"> @@ -1131,28 +1129,16 @@ const TeamDetailsV1 = ({ - - {t('label.description')} - - {editDescriptionPermission && ( - descriptionHandler(true)} - /> - )} - - }> - + descriptionHandler(false)} + onDescriptionEdit={() => descriptionHandler(true)} onDescriptionUpdate={onDescriptionUpdate} /> diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Settings/Team/TeamDetails/TeamsHeaderSection/TeamsHeadingLabel.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Settings/Team/TeamDetails/TeamsHeaderSection/TeamsHeadingLabel.component.tsx index 40e6c9dbcc05..df508602e77a 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Settings/Team/TeamDetails/TeamsHeaderSection/TeamsHeadingLabel.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Settings/Team/TeamDetails/TeamsHeaderSection/TeamsHeadingLabel.component.tsx @@ -22,9 +22,10 @@ import { useTranslation } from 'react-i18next'; import { ReactComponent as EditIcon } from '../../../../../assets/svg/edit-new.svg'; import { Team } from '../../../../../generated/entity/teams/team'; import { useAuth } from '../../../../../hooks/authHooks'; +import { useApplicationStore } from '../../../../../hooks/useApplicationStore'; import { hasEditAccess } from '../../../../../utils/CommonUtils'; +import { getEntityName } from '../../../../../utils/EntityUtils'; import { showErrorToast } from '../../../../../utils/ToastUtils'; -import { useAuthContext } from '../../../../Auth/AuthProviders/AuthProvider'; import { TeamsHeadingLabelProps } from '../team.interface'; const TeamsHeadingLabel = ({ @@ -39,7 +40,7 @@ const TeamsHeadingLabel = ({ currentTeam ? currentTeam.displayName : '' ); const { isAdminUser } = useAuth(); - const { currentUser } = useAuthContext(); + const { currentUser } = useApplicationStore(); const { owner } = useMemo(() => currentTeam, [currentTeam]); const isCurrentTeamOwner = useMemo( @@ -82,14 +83,16 @@ const TeamsHeadingLabel = ({ }; const handleClose = useCallback(() => { - setHeading(currentTeam ? currentTeam.displayName : ''); + setHeading(currentTeam ? getEntityName(currentTeam) : ''); setIsHeadingEditing(false); - }, [currentTeam.displayName]); + }, [currentTeam]); const teamHeadingRender = useMemo( () => isHeadingEditing ? ( - + // Used onClick stop click propagation event anywhere in the component to parent + // TeamDetailsV1 component collapsible panel + e.stopPropagation()}> setIsHeadingEditing(true)} + onClick={(e) => { + // Used to stop click propagation event to parent TeamDetailV1 collapsible panel + e.stopPropagation(); + setIsHeadingEditing(true); + }} /> )} diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Settings/Team/TeamDetails/TeamsHeaderSection/TeamsHeadingLabel.test.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Settings/Team/TeamDetails/TeamsHeaderSection/TeamsHeadingLabel.test.tsx index 6af0edd62f30..d341e8eb079d 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Settings/Team/TeamDetails/TeamsHeaderSection/TeamsHeadingLabel.test.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Settings/Team/TeamDetails/TeamsHeaderSection/TeamsHeadingLabel.test.tsx @@ -22,8 +22,8 @@ jest.mock('../../../../../hooks/authHooks', () => ({ useAuth: jest.fn().mockReturnValue({ isAdminUser: true }), })); -jest.mock('../../../../Auth/AuthProviders/AuthProvider', () => ({ - useAuthContext: jest.fn().mockReturnValue({ +jest.mock('../../../../../hooks/useApplicationStore', () => ({ + useApplicationStore: jest.fn().mockReturnValue({ currentUser: { userId: 'test-user' }, }), })); diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Settings/Team/TeamDetails/TeamsHeaderSection/TeamsInfo.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Settings/Team/TeamDetails/TeamsHeaderSection/TeamsInfo.component.tsx index 3d9e2745b6ff..43698ac37a4f 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Settings/Team/TeamDetails/TeamsHeaderSection/TeamsInfo.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Settings/Team/TeamDetails/TeamsHeaderSection/TeamsInfo.component.tsx @@ -24,7 +24,7 @@ import { EntityType } from '../../../../../enums/entity.enum'; import { Team, TeamType } from '../../../../../generated/entity/teams/team'; import { EntityReference } from '../../../../../generated/entity/type'; import { useAuth } from '../../../../../hooks/authHooks'; -import { useAuthContext } from '../../../../Auth/AuthProviders/AuthProvider'; +import { useApplicationStore } from '../../../../../hooks/useApplicationStore'; import { DomainLabel } from '../../../../common/DomainLabel/DomainLabel.component'; import { OwnerLabel } from '../../../../common/OwnerLabel/OwnerLabel.component'; import TeamTypeSelect from '../../../../common/TeamTypeSelect/TeamTypeSelect.component'; @@ -48,7 +48,7 @@ const TeamsInfo = ({ const [showTypeSelector, setShowTypeSelector] = useState(false); const [isLoading, setIsLoading] = useState(false); - const { currentUser } = useAuthContext(); + const { currentUser } = useApplicationStore(); const { email, owner, teamType, id, fullyQualifiedName } = useMemo( () => currentTeam, @@ -138,7 +138,12 @@ const TeamsInfo = ({ 'label.email' )} :`} {isEmailEdit ? ( -
    + e.stopPropagation()} + onFinish={onEmailSave}> {hasEditPermission && ( setIsEmailEdit(true)} + onClick={(e) => { + // Used to stop click propagation event to parent TeamDetailV1 collapsible panel + e.stopPropagation(); + setIsEmailEdit(true); + }} /> )} @@ -241,23 +249,33 @@ const TeamsInfo = ({ {hasEditPermission && ( - setShowTypeSelector(true) - }> - - + + { + // Used to stop click propagation event to parent TeamDetailV1 collapsible panel + e.stopPropagation(); + if (isGroupType) { + return; + } + setShowTypeSelector(true); + }}> + + + )} )} diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Settings/Team/TeamDetails/TeamsHeaderSection/TeamsInfo.test.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Settings/Team/TeamDetails/TeamsHeaderSection/TeamsInfo.test.tsx index e1e76cd2a863..2fe6416cabb0 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Settings/Team/TeamDetails/TeamsHeaderSection/TeamsInfo.test.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Settings/Team/TeamDetails/TeamsHeaderSection/TeamsInfo.test.tsx @@ -68,8 +68,8 @@ jest.mock('../../../../common/TeamTypeSelect/TeamTypeSelect.component', () => ({ default: jest.fn().mockImplementation(() =>
    TeamTypeSelect
    ), })); -jest.mock('../../../../Auth/AuthProviders/AuthProvider', () => ({ - useAuthContext: jest.fn().mockReturnValue({ +jest.mock('../../../../../hooks/useApplicationStore', () => ({ + useApplicationStore: jest.fn().mockReturnValue({ currentUser: { id: 'test-user' }, }), })); diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Settings/Team/TeamDetails/TeamsHeaderSection/TeamsSubscription.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Settings/Team/TeamDetails/TeamsHeaderSection/TeamsSubscription.component.tsx index 6437e266bb6b..841c0b7265a3 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Settings/Team/TeamDetails/TeamsHeaderSection/TeamsSubscription.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Settings/Team/TeamDetails/TeamsHeaderSection/TeamsSubscription.component.tsx @@ -10,7 +10,16 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import { Form, Input, Modal, Select, Space, Typography } from 'antd'; +import { + Button, + Form, + Input, + Modal, + Select, + Space, + Tooltip, + Typography, +} from 'antd'; import { useForm } from 'antd/lib/form/Form'; import { isEmpty } from 'lodash'; import React, { useCallback, useEffect, useMemo, useState } from 'react'; @@ -76,13 +85,22 @@ const TeamsSubscription = ({ data-testid="subscription-no-data"> {t('label.none')} - setEditSubscription(true)} - /> + + { + // Used to stop click propagation event to parent TeamDetailV1 collapsible panel + e.stopPropagation(); + setEditSubscription(true); + }} + /> + ); } @@ -97,7 +115,7 @@ const TeamsSubscription = ({ } return cellItem(webhook[0], webhook[1]); - }, [subscription, hasEditPermission, setEditSubscription]); + }, [subscription, hasEditPermission]); const handleSave = async (values: SubscriptionWebhook) => { setIsLoading(true); @@ -136,70 +154,84 @@ const TeamsSubscription = ({ {subscriptionRenderElement} {!editSubscription && !isEmpty(subscription) && hasEditPermission && ( - setEditSubscription(true)} - /> + + { + // Used to stop click propagation event to parent TeamDetailV1 collapsible panel + e.stopPropagation(); + setEditSubscription(true); + }} + /> + )} {editSubscription && ( - setEditSubscription(false)}> - - - - - - + // Used Button to stop click propagation event anywhere in the form to parent TeamDetailV1 collapsible panel + )}     ); diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Settings/Team/TeamDetails/UserTab/UserTab.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Settings/Team/TeamDetails/UserTab/UserTab.component.tsx index 6802d0fa8e3f..9079e61c3a55 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Settings/Team/TeamDetails/UserTab/UserTab.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Settings/Team/TeamDetails/UserTab/UserTab.component.tsx @@ -191,7 +191,7 @@ export const UserTab = ({ className="w-full justify-center remove-icon" size={8}> void; onSave: (data: ChangePasswordRequest) => void; isLoggedInUser: boolean; isLoading: boolean; -}; +} -const ChangePasswordForm: React.FC = ({ +const ChangePasswordForm: React.FC = ({ visible, onCancel, onSave, @@ -38,95 +38,101 @@ const ChangePasswordForm: React.FC = ({ const newPassword = Form.useWatch('newPassword', form); return ( - { - form.resetFields(); - onCancel(); - }}> -
    - {isLoggedInUser && ( + // Used Button to stop click propagation event anywhere in the component to parent + // Users.Component collapsible panel + ); }; diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Settings/Users/CreateUser/CreateUser.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Settings/Users/CreateUser/CreateUser.component.tsx index cc644aa71739..e5e017f603d0 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Settings/Users/CreateUser/CreateUser.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Settings/Users/CreateUser/CreateUser.component.tsx @@ -40,12 +40,12 @@ import { } from '../../../../generated/api/teams/createUser'; import { EntityReference } from '../../../../generated/entity/type'; import { AuthProvider } from '../../../../generated/settings/settings'; +import { useApplicationStore } from '../../../../hooks/useApplicationStore'; import { checkEmailInUse, generateRandomPwd } from '../../../../rest/auth-API'; import { getJWTTokenExpiryOptions } from '../../../../utils/BotsUtils'; import { handleSearchFilterOption } from '../../../../utils/CommonUtils'; import { getEntityName } from '../../../../utils/EntityUtils'; import { showErrorToast } from '../../../../utils/ToastUtils'; -import { useAuthContext } from '../../../Auth/AuthProviders/AuthProvider'; import CopyToClipboardButton from '../../../common/CopyToClipboardButton/CopyToClipboardButton'; import Loader from '../../../common/Loader/Loader'; import RichTextEditor from '../../../common/RichTextEditor/RichTextEditor'; @@ -63,7 +63,7 @@ const CreateUser = ({ }: CreateUserProps) => { const { t } = useTranslation(); const [form] = Form.useForm(); - const { authConfig } = useAuthContext(); + const { authConfig } = useApplicationStore(); const [isAdmin, setIsAdmin] = useState(false); const [isBot, setIsBot] = useState(forceBot); const [selectedTeams, setSelectedTeams] = useState< diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Settings/Users/UserProfileIcon/UserProfileIcon.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Settings/Users/UserProfileIcon/UserProfileIcon.component.tsx index e4b457d92487..c78387473426 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Settings/Users/UserProfileIcon/UserProfileIcon.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Settings/Users/UserProfileIcon/UserProfileIcon.component.tsx @@ -32,15 +32,14 @@ import { TERM_ADMIN, TERM_USER, } from '../../../../constants/constants'; -import { useApplicationConfigContext } from '../../../../context/ApplicationConfigProvider/ApplicationConfigProvider'; import { EntityReference } from '../../../../generated/entity/type'; +import { useApplicationStore } from '../../../../hooks/useApplicationStore'; import { getEntityName } from '../../../../utils/EntityUtils'; import i18n from '../../../../utils/i18next/LocalUtil'; import { getImageWithResolutionAndFallback, ImageQuality, } from '../../../../utils/ProfilerUtils'; -import { useAuthContext } from '../../../Auth/AuthProviders/AuthProvider'; import Avatar from '../../../common/AvatarComponent/Avatar'; import './user-profile-icon.less'; @@ -85,9 +84,13 @@ const renderLimitedListMenuItem = ({ }; export const UserProfileIcon = () => { - const { currentUser, onLogoutHandler } = useAuthContext(); - const { selectedPersona, updateSelectedPersona } = - useApplicationConfigContext(); + const { + currentUser, + onLogoutHandler, + selectedPersona, + setSelectedPersona: updateSelectedPersona, + } = useApplicationStore(); + const [isImgUrlValid, setIsImgUrlValid] = useState(true); const { t } = useTranslation(); const profilePicture = getImageWithResolutionAndFallback( diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Settings/Users/UserProfileIcon/UserProfileIcon.test.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Settings/Users/UserProfileIcon/UserProfileIcon.test.tsx index 99b4797f87d7..de574f31cd3d 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Settings/Users/UserProfileIcon/UserProfileIcon.test.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Settings/Users/UserProfileIcon/UserProfileIcon.test.tsx @@ -13,24 +13,21 @@ import { act, fireEvent, render, screen } from '@testing-library/react'; import userEvent from '@testing-library/user-event'; import React from 'react'; -import { useApplicationConfigContext } from '../../../../context/ApplicationConfigProvider/ApplicationConfigProvider'; +import { useApplicationStore } from '../../../../hooks/useApplicationStore'; import { getImageWithResolutionAndFallback } from '../../../../utils/ProfilerUtils'; -import { useAuthContext } from '../../../Auth/AuthProviders/AuthProvider'; import { mockPersonaData, mockUserData } from '../mocks/User.mocks'; import { UserProfileIcon } from './UserProfileIcon.component'; const mockLogout = jest.fn(); -const mockUpdateSelectedPersona = jest.fn(); - -jest.mock( - '../../../../context/ApplicationConfigProvider/ApplicationConfigProvider', - () => ({ - useApplicationConfigContext: jest.fn().mockImplementation(() => ({ - selectedPersona: {}, - updateSelectedPersona: mockUpdateSelectedPersona, - })), - }) -); + +jest.mock('../../../../hooks/useApplicationStore', () => ({ + useApplicationStore: jest.fn().mockImplementation(() => ({ + selectedPersona: {}, + setSelectedPersona: jest.fn(), + onLogoutHandler: mockLogout, + currentUser: mockUserData, + })), +})); jest.mock('../../../../utils/EntityUtils', () => ({ getEntityName: jest.fn().mockReturnValue('Test User'), @@ -55,13 +52,6 @@ jest.mock('react-router-dom', () => ({ )), })); -jest.mock('../../../Auth/AuthProviders/AuthProvider', () => ({ - useAuthContext: jest.fn(() => ({ - currentUser: mockUserData, - })), - onLogoutHandler: mockLogout, -})); - describe('UserProfileIcon', () => { it('should render User Profile Icon', () => { const { getByTestId } = render(); @@ -98,12 +88,12 @@ describe('UserProfileIcon', () => { }); it('should display the user team', () => { - (useApplicationConfigContext as jest.Mock).mockImplementation(() => ({ + (useApplicationStore as unknown as jest.Mock).mockImplementation(() => ({ selectedPersona: { id: '3362fe18-05ad-4457-9632-84f22887dda6', type: 'team', }, - updateSelectedPersona: jest.fn(), + setSelectedPersona: jest.fn(), })); const { getByTestId } = render(); @@ -111,7 +101,7 @@ describe('UserProfileIcon', () => { }); it('should show empty placeholder when no teams data', async () => { - (useAuthContext as jest.Mock).mockImplementation(() => ({ + (useApplicationStore as unknown as jest.Mock).mockImplementation(() => ({ currentUser: { ...mockUserData, teams: [] }, onLogoutHandler: mockLogout, })); @@ -123,20 +113,19 @@ describe('UserProfileIcon', () => { }); it('should show checked if selected persona is true', async () => { - (useAuthContext as jest.Mock).mockImplementation(() => ({ + (useApplicationStore as unknown as jest.Mock).mockImplementation(() => ({ currentUser: { ...mockUserData, personas: mockPersonaData, }, onLogoutHandler: mockLogout, - })); - (useApplicationConfigContext as jest.Mock).mockImplementation(() => ({ selectedPersona: { id: '0430976d-092a-46c9-90a8-61c6091a6f38', type: 'persona', }, - updateSelectedPersona: jest.fn(), + setSelectedPersona: jest.fn(), })); + const { getByTestId } = render(); await act(async () => { userEvent.click(getByTestId('dropdown-profile')); @@ -149,20 +138,19 @@ describe('UserProfileIcon', () => { }); it('should not show checked if selected persona is true', async () => { - (useAuthContext as jest.Mock).mockImplementation(() => ({ + (useApplicationStore as unknown as jest.Mock).mockImplementation(() => ({ currentUser: { ...mockUserData, personas: mockPersonaData, }, onLogoutHandler: mockLogout, - })); - (useApplicationConfigContext as jest.Mock).mockImplementation(() => ({ selectedPersona: { id: 'test', type: 'persona', }, - updateSelectedPersona: jest.fn(), + setSelectedPersona: jest.fn(), })); + const { getByTestId, queryByTestId } = render(); await act(async () => { userEvent.click(getByTestId('dropdown-profile')); diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Settings/Users/Users.component.test.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Settings/Users/Users.component.test.tsx index 1a75203a747d..872ad133d362 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Settings/Users/Users.component.test.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Settings/Users/Users.component.test.tsx @@ -23,7 +23,7 @@ import userEvent from '@testing-library/user-event'; import React, { ReactNode } from 'react'; import { MemoryRouter } from 'react-router-dom'; import { AuthProvider } from '../../../generated/settings/settings'; -import { useAuthContext } from '../../Auth/AuthProviders/AuthProvider'; +import { useApplicationStore } from '../../../hooks/useApplicationStore'; import { mockAccessData, mockUserData, mockUserRole } from './mocks/User.mocks'; import Users from './Users.component'; import { UserPageTabs } from './Users.interface'; @@ -99,8 +99,8 @@ jest.mock( .mockImplementation(() => <>ActivityFeedTabTest), }) ); -jest.mock('../../Auth/AuthProviders/AuthProvider', () => ({ - useAuthContext: jest.fn(() => ({ +jest.mock('../../../hooks/useApplicationStore', () => ({ + useApplicationStore: jest.fn(() => ({ authConfig: { provider: AuthProvider.Basic, }, @@ -252,11 +252,13 @@ describe('Test User Component', () => { }); it('Access Token tab should show user access component', async () => { - (useAuthContext as jest.Mock).mockImplementationOnce(() => ({ - currentUser: { - name: 'test', - }, - })); + (useApplicationStore as unknown as jest.Mock).mockImplementationOnce( + () => ({ + currentUser: { + name: 'test', + }, + }) + ); mockParams.tab = UserPageTabs.ACCESS_TOKEN; render( { const { isAdminUser } = useAuth(); const history = useHistory(); const location = useLocation(); - const { currentUser } = useAuthContext(); + const { currentUser } = useApplicationStore(); const [previewAsset, setPreviewAsset] = useState(); const [isDescriptionEdit, setIsDescriptionEdit] = useState(false); - const { t } = useTranslation(); const isLoggedInUser = useMemo( @@ -304,11 +303,9 @@ const Users = ({ userData, queryFilters, updateUserDetails }: Props) => { + className="header-collapse-custom-collapse user-profile-container"> diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Settings/Users/UsersProfile/UserProfileDetails/UserProfileDetails.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Settings/Users/UsersProfile/UserProfileDetails/UserProfileDetails.component.tsx index 3db0312d1c41..ec90bf8658a9 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Settings/Users/UsersProfile/UserProfileDetails/UserProfileDetails.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Settings/Users/UsersProfile/UserProfileDetails/UserProfileDetails.component.tsx @@ -11,7 +11,7 @@ * limitations under the License. */ -import { Button, Divider, Input, Space, Typography } from 'antd'; +import { Button, Divider, Input, Space, Tooltip, Typography } from 'antd'; import { AxiosError } from 'axios'; import React, { useCallback, useMemo, useState } from 'react'; import { useTranslation } from 'react-i18next'; @@ -29,6 +29,7 @@ import { import { EntityReference } from '../../../../../generated/entity/type'; import { AuthProvider } from '../../../../../generated/settings/settings'; import { useAuth } from '../../../../../hooks/authHooks'; +import { useApplicationStore } from '../../../../../hooks/useApplicationStore'; import { useFqn } from '../../../../../hooks/useFqn'; import { changePassword } from '../../../../../rest/auth-API'; import { getEntityName } from '../../../../../utils/EntityUtils'; @@ -36,7 +37,6 @@ import { showErrorToast, showSuccessToast, } from '../../../../../utils/ToastUtils'; -import { useAuthContext } from '../../../../Auth/AuthProviders/AuthProvider'; import Chip from '../../../../common/Chip/Chip.component'; import { DomainLabel } from '../../../../common/DomainLabel/DomainLabel.component'; import InlineEdit from '../../../../common/InlineEdit/InlineEdit.component'; @@ -52,7 +52,7 @@ const UserProfileDetails = ({ const { t } = useTranslation(); const { fqn: username } = useFqn(); const { isAdminUser } = useAuth(); - const { authConfig, currentUser } = useAuthContext(); + const { authConfig, currentUser } = useApplicationStore(); const [isLoading, setIsLoading] = useState(false); const [isChangePassword, setIsChangePassword] = useState(false); @@ -142,13 +142,22 @@ const UserProfileDetails = ({ : getEntityName(userData)} {hasEditPermission && ( - setIsDisplayNameEdit(true)} - /> + + { + // Used to stop click propagation event to parent User.component collapsible panel + e.stopPropagation(); + setIsDisplayNameEdit(true); + }} + /> + )}     ), @@ -170,7 +179,11 @@ const UserProfileDetails = ({ className="w-full text-xs" data-testid="change-password-button" type="primary" - onClick={() => setIsChangePassword(true)}> + onClick={(e) => { + // Used to stop click propagation event to parent User.component collapsible panel + e.stopPropagation(); + setIsChangePassword(true); + }}> {t('label.change-entity', { entity: t('label.password-lowercase'), })} diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Settings/Users/UsersProfile/UserProfileDetails/UserProfileDetails.test.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Settings/Users/UsersProfile/UserProfileDetails/UserProfileDetails.test.tsx index bbeed1f34d42..5ba0452a6b32 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Settings/Users/UsersProfile/UserProfileDetails/UserProfileDetails.test.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Settings/Users/UsersProfile/UserProfileDetails/UserProfileDetails.test.tsx @@ -15,8 +15,8 @@ import React from 'react'; import { MemoryRouter } from 'react-router-dom'; import { AuthProvider } from '../../../../../generated/settings/settings'; import { useAuth } from '../../../../../hooks/authHooks'; +import { useApplicationStore } from '../../../../../hooks/useApplicationStore'; import { USER_DATA } from '../../../../../mocks/User.mock'; -import { useAuthContext } from '../../../../Auth/AuthProviders/AuthProvider'; import UserProfileDetails from './UserProfileDetails.component'; import { UserProfileDetailsProps } from './UserProfileDetails.interface'; @@ -34,8 +34,8 @@ jest.mock('react-router-dom', () => ({ useParams: jest.fn().mockImplementation(() => mockParams), })); -jest.mock('../../../../Auth/AuthProviders/AuthProvider', () => ({ - useAuthContext: jest.fn(() => ({ +jest.mock('../../../../../hooks/useApplicationStore', () => ({ + useApplicationStore: jest.fn(() => ({ authConfig: { provider: AuthProvider.Basic, }, @@ -137,11 +137,13 @@ describe('Test User Profile Details Component', () => { }); it('should not render change password button and component in case of SSO', async () => { - (useAuthContext as jest.Mock).mockImplementationOnce(() => ({ - authConfig: jest.fn().mockImplementationOnce(() => ({ - provider: AuthProvider.Google, - })), - })); + (useApplicationStore as unknown as jest.Mock).mockImplementationOnce( + () => ({ + authConfig: jest.fn().mockImplementationOnce(() => ({ + provider: AuthProvider.Google, + })), + }) + ); render(, { wrapper: MemoryRouter, @@ -159,12 +161,14 @@ describe('Test User Profile Details Component', () => { isAdminUser: false, })); - (useAuthContext as jest.Mock).mockImplementationOnce(() => ({ - currentUser: { - name: 'admin', - id: '1234', - }, - })); + (useApplicationStore as unknown as jest.Mock).mockImplementationOnce( + () => ({ + currentUser: { + name: 'admin', + id: '1234', + }, + }) + ); render(, { wrapper: MemoryRouter, diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Settings/Users/UsersProfile/UserProfileRoles/UserProfileRoles.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Settings/Users/UsersProfile/UserProfileRoles/UserProfileRoles.component.tsx index d2eab847076b..1a4592d7fa44 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Settings/Users/UsersProfile/UserProfileRoles/UserProfileRoles.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Settings/Users/UsersProfile/UserProfileRoles/UserProfileRoles.component.tsx @@ -11,7 +11,7 @@ * limitations under the License. */ -import { Card, Select, Space, Typography } from 'antd'; +import { Card, Select, Space, Tooltip, Typography } from 'antd'; import { AxiosError } from 'axios'; import { isEmpty, toLower } from 'lodash'; import React, { useEffect, useMemo, useState } from 'react'; @@ -153,13 +153,18 @@ const UserProfileRoles = ({ {t('label.role-plural')} {!isRolesEdit && isAdminUser && ( - setIsRolesEdit(true)} - /> + + setIsRolesEdit(true)} + /> + )}     }> diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Settings/Users/UsersProfile/UserProfileTeams/UserProfileTeams.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Settings/Users/UsersProfile/UserProfileTeams/UserProfileTeams.component.tsx index b37787238456..707eaf14cebe 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Settings/Users/UsersProfile/UserProfileTeams/UserProfileTeams.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Settings/Users/UsersProfile/UserProfileTeams/UserProfileTeams.component.tsx @@ -11,7 +11,7 @@ * limitations under the License. */ -import { Card, Space, Typography } from 'antd'; +import { Card, Space, Tooltip, Typography } from 'antd'; import React, { useEffect, useMemo, useState } from 'react'; import { useTranslation } from 'react-i18next'; import { ReactComponent as EditIcon } from '../../../../../assets/svg/edit-new.svg'; @@ -75,13 +75,18 @@ const UserProfileTeams = ({ {!isTeamsEdit && isAdminUser && ( - setIsTeamsEdit(true)} - /> + + setIsTeamsEdit(true)} + /> + )} }> diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Settings/Users/UsersTab/UsersTabs.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Settings/Users/UsersTab/UsersTabs.component.tsx index b05dd748c991..cf55ca90c904 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Settings/Users/UsersTab/UsersTabs.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Settings/Users/UsersTab/UsersTabs.component.tsx @@ -10,7 +10,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import { Button, Modal } from 'antd'; +import { Button, Modal, Tooltip } from 'antd'; import { isNil } from 'lodash'; import React, { useCallback, useEffect, useMemo, useState } from 'react'; import { useTranslation } from 'react-i18next'; @@ -90,14 +90,19 @@ export const UsersTab = ({ users, onRemoveUser }: UsersTabProps) => { render: (_: string, record: User) => { return ( onRemoveUser && ( - + + + + + + ); +} diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Suggestions/SuggestionsProvider/SuggestionsProvider.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Suggestions/SuggestionsProvider/SuggestionsProvider.tsx new file mode 100644 index 000000000000..05e8f4b87e08 --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/components/Suggestions/SuggestionsProvider/SuggestionsProvider.tsx @@ -0,0 +1,209 @@ +/* + * Copyright 2024 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { AxiosError } from 'axios'; +import { isEmpty, isEqual, uniqWith } from 'lodash'; + +import React, { + createContext, + ReactNode, + useCallback, + useContext, + useEffect, + useMemo, + useRef, + useState, +} from 'react'; +import { useTranslation } from 'react-i18next'; +import { usePermissionProvider } from '../../../context/PermissionProvider/PermissionProvider'; +import { + Suggestion, + SuggestionType, +} from '../../../generated/entity/feed/suggestion'; +import { EntityReference } from '../../../generated/entity/type'; +import { useFqn } from '../../../hooks/useFqn'; +import { usePub } from '../../../hooks/usePubSub'; +import { + aproveRejectAllSuggestions, + getSuggestionsList, + updateSuggestionStatus, +} from '../../../rest/suggestionsAPI'; +import { showErrorToast } from '../../../utils/ToastUtils'; +import { + SuggestionAction, + SuggestionsContextType, +} from './SuggestionsProvider.interface'; + +export const SuggestionsContext = createContext({} as SuggestionsContextType); + +const SuggestionsProvider = ({ children }: { children?: ReactNode }) => { + const { t } = useTranslation(); + const { fqn: entityFqn } = useFqn(); + const [activeUser, setActiveUser] = useState(); + const [loadingAccept, setLoadingAccept] = useState(false); + const [loadingReject, setLoadingReject] = useState(false); + + const [allSuggestionsUsers, setAllSuggestionsUsers] = useState< + EntityReference[] + >([]); + const [suggestions, setSuggestions] = useState([]); + const [suggestionsByUser, setSuggestionsByUser] = useState< + Map + >(new Map()); + const publish = usePub(); + + const [loading, setLoading] = useState(false); + const refreshEntity = useRef<(suggestion: Suggestion) => void>(); + const { permissions } = usePermissionProvider(); + + const fetchSuggestions = useCallback(async (entityFQN: string) => { + setLoading(true); + try { + const { data } = await getSuggestionsList({ + entityFQN, + }); + setSuggestions(data); + + const allUsersData = data.map( + (suggestion) => suggestion.createdBy as EntityReference + ); + const uniqueUsers = uniqWith(allUsersData, isEqual); + setAllSuggestionsUsers(uniqueUsers); + + const groupedSuggestions = data.reduce((acc, suggestion) => { + const createdBy = suggestion?.createdBy?.name ?? ''; + if (!acc.has(createdBy)) { + acc.set(createdBy, []); + } + acc.get(createdBy)?.push(suggestion); + + return acc; + }, new Map() as Map); + + setSuggestionsByUser(groupedSuggestions); + } catch (err) { + showErrorToast( + err as AxiosError, + t('server.entity-fetch-error', { + entity: t('label.lineage-data-lowercase'), + }) + ); + } finally { + setLoading(false); + } + }, []); + + const acceptRejectSuggestion = useCallback( + async (suggestion: Suggestion, status: SuggestionAction) => { + try { + await updateSuggestionStatus(suggestion, status); + await fetchSuggestions(entityFqn); + if (status === SuggestionAction.Accept) { + // call component refresh function + publish('updateDetails', suggestion); + } + } catch (err) { + showErrorToast(err as AxiosError); + } + }, + [entityFqn, refreshEntity] + ); + + const onUpdateActiveUser = useCallback( + (user?: EntityReference) => { + setActiveUser(user); + }, + [suggestionsByUser] + ); + + const selectedUserSuggestions = useMemo(() => { + return suggestionsByUser.get(activeUser?.name ?? '') ?? []; + }, [activeUser, suggestionsByUser]); + + const acceptRejectAllSuggestions = useCallback( + async (suggestionType: SuggestionType, status: SuggestionAction) => { + if (status === SuggestionAction.Accept) { + setLoadingAccept(true); + } else { + setLoadingReject(true); + } + try { + await aproveRejectAllSuggestions( + activeUser?.id ?? '', + entityFqn, + suggestionType, + status + ); + + await fetchSuggestions(entityFqn); + if (status === SuggestionAction.Accept) { + selectedUserSuggestions.forEach((suggestion) => { + publish('updateDetails', suggestion); + }); + } + setActiveUser(undefined); + } catch (err) { + showErrorToast(err as AxiosError); + } finally { + setLoadingAccept(false); + setLoadingReject(false); + } + }, + [activeUser, entityFqn, selectedUserSuggestions] + ); + + useEffect(() => { + if (!isEmpty(permissions) && !isEmpty(entityFqn)) { + fetchSuggestions(entityFqn); + } + }, [entityFqn, permissions]); + + const suggestionsContextObj = useMemo(() => { + return { + suggestions, + suggestionsByUser, + selectedUserSuggestions, + entityFqn, + loading, + loadingAccept, + loadingReject, + allSuggestionsUsers, + onUpdateActiveUser, + fetchSuggestions, + acceptRejectSuggestion, + acceptRejectAllSuggestions, + }; + }, [ + suggestions, + suggestionsByUser, + selectedUserSuggestions, + entityFqn, + loading, + loadingAccept, + loadingReject, + allSuggestionsUsers, + onUpdateActiveUser, + fetchSuggestions, + acceptRejectSuggestion, + acceptRejectAllSuggestions, + ]); + + return ( + + {children} + + ); +}; + +export const useSuggestionsContext = () => useContext(SuggestionsContext); + +export default SuggestionsProvider; diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Suggestions/SuggestionsSlider/SuggestionsSlider.test.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Suggestions/SuggestionsSlider/SuggestionsSlider.test.tsx new file mode 100644 index 000000000000..dbe01b8f98ec --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/components/Suggestions/SuggestionsSlider/SuggestionsSlider.test.tsx @@ -0,0 +1,55 @@ +/* + * Copyright 2024 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { fireEvent, render, screen } from '@testing-library/react'; +import React from 'react'; +import { useSuggestionsContext } from '../SuggestionsProvider/SuggestionsProvider'; +import SuggestionsSlider from './SuggestionsSlider'; + +jest.mock('../SuggestionsProvider/SuggestionsProvider', () => ({ + useSuggestionsContext: jest.fn(), +})); + +jest.mock('../../common/AvatarCarousel/AvatarCarousel', () => { + return jest.fn(() =>

    Avatar Carousel

    ); +}); + +describe('SuggestionsSlider', () => { + it('renders buttons when there are selected user suggestions', () => { + (useSuggestionsContext as jest.Mock).mockReturnValue({ + selectedUserSuggestions: [{ id: '1' }, { id: '2' }], + acceptRejectAllSuggestions: jest.fn(), + loadingAccept: false, + loadingReject: false, + }); + + render(); + + expect(screen.getByTestId('accept-all-suggestions')).toBeInTheDocument(); + expect(screen.getByTestId('reject-all-suggestions')).toBeInTheDocument(); + }); + + it('calls acceptRejectAllSuggestions on button click', () => { + const acceptRejectAllSuggestions = jest.fn(); + (useSuggestionsContext as jest.Mock).mockReturnValue({ + selectedUserSuggestions: [{ id: '1' }, { id: '2' }], + acceptRejectAllSuggestions, + loadingAccept: false, + loadingReject: false, + }); + + render(); + fireEvent.click(screen.getByTestId('accept-all-suggestions')); + + expect(acceptRejectAllSuggestions).toHaveBeenCalled(); + }); +}); diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Suggestions/SuggestionsSlider/SuggestionsSlider.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Suggestions/SuggestionsSlider/SuggestionsSlider.tsx new file mode 100644 index 000000000000..d7b075732182 --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/components/Suggestions/SuggestionsSlider/SuggestionsSlider.tsx @@ -0,0 +1,85 @@ +/* + * Copyright 2024 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { CheckOutlined, CloseOutlined } from '@ant-design/icons'; +import { Button, Space, Typography } from 'antd'; +import { t } from 'i18next'; +import React from 'react'; +import { ReactComponent as ExitIcon } from '../../../assets/svg/ic-exit.svg'; +import { SuggestionType } from '../../../generated/entity/feed/suggestion'; +import AvatarCarousel from '../../common/AvatarCarousel/AvatarCarousel'; +import { useSuggestionsContext } from '../SuggestionsProvider/SuggestionsProvider'; +import { SuggestionAction } from '../SuggestionsProvider/SuggestionsProvider.interface'; + +const SuggestionsSlider = () => { + const { + selectedUserSuggestions, + acceptRejectAllSuggestions, + loadingAccept, + loadingReject, + onUpdateActiveUser, + } = useSuggestionsContext(); + + return ( +
    + + {t('label.suggested-description-plural')} + + + {selectedUserSuggestions.length > 0 && ( + + + + + + )} +
    + ); +}; + +export default SuggestionsSlider; diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Tag/TagsContainerV2/TagsContainerV2.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Tag/TagsContainerV2/TagsContainerV2.tsx index 1192828db5cd..c00e15f22e28 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Tag/TagsContainerV2/TagsContainerV2.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Tag/TagsContainerV2/TagsContainerV2.tsx @@ -21,7 +21,7 @@ import { useHistory } from 'react-router-dom'; import { ReactComponent as IconComments } from '../../../assets/svg/comment.svg'; import { ReactComponent as EditIcon } from '../../../assets/svg/edit-new.svg'; import { ReactComponent as IconRequest } from '../../../assets/svg/request-icon.svg'; -import { DE_ACTIVE_COLOR } from '../../../constants/constants'; +import { DE_ACTIVE_COLOR, ICON_DIMENSION } from '../../../constants/constants'; import { GLOSSARY_CONSTANT, TAG_CONSTANT, @@ -273,13 +273,18 @@ const TagsContainerV2 = ({ {!isEmpty(tags?.[tagType]) && !isEditTags && (     - + + + )} {showTaskHandler && ( @@ -308,16 +313,21 @@ const TagsContainerV2 = ({ const editTagButton = useMemo( () => permission && !isEmpty(tags?.[tagType]) ? ( - + + + ) : null, [permission, tags, tagType, handleAddClick] ); diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Topic/TopicDetails/TopicDetails.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Topic/TopicDetails/TopicDetails.component.tsx index 5851a538514a..96255bc3f3fe 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Topic/TopicDetails/TopicDetails.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Topic/TopicDetails/TopicDetails.component.tsx @@ -27,6 +27,7 @@ import { Topic } from '../../../generated/entity/data/topic'; import { DataProduct } from '../../../generated/entity/domains/dataProduct'; import { ThreadType } from '../../../generated/entity/feed/thread'; import { TagLabel } from '../../../generated/type/schema'; +import { useApplicationStore } from '../../../hooks/useApplicationStore'; import { useFqn } from '../../../hooks/useFqn'; import { FeedCounts } from '../../../interface/feed.interface'; import { restoreTopic } from '../../../rest/topicsAPI'; @@ -42,7 +43,6 @@ import { useActivityFeedProvider } from '../../ActivityFeed/ActivityFeedProvider import { ActivityFeedTab } from '../../ActivityFeed/ActivityFeedTab/ActivityFeedTab.component'; import ActivityThreadPanel from '../../ActivityFeed/ActivityThreadPanel/ActivityThreadPanel'; import { withActivityFeed } from '../../AppRouter/withActivityFeed'; -import { useAuthContext } from '../../Auth/AuthProviders/AuthProvider'; import { CustomPropertyTable } from '../../common/CustomPropertyTable/CustomPropertyTable'; import DescriptionV1 from '../../common/EntityDescription/DescriptionV1'; import ErrorPlaceHolder from '../../common/ErrorWithPlaceholder/ErrorPlaceHolder'; @@ -72,7 +72,7 @@ const TopicDetails: React.FC = ({ onUpdateVote, }: TopicDetailsProps) => { const { t } = useTranslation(); - const { currentUser } = useAuthContext(); + const { currentUser } = useApplicationStore(); const { postFeed, deleteFeed, updateFeed } = useActivityFeedProvider(); const { tab: activeTab = EntityTabs.SCHEMA } = useParams<{ tab: EntityTabs }>(); diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Topic/TopicDetails/TopicDetails.test.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Topic/TopicDetails/TopicDetails.test.tsx index c3ec8af9a0cf..e058496adaf0 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Topic/TopicDetails/TopicDetails.test.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Topic/TopicDetails/TopicDetails.test.tsx @@ -77,7 +77,7 @@ jest.mock('../../Lineage/Lineage.component', () => { return jest.fn().mockReturnValue(

    EntityLineage.component

    ); }); -jest.mock('../../common/EntityDescription/Description', () => { +jest.mock('../../common/EntityDescription/DescriptionV1', () => { return jest.fn().mockReturnValue(

    Description Component

    ); }); diff --git a/openmetadata-ui/src/main/resources/ui/src/components/WebAnalytics/WebAnalyticsProvider.tsx b/openmetadata-ui/src/main/resources/ui/src/components/WebAnalytics/WebAnalyticsProvider.tsx index 735e35ef5aca..31454f499327 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/WebAnalytics/WebAnalyticsProvider.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/WebAnalytics/WebAnalyticsProvider.tsx @@ -13,15 +13,15 @@ import React, { ReactNode } from 'react'; import { AnalyticsProvider } from 'use-analytics'; +import { useApplicationStore } from '../../hooks/useApplicationStore'; import { getAnalyticInstance } from '../../utils/WebAnalyticsUtils'; -import { useAuthContext } from '../Auth/AuthProviders/AuthProvider'; interface WebAnalyticsProps { children: ReactNode; } const WebAnalyticsProvider = ({ children }: WebAnalyticsProps) => { - const { currentUser } = useAuthContext(); + const { currentUser } = useApplicationStore(); return ( diff --git a/openmetadata-ui/src/main/resources/ui/src/components/common/AvatarCarousel/AvatarCarousel.test.tsx b/openmetadata-ui/src/main/resources/ui/src/components/common/AvatarCarousel/AvatarCarousel.test.tsx new file mode 100644 index 000000000000..ebdddcf238cd --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/components/common/AvatarCarousel/AvatarCarousel.test.tsx @@ -0,0 +1,75 @@ +/* + * Copyright 2024 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { render, screen } from '@testing-library/react'; +import React from 'react'; +import AvatarCarousel from './AvatarCarousel'; + +const suggestions = [ + { + id: '1', + description: 'Test suggestion', + createdBy: { id: '1', name: 'Avatar 1', type: 'user' }, + entityLink: '<#E::table::sample_data.ecommerce_db.shopify.dim_address>', + }, + { + id: '2', + description: 'Test suggestion', + createdBy: { id: '2', name: 'Avatar 2', type: 'user' }, + entityLink: '<#E::table::sample_data.ecommerce_db.shopify.dim_address>', + }, +]; + +const suggByUser = new Map([ + ['Avatar 1', [suggestions[0]]], + ['Avatar 2', [suggestions[1]]], +]); + +jest.mock('../../Suggestions/SuggestionsProvider/SuggestionsProvider', () => ({ + useSuggestionsContext: jest.fn().mockImplementation(() => ({ + suggestions: suggestions, + suggestionsByUser: suggByUser, + allSuggestionsUsers: [ + { id: '1', name: 'Avatar 1', type: 'user' }, + { id: '2', name: 'Avatar 2', type: 'user' }, + ], + acceptRejectSuggestion: jest.fn(), + selectedUserSuggestions: [], + onUpdateActiveUser: jest.fn(), + })), + __esModule: true, + default: 'SuggestionsProvider', +})); + +jest.mock('../ProfilePicture/ProfilePicture', () => + jest + .fn() + .mockImplementation(({ name }) => ( + {name} + )) +); + +jest.mock('../../../rest/suggestionsAPI', () => ({ + getSuggestionsList: jest + .fn() + .mockImplementation(() => Promise.resolve(suggestions)), +})); + +describe('AvatarCarousel', () => { + it('renders without crashing', () => { + render(); + + expect(screen.getByText(/Avatar 1/i)).toBeInTheDocument(); + expect(screen.getByText(/Avatar 2/i)).toBeInTheDocument(); + expect(screen.getByTestId('prev-slide')).toBeDisabled(); + }); +}); diff --git a/openmetadata-ui/src/main/resources/ui/src/components/common/AvatarCarousel/AvatarCarousel.tsx b/openmetadata-ui/src/main/resources/ui/src/components/common/AvatarCarousel/AvatarCarousel.tsx new file mode 100644 index 000000000000..21e95fd74ad5 --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/components/common/AvatarCarousel/AvatarCarousel.tsx @@ -0,0 +1,119 @@ +/* + * Copyright 2024 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { LeftOutlined, RightOutlined } from '@ant-design/icons'; +import { Button, Carousel } from 'antd'; +import React, { + RefObject, + useCallback, + useEffect, + useRef, + useState, +} from 'react'; +import { useSuggestionsContext } from '../../Suggestions/SuggestionsProvider/SuggestionsProvider'; +import AvatarCarouselItem from '../AvatarCarouselItem/AvatarCarouselItem'; +import './avatar-carousel.less'; + +interface AvatarCarouselProps { + showArrows?: boolean; +} + +const AvatarCarousel = ({ showArrows = false }: AvatarCarouselProps) => { + const { + allSuggestionsUsers: avatarList, + onUpdateActiveUser, + selectedUserSuggestions, + } = useSuggestionsContext(); + const [currentSlide, setCurrentSlide] = useState(-1); + const avatarBtnRefs = useRef[]>([]); + + const prevSlide = useCallback(() => { + setCurrentSlide((prev) => (prev === 0 ? avatarList.length - 1 : prev - 1)); + }, [avatarList]); + + const nextSlide = useCallback(() => { + setCurrentSlide((prev) => (prev === avatarList.length - 1 ? 0 : prev + 1)); + }, [avatarList]); + + const handleMouseOut = useCallback(() => { + avatarBtnRefs.current.forEach((ref: any) => { + ref.current?.dispatchEvent(new MouseEvent('mouseout', { bubbles: true })); + }); + }, [avatarBtnRefs]); + + const onProfileClick = useCallback( + (index: number) => { + const activeUser = avatarList[index]; + onUpdateActiveUser(activeUser); + handleMouseOut(); + }, + [avatarList] + ); + + useEffect(() => { + onProfileClick(currentSlide); + }, [currentSlide]); + + useEffect(() => { + if (selectedUserSuggestions.length === 0) { + setCurrentSlide(-1); + } + }, [selectedUserSuggestions]); + + return ( +
    + {showArrows && ( +
    + ); +}; + +export default AvatarCarousel; diff --git a/openmetadata-ui/src/main/resources/ui/src/components/common/AvatarCarousel/avatar-carousel.less b/openmetadata-ui/src/main/resources/ui/src/components/common/AvatarCarousel/avatar-carousel.less new file mode 100644 index 000000000000..52be99a10d64 --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/components/common/AvatarCarousel/avatar-carousel.less @@ -0,0 +1,54 @@ +/* + * Copyright 2024 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +@import url('../../../styles/variables.less'); + +.avatar-item { + position: relative; + &:hover { + border-color: @border-color !important; + } + &:focus { + border-color: @border-color !important; + } + &.ant-btn { + width: 28px; + height: 28px; + min-width: 28px; + } +} + +.avatar-carousel-container { + .slick-slide { + width: 28px !important; + } + .slick-list { + overflow: visible !important; + } + .ant-badge-count { + right: 4px; + background-color: @red-3; + } +} + +.slider-btn-container { + .ant-btn { + padding: 0 10px; + height: 30px; + } + .ant-btn.exit-suggestion { + color: @grey-4; + border-color: @grey-4; + padding: 0; + width: 30px; + } +} diff --git a/openmetadata-ui/src/main/resources/ui/src/components/common/AvatarCarouselItem/AvatarCarouselItem.test.tsx b/openmetadata-ui/src/main/resources/ui/src/components/common/AvatarCarouselItem/AvatarCarouselItem.test.tsx new file mode 100644 index 000000000000..d0eefcd2203a --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/components/common/AvatarCarouselItem/AvatarCarouselItem.test.tsx @@ -0,0 +1,119 @@ +/* + * Copyright 2024 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { render } from '@testing-library/react'; +import React from 'react'; +import { EntityReference } from '../../../generated/entity/type'; +import AvatarCarouselItem from './AvatarCarouselItem'; + +const suggestions = [ + { + id: '1', + description: 'Test suggestion', + createdBy: { id: '1', name: 'Avatar 1', type: 'user' }, + entityLink: '<#E::table::sample_data.ecommerce_db.shopify.dim_address>', + }, + { + id: '2', + description: 'Test suggestion', + createdBy: { id: '2', name: 'Avatar 2', type: 'user' }, + entityLink: '<#E::table::sample_data.ecommerce_db.shopify.dim_address>', + }, +]; + +const suggByUser = new Map([ + ['Avatar 1', [suggestions[0]]], + ['Avatar 2', [suggestions[1]]], +]); + +jest.mock('../../Suggestions/SuggestionsProvider/SuggestionsProvider', () => ({ + useSuggestionsContext: jest.fn().mockImplementation(() => ({ + suggestions: suggestions, + suggestionsByUser: suggByUser, + allSuggestionsUsers: [ + { id: '1', name: 'Avatar 1', type: 'user' }, + { id: '2', name: 'Avatar 2', type: 'user' }, + ], + acceptRejectSuggestion: jest.fn(), + selectedUserSuggestions: [], + onUpdateActiveUser: jest.fn(), + })), + __esModule: true, + default: 'SuggestionsProvider', +})); + +jest.mock('../../../rest/suggestionsAPI', () => ({ + getSuggestionsList: jest + .fn() + .mockImplementation(() => Promise.resolve(suggestions)), +})); + +describe('AvatarCarouselItem', () => { + const avatar: EntityReference = { + id: '1', + name: 'Test Avatar', + type: 'user', + }; + const index = 0; + const onAvatarClick = jest.fn(); + const avatarBtnRefs = { current: [] }; + const isActive = false; + + it('renders AvatarCarouselItem with ProfilePicture component', () => { + const { getByTestId } = render( + + ); + + expect( + getByTestId(`avatar-carousel-item-${avatar.id}`) + ).toBeInTheDocument(); + }); + + it('calls onAvatarClick function when clicked', () => { + const { getByTestId } = render( + + ); + + const button = getByTestId(`avatar-carousel-item-${avatar.id}`); + button.click(); + + expect(onAvatarClick).toHaveBeenCalledWith(index); + }); + + it('sets isActive class when isActive is true', () => { + const { getByTestId } = render( + + ); + + expect(getByTestId(`avatar-carousel-item-${avatar.id}`)).toHaveClass( + 'active' + ); + }); +}); diff --git a/openmetadata-ui/src/main/resources/ui/src/components/common/AvatarCarouselItem/AvatarCarouselItem.tsx b/openmetadata-ui/src/main/resources/ui/src/components/common/AvatarCarouselItem/AvatarCarouselItem.tsx new file mode 100644 index 000000000000..0863b84970fc --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/components/common/AvatarCarouselItem/AvatarCarouselItem.tsx @@ -0,0 +1,68 @@ +/* + * Copyright 2024 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { Badge, Button } from 'antd'; +import classNames from 'classnames'; +import React, { RefObject, useCallback, useRef } from 'react'; +import { EntityReference } from '../../../generated/entity/type'; +import { useSuggestionsContext } from '../../Suggestions/SuggestionsProvider/SuggestionsProvider'; +import UserPopOverCard from '../PopOverCard/UserPopOverCard'; +import ProfilePicture from '../ProfilePicture/ProfilePicture'; + +interface AvatarCarouselItemProps { + avatar: EntityReference; + index: number; + onAvatarClick: (index: number) => void; + avatarBtnRefs: React.MutableRefObject[]>; + isActive: boolean; +} + +const AvatarCarouselItem = ({ + avatar, + index, + avatarBtnRefs, + onAvatarClick, + isActive, +}: AvatarCarouselItemProps) => { + const { suggestionsByUser } = useSuggestionsContext(); + const buttonRef = useRef(null); + avatarBtnRefs.current[index] = buttonRef; + const getUserSuggestionsCount = useCallback( + (userName: string) => { + return suggestionsByUser.get(userName) ?? []; + }, + [suggestionsByUser] + ); + + const button = ( + + ); + + return ( + + + {button} + + + ); +}; + +export default AvatarCarouselItem; diff --git a/openmetadata-ui/src/main/resources/ui/src/components/common/BrandImage/BrandImage.test.tsx b/openmetadata-ui/src/main/resources/ui/src/components/common/BrandImage/BrandImage.test.tsx index 4b7c004e0222..f1f0d49ac44d 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/common/BrandImage/BrandImage.test.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/common/BrandImage/BrandImage.test.tsx @@ -14,15 +14,14 @@ import { render, screen } from '@testing-library/react'; import React from 'react'; import BrandImage from './BrandImage'; -jest.mock( - '../../../context/ApplicationConfigProvider/ApplicationConfigProvider', - () => ({ - useApplicationConfigContext: jest.fn().mockImplementation(() => ({ +jest.mock('../../../hooks/useApplicationStore', () => ({ + useApplicationStore: jest.fn().mockImplementation(() => ({ + applicationConfig: { customLogoUrlPath: 'https://custom-logo.png', customMonogramUrlPath: 'https://custom-monogram.png', - })), - }) -); + }, + })), +})); describe('Test Brand Logo', () => { it('Should render the brand logo with default props value', () => { diff --git a/openmetadata-ui/src/main/resources/ui/src/components/common/BrandImage/BrandImage.tsx b/openmetadata-ui/src/main/resources/ui/src/components/common/BrandImage/BrandImage.tsx index 1272493d6bdc..99ca0426d7df 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/common/BrandImage/BrandImage.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/common/BrandImage/BrandImage.tsx @@ -11,7 +11,7 @@ * limitations under the License. */ import React, { FC, useMemo } from 'react'; -import { useApplicationConfigContext } from '../../../context/ApplicationConfigProvider/ApplicationConfigProvider'; +import { useApplicationStore } from '../../../hooks/useApplicationStore'; import brandImageClassBase from '../../../utils/BrandImage/BrandImageClassBase'; interface BrandImageProps { @@ -39,8 +39,9 @@ const BrandImage: FC = ({ [] ); + const { applicationConfig } = useApplicationStore(); const { customLogoUrlPath = '', customMonogramUrlPath = '' } = - useApplicationConfigContext(); + applicationConfig ?? {}; const logoSource = isMonoGram ? customMonogramUrlPath || MonoGram diff --git a/openmetadata-ui/src/main/resources/ui/src/components/common/CopyToClipboardButton/CopyToClipboardButton.test.tsx b/openmetadata-ui/src/main/resources/ui/src/components/common/CopyToClipboardButton/CopyToClipboardButton.test.tsx index ec3519d59fb8..61aa0d86c586 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/common/CopyToClipboardButton/CopyToClipboardButton.test.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/common/CopyToClipboardButton/CopyToClipboardButton.test.tsx @@ -46,22 +46,18 @@ describe('Test CopyToClipboardButton Component', () => { expect(callBack).toHaveBeenCalled(); }); - it('Should show success message and hide after timeout', async () => { + it('Should show success message on clipboard click', async () => { jest.useFakeTimers(); - await act(async () => { - render(); - }); + render(); await act(async () => { fireEvent.click(screen.getByTestId('copy-secret')); }); - expect(screen.getByTestId('copy-success')).toBeInTheDocument(); + fireEvent.mouseOver(screen.getByTestId('copy-secret')); + jest.advanceTimersByTime(1000); - jest.advanceTimersByTime(1500); - - // success message should not be in the dom after timeout - expect(screen.queryByTestId('copy-success')).not.toBeInTheDocument(); + expect(screen.getByTestId('copy-success')).toBeInTheDocument(); }); it('Should have copied text in clipboard', async () => { diff --git a/openmetadata-ui/src/main/resources/ui/src/components/common/CopyToClipboardButton/CopyToClipboardButton.tsx b/openmetadata-ui/src/main/resources/ui/src/components/common/CopyToClipboardButton/CopyToClipboardButton.tsx index 0adda05a08c6..a0a7d5254ee2 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/common/CopyToClipboardButton/CopyToClipboardButton.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/common/CopyToClipboardButton/CopyToClipboardButton.tsx @@ -40,14 +40,18 @@ export const CopyToClipboardButton: FunctionComponent = ({ return ( - {t('message.copied-to-clipboard')} - - } - trigger="click"> + hasCopied ? ( + + {t('message.copied-to-clipboard')} + + ) : ( + + {t('message.copy-to-clipboard')} + + ) + }>     {loginConfig?.jwtTokenExpiryTime ?? NO_DATA_PLACEHOLDER}{' '} - {t('label.ms')} + {t('label.seconds')} diff --git a/openmetadata-ui/src/main/resources/ui/src/pages/ContainerPage/ContainerPage.test.tsx b/openmetadata-ui/src/main/resources/ui/src/pages/ContainerPage/ContainerPage.test.tsx index 3747f0826684..73622cca16d0 100644 --- a/openmetadata-ui/src/main/resources/ui/src/pages/ContainerPage/ContainerPage.test.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/pages/ContainerPage/ContainerPage.test.tsx @@ -59,8 +59,8 @@ jest.mock( () => jest.fn().mockImplementation(() => <>ActivityThreadPanel) ); -jest.mock('../../components/Auth/AuthProviders/AuthProvider', () => ({ - useAuthContext: jest.fn().mockReturnValue({ +jest.mock('../../hooks/useApplicationStore', () => ({ + useApplicationStore: jest.fn().mockReturnValue({ id: 'userid', }), })); diff --git a/openmetadata-ui/src/main/resources/ui/src/pages/ContainerPage/ContainerPage.tsx b/openmetadata-ui/src/main/resources/ui/src/pages/ContainerPage/ContainerPage.tsx index e636e6ab54b5..1a67d333f3cc 100644 --- a/openmetadata-ui/src/main/resources/ui/src/pages/ContainerPage/ContainerPage.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/pages/ContainerPage/ContainerPage.tsx @@ -22,7 +22,7 @@ import { useActivityFeedProvider } from '../../components/ActivityFeed/ActivityF import { ActivityFeedTab } from '../../components/ActivityFeed/ActivityFeedTab/ActivityFeedTab.component'; import ActivityThreadPanel from '../../components/ActivityFeed/ActivityThreadPanel/ActivityThreadPanel'; import { withActivityFeed } from '../../components/AppRouter/withActivityFeed'; -import { useAuthContext } from '../../components/Auth/AuthProviders/AuthProvider'; + import { CustomPropertyTable } from '../../components/common/CustomPropertyTable/CustomPropertyTable'; import DescriptionV1 from '../../components/common/EntityDescription/DescriptionV1'; import ErrorPlaceHolder from '../../components/common/ErrorWithPlaceholder/ErrorPlaceHolder'; @@ -56,6 +56,7 @@ import { Container } from '../../generated/entity/data/container'; import { ThreadType } from '../../generated/entity/feed/thread'; import { Include } from '../../generated/type/include'; import { TagLabel } from '../../generated/type/tagLabel'; +import { useApplicationStore } from '../../hooks/useApplicationStore'; import { useFqn } from '../../hooks/useFqn'; import { FeedCounts } from '../../interface/feed.interface'; import { postThread } from '../../rest/feedsAPI'; @@ -82,7 +83,7 @@ import { showErrorToast, showSuccessToast } from '../../utils/ToastUtils'; const ContainerPage = () => { const history = useHistory(); const { t } = useTranslation(); - const { currentUser } = useAuthContext(); + const { currentUser } = useApplicationStore(); const { getEntityPermissionByFqn } = usePermissionProvider(); const { postFeed, deleteFeed, updateFeed } = useActivityFeedProvider(); const { tab } = useParams<{ tab: EntityTabs }>(); diff --git a/openmetadata-ui/src/main/resources/ui/src/pages/CustomLogoConfigSettingsPage/CustomLogoConfigSettingsPage.tsx b/openmetadata-ui/src/main/resources/ui/src/pages/CustomLogoConfigSettingsPage/CustomLogoConfigSettingsPage.tsx index 5315b4aca53c..35ee2f06bc35 100644 --- a/openmetadata-ui/src/main/resources/ui/src/pages/CustomLogoConfigSettingsPage/CustomLogoConfigSettingsPage.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/pages/CustomLogoConfigSettingsPage/CustomLogoConfigSettingsPage.tsx @@ -48,7 +48,7 @@ const CustomLogoConfigSettingsPage = () => { const breadcrumbs: TitleBreadcrumbProps['titleLinks'] = useMemo( () => getSettingPageEntityBreadCrumb( - GlobalSettingsMenuCategory.OPEN_METADATA, + GlobalSettingsMenuCategory.PREFERENCES, t('label.custom-logo') ), [] diff --git a/openmetadata-ui/src/main/resources/ui/src/pages/CustomPageSettings/CustomPageSettings.tsx b/openmetadata-ui/src/main/resources/ui/src/pages/CustomPageSettings/CustomPageSettings.tsx index e2c8d1b5375e..8cefe05e7836 100644 --- a/openmetadata-ui/src/main/resources/ui/src/pages/CustomPageSettings/CustomPageSettings.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/pages/CustomPageSettings/CustomPageSettings.tsx @@ -66,7 +66,7 @@ export const CustomPageSettings = () => { const breadcrumbs: TitleBreadcrumbProps['titleLinks'] = useMemo( () => getSettingPageEntityBreadCrumb( - GlobalSettingsMenuCategory.OPEN_METADATA, + GlobalSettingsMenuCategory.PREFERENCES, t('label.customize-entity', { entity: t('label.landing-page'), }) diff --git a/openmetadata-ui/src/main/resources/ui/src/pages/DashboardDetailsPage/DashboardDetailsPage.component.tsx b/openmetadata-ui/src/main/resources/ui/src/pages/DashboardDetailsPage/DashboardDetailsPage.component.tsx index 8b310b7efdb6..6b20b0c1666a 100644 --- a/openmetadata-ui/src/main/resources/ui/src/pages/DashboardDetailsPage/DashboardDetailsPage.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/pages/DashboardDetailsPage/DashboardDetailsPage.component.tsx @@ -17,7 +17,7 @@ import { isUndefined, omitBy, toString } from 'lodash'; import React, { useCallback, useEffect, useMemo, useState } from 'react'; import { useTranslation } from 'react-i18next'; import { useHistory } from 'react-router-dom'; -import { useAuthContext } from '../../components/Auth/AuthProviders/AuthProvider'; + import ErrorPlaceHolder from '../../components/common/ErrorWithPlaceholder/ErrorPlaceHolder'; import Loader from '../../components/common/Loader/Loader'; import DashboardDetails from '../../components/Dashboard/DashboardDetails/DashboardDetails.component'; @@ -30,6 +30,7 @@ import { EntityType, TabSpecificField } from '../../enums/entity.enum'; import { CreateThread } from '../../generated/api/feed/createThread'; import { Chart } from '../../generated/entity/data/chart'; import { Dashboard } from '../../generated/entity/data/dashboard'; +import { useApplicationStore } from '../../hooks/useApplicationStore'; import { useFqn } from '../../hooks/useFqn'; import { updateChart } from '../../rest/chartAPI'; import { @@ -60,7 +61,7 @@ export type ChartType = { const DashboardDetailsPage = () => { const { t } = useTranslation(); - const { currentUser } = useAuthContext(); + const { currentUser } = useApplicationStore(); const USERId = currentUser?.id ?? ''; const history = useHistory(); const { getEntityPermissionByFqn } = usePermissionProvider(); diff --git a/openmetadata-ui/src/main/resources/ui/src/pages/DashboardDetailsPage/DashboardDetailsPage.test.tsx b/openmetadata-ui/src/main/resources/ui/src/pages/DashboardDetailsPage/DashboardDetailsPage.test.tsx index 341d03393973..6627420ecfe4 100644 --- a/openmetadata-ui/src/main/resources/ui/src/pages/DashboardDetailsPage/DashboardDetailsPage.test.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/pages/DashboardDetailsPage/DashboardDetailsPage.test.tsx @@ -55,8 +55,8 @@ jest.mock('react-router-dom', () => ({ })), })); -jest.mock('../../components/Auth/AuthProviders/AuthProvider', () => ({ - useAuthContext: jest.fn(() => ({ currentUser: mockUserData })), +jest.mock('../../hooks/useApplicationStore', () => ({ + useApplicationStore: jest.fn(() => ({ currentUser: mockUserData })), })); jest.mock('../../components/common/ErrorWithPlaceholder/ErrorPlaceHolder', () => diff --git a/openmetadata-ui/src/main/resources/ui/src/pages/DataInsightPage/DataInsightProvider.tsx b/openmetadata-ui/src/main/resources/ui/src/pages/DataInsightPage/DataInsightProvider.tsx index 81a9fc18127c..425edaf554e8 100644 --- a/openmetadata-ui/src/main/resources/ui/src/pages/DataInsightPage/DataInsightProvider.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/pages/DataInsightPage/DataInsightProvider.tsx @@ -19,11 +19,11 @@ import React, { useMemo, useState, } from 'react'; -import { ListItem } from 'react-awesome-query-builder'; import Loader from '../../components/common/Loader/Loader'; import { SearchDropdownOption } from '../../components/SearchDropdown/SearchDropdown.interface'; import { autocomplete } from '../../constants/AdvancedSearch.constants'; -import { PAGE_SIZE } from '../../constants/constants'; +import { WILD_CARD_CHAR } from '../../constants/char.constants'; +import { PAGE_SIZE_BASE } from '../../constants/constants'; import { INITIAL_CHART_FILTER } from '../../constants/DataInsight.constants'; import { DEFAULT_RANGE_DATA, @@ -37,7 +37,6 @@ import { ChartFilter } from '../../interface/data-insight.interface'; import { getListKPIs } from '../../rest/KpiAPI'; import { searchQuery } from '../../rest/searchAPI'; import { getTags } from '../../rest/tagAPI'; -import { getTeamFilter } from '../../utils/DataInsightUtils'; import { getEntityName } from '../../utils/EntityUtils'; import { DataInsightContextType, @@ -60,6 +59,7 @@ const DataInsightProvider = ({ children }: DataInsightProviderProps) => { selectedOptions: [], options: [], }); + const [isTeamLoading, setIsTeamLoading] = useState(false); const [tierOptions, setTierOptions] = useState({ selectedOptions: [], options: [], @@ -120,16 +120,35 @@ const DataInsightProvider = ({ children }: DataInsightProviderProps) => { })); }; + const fetchTeamOptions = async (query = WILD_CARD_CHAR) => { + const response = await searchQuery({ + searchIndex: SearchIndex.TEAM, + query: query, + pageSize: PAGE_SIZE_BASE, + }); + const hits = response.hits.hits; + const teamFilterOptions = hits.map((hit) => { + const source = hit._source; + + return { key: source.name, label: source.displayName ?? source.name }; + }); + + return teamFilterOptions; + }; + const handleTeamSearch = async (query: string) => { if (fetchTeamSuggestions && !isEmpty(query)) { + setIsTeamLoading(true); try { - const response = await fetchTeamSuggestions(query, PAGE_SIZE); + const response = await fetchTeamOptions(query); setTeamOptions((prev) => ({ ...prev, - options: getTeamFilter(response.values as ListItem[]), + options: response, })); } catch (_error) { // we will not show the toast error message for suggestion API + } finally { + setIsTeamLoading(false); } } else { setTeamOptions((prev) => ({ @@ -163,31 +182,24 @@ const DataInsightProvider = ({ children }: DataInsightProviderProps) => { if (teamsOptions.defaultOptions.length) { setTeamOptions((prev) => ({ ...prev, - options: prev.defaultOptions, + options: [...prev.selectedOptions, ...prev.defaultOptions], })); return; } try { - const response = await searchQuery({ - searchIndex: SearchIndex.TEAM, - query: '*', - pageSize: PAGE_SIZE, - }); - const hits = response.hits.hits; - const teamFilterOptions = hits.map((hit) => { - const source = hit._source; - - return { key: source.name, label: source.displayName ?? source.name }; - }); + setIsTeamLoading(true); + const response = await fetchTeamOptions(); setTeamOptions((prev) => ({ ...prev, - defaultOptions: teamFilterOptions, - options: teamFilterOptions, + defaultOptions: response, + options: response, })); } catch (_error) { // we will not show the toast error message for search API + } finally { + setIsTeamLoading(false); } }; @@ -247,6 +259,7 @@ const DataInsightProvider = ({ children }: DataInsightProviderProps) => { onChange: handleTeamChange, onGetInitialOptions: fetchDefaultTeamOptions, onSearch: handleTeamSearch, + isSuggestionsLoading: isTeamLoading, }, tierFilter: { options: tierOptions.options, @@ -270,6 +283,7 @@ const DataInsightProvider = ({ children }: DataInsightProviderProps) => { fetchDefaultTeamOptions, handleTeamChange, teamsOptions, + isTeamLoading, ] ); diff --git a/openmetadata-ui/src/main/resources/ui/src/pages/DataInsightPage/KPIList.test.tsx b/openmetadata-ui/src/main/resources/ui/src/pages/DataInsightPage/KPIList.test.tsx index 11fed4923f7a..6fe752145bbe 100644 --- a/openmetadata-ui/src/main/resources/ui/src/pages/DataInsightPage/KPIList.test.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/pages/DataInsightPage/KPIList.test.tsx @@ -20,8 +20,8 @@ import KPIList from './KPIList'; import { KPI_DATA } from './mocks/KPIList'; const mockPush = jest.fn(); -jest.mock('../../components/Auth/AuthProviders/AuthProvider', () => ({ - useAuthContext: jest.fn(() => ({ +jest.mock('../../hooks/useApplicationStore', () => ({ + useApplicationStore: jest.fn(() => ({ currentUser: { ...mockUserData, isAdmin: true }, })), })); diff --git a/openmetadata-ui/src/main/resources/ui/src/pages/DataInsightPage/KPIList.tsx b/openmetadata-ui/src/main/resources/ui/src/pages/DataInsightPage/KPIList.tsx index 3a3cd23a3cb7..4d143151f0c8 100644 --- a/openmetadata-ui/src/main/resources/ui/src/pages/DataInsightPage/KPIList.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/pages/DataInsightPage/KPIList.tsx @@ -20,7 +20,6 @@ import { useTranslation } from 'react-i18next'; import { Link, useHistory } from 'react-router-dom'; import { ReactComponent as EditIcon } from '../../assets/svg/edit-new.svg'; import { ReactComponent as IconDelete } from '../../assets/svg/ic-delete.svg'; -import { useAuthContext } from '../../components/Auth/AuthProviders/AuthProvider'; import DeleteWidgetModal from '../../components/common/DeleteWidget/DeleteWidgetModal'; import ErrorPlaceHolder from '../../components/common/ErrorWithPlaceholder/ErrorPlaceHolder'; import NextPrevious from '../../components/common/NextPrevious/NextPrevious'; @@ -41,6 +40,7 @@ import { EntityType } from '../../enums/entity.enum'; import { Kpi, KpiTargetType } from '../../generated/dataInsight/kpi/kpi'; import { Operation } from '../../generated/entity/policies/policy'; import { Paging } from '../../generated/type/paging'; +import { useApplicationStore } from '../../hooks/useApplicationStore'; import { getListKPIs } from '../../rest/KpiAPI'; import { formatDateTime } from '../../utils/date-time/DateTimeUtils'; import { getEntityName } from '../../utils/EntityUtils'; @@ -48,7 +48,7 @@ import { checkPermission } from '../../utils/PermissionsUtils'; const KPIList = () => { const history = useHistory(); - const { currentUser } = useAuthContext(); + const { currentUser } = useApplicationStore(); const isAdminUser = currentUser?.isAdmin ?? false; const { t } = useTranslation(); const { permissions } = usePermissionProvider(); diff --git a/openmetadata-ui/src/main/resources/ui/src/pages/DataModelPage/DataModelPage.component.tsx b/openmetadata-ui/src/main/resources/ui/src/pages/DataModelPage/DataModelPage.component.tsx index df2d637871fb..3bd22b882647 100644 --- a/openmetadata-ui/src/main/resources/ui/src/pages/DataModelPage/DataModelPage.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/pages/DataModelPage/DataModelPage.component.tsx @@ -23,7 +23,7 @@ import { useState, } from 'react'; import { useTranslation } from 'react-i18next'; -import { useAuthContext } from '../../components/Auth/AuthProviders/AuthProvider'; + import ErrorPlaceHolder from '../../components/common/ErrorWithPlaceholder/ErrorPlaceHolder'; import Loader from '../../components/common/Loader/Loader'; import DataModelDetails from '../../components/Dashboard/DataModel/DataModels/DataModelDetails.component'; @@ -38,6 +38,7 @@ import { CreateThread } from '../../generated/api/feed/createThread'; import { Tag } from '../../generated/entity/classification/tag'; import { DashboardDataModel } from '../../generated/entity/data/dashboardDataModel'; import { Include } from '../../generated/type/include'; +import { useApplicationStore } from '../../hooks/useApplicationStore'; import { useFqn } from '../../hooks/useFqn'; import { addDataModelFollower, @@ -59,7 +60,7 @@ import { showErrorToast } from '../../utils/ToastUtils'; const DataModelsPage = () => { const { t } = useTranslation(); - const { currentUser } = useAuthContext(); + const { currentUser } = useApplicationStore(); const { getEntityPermissionByFqn } = usePermissionProvider(); const { fqn: dashboardDataModelFQN } = useFqn(); diff --git a/openmetadata-ui/src/main/resources/ui/src/pages/DataModelPage/DataModelPage.test.tsx b/openmetadata-ui/src/main/resources/ui/src/pages/DataModelPage/DataModelPage.test.tsx index e38c0eb65b0f..84ad84e97427 100644 --- a/openmetadata-ui/src/main/resources/ui/src/pages/DataModelPage/DataModelPage.test.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/pages/DataModelPage/DataModelPage.test.tsx @@ -46,8 +46,8 @@ const mockUpdateTierTag = jest.fn(); const mockShowErrorToast = jest.fn(); const ENTITY_MISSING_ERROR = 'Entity missing error.'; -jest.mock('../../components/Auth/AuthProviders/AuthProvider', () => ({ - useAuthContext: jest.fn(() => ({ +jest.mock('../../hooks/useApplicationStore', () => ({ + useApplicationStore: jest.fn().mockImplementation(() => ({ currentUser: mockUserData, })), })); diff --git a/openmetadata-ui/src/main/resources/ui/src/pages/EditCustomLogoConfig/EditCustomLogoConfig.tsx b/openmetadata-ui/src/main/resources/ui/src/pages/EditCustomLogoConfig/EditCustomLogoConfig.tsx index b560e14fa3ae..8b64b52d988f 100644 --- a/openmetadata-ui/src/main/resources/ui/src/pages/EditCustomLogoConfig/EditCustomLogoConfig.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/pages/EditCustomLogoConfig/EditCustomLogoConfig.tsx @@ -72,7 +72,7 @@ const EditCustomLogoConfig = () => { { name: t('label.custom-logo'), url: getSettingPath( - GlobalSettingsMenuCategory.OPEN_METADATA, + GlobalSettingsMenuCategory.PREFERENCES, GlobalSettingOptions.CUSTOM_LOGO ), }, diff --git a/openmetadata-ui/src/main/resources/ui/src/pages/EditEmailConfigPage/EditEmailConfigPage.component.tsx b/openmetadata-ui/src/main/resources/ui/src/pages/EditEmailConfigPage/EditEmailConfigPage.component.tsx index d4cea4389dde..9425c2256fc5 100644 --- a/openmetadata-ui/src/main/resources/ui/src/pages/EditEmailConfigPage/EditEmailConfigPage.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/pages/EditEmailConfigPage/EditEmailConfigPage.component.tsx @@ -61,7 +61,7 @@ function EditEmailConfigPage() { { name: t('label.email'), url: getSettingPath( - GlobalSettingsMenuCategory.OPEN_METADATA, + GlobalSettingsMenuCategory.PREFERENCES, GlobalSettingOptions.EMAIL ), }, @@ -99,7 +99,7 @@ function EditEmailConfigPage() { const handleRedirectionToSettingsPage = useCallback(() => { history.push( getSettingPath( - GlobalSettingsMenuCategory.OPEN_METADATA, + GlobalSettingsMenuCategory.PREFERENCES, GlobalSettingOptions.EMAIL ) ); diff --git a/openmetadata-ui/src/main/resources/ui/src/pages/EmailConfigSettingsPage/EmailConfigSettingsPage.component.tsx b/openmetadata-ui/src/main/resources/ui/src/pages/EmailConfigSettingsPage/EmailConfigSettingsPage.component.tsx index 8f26396925c1..a8786791f317 100644 --- a/openmetadata-ui/src/main/resources/ui/src/pages/EmailConfigSettingsPage/EmailConfigSettingsPage.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/pages/EmailConfigSettingsPage/EmailConfigSettingsPage.component.tsx @@ -48,7 +48,7 @@ function EmailConfigSettingsPage() { const breadcrumbs: TitleBreadcrumbProps['titleLinks'] = useMemo( () => getSettingPageEntityBreadCrumb( - GlobalSettingsMenuCategory.OPEN_METADATA, + GlobalSettingsMenuCategory.PREFERENCES, t('label.email') ), [] diff --git a/openmetadata-ui/src/main/resources/ui/src/pages/GlobalSettingPage/GlobalSettingCategory/GlobalSettingCategoryPage.tsx b/openmetadata-ui/src/main/resources/ui/src/pages/GlobalSettingPage/GlobalSettingCategory/GlobalSettingCategoryPage.tsx index 34455e90bfc8..95a69f3df400 100644 --- a/openmetadata-ui/src/main/resources/ui/src/pages/GlobalSettingPage/GlobalSettingCategory/GlobalSettingCategoryPage.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/pages/GlobalSettingPage/GlobalSettingCategory/GlobalSettingCategoryPage.tsx @@ -76,7 +76,7 @@ const GlobalSettingCategoryPage = () => { break; case GlobalSettingOptions.SEARCH: - if (category === GlobalSettingsMenuCategory.OPEN_METADATA) { + if (category === GlobalSettingsMenuCategory.PREFERENCES) { history.push( getSettingsPathWithFqn( category, diff --git a/openmetadata-ui/src/main/resources/ui/src/pages/Glossary/GlossaryPage/GlossaryPage.component.tsx b/openmetadata-ui/src/main/resources/ui/src/pages/Glossary/GlossaryPage/GlossaryPage.component.tsx index 772aef1583d9..51cb7c75b47f 100644 --- a/openmetadata-ui/src/main/resources/ui/src/pages/Glossary/GlossaryPage/GlossaryPage.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/pages/Glossary/GlossaryPage/GlossaryPage.component.tsx @@ -13,6 +13,7 @@ import { AxiosError } from 'axios'; import { compare } from 'fast-json-patch'; +import { isEmpty } from 'lodash'; import React, { useCallback, useEffect, useMemo, useState } from 'react'; import { useTranslation } from 'react-i18next'; import { useHistory } from 'react-router-dom'; @@ -257,7 +258,7 @@ const GlossaryPage = () => { fetchGlossaryList(); } catch (error) { showErrorToast( - error, + error as AxiosError, t('server.delete-entity-error', { entity: t('label.glossary'), }) @@ -270,6 +271,9 @@ const GlossaryPage = () => { const handleGlossaryTermUpdate = useCallback( async (updatedData: GlossaryTerm) => { const jsonPatch = compare(selectedData as GlossaryTerm, updatedData); + if (isEmpty(jsonPatch)) { + return; + } try { const response = await patchGlossaryTerm( selectedData?.id as string, diff --git a/openmetadata-ui/src/main/resources/ui/src/pages/IncidentManager/IncidentManagerDetailPage/IncidentManagerDetailPage.tsx b/openmetadata-ui/src/main/resources/ui/src/pages/IncidentManager/IncidentManagerDetailPage/IncidentManagerDetailPage.tsx index 862d533e8a52..6c4748ac30fd 100644 --- a/openmetadata-ui/src/main/resources/ui/src/pages/IncidentManager/IncidentManagerDetailPage/IncidentManagerDetailPage.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/pages/IncidentManager/IncidentManagerDetailPage/IncidentManagerDetailPage.tsx @@ -45,7 +45,6 @@ import { getTestCaseByFqn, updateTestCaseById } from '../../../rest/testAPI'; import { getFeedCounts } from '../../../utils/CommonUtils'; import { checkPermission } from '../../../utils/PermissionsUtils'; import { getIncidentManagerDetailPagePath } from '../../../utils/RouterUtils'; -import { getDecodedFqn } from '../../../utils/StringsUtils'; import { showErrorToast } from '../../../utils/ToastUtils'; import { IncidentManagerTabs } from '../IncidentManager.interface'; import { TestCaseData } from './IncidentManagerDetailPage.interface'; @@ -61,11 +60,6 @@ const IncidentManagerDetailPage = () => { const { fqn: testCaseFQN } = useFqn(); - const decodedTestCaseFQN = useMemo( - () => getDecodedFqn(testCaseFQN), - [testCaseFQN] - ); - const [testCaseData, setTestCaseData] = useState({ data: undefined, isLoading: true, @@ -132,7 +126,7 @@ const IncidentManagerDetailPage = () => { const fetchTestCaseData = async () => { setTestCaseData((prev) => ({ ...prev, isLoading: true })); try { - const response = await getTestCaseByFqn(decodedTestCaseFQN, { + const response = await getTestCaseByFqn(testCaseFQN, { fields: [ 'testSuite', 'testCaseResult', @@ -177,7 +171,7 @@ const IncidentManagerDetailPage = () => { if (activeKey !== activeTab) { history.push( getIncidentManagerDetailPagePath( - decodedTestCaseFQN, + testCaseFQN, activeKey as IncidentManagerTabs ) ); @@ -230,17 +224,17 @@ const IncidentManagerDetailPage = () => { }, []); const getEntityFeedCount = useCallback(() => { - getFeedCounts(EntityType.TEST_CASE, decodedTestCaseFQN, handleFeedCount); - }, [decodedTestCaseFQN]); + getFeedCounts(EntityType.TEST_CASE, testCaseFQN, handleFeedCount); + }, [testCaseFQN]); useEffect(() => { - if (hasViewPermission && decodedTestCaseFQN) { + if (hasViewPermission && testCaseFQN) { fetchTestCaseData(); getEntityFeedCount(); } else { setTestCaseData((prev) => ({ ...prev, isLoading: false })); } - }, [decodedTestCaseFQN, hasViewPermission]); + }, [testCaseFQN, hasViewPermission]); if (testCaseData.isLoading) { return ; diff --git a/openmetadata-ui/src/main/resources/ui/src/pages/IncidentManager/IncidentManagerPage.test.tsx b/openmetadata-ui/src/main/resources/ui/src/pages/IncidentManager/IncidentManagerPage.test.tsx index fb4e1e2412ce..1dc858df84b0 100644 --- a/openmetadata-ui/src/main/resources/ui/src/pages/IncidentManager/IncidentManagerPage.test.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/pages/IncidentManager/IncidentManagerPage.test.tsx @@ -10,8 +10,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import { render, screen } from '@testing-library/react'; +import { act, fireEvent, render, screen } from '@testing-library/react'; import React from 'react'; +import { getListTestCaseIncidentStatus } from '../../rest/incidentManagerAPI'; import IncidentManagerPage from './IncidentManagerPage'; jest.mock('../../components/common/NextPrevious/NextPrevious', () => { @@ -20,9 +21,21 @@ jest.mock('../../components/common/NextPrevious/NextPrevious', () => { jest.mock( '../../components/common/DatePickerMenu/DatePickerMenu.component', () => { - return jest - .fn() - .mockImplementation(() =>
    DatePickerMenu.component
    ); + return jest.fn().mockImplementation(({ handleDateRangeChange }) => ( +
    +

    DatePickerMenu.component

    + +
    + )); } ); jest.mock('../../components/PageLayoutV1/PageLayoutV1', () => { @@ -98,4 +111,23 @@ describe('IncidentManagerPage', () => { await screen.findByText('NextPrevious.component') ).toBeInTheDocument(); }); + + it('Incident should be fetch with updated time', async () => { + const mockGetListTestCaseIncidentStatus = + getListTestCaseIncidentStatus as jest.Mock; + render(); + + const timeFilterButton = await screen.findByTestId('time-filter'); + + act(() => { + fireEvent.click(timeFilterButton); + }); + + expect(mockGetListTestCaseIncidentStatus).toHaveBeenCalledWith({ + endTs: 1710161424255, + latest: true, + limit: 10, + startTs: 1709556624254, + }); + }); }); diff --git a/openmetadata-ui/src/main/resources/ui/src/pages/IncidentManager/IncidentManagerPage.tsx b/openmetadata-ui/src/main/resources/ui/src/pages/IncidentManager/IncidentManagerPage.tsx index 3578ee59bd93..4bbf109a8a61 100644 --- a/openmetadata-ui/src/main/resources/ui/src/pages/IncidentManager/IncidentManagerPage.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/pages/IncidentManager/IncidentManagerPage.tsx @@ -15,7 +15,7 @@ import { DefaultOptionType } from 'antd/lib/select'; import { ColumnsType } from 'antd/lib/table'; import { AxiosError } from 'axios'; import { compare } from 'fast-json-patch'; -import { isEqual, startCase } from 'lodash'; +import { isEqual, pick, startCase } from 'lodash'; import { DateRangeObject } from 'Models'; import QueryString from 'qs'; import React, { useCallback, useEffect, useMemo, useState } from 'react'; @@ -242,13 +242,11 @@ const IncidentManagerPage = () => { }; const handleDateRangeChange = (value: DateRangeObject) => { - const dateRangeObject = { - startTs: filters.startTs, - endTs: filters.endTs, - }; + const updatedFilter = pick(value, ['startTs', 'endTs']); + const existingFilters = pick(filters, ['startTs', 'endTs']); - if (!isEqual(value, dateRangeObject)) { - setFilters((pre) => ({ ...pre, ...dateRangeObject })); + if (!isEqual(existingFilters, updatedFilter)) { + setFilters((pre) => ({ ...pre, ...updatedFilter })); } }; diff --git a/openmetadata-ui/src/main/resources/ui/src/pages/LoginPage/SignInPage.test.tsx b/openmetadata-ui/src/main/resources/ui/src/pages/LoginPage/SignInPage.test.tsx index b787c7c6f5a5..b3fe1f301994 100644 --- a/openmetadata-ui/src/main/resources/ui/src/pages/LoginPage/SignInPage.test.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/pages/LoginPage/SignInPage.test.tsx @@ -19,17 +19,24 @@ import { } from '@testing-library/react'; import React from 'react'; import { MemoryRouter } from 'react-router-dom'; -import { useAuthContext } from '../../components/Auth/AuthProviders/AuthProvider'; + +import { useApplicationStore } from '../../hooks/useApplicationStore'; import SignInPage from './SignInPage'; -const mockUseAuthContext = useAuthContext as jest.Mock; +const mockuseApplicationStore = useApplicationStore as unknown as jest.Mock; jest.mock('react-router-dom', () => ({ useHistory: jest.fn(), })); -jest.mock('../../components/Auth/AuthProviders/AuthProvider', () => ({ - useAuthContext: jest.fn(), +jest.mock('../../hooks/useApplicationStore', () => ({ + useApplicationStore: jest.fn().mockImplementation(() => ({ + applicationConfig: { + customLogoUrlPath: 'https://custom-logo.png', + customMonogramUrlPath: 'https://custom-monogram.png', + }, + getOidcToken: jest.fn(), + })), })); jest.mock('./LoginCarousel', () => @@ -50,11 +57,12 @@ describe('Test SignInPage Component', () => { }); it('Component should render', async () => { - mockUseAuthContext.mockReturnValue({ + mockuseApplicationStore.mockReturnValue({ isAuthDisabled: false, authConfig: { provider: 'google' }, onLoginHandler: jest.fn(), onLogoutHandler: jest.fn(), + getOidcToken: jest.fn(), }); const { container } = render(, { wrapper: MemoryRouter, @@ -77,11 +85,12 @@ describe('Test SignInPage Component', () => { ['aws-cognito', 'Sign in with aws cognito'], ['unknown-provider', 'SSO Provider unknown-provider is not supported'], ])('Sign in button should render correctly for %s', async (provider) => { - mockUseAuthContext.mockReturnValue({ + mockuseApplicationStore.mockReturnValue({ isAuthDisabled: false, authConfig: { provider }, onLoginHandler: jest.fn(), onLogoutHandler: jest.fn(), + getOidcToken: jest.fn(), }); const { container } = render(, { wrapper: MemoryRouter, @@ -99,11 +108,12 @@ describe('Test SignInPage Component', () => { }); it('Sign in button should render correctly with custom provider name', async () => { - mockUseAuthContext.mockReturnValue({ + mockuseApplicationStore.mockReturnValue({ isAuthDisabled: false, authConfig: { provider: 'custom-oidc', providerName: 'Custom OIDC' }, onLoginHandler: jest.fn(), onLogoutHandler: jest.fn(), + getOidcToken: jest.fn(), }); const { container } = render(, { wrapper: MemoryRouter, @@ -113,12 +123,17 @@ describe('Test SignInPage Component', () => { expect(signinButton).toBeInTheDocument(); }); - it('Page should render the brand logo', async () => { - mockUseAuthContext.mockReturnValue({ + it('Page should render the correct logo image', async () => { + mockuseApplicationStore.mockReturnValue({ isAuthDisabled: false, authConfig: { provider: 'custom-oidc', providerName: 'Custom OIDC' }, onLoginHandler: jest.fn(), onLogoutHandler: jest.fn(), + getOidcToken: jest.fn(), + applicationConfig: { + customLogoUrlPath: 'https://custom-logo.png', + customMonogramUrlPath: 'https://custom-monogram.png', + }, }); render(, { wrapper: MemoryRouter, diff --git a/openmetadata-ui/src/main/resources/ui/src/pages/LoginPage/SignInPage.tsx b/openmetadata-ui/src/main/resources/ui/src/pages/LoginPage/SignInPage.tsx index 3e4a298ea0d7..338571886697 100644 --- a/openmetadata-ui/src/main/resources/ui/src/pages/LoginPage/SignInPage.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/pages/LoginPage/SignInPage.tsx @@ -25,14 +25,14 @@ import IconGoogle from '../../assets/img/icon-google.png'; import IconOkta from '../../assets/img/icon-okta.png'; import loginBG from '../../assets/img/login-bg.png'; import { ReactComponent as IconFailBadge } from '../../assets/svg/fail-badge.svg'; -import { useAuthContext } from '../../components/Auth/AuthProviders/AuthProvider'; + import { useBasicAuth } from '../../components/Auth/AuthProviders/BasicAuthProvider'; import BrandImage from '../../components/common/BrandImage/BrandImage'; import Loader from '../../components/common/Loader/Loader'; import LoginButton from '../../components/common/LoginButton/LoginButton'; import { ROUTES, VALIDATION_MESSAGES } from '../../constants/constants'; import { AuthProvider } from '../../generated/settings/settings'; -import localState from '../../utils/LocalStorageUtils'; +import { useApplicationStore } from '../../hooks/useApplicationStore'; import './login.style.less'; import LoginCarousel from './LoginCarousel'; @@ -41,21 +41,21 @@ const SignInPage = () => { const [form] = Form.useForm(); const history = useHistory(); - const { authConfig, onLoginHandler, onLogoutHandler, isAuthenticated } = - useAuthContext(); + const { + authConfig, + onLoginHandler, + onLogoutHandler, + isAuthenticated, + getOidcToken, + } = useApplicationStore(); const { t } = useTranslation(); - const { isAuthProviderBasic } = useMemo(() => { + const { isAuthProviderBasic, isAuthProviderLDAP } = useMemo(() => { return { isAuthProviderBasic: authConfig?.provider === AuthProvider.Basic || authConfig?.provider === AuthProvider.LDAP, - }; - }, [authConfig]); - - const { isAuthProviderLDAP } = useMemo(() => { - return { isAuthProviderLDAP: authConfig?.provider === AuthProvider.LDAP, }; }, [authConfig]); @@ -63,7 +63,7 @@ const SignInPage = () => { const { handleLogin, loginError } = useBasicAuth(); const isTokenExpired = () => { - const token = localState.getOidcToken(); + const token = getOidcToken(); if (token) { try { const { exp } = jwtDecode(token); @@ -268,13 +268,14 @@ const SignInPage = () => { )} -
    - - {t('label.forgot-password')} - -
    - - {(authConfig?.enableSelfSignup || isAuthProviderLDAP) && ( + {!isAuthProviderLDAP && ( +
    + + {t('label.forgot-password')} + +
    + )} + {authConfig?.enableSelfSignup && !isAuthProviderLDAP && ( <> diff --git a/openmetadata-ui/src/main/resources/ui/src/pages/LogsViewer/LogsViewer.component.tsx b/openmetadata-ui/src/main/resources/ui/src/pages/LogsViewer/LogsViewer.component.tsx index d716729a7e07..489b6870fef9 100644 --- a/openmetadata-ui/src/main/resources/ui/src/pages/LogsViewer/LogsViewer.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/pages/LogsViewer/LogsViewer.component.tsx @@ -91,7 +91,7 @@ const LogsViewer = () => { const logs = await getLatestApplicationRuns(ingestionName); setAppLatestRun(data[0]); - setLogs(logs.data_insight_task); + setLogs(logs.data_insight_task || logs.application_task); return; } @@ -295,7 +295,9 @@ const LogsViewer = () => { return { Type: - ingestionDetails?.pipelineType ?? scheduleClass?.scheduleType ?? '--', + ingestionDetails?.pipelineType ?? + scheduleClass?.scheduleTimeline ?? + '--', Schedule: ingestionDetails?.airflowConfig.scheduleInterval ?? scheduleClass?.cronExpression ?? diff --git a/openmetadata-ui/src/main/resources/ui/src/pages/LogsViewer/LogsViewer.interfaces.ts b/openmetadata-ui/src/main/resources/ui/src/pages/LogsViewer/LogsViewer.interfaces.ts index e54437c8551b..97dbe1e7c43a 100644 --- a/openmetadata-ui/src/main/resources/ui/src/pages/LogsViewer/LogsViewer.interfaces.ts +++ b/openmetadata-ui/src/main/resources/ui/src/pages/LogsViewer/LogsViewer.interfaces.ts @@ -18,8 +18,8 @@ export interface IngestionPipelineLogByIdInterface { lineage_task?: string; test_suite_task?: string; data_insight_task?: string; - elasticsearch_reindex_task?: string; dbt_task?: string; + elasticsearch_reindex_task?: string; total?: string; after?: string; } diff --git a/openmetadata-ui/src/main/resources/ui/src/pages/MlModelPage/MlModelPage.component.tsx b/openmetadata-ui/src/main/resources/ui/src/pages/MlModelPage/MlModelPage.component.tsx index 5d1016e1b1e1..b5225caaa6b0 100644 --- a/openmetadata-ui/src/main/resources/ui/src/pages/MlModelPage/MlModelPage.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/pages/MlModelPage/MlModelPage.component.tsx @@ -17,7 +17,7 @@ import { isEmpty, isNil, isUndefined, omitBy, toString } from 'lodash'; import React, { useCallback, useEffect, useMemo, useState } from 'react'; import { useTranslation } from 'react-i18next'; import { useHistory } from 'react-router-dom'; -import { useAuthContext } from '../../components/Auth/AuthProviders/AuthProvider'; + import ErrorPlaceHolder from '../../components/common/ErrorWithPlaceholder/ErrorPlaceHolder'; import Loader from '../../components/common/Loader/Loader'; import { QueryVote } from '../../components/Database/TableQueries/TableQueries.interface'; @@ -29,6 +29,7 @@ import { ERROR_PLACEHOLDER_TYPE } from '../../enums/common.enum'; import { EntityType, TabSpecificField } from '../../enums/entity.enum'; import { CreateThread } from '../../generated/api/feed/createThread'; import { Mlmodel } from '../../generated/entity/data/mlmodel'; +import { useApplicationStore } from '../../hooks/useApplicationStore'; import { useFqn } from '../../hooks/useFqn'; import { postThread } from '../../rest/feedsAPI'; import { @@ -50,7 +51,7 @@ import { showErrorToast } from '../../utils/ToastUtils'; const MlModelPage = () => { const { t } = useTranslation(); - const { currentUser } = useAuthContext(); + const { currentUser } = useApplicationStore(); const history = useHistory(); const { fqn: mlModelFqn } = useFqn(); const [mlModelDetail, setMlModelDetail] = useState({} as Mlmodel); diff --git a/openmetadata-ui/src/main/resources/ui/src/pages/MyDataPage/MyDataPage.component.tsx b/openmetadata-ui/src/main/resources/ui/src/pages/MyDataPage/MyDataPage.component.tsx index 72a2aa966cc7..13b404e216fb 100644 --- a/openmetadata-ui/src/main/resources/ui/src/pages/MyDataPage/MyDataPage.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/pages/MyDataPage/MyDataPage.component.tsx @@ -23,16 +23,15 @@ import React, { import RGL, { WidthProvider } from 'react-grid-layout'; import { useTranslation } from 'react-i18next'; import ActivityFeedProvider from '../../components/ActivityFeed/ActivityFeedProvider/ActivityFeedProvider'; -import { useAuthContext } from '../../components/Auth/AuthProviders/AuthProvider'; import Loader from '../../components/common/Loader/Loader'; import WelcomeScreen from '../../components/MyData/WelcomeScreen/WelcomeScreen.component'; import PageLayoutV1 from '../../components/PageLayoutV1/PageLayoutV1'; import { LOGGED_IN_USER_STORAGE_KEY } from '../../constants/constants'; -import { useApplicationConfigContext } from '../../context/ApplicationConfigProvider/ApplicationConfigProvider'; import { AssetsType, EntityType } from '../../enums/entity.enum'; import { Thread } from '../../generated/entity/feed/thread'; import { PageType } from '../../generated/system/ui/page'; import { EntityReference } from '../../generated/type/entityReference'; +import { useApplicationStore } from '../../hooks/useApplicationStore'; import { useGridLayoutDirection } from '../../hooks/useGridLayoutDirection'; import { getDocumentByFQN } from '../../rest/DocStoreAPI'; import { getActiveAnnouncement } from '../../rest/feedsAPI'; @@ -47,8 +46,7 @@ const ReactGridLayout = WidthProvider(RGL); const MyDataPage = () => { const { t } = useTranslation(); - const { currentUser } = useAuthContext(); - const { selectedPersona } = useApplicationConfigContext(); + const { currentUser, selectedPersona } = useApplicationStore(); const [followedData, setFollowedData] = useState>(); const [followedDataCount, setFollowedDataCount] = useState(0); const [isLoadingOwnedData, setIsLoadingOwnedData] = useState(false); diff --git a/openmetadata-ui/src/main/resources/ui/src/pages/MyDataPage/MyDataPage.test.tsx b/openmetadata-ui/src/main/resources/ui/src/pages/MyDataPage/MyDataPage.test.tsx index 945c011e8bb0..8f0fcab3afb1 100644 --- a/openmetadata-ui/src/main/resources/ui/src/pages/MyDataPage/MyDataPage.test.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/pages/MyDataPage/MyDataPage.test.tsx @@ -10,11 +10,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - import { act, render, screen } from '@testing-library/react'; import userEvent from '@testing-library/user-event'; import React from 'react'; -import { useApplicationConfigContext } from '../../context/ApplicationConfigProvider/ApplicationConfigProvider'; import { mockActiveAnnouncementData, mockCustomizePageClassBase, @@ -25,7 +23,6 @@ import { import { getDocumentByFQN } from '../../rest/DocStoreAPI'; import { getActiveAnnouncement } from '../../rest/feedsAPI'; import MyDataPage from './MyDataPage.component'; - const mockLocalStorage = (() => { let store: Record = {}; @@ -41,7 +38,6 @@ const mockLocalStorage = (() => { }, }; })(); - Object.defineProperty(window, 'localStorage', { value: mockLocalStorage, }); @@ -56,15 +52,12 @@ jest.mock( )); } ); - jest.mock('../../components/common/Loader/Loader', () => { return jest.fn().mockImplementation(() =>
    Loader
    ); }); - jest.mock('../../utils/CustomizePageClassBase', () => { return mockCustomizePageClassBase; }); - jest.mock('../../components/PageLayoutV1/PageLayoutV1', () => { return jest .fn() @@ -72,7 +65,6 @@ jest.mock('../../components/PageLayoutV1/PageLayoutV1', () => {
    {children}
    )); }); - jest.mock( '../../components/MyData/WelcomeScreen/WelcomeScreen.component', () => { @@ -84,19 +76,19 @@ jest.mock( } ); -jest.mock( - '../../context/ApplicationConfigProvider/ApplicationConfigProvider', - () => ({ - useApplicationConfigContext: jest - .fn() - .mockImplementation(() => ({ selectedPersona: mockPersonaName })), - }) -); +let mockSelectedPersona: Record = { + fullyQualifiedName: mockPersonaName, +}; -jest.mock('../../components/Auth/AuthProviders/AuthProvider', () => ({ - useAuthContext: jest - .fn() - .mockImplementation(() => ({ currentUser: mockUserData })), +jest.mock('../../hooks/useApplicationStore', () => ({ + useApplicationStore: jest.fn().mockImplementation(() => ({ + currentUser: mockUserData, + selectedPersona: mockSelectedPersona, + })), +})); + +jest.mock('../../hooks/useGridLayoutDirection', () => ({ + useGridLayoutDirection: jest.fn().mockImplementation(() => 'ltr'), })); jest.mock('../../rest/DocStoreAPI', () => ({ @@ -104,22 +96,21 @@ jest.mock('../../rest/DocStoreAPI', () => ({ .fn() .mockImplementation(() => Promise.resolve(mockDocumentData)), })); - jest.mock('../../rest/feedsAPI', () => ({ getActiveAnnouncement: jest .fn() - .mockImplementation(() => mockActiveAnnouncementData), + .mockImplementation(() => Promise.resolve(mockActiveAnnouncementData)), })); - jest.mock('../../rest/userAPI', () => ({ - getUserById: jest.fn().mockImplementation(() => mockUserData), + getUserById: jest + .fn() + .mockImplementation(() => Promise.resolve(mockUserData)), })); - jest.mock('react-router-dom', () => ({ useLocation: jest.fn().mockImplementation(() => ({ pathname: '' })), })); - jest.mock('react-grid-layout', () => ({ + ...jest.requireActual('react-grid-layout'), WidthProvider: jest .fn() .mockImplementation(() => @@ -133,10 +124,6 @@ jest.mock('react-grid-layout', () => ({ default: '', })); -jest.mock('../../hooks/authHooks', () => ({ - useAuth: jest.fn().mockImplementation(() => ({ isAuthDisabled: false })), -})); - describe('MyDataPage component', () => { beforeEach(() => { localStorage.setItem('loggedInUsers', mockUserData.name); @@ -145,7 +132,6 @@ describe('MyDataPage component', () => { it('MyDataPage should only display WelcomeScreen when user logs in for the first time', async () => { // Simulate no user is logged in condition localStorage.clear(); - await act(async () => { render(); }); @@ -157,7 +143,6 @@ describe('MyDataPage component', () => { it('MyDataPage should display the main content after the WelcomeScreen is closed', async () => { // Simulate no user is logged in condition localStorage.clear(); - await act(async () => { render(); }); @@ -205,10 +190,10 @@ describe('MyDataPage component', () => { it('MyDataPage should not render announcement widget if there are no announcements', async () => { (getActiveAnnouncement as jest.Mock).mockImplementationOnce(() => Promise.resolve({ - response: { ...mockActiveAnnouncementData, data: [] }, + ...mockActiveAnnouncementData, + data: [], }) ); - await act(async () => { render(); }); @@ -228,7 +213,6 @@ describe('MyDataPage component', () => { (getDocumentByFQN as jest.Mock).mockImplementationOnce(() => Promise.reject(new Error('API failure')) ); - await act(async () => { render(); }); @@ -247,10 +231,7 @@ describe('MyDataPage component', () => { }); it('MyDataPage should render default widgets when there is no selected persona', async () => { - (useApplicationConfigContext as jest.Mock).mockImplementation(() => ({ - selectedPersona: {}, - })); - + mockSelectedPersona = {}; await act(async () => { render(); }); diff --git a/openmetadata-ui/src/main/resources/ui/src/pages/OmHealth/OmHealthPage.test.tsx b/openmetadata-ui/src/main/resources/ui/src/pages/OmHealth/OmHealthPage.test.tsx new file mode 100644 index 000000000000..ef7ee9974ef7 --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/pages/OmHealth/OmHealthPage.test.tsx @@ -0,0 +1,136 @@ +/* + * Copyright 2024 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { act, render, screen } from '@testing-library/react'; +import React from 'react'; +import { MemoryRouter } from 'react-router-dom'; +import ConnectionStepCard from '../../components/common/TestConnection/ConnectionStepCard/ConnectionStepCard'; +import { fetchOMStatus } from '../../rest/miscAPI'; +import { showErrorToast } from '../../utils/ToastUtils'; +import OmHealthPage from './OmHealthPage'; + +jest.mock('../../components/common/Loader/Loader', () => { + return jest.fn(() =>

    Loader

    ); +}); + +jest.mock( + '../../components/common/TestConnection/ConnectionStepCard/ConnectionStepCard', + () => { + return jest.fn().mockImplementation(() => jest.fn()); + } +); +jest.mock( + '../../components/common/TitleBreadcrumb/TitleBreadcrumb.component', + () => { + return jest.fn(() =>

    TitleBreadcrumb

    ); + } +); +jest.mock('../../components/PageHeader/PageHeader.component', () => { + return jest.fn(() =>

    PageHeader

    ); +}); +jest.mock('../../components/PageLayoutV1/PageLayoutV1', () => { + return jest.fn(({ children }) =>
    {children}
    ); +}); + +jest.mock('../../rest/miscAPI'); +jest.mock('../../utils/GlobalSettingsUtils'); +jest.mock('../../utils/ToastUtils', () => ({ + showErrorToast: jest.fn(), +})); + +describe('OmHealthPage', () => { + it('should render loader while loading', () => { + render(, { wrapper: MemoryRouter }); + + expect(screen.getByText('Loader')).toBeInTheDocument(); + }); + + it('should render health data after loading', async () => { + const mockData = { + validation1: { + description: 'Validation 1', + passed: true, + }, + validation2: { + description: 'Validation 2', + passed: false, + }, + }; + (fetchOMStatus as jest.Mock).mockResolvedValueOnce({ data: mockData }); + + await act(async () => { + render(, { wrapper: MemoryRouter }); + }); + + expect(ConnectionStepCard).toHaveBeenCalledWith( + { + isTestingConnection: false, + testConnectionStep: { description: '', mandatory: true, name: 'Data' }, + testConnectionStepResult: { + errorLog: undefined, + mandatory: true, + message: undefined, + name: 'Data', + passed: false, + }, + }, + {} + ); + }); + + it('should handle error while fetching health data', async () => { + const mockError = new Error('Failed to fetch health data'); + (fetchOMStatus as jest.Mock).mockRejectedValueOnce(mockError); + + await act(async () => { + render(, { wrapper: MemoryRouter }); + }); + + expect(showErrorToast).toHaveBeenCalledWith(mockError); + }); + + it('should refresh health data on button click', async () => { + const mockData = { + validation1: { + description: 'Validation 1', + passed: true, + }, + }; + (fetchOMStatus as jest.Mock) + .mockResolvedValueOnce({ data: mockData }) + .mockResolvedValueOnce({ data: {} }); + + await act(async () => { + render(, { wrapper: MemoryRouter }); + }); + + expect(ConnectionStepCard).toHaveBeenCalledWith( + { + isTestingConnection: false, + testConnectionStep: { description: '', mandatory: true, name: 'Data' }, + testConnectionStepResult: { + errorLog: undefined, + mandatory: true, + message: undefined, + name: 'Data', + passed: false, + }, + }, + {} + ); + + const refreshButton = screen.getByText('label.refresh'); + refreshButton.click(); + + expect(fetchOMStatus).toHaveBeenCalledTimes(2); + }); +}); diff --git a/openmetadata-ui/src/main/resources/ui/src/pages/OmHealth/OmHealthPage.tsx b/openmetadata-ui/src/main/resources/ui/src/pages/OmHealth/OmHealthPage.tsx new file mode 100644 index 000000000000..a23b32bd9aec --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/pages/OmHealth/OmHealthPage.tsx @@ -0,0 +1,112 @@ +/* + * Copyright 2024 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { Button, Col, Row } from 'antd'; +import { AxiosError } from 'axios'; +import { map, startCase } from 'lodash'; +import React, { useEffect, useMemo, useState } from 'react'; +import { useTranslation } from 'react-i18next'; +import Loader from '../../components/common/Loader/Loader'; +import ConnectionStepCard from '../../components/common/TestConnection/ConnectionStepCard/ConnectionStepCard'; +import TitleBreadcrumb from '../../components/common/TitleBreadcrumb/TitleBreadcrumb.component'; +import { TitleBreadcrumbProps } from '../../components/common/TitleBreadcrumb/TitleBreadcrumb.interface'; +import PageHeader from '../../components/PageHeader/PageHeader.component'; +import PageLayoutV1 from '../../components/PageLayoutV1/PageLayoutV1'; +import { GlobalSettingsMenuCategory } from '../../constants/GlobalSettings.constants'; +import { PAGE_HEADERS } from '../../constants/PageHeaders.constant'; +import { ValidationResponse } from '../../generated/system/validationResponse'; +import { fetchOMStatus } from '../../rest/miscAPI'; +import { getSettingPageEntityBreadCrumb } from '../../utils/GlobalSettingsUtils'; +import { showErrorToast } from '../../utils/ToastUtils'; + +const OmHealthPage = () => { + const { t } = useTranslation(); + const [loading, setLoading] = useState(false); + const [validationStatus, setValidationStatus] = + useState(); + const breadcrumbs: TitleBreadcrumbProps['titleLinks'] = useMemo( + () => + getSettingPageEntityBreadCrumb( + GlobalSettingsMenuCategory.PREFERENCES, + t('label.om-status') + ), + [] + ); + + const getHealthData = async () => { + setLoading(true); + try { + const response = await fetchOMStatus(); + setValidationStatus(response); + } catch (error) { + showErrorToast(error as AxiosError); + } finally { + setLoading(false); + } + }; + useEffect(() => { + getHealthData(); + }, []); + + if (loading) { + return ; + } + + return ( + + +     + + + + + + + + + + + + + + {validationStatus && + map( + validationStatus, + (validation, key) => + validation && ( + + + + ) + )} + + + ); +}; + +export default OmHealthPage; diff --git a/openmetadata-ui/src/main/resources/ui/src/pages/Persona/PersonaDetailsPage/PersonaDetailsPage.test.tsx b/openmetadata-ui/src/main/resources/ui/src/pages/Persona/PersonaDetailsPage/PersonaDetailsPage.test.tsx index 09f3b611004d..b42686c40c9a 100644 --- a/openmetadata-ui/src/main/resources/ui/src/pages/Persona/PersonaDetailsPage/PersonaDetailsPage.test.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/pages/Persona/PersonaDetailsPage/PersonaDetailsPage.test.tsx @@ -126,7 +126,7 @@ describe('PersonaDetailsPage', () => { expect( await screen.findByText('DescriptionV1.component') ).toBeInTheDocument(); - expect(await screen.findByTestId('add-user-button')).toBeInTheDocument(); + expect(await screen.findByTestId('add-persona-button')).toBeInTheDocument(); }); it('NoDataPlaceholder', async () => { diff --git a/openmetadata-ui/src/main/resources/ui/src/pages/Persona/PersonaDetailsPage/PersonaDetailsPage.tsx b/openmetadata-ui/src/main/resources/ui/src/pages/Persona/PersonaDetailsPage/PersonaDetailsPage.tsx index 4d47a9531a78..c67669178874 100644 --- a/openmetadata-ui/src/main/resources/ui/src/pages/Persona/PersonaDetailsPage/PersonaDetailsPage.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/pages/Persona/PersonaDetailsPage/PersonaDetailsPage.tsx @@ -222,7 +222,7 @@ export const PersonaDetailsPage = () => { selectedUsers={personaDetails.users ?? []} onUpdate={(users) => handlePersonaUpdate({ users })}>
    (); const { tab: activeTab = EntityTabs.SCHEMA } = useParams<{ tab: EntityTabs }>(); @@ -498,6 +502,7 @@ const TableDetailsPageV1 = () => {
    { })); }, []); + const updateDescriptionFromSuggestions = useCallback( + (suggestion: Suggestion) => { + setTableDetails((prev) => { + if (!prev) { + return; + } + + const activeCol = prev?.columns.find((column) => { + return ( + EntityLink.getTableEntityLink( + prev.fullyQualifiedName ?? '', + column.name ?? '' + ) === suggestion.entityLink + ); + }); + + if (!activeCol) { + return { + ...prev, + description: suggestion.description, + }; + } else { + const updatedColumns = prev.columns.map((column) => { + if (column.fullyQualifiedName === activeCol.fullyQualifiedName) { + return { + ...column, + description: suggestion.description, + }; + } else { + return column; + } + }); + + return { + ...prev, + columns: updatedColumns, + }; + } + }); + }, + [] + ); + useEffect(() => { if (isTourOpen || isTourPage) { setTableDetails(mockDatasetData.tableDetails as unknown as Table); @@ -913,6 +961,14 @@ const TableDetailsPageV1 = () => { } }, [tableDetails?.fullyQualifiedName]); + useSub( + 'updateDetails', + (suggestion: Suggestion) => { + updateDescriptionFromSuggestions(suggestion); + }, + [tableDetails] + ); + const onThreadPanelClose = () => { setThreadLink(''); }; @@ -1015,4 +1071,4 @@ const TableDetailsPageV1 = () => { ); }; -export default withActivityFeed(TableDetailsPageV1); +export default withSuggestions(withActivityFeed(TableDetailsPageV1)); diff --git a/openmetadata-ui/src/main/resources/ui/src/pages/TagsPage/TagsPage.test.tsx b/openmetadata-ui/src/main/resources/ui/src/pages/TagsPage/TagsPage.test.tsx index 4826c1bda7e2..17b54e85fed3 100644 --- a/openmetadata-ui/src/main/resources/ui/src/pages/TagsPage/TagsPage.test.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/pages/TagsPage/TagsPage.test.tsx @@ -272,7 +272,7 @@ jest.mock('../../components/Modals/FormModal', () => { .mockReturnValue(

    FormModal

    ); }); -jest.mock('../../components/common/EntityDescription/Description', () => { +jest.mock('../../components/common/EntityDescription/DescriptionV1', () => { return jest.fn().mockReturnValue(

    DescriptionComponent

    ); }); diff --git a/openmetadata-ui/src/main/resources/ui/src/pages/TasksPage/RequestDescriptionPage/RequestDescriptionPage.tsx b/openmetadata-ui/src/main/resources/ui/src/pages/TasksPage/RequestDescriptionPage/RequestDescriptionPage.tsx index 8c0a495b9a39..cbd2f9eaee87 100644 --- a/openmetadata-ui/src/main/resources/ui/src/pages/TasksPage/RequestDescriptionPage/RequestDescriptionPage.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/pages/TasksPage/RequestDescriptionPage/RequestDescriptionPage.tsx @@ -19,7 +19,6 @@ import React, { useEffect, useMemo, useRef, useState } from 'react'; import { useTranslation } from 'react-i18next'; import { useHistory, useLocation, useParams } from 'react-router-dom'; import { ActivityFeedTabs } from '../../../components/ActivityFeed/ActivityFeedTab/ActivityFeedTab.interface'; -import { useAuthContext } from '../../../components/Auth/AuthProviders/AuthProvider'; import Loader from '../../../components/common/Loader/Loader'; import ResizablePanels from '../../../components/common/ResizablePanels/ResizablePanels'; import RichTextEditor from '../../../components/common/RichTextEditor/RichTextEditor'; @@ -34,6 +33,7 @@ import { TaskType, } from '../../../generated/api/feed/createThread'; import { ThreadType } from '../../../generated/entity/feed/thread'; +import { useApplicationStore } from '../../../hooks/useApplicationStore'; import { useFqn } from '../../../hooks/useFqn'; import { postThread } from '../../../rest/feedsAPI'; import { getEntityDetailLink } from '../../../utils/CommonUtils'; @@ -55,7 +55,7 @@ import { EntityData, Option } from '../TasksPage.interface'; const RequestDescription = () => { const { t } = useTranslation(); - const { currentUser } = useAuthContext(); + const { currentUser } = useApplicationStore(); const location = useLocation(); const history = useHistory(); const [form] = useForm(); @@ -240,7 +240,6 @@ const RequestDescription = () => { }, ]}> { const { t } = useTranslation(); - const { currentUser } = useAuthContext(); + const { currentUser } = useApplicationStore(); const location = useLocation(); const history = useHistory(); const [form] = useForm(); @@ -229,7 +229,6 @@ const RequestTag = () => { }, ]}> { const { t } = useTranslation(); - const { currentUser } = useAuthContext(); + const { currentUser } = useApplicationStore(); const location = useLocation(); const history = useHistory(); const [form] = useForm(); @@ -257,7 +257,6 @@ const UpdateDescription = () => { }, ]}> { const location = useLocation(); const history = useHistory(); const [form] = useForm(); - const { currentUser } = useAuthContext(); + const { currentUser } = useApplicationStore(); const { entityType } = useParams<{ entityType: EntityType }>(); @@ -269,7 +269,6 @@ const UpdateTag = () => { }, ]}> { const [isLoading, setIsLoading] = useState(false); const [assets, setAssets] = useState(0); const [parentTeams, setParentTeams] = useState([]); - const { updateCurrentUser } = useAuthContext(); + const { updateCurrentUser } = useApplicationStore(); const [entityPermissions, setEntityPermissions] = useState(DEFAULT_ENTITY_PERMISSION); diff --git a/openmetadata-ui/src/main/resources/ui/src/pages/TestSuiteDetailsPage/TestSuiteDetailsPage.component.tsx b/openmetadata-ui/src/main/resources/ui/src/pages/TestSuiteDetailsPage/TestSuiteDetailsPage.component.tsx index a01fdd58f17e..a7c012cc8645 100644 --- a/openmetadata-ui/src/main/resources/ui/src/pages/TestSuiteDetailsPage/TestSuiteDetailsPage.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/pages/TestSuiteDetailsPage/TestSuiteDetailsPage.component.tsx @@ -17,7 +17,7 @@ import { compare } from 'fast-json-patch'; import React, { useCallback, useEffect, useMemo, useState } from 'react'; import { useTranslation } from 'react-i18next'; import { useHistory } from 'react-router-dom'; -import Description from '../../components/common/EntityDescription/Description'; +import DescriptionV1 from '../../components/common/EntityDescription/DescriptionV1'; import ManageButton from '../../components/common/EntityPageInfos/ManageButton/ManageButton'; import ErrorPlaceHolder from '../../components/common/ErrorWithPlaceholder/ErrorPlaceHolder'; import Loader from '../../components/common/Loader/Loader'; @@ -373,12 +373,14 @@ const TestSuiteDetailsPage = () => { />
    - descriptionHandler(false)} onDescriptionEdit={() => descriptionHandler(true)} onDescriptionUpdate={onDescriptionUpdate} diff --git a/openmetadata-ui/src/main/resources/ui/src/pages/TestSuiteDetailsPage/TestSuiteDetailsPage.test.tsx b/openmetadata-ui/src/main/resources/ui/src/pages/TestSuiteDetailsPage/TestSuiteDetailsPage.test.tsx index 0d25e0e6ff8a..fb2d9808679d 100644 --- a/openmetadata-ui/src/main/resources/ui/src/pages/TestSuiteDetailsPage/TestSuiteDetailsPage.test.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/pages/TestSuiteDetailsPage/TestSuiteDetailsPage.test.tsx @@ -55,7 +55,7 @@ jest.mock( .mockImplementation(() =>
    ManageButton.component
    ); } ); -jest.mock('../../components/common/EntityDescription/Description', () => { +jest.mock('../../components/common/EntityDescription/DescriptionV1', () => { return jest.fn().mockImplementation(() =>
    Description.component
    ); }); jest.mock( @@ -66,18 +66,14 @@ jest.mock( .mockImplementation(() =>
    DataQualityTab.component
    ); } ); -jest.mock('../../components/Auth/AuthProviders/AuthProvider', () => { +jest.mock('../../hooks/useApplicationStore', () => { return { - useAuthContext: jest + useApplicationStore: jest .fn() .mockImplementation(() => ({ isAuthDisabled: true })), }; }); -jest.mock('../../hooks/authHooks', () => { - return { - useAuth: jest.fn().mockImplementation(() => ({ isAdminUser: true })), - }; -}); + jest.mock('react-router-dom', () => { return { useHistory: jest.fn().mockImplementation(() => ({ push: jest.fn() })), diff --git a/openmetadata-ui/src/main/resources/ui/src/pages/TopicDetails/TopicDetailsPage.component.tsx b/openmetadata-ui/src/main/resources/ui/src/pages/TopicDetails/TopicDetailsPage.component.tsx index 36810c5f7329..426a93cd71b9 100644 --- a/openmetadata-ui/src/main/resources/ui/src/pages/TopicDetails/TopicDetailsPage.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/pages/TopicDetails/TopicDetailsPage.component.tsx @@ -22,7 +22,7 @@ import React, { } from 'react'; import { useTranslation } from 'react-i18next'; import { useHistory } from 'react-router-dom'; -import { useAuthContext } from '../../components/Auth/AuthProviders/AuthProvider'; + import ErrorPlaceHolder from '../../components/common/ErrorWithPlaceholder/ErrorPlaceHolder'; import Loader from '../../components/common/Loader/Loader'; import { QueryVote } from '../../components/Database/TableQueries/TableQueries.interface'; @@ -37,6 +37,7 @@ import { ERROR_PLACEHOLDER_TYPE } from '../../enums/common.enum'; import { EntityType, TabSpecificField } from '../../enums/entity.enum'; import { CreateThread } from '../../generated/api/feed/createThread'; import { Topic } from '../../generated/entity/data/topic'; +import { useApplicationStore } from '../../hooks/useApplicationStore'; import { useFqn } from '../../hooks/useFqn'; import { postThread } from '../../rest/feedsAPI'; import { @@ -57,7 +58,7 @@ import { showErrorToast } from '../../utils/ToastUtils'; const TopicDetailsPage: FunctionComponent = () => { const { t } = useTranslation(); - const { currentUser } = useAuthContext(); + const { currentUser } = useApplicationStore(); const USERId = currentUser?.id ?? ''; const history = useHistory(); const { getEntityPermissionByFqn } = usePermissionProvider(); diff --git a/openmetadata-ui/src/main/resources/ui/src/pages/UserListPage/UserListPageV1.tsx b/openmetadata-ui/src/main/resources/ui/src/pages/UserListPage/UserListPageV1.tsx index d06000b93b34..35d3d7949c9d 100644 --- a/openmetadata-ui/src/main/resources/ui/src/pages/UserListPage/UserListPageV1.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/pages/UserListPage/UserListPageV1.tsx @@ -313,9 +313,18 @@ const UserListPageV1 = () => { className="w-full justify-center action-icons" size={8}> {showRestore && ( - + + )), +})); + +jest.mock('../../rest/miscAPI', () => ({ + searchData: jest.fn(), +})); + describe('AlertsUtil tests', () => { it('getFunctionDisplayName should return correct text for matchAnyEntityFqn', () => { expect(getFunctionDisplayName('matchAnyEntityFqn')).toBe( @@ -213,3 +229,279 @@ describe('AlertsUtil tests', () => { }); }); }); + +describe('getFieldByArgumentType tests', () => { + it('should return correct fields for argumentType fqnList', async () => { + const field = getFieldByArgumentType(0, 'fqnList', 0, 'table'); + + render(field); + + const selectDiv = screen.getByText('AsyncSelect'); + + await act(async () => { + userEvent.click(selectDiv); + }); + + expect(searchData).toHaveBeenCalledWith( + undefined, + 1, + 50, + '', + '', + '', + 'table_search_index' + ); + }); + + it('should return correct fields for argumentType domainList', async () => { + const field = getFieldByArgumentType(0, 'domainList', 0, 'container'); + + render(field); + + const selectDiv = screen.getByText('AsyncSelect'); + + await act(async () => { + userEvent.click(selectDiv); + }); + + expect(searchData).toHaveBeenCalledWith( + undefined, + 1, + 50, + '', + '', + '', + 'domain_search_index' + ); + }); + + it('should return correct fields for argumentType tableNameList', async () => { + const field = getFieldByArgumentType( + 0, + 'tableNameList', + 0, + 'selectedTrigger' + ); + + render(field); + + const selectDiv = screen.getByText('AsyncSelect'); + + await act(async () => { + userEvent.click(selectDiv); + }); + + expect(searchData).toHaveBeenCalledWith( + undefined, + 1, + 50, + '', + '', + '', + 'table_search_index' + ); + }); + + it('should return correct fields for argumentType ownerNameList', async () => { + const field = getFieldByArgumentType( + 0, + 'ownerNameList', + 0, + 'selectedTrigger' + ); + + render(field); + + const selectDiv = screen.getByText('AsyncSelect'); + + await act(async () => { + userEvent.click(selectDiv); + }); + + expect(searchData).toHaveBeenCalledWith( + undefined, + 1, + 50, + 'isBot:false', + '', + '', + ['team_search_index', 'user_search_index'] + ); + }); + + it('should return correct fields for argumentType updateByUserList', async () => { + const field = getFieldByArgumentType( + 0, + 'updateByUserList', + 0, + 'selectedTrigger' + ); + + render(field); + + const selectDiv = screen.getByText('AsyncSelect'); + + await act(async () => { + userEvent.click(selectDiv); + }); + + expect(searchData).toHaveBeenCalledWith( + undefined, + 1, + 50, + '', + '', + '', + 'user_search_index' + ); + }); + + it('should return correct fields for argumentType userList', async () => { + const field = getFieldByArgumentType(0, 'userList', 0, 'selectedTrigger'); + + render(field); + + const selectDiv = screen.getByText('AsyncSelect'); + + await act(async () => { + userEvent.click(selectDiv); + }); + + expect(searchData).toHaveBeenCalledWith( + undefined, + 1, + 50, + 'isBot:false', + '', + '', + 'user_search_index' + ); + }); + + it('should return correct fields for argumentType eventTypeList', async () => { + const field = getFieldByArgumentType( + 0, + 'eventTypeList', + 0, + 'selectedTrigger' + ); + + render(field); + + const selectDiv = screen.getByTestId('event-type-select'); + + expect(selectDiv).toBeInTheDocument(); + }); + + it('should return correct fields for argumentType entityIdList', () => { + const field = getFieldByArgumentType( + 0, + 'entityIdList', + 0, + 'selectedTrigger' + ); + + render(field); + + const selectDiv = screen.getByTestId('entity-id-select'); + + expect(selectDiv).toBeInTheDocument(); + }); + + it('should return correct fields for argumentType pipelineStateList', () => { + const field = getFieldByArgumentType( + 0, + 'pipelineStateList', + 0, + 'selectedTrigger' + ); + + render(field); + + const selectDiv = screen.getByTestId('pipeline-status-select'); + + expect(selectDiv).toBeInTheDocument(); + }); + + it('should return correct fields for argumentType ingestionPipelineStateList', () => { + const field = getFieldByArgumentType( + 0, + 'ingestionPipelineStateList', + 0, + 'selectedTrigger' + ); + + render(field); + + const selectDiv = screen.getByTestId('pipeline-status-select'); + + expect(selectDiv).toBeInTheDocument(); + }); + + it('should return correct fields for argumentType testStatusList', () => { + const field = getFieldByArgumentType( + 0, + 'testStatusList', + 0, + 'selectedTrigger' + ); + + render(field); + + const selectDiv = screen.getByTestId('test-status-select'); + + expect(selectDiv).toBeInTheDocument(); + }); + + it('should return correct fields for argumentType testResultList', () => { + const field = getFieldByArgumentType( + 0, + 'testResultList', + 0, + 'selectedTrigger' + ); + + render(field); + + const selectDiv = screen.getByTestId('test-result-select'); + + expect(selectDiv).toBeInTheDocument(); + }); + + it('should return correct fields for argumentType testSuiteList', async () => { + const field = getFieldByArgumentType( + 0, + 'testSuiteList', + 0, + 'selectedTrigger' + ); + + render(field); + + const selectDiv = screen.getByText('AsyncSelect'); + + await act(async () => { + userEvent.click(selectDiv); + }); + + expect(searchData).toHaveBeenCalledWith( + undefined, + 1, + 50, + '', + '', + '', + 'test_suite_search_index' + ); + }); + + it('should not return select component for random argumentType', () => { + const field = getFieldByArgumentType(0, 'unknown', 0, 'selectedTrigger'); + + render(field); + + const selectDiv = screen.queryByText('AsyncSelect'); + + expect(selectDiv).toBeNull(); + }); +}); diff --git a/openmetadata-ui/src/main/resources/ui/src/utils/Alerts/AlertsUtil.tsx b/openmetadata-ui/src/main/resources/ui/src/utils/Alerts/AlertsUtil.tsx index b334607ec592..cc5cb28b1ea2 100644 --- a/openmetadata-ui/src/main/resources/ui/src/utils/Alerts/AlertsUtil.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/utils/Alerts/AlertsUtil.tsx @@ -230,6 +230,13 @@ const getUserOptions = async (searchText: string) => { }); }; +const getUserBotOptions = async (searchText: string) => { + return searchEntity({ + searchText, + searchIndex: SearchIndex.USER, + }); +}; + const getTeamOptions = async (searchText: string) => { return searchEntity({ searchText, searchIndex: SearchIndex.TEAM }); }; @@ -378,6 +385,91 @@ export const getDestinationConfigField = ( } }; +export const getMessageFromArgumentName = (argumentName: string) => { + switch (argumentName) { + case 'fqnList': + return t('message.field-text-is-required', { + fieldText: t('label.entity-list', { + entity: t('label.fqn-uppercase'), + }), + }); + case 'domainList': + return t('message.field-text-is-required', { + fieldText: t('label.entity-list', { + entity: t('label.domain'), + }), + }); + case 'tableNameList': + return t('message.field-text-is-required', { + fieldText: t('label.entity-list', { + entity: t('label.entity-name', { + entity: t('label.table'), + }), + }), + }); + case 'ownerNameList': + return t('message.field-text-is-required', { + fieldText: t('label.entity-list', { + entity: t('label.entity-name', { + entity: t('label.owner'), + }), + }), + }); + case 'updateByUserList': + case 'userList': + return t('message.field-text-is-required', { + fieldText: t('label.entity-list', { + entity: t('label.entity-name', { + entity: t('label.user'), + }), + }), + }); + case 'eventTypeList': + return t('message.field-text-is-required', { + fieldText: t('label.entity-list', { + entity: t('label.entity-name', { + entity: t('label.event'), + }), + }), + }); + case 'entityIdList': + return t('message.field-text-is-required', { + fieldText: t('label.entity-list', { + entity: t('label.entity-id', { + entity: t('label.data-asset'), + }), + }), + }); + case 'pipelineStateList': + case 'ingestionPipelineStateList': + return t('message.field-text-is-required', { + fieldText: t('label.entity-list', { + entity: t('label.pipeline-state'), + }), + }); + case 'testStatusList': + return t('message.field-text-is-required', { + fieldText: t('label.entity-list', { + entity: t('label.test-suite-status'), + }), + }); + case 'testResultList': + return t('message.field-text-is-required', { + fieldText: t('label.entity-list', { + entity: t('label.test-case-result'), + }), + }); + case 'testSuiteList': + return t('message.field-text-is-required', { + fieldText: t('label.entity-list', { + entity: t('label.test-suite'), + }), + }); + default: + return ''; + } +}; + export const getFieldByArgumentType = ( fieldName: number, argument: string, @@ -400,377 +492,192 @@ export const getFieldByArgumentType = ( switch (argument) { case 'fqnList': field = ( -     - - - - + ); break; case 'domainList': field = ( - - - - - + ); break; case 'tableNameList': field = ( - - - - - + ); break; case 'ownerNameList': field = ( - - - - - + ); break; case 'updateByUserList': + case 'userList': field = ( - - - - - + ); break; case 'eventTypeList': field = ( - - - ); break; case 'entityIdList': field = ( - - - ); break; case 'pipelineStateList': field = ( - - - ); break; case 'ingestionPipelineStateList': field = ( - - - ); break; case 'testStatusList': field = ( - - - ); break; case 'testResultList': field = ( - - - ); break; case 'testSuiteList': field = ( - - - - - + ); break; @@ -780,7 +687,18 @@ export const getFieldByArgumentType = ( return ( <> - {field} + + + {field} + + + + + + + {item.label}{' '} + {item.key === HELP_ITEMS_ENUM.VERSION && + (version ?? '?').split('-')[0]} + + + {item.isExternal && ( + + )} + + + ); +}; + +const getHelpDropdownLabel = (item: SupportItem, version?: string) => { + if (item.isExternal) { + return ( + + {getHelpDropdownLabelContentRenderer(item, version)} + + ); + } else if (item.link) { + return ( + + {getHelpDropdownLabelContentRenderer(item)} + + ); + } else { + return getHelpDropdownLabelContentRenderer(item); + } +}; + +export const getHelpDropdownItems = (version?: string) => + HELP_ITEMS.map((item) => ({ + label: getHelpDropdownLabel(item, version), + key: item.key, + })); diff --git a/openmetadata-ui/src/main/resources/ui/src/utils/ObservabilityUtils.tsx b/openmetadata-ui/src/main/resources/ui/src/utils/ObservabilityUtils.tsx index 7e61df1db2f4..4ba44b72f790 100644 --- a/openmetadata-ui/src/main/resources/ui/src/utils/ObservabilityUtils.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/utils/ObservabilityUtils.tsx @@ -15,6 +15,7 @@ import { includes, isNil } from 'lodash'; import React from 'react'; import { ReactComponent as AdminIcon } from '../assets/svg/admin-colored-icon.svg'; import { ReactComponent as GChatIcon } from '../assets/svg/gchat.svg'; +import { ReactComponent as MentionIcon } from '../assets/svg/ic-mentions.svg'; import { ReactComponent as MSTeamsIcon } from '../assets/svg/ms-teams.svg'; import { ReactComponent as SlackIcon } from '../assets/svg/slack.svg'; import { ReactComponent as TeamIcon } from '../assets/svg/team-colored-icon.svg'; @@ -42,6 +43,10 @@ export const getAlertDestinationCategoryIcons = (type: string) => { case 'Assignees': Icon = AssigneeIcon; + break; + case 'Mentions': + Icon = MentionIcon; + break; case 'GChat': Icon = GChatIcon; diff --git a/openmetadata-ui/src/main/resources/ui/src/utils/QueryClassBase.test.ts b/openmetadata-ui/src/main/resources/ui/src/utils/QueryClassBase.test.ts new file mode 100644 index 000000000000..7293f5b6fb37 --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/utils/QueryClassBase.test.ts @@ -0,0 +1,27 @@ +/* + * Copyright 2024 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import queryClassBase from './QueryClassBase'; + +describe('QueryClassBase', () => { + it('should return null from getQueryExtras', () => { + const result = queryClassBase.getQueryExtras(); + + expect(result).toBeNull(); + }); + + it('should return null from getQueryHeaderActionsButtons', () => { + const result = queryClassBase.getQueryHeaderActionsButtons(); + + expect(result).toBeNull(); + }); +}); diff --git a/openmetadata-ui/src/main/resources/ui/src/utils/QueryClassBase.ts b/openmetadata-ui/src/main/resources/ui/src/utils/QueryClassBase.ts new file mode 100644 index 000000000000..372ee4f2e846 --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/utils/QueryClassBase.ts @@ -0,0 +1,30 @@ +/* + * Copyright 2024 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { FC } from 'react'; + +class QueryClassBase { + public getQueryExtras(): FC | null { + return null; + } + + public getQueryHeaderActionsButtons(): FC<{ + onClickHandler: () => void; + }> | null { + return null; + } +} + +const queryClassBase = new QueryClassBase(); + +export default queryClassBase; +export { QueryClassBase }; diff --git a/openmetadata-ui/src/main/resources/ui/src/utils/SearchClassBase.test.ts b/openmetadata-ui/src/main/resources/ui/src/utils/SearchClassBase.test.ts index 0e9ec03c35a8..7b9d534525fc 100644 --- a/openmetadata-ui/src/main/resources/ui/src/utils/SearchClassBase.test.ts +++ b/openmetadata-ui/src/main/resources/ui/src/utils/SearchClassBase.test.ts @@ -10,6 +10,19 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +import { + COMMON_DROPDOWN_ITEMS, + CONTAINER_DROPDOWN_ITEMS, + DASHBOARD_DATA_MODEL_TYPE, + DASHBOARD_DROPDOWN_ITEMS, + DATA_PRODUCT_DROPDOWN_ITEMS, + GLOSSARY_DROPDOWN_ITEMS, + PIPELINE_DROPDOWN_ITEMS, + SEARCH_INDEX_DROPDOWN_ITEMS, + TABLE_DROPDOWN_ITEMS, + TAG_DROPDOWN_ITEMS, + TOPIC_DROPDOWN_ITEMS, +} from '../constants/AdvancedSearch.constants'; import { EntityType } from '../enums/entity.enum'; import { SearchIndex } from '../enums/search.enum'; import { SearchClassBase } from './SearchClassBase'; @@ -40,7 +53,7 @@ describe('SearchClassBase', () => { ); expect(searchIndexMapping[EntityType.TAG]).toEqual(SearchIndex.TAG); expect(searchIndexMapping[EntityType.GLOSSARY_TERM]).toEqual( - SearchIndex.GLOSSARY + SearchIndex.GLOSSARY_TERM ); expect(searchIndexMapping[EntityType.STORED_PROCEDURE]).toEqual( SearchIndex.STORED_PROCEDURE @@ -83,4 +96,94 @@ describe('SearchClassBase', () => { SearchIndex.DATABASE_SCHEMA ); }); + + it('should return dropdown item based on entity type', () => { + const tableItems = searchClassBase.getDropDownItems(SearchIndex.TABLE); + const topicItems = searchClassBase.getDropDownItems(SearchIndex.TOPIC); + const dashboardItems = searchClassBase.getDropDownItems( + SearchIndex.DASHBOARD + ); + const pipelineItems = searchClassBase.getDropDownItems( + SearchIndex.PIPELINE + ); + const searchIndexItems = searchClassBase.getDropDownItems( + SearchIndex.SEARCH_INDEX + ); + const mlmodelsItems = searchClassBase.getDropDownItems(SearchIndex.MLMODEL); + const containerItems = searchClassBase.getDropDownItems( + SearchIndex.CONTAINER + ); + const storedProcedureItems = searchClassBase.getDropDownItems( + SearchIndex.STORED_PROCEDURE + ); + const dashboardDataModelItems = searchClassBase.getDropDownItems( + SearchIndex.DASHBOARD_DATA_MODEL + ); + + const glossaryTermItems = searchClassBase.getDropDownItems( + SearchIndex.GLOSSARY_TERM + ); + const tagItems = searchClassBase.getDropDownItems(SearchIndex.TAG); + const dataProductItems = searchClassBase.getDropDownItems( + SearchIndex.DATA_PRODUCT + ); + const databaseItems = searchClassBase.getDropDownItems( + SearchIndex.DATABASE + ); + const databaseSchemaItems = searchClassBase.getDropDownItems( + SearchIndex.DATABASE_SCHEMA + ); + + expect(tableItems).toEqual([ + ...COMMON_DROPDOWN_ITEMS, + ...TABLE_DROPDOWN_ITEMS, + ]); + + expect(topicItems).toEqual([ + ...COMMON_DROPDOWN_ITEMS, + ...TOPIC_DROPDOWN_ITEMS, + ]); + expect(dashboardItems).toEqual([ + ...COMMON_DROPDOWN_ITEMS, + ...DASHBOARD_DROPDOWN_ITEMS, + ]); + expect(pipelineItems).toEqual([ + ...COMMON_DROPDOWN_ITEMS, + ...PIPELINE_DROPDOWN_ITEMS, + ]); + + expect(mlmodelsItems).toEqual([ + ...COMMON_DROPDOWN_ITEMS.filter((item) => item.key !== 'service_type'), + ]); + + expect(searchIndexItems).toEqual([ + ...COMMON_DROPDOWN_ITEMS, + ...SEARCH_INDEX_DROPDOWN_ITEMS, + ]); + + expect(containerItems).toEqual([ + ...COMMON_DROPDOWN_ITEMS, + ...CONTAINER_DROPDOWN_ITEMS, + ]); + expect(storedProcedureItems).toEqual([...COMMON_DROPDOWN_ITEMS]); + + expect(dashboardDataModelItems).toEqual([ + ...COMMON_DROPDOWN_ITEMS, + ...DASHBOARD_DATA_MODEL_TYPE, + ]); + expect(glossaryTermItems).toEqual(GLOSSARY_DROPDOWN_ITEMS); + + expect(tagItems).toEqual(TAG_DROPDOWN_ITEMS); + + expect(dataProductItems).toEqual(DATA_PRODUCT_DROPDOWN_ITEMS); + + expect(databaseItems).toEqual(COMMON_DROPDOWN_ITEMS); + expect(databaseSchemaItems).toEqual(COMMON_DROPDOWN_ITEMS); + }); + + it('should return empty dropdown item based if index not related to explore items', () => { + const dropdownItem = searchClassBase.getDropDownItems(SearchIndex.DOMAIN); + + expect(dropdownItem).toEqual([]); + }); }); diff --git a/openmetadata-ui/src/main/resources/ui/src/utils/SearchClassBase.ts b/openmetadata-ui/src/main/resources/ui/src/utils/SearchClassBase.ts index 2eee6e70a399..cae4069810d6 100644 --- a/openmetadata-ui/src/main/resources/ui/src/utils/SearchClassBase.ts +++ b/openmetadata-ui/src/main/resources/ui/src/utils/SearchClassBase.ts @@ -76,7 +76,7 @@ class SearchClassBase { [EntityType.TOPIC]: SearchIndex.TOPIC, [EntityType.CONTAINER]: SearchIndex.CONTAINER, [EntityType.TAG]: SearchIndex.TAG, - [EntityType.GLOSSARY_TERM]: SearchIndex.GLOSSARY, + [EntityType.GLOSSARY_TERM]: SearchIndex.GLOSSARY_TERM, [EntityType.STORED_PROCEDURE]: SearchIndex.STORED_PROCEDURE, [EntityType.DASHBOARD_DATA_MODEL]: SearchIndex.DASHBOARD_DATA_MODEL, [EntityType.SEARCH_INDEX]: SearchIndex.SEARCH_INDEX, @@ -117,7 +117,7 @@ class SearchClassBase { value: SearchIndex.DASHBOARD_DATA_MODEL, label: i18n.t('label.data-model'), }, - { value: SearchIndex.GLOSSARY, label: i18n.t('label.glossary') }, + { value: SearchIndex.GLOSSARY_TERM, label: i18n.t('label.glossary') }, { value: SearchIndex.TAG, label: i18n.t('label.tag') }, { value: SearchIndex.SEARCH_INDEX, label: i18n.t('label.search-index') }, { value: SearchIndex.DATA_PRODUCT, label: i18n.t('label.data-product') }, @@ -203,8 +203,8 @@ class SearchClassBase { path: 'searchIndexes', icon: SearchOutlined, }, - [SearchIndex.GLOSSARY]: { - label: i18n.t('label.glossary-plural'), + [SearchIndex.GLOSSARY_TERM]: { + label: i18n.t('label.glossary-term-plural'), sortingFields: entitySortingFields, sortField: INITIAL_SORT_FIELD, path: 'glossaries', @@ -253,19 +253,18 @@ class SearchClassBase { ]; case SearchIndex.CONTAINER: return [...COMMON_DROPDOWN_ITEMS, ...CONTAINER_DROPDOWN_ITEMS]; - case SearchIndex.STORED_PROCEDURE: - return [...COMMON_DROPDOWN_ITEMS]; case SearchIndex.DASHBOARD_DATA_MODEL: return [...COMMON_DROPDOWN_ITEMS, ...DASHBOARD_DATA_MODEL_TYPE]; - case SearchIndex.GLOSSARY: - return [...GLOSSARY_DROPDOWN_ITEMS]; + case SearchIndex.GLOSSARY_TERM: + return GLOSSARY_DROPDOWN_ITEMS; case SearchIndex.TAG: - return [...TAG_DROPDOWN_ITEMS]; + return TAG_DROPDOWN_ITEMS; case SearchIndex.DATA_PRODUCT: - return [...DATA_PRODUCT_DROPDOWN_ITEMS]; + return DATA_PRODUCT_DROPDOWN_ITEMS; + case SearchIndex.STORED_PROCEDURE: case SearchIndex.DATABASE: case SearchIndex.DATABASE_SCHEMA: - return [...COMMON_DROPDOWN_ITEMS]; + return COMMON_DROPDOWN_ITEMS; default: return []; diff --git a/openmetadata-ui/src/main/resources/ui/src/utils/SearchUtils.test.ts b/openmetadata-ui/src/main/resources/ui/src/utils/SearchUtils.test.ts index a6a9c41579a4..035472ae1c5f 100644 --- a/openmetadata-ui/src/main/resources/ui/src/utils/SearchUtils.test.ts +++ b/openmetadata-ui/src/main/resources/ui/src/utils/SearchUtils.test.ts @@ -33,7 +33,7 @@ describe('getEntityTypeFromSearchIndex', () => { [SearchIndex.ML_MODEL_SERVICE, EntityType.MLMODEL_SERVICE], [SearchIndex.STORAGE_SERVICE, EntityType.STORAGE_SERVICE], [SearchIndex.SEARCH_SERVICE, EntityType.SEARCH_SERVICE], - [SearchIndex.GLOSSARY, EntityType.GLOSSARY_TERM], + [SearchIndex.GLOSSARY_TERM, EntityType.GLOSSARY_TERM], [SearchIndex.TAG, EntityType.TAG], [SearchIndex.DATABASE, EntityType.DATABASE], [SearchIndex.DOMAIN, EntityType.DOMAIN], diff --git a/openmetadata-ui/src/main/resources/ui/src/utils/SearchUtils.tsx b/openmetadata-ui/src/main/resources/ui/src/utils/SearchUtils.tsx index 641921057068..19ac141f9f18 100644 --- a/openmetadata-ui/src/main/resources/ui/src/utils/SearchUtils.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/utils/SearchUtils.tsx @@ -119,7 +119,7 @@ export const getGroupLabel = (index: string) => { GroupIcon = IconMlModal; break; - case SearchIndex.GLOSSARY: + case SearchIndex.GLOSSARY_TERM: label = i18next.t('label.glossary-term-plural'); GroupIcon = IconTable; @@ -264,7 +264,7 @@ export const getEntityTypeFromSearchIndex = (searchIndex: string) => { [SearchIndex.ML_MODEL_SERVICE]: EntityType.MLMODEL_SERVICE, [SearchIndex.STORAGE_SERVICE]: EntityType.STORAGE_SERVICE, [SearchIndex.SEARCH_SERVICE]: EntityType.SEARCH_SERVICE, - [SearchIndex.GLOSSARY]: EntityType.GLOSSARY_TERM, + [SearchIndex.GLOSSARY_TERM]: EntityType.GLOSSARY_TERM, [SearchIndex.TAG]: EntityType.TAG, [SearchIndex.DATABASE]: EntityType.DATABASE, [SearchIndex.DOMAIN]: EntityType.DOMAIN, diff --git a/openmetadata-ui/src/main/resources/ui/src/utils/TableUtils.tsx b/openmetadata-ui/src/main/resources/ui/src/utils/TableUtils.tsx index d8c7b1a64624..4917f8d3282c 100644 --- a/openmetadata-ui/src/main/resources/ui/src/utils/TableUtils.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/utils/TableUtils.tsx @@ -299,8 +299,8 @@ export const getEntityIcon = (indexType: string) => { case EntityType.GLOSSARY: return ; case EntityType.GLOSSARY_TERM: + case SearchIndex.GLOSSARY_TERM: return ; - case EntityType.SEARCH_INDEX: case SearchIndex.SEARCH_INDEX: case EntityType.SEARCH_SERVICE: diff --git a/openmetadata-ui/src/main/resources/ui/src/utils/TagsUtils.tsx b/openmetadata-ui/src/main/resources/ui/src/utils/TagsUtils.tsx index a6ad13edb9b1..932e0fe4bcf1 100644 --- a/openmetadata-ui/src/main/resources/ui/src/utils/TagsUtils.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/utils/TagsUtils.tsx @@ -16,7 +16,7 @@ import { Tag as AntdTag, Tooltip, Typography } from 'antd'; import { AxiosError } from 'axios'; import i18next from 'i18next'; import { omit } from 'lodash'; -import { EntityTags, TagOption } from 'Models'; +import { EntityTags } from 'Models'; import type { CustomTagProps } from 'rc-select/lib/BaseSelect'; import React from 'react'; import { ReactComponent as DeleteIcon } from '../assets/svg/ic-delete.svg'; @@ -39,7 +39,6 @@ import { getClassificationByName, getTags, } from '../rest/tagAPI'; -import { fetchGlossaryTerms, getGlossaryTermlist } from './GlossaryUtils'; import { getTagsWithoutTier } from './TableUtils'; export const getClassifications = async ( @@ -80,24 +79,6 @@ export const getClassifications = async ( } }; -/** - * This method returns all the tags present in the system - * @returns tags: Tag[] - */ -export const getAllTagsForOptions = async () => { - let tags: Tag[] = []; - try { - const { data } = await getTags({ limit: 1000 }); - - tags = data; - } catch (error) { - // eslint-disable-next-line no-console - console.error(error); - } - - return tags; -}; - /** * Return tags based on classifications * @param classifications -- Parent for tags @@ -171,37 +152,6 @@ export const getTagDisplay = (tag: string) => { return tag; }; -export const fetchTagsAndGlossaryTerms = async () => { - const responses = await Promise.allSettled([ - getAllTagsForOptions(), - fetchGlossaryTerms(), - ]); - - let tagsAndTerms: TagOption[] = []; - if (responses[0].status === SettledStatus.FULFILLED && responses[0].value) { - tagsAndTerms = responses[0].value.map((tag) => { - return { - fqn: tag.fullyQualifiedName ?? tag.name, - source: 'Classification', - }; - }); - } - if ( - responses[1].status === SettledStatus.FULFILLED && - responses[1].value && - responses[1].value.length > 0 - ) { - const glossaryTerms: TagOption[] = getGlossaryTermlist( - responses[1].value - ).map((tag) => { - return { fqn: tag, source: 'Glossary' }; - }); - tagsAndTerms = [...tagsAndTerms, ...glossaryTerms]; - } - - return tagsAndTerms; -}; - export const getTagTooltip = (fqn: string, description?: string) => (
    @@ -320,7 +270,7 @@ export const fetchGlossaryList = async ( pageNumber: page, pageSize: 10, queryFilter: {}, - searchIndex: SearchIndex.GLOSSARY, + searchIndex: SearchIndex.GLOSSARY_TERM, }); const hits = glossaryResponse.hits.hits; diff --git a/openmetadata-ui/src/main/resources/ui/src/utils/TasksUtils.ts b/openmetadata-ui/src/main/resources/ui/src/utils/TasksUtils.ts index b03727f35a22..994bd42d1e2d 100644 --- a/openmetadata-ui/src/main/resources/ui/src/utils/TasksUtils.ts +++ b/openmetadata-ui/src/main/resources/ui/src/utils/TasksUtils.ts @@ -535,7 +535,9 @@ export const fetchEntityDetail = ( break; case EntityType.GLOSSARY: - getGlossariesByName(entityFQN, { fields: TabSpecificField.TAGS }) + getGlossariesByName(entityFQN, { + fields: [TabSpecificField.OWNER, TabSpecificField.TAGS].join(','), + }) .then((res) => { setEntityData(res); }) @@ -544,7 +546,7 @@ export const fetchEntityDetail = ( break; case EntityType.GLOSSARY_TERM: getGlossaryTermByFQN(entityFQN, { - fields: TabSpecificField.TAGS, + fields: [TabSpecificField.OWNER, TabSpecificField.TAGS].join(','), }) .then((res) => { setEntityData(res); diff --git a/openmetadata-ui/src/main/resources/ui/src/utils/UserClassBase.test.ts b/openmetadata-ui/src/main/resources/ui/src/utils/UserClassBase.test.ts new file mode 100644 index 000000000000..9f40d5098b2d --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/utils/UserClassBase.test.ts @@ -0,0 +1,25 @@ +/* + * Copyright 2024 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import userClassBase from './UserClassBase'; + +describe('UserClassBase', () => { + it('should return empty string from getBotLogo when botName is empty', () => { + let result = userClassBase.getBotLogo(''); + + expect(result).toBeUndefined(); + + result = userClassBase.getBotLogo('unknown'); + + expect(result).toBeUndefined(); + }); +}); diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Settings/Applications/AppDetails/ApplicationSchemaClassBase.ts b/openmetadata-ui/src/main/resources/ui/src/utils/UserClassBase.ts similarity index 63% rename from openmetadata-ui/src/main/resources/ui/src/components/Settings/Applications/AppDetails/ApplicationSchemaClassBase.ts rename to openmetadata-ui/src/main/resources/ui/src/utils/UserClassBase.ts index fbd8e469682b..099e7c69f66b 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Settings/Applications/AppDetails/ApplicationSchemaClassBase.ts +++ b/openmetadata-ui/src/main/resources/ui/src/utils/UserClassBase.ts @@ -1,5 +1,5 @@ /* - * Copyright 2023 Collate. + * Copyright 2024 Collate. * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at @@ -10,14 +10,14 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - -class ApplicationSchemaClassBase { - public importSchema(fqn: string) { - return import(`../../../../utils/ApplicationSchemas/${fqn}.json`); +class UserClassBase { + protected botLogos: Record = {}; + public getBotLogo(botName: string) { + return this.botLogos[botName]; } } -const applicationSchemaClassBase = new ApplicationSchemaClassBase(); +const userClassBase = new UserClassBase(); -export default applicationSchemaClassBase; -export { ApplicationSchemaClassBase }; +export default userClassBase; +export { UserClassBase }; diff --git a/openmetadata-ui/src/main/resources/ui/src/utils/UserDataUtils.test.ts b/openmetadata-ui/src/main/resources/ui/src/utils/UserDataUtils.test.ts new file mode 100644 index 000000000000..5a7599d60de4 --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/utils/UserDataUtils.test.ts @@ -0,0 +1,90 @@ +/* + * Copyright 2024 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { User } from '../generated/entity/teams/user'; +import { getUserWithImage } from './UserDataUtils'; + +describe('getUserWithImage', () => { + it('should return the correct user based on profile and isBot status', () => { + const userWithProfileImage: User = { + email: 'a@a.com', + id: '1', + name: 'user', + profile: { + images: { + image: 'profileImage', + }, + }, + isBot: false, + }; + + const userWithEmptyProfileImage: User = { + email: 'a@a.com', + id: '1', + name: 'user', + profile: { + images: { + image: '', + }, + }, + isBot: false, + }; + + const botUser: User = { + email: 'a@a.com', + id: '1', + name: 'user', + isBot: true, + }; + + const userWithoutProfile: User = { + email: 'a@a.com', + id: '1', + name: 'user', + isBot: false, + }; + + // Test user with profile image + let result = getUserWithImage(userWithProfileImage); + + expect(result).toEqual(userWithProfileImage); + + // Test user with empty profile image + result = getUserWithImage(userWithEmptyProfileImage); + + expect(result).toEqual({ + ...userWithEmptyProfileImage, + profile: { + images: { + image: '', + }, + }, + }); + + // Test bot user + result = getUserWithImage(botUser); + + expect(result).toEqual({ + ...botUser, + profile: { + images: { + image: '', + }, + }, + }); + + // Test user without profile + result = getUserWithImage(userWithoutProfile); + + expect(result).toEqual(userWithoutProfile); + }); +}); diff --git a/openmetadata-ui/src/main/resources/ui/src/utils/UserDataUtils.ts b/openmetadata-ui/src/main/resources/ui/src/utils/UserDataUtils.ts index 7fc5736de8e1..c02e8a0629c1 100644 --- a/openmetadata-ui/src/main/resources/ui/src/utils/UserDataUtils.ts +++ b/openmetadata-ui/src/main/resources/ui/src/utils/UserDataUtils.ts @@ -21,6 +21,11 @@ import { getSearchedTeams, getSearchedUsers } from '../rest/miscAPI'; import { User } from './../generated/entity/teams/user'; import { formatTeamsResponse, formatUsersResponse } from './APIUtils'; import { getImages } from './CommonUtils'; +import { + getImageWithResolutionAndFallback, + ImageQuality, +} from './ProfilerUtils'; +import userClassBase from './UserClassBase'; export const getUserDataFromOidc = ( userData: User, @@ -98,3 +103,24 @@ export const searchFormattedUsersAndTeams = async ( return { users: [], teams: [], usersTotal: 0, teamsTotal: 0 }; } }; + +export const getUserWithImage = (user: User) => { + const profile = + getImageWithResolutionAndFallback( + ImageQuality['6x'], + user.profile?.images + ) ?? ''; + + if (!profile && user.isBot) { + user = { + ...user, + profile: { + images: { + image: userClassBase.getBotLogo(user.name) ?? '', + }, + }, + }; + } + + return user; +}; diff --git a/openmetadata-ui/src/main/resources/ui/src/utils/mocks/ApplicationUtils.mock.ts b/openmetadata-ui/src/main/resources/ui/src/utils/mocks/ApplicationUtils.mock.ts new file mode 100644 index 000000000000..e6762cc6c25f --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/utils/mocks/ApplicationUtils.mock.ts @@ -0,0 +1,404 @@ +/* + * Copyright 2024 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { EntityType } from '../../enums/entity.enum'; + +export const MOCK_APPLICATION_ENTITY_STATS = { + [EntityType.TAG]: { + totalRecords: 10, + failedRecords: 0, + successRecords: 10, + }, + [EntityType.TEAM]: { + totalRecords: 17, + failedRecords: 0, + successRecords: 17, + }, + [EntityType.USER]: { + totalRecords: 105, + failedRecords: 0, + successRecords: 105, + }, + [EntityType.CHART]: { + totalRecords: 16, + failedRecords: 0, + successRecords: 16, + }, + [EntityType.QUERY]: { + totalRecords: 8, + failedRecords: 0, + successRecords: 8, + }, + [EntityType.TABLE]: { + totalRecords: 180, + failedRecords: 0, + successRecords: 180, + }, + [EntityType.TOPIC]: { + totalRecords: 10, + failedRecords: 0, + successRecords: 10, + }, + [EntityType.DOMAIN]: { + totalRecords: 0, + failedRecords: 0, + successRecords: 0, + }, + [EntityType.MLMODEL]: { + totalRecords: 2, + failedRecords: 0, + successRecords: 2, + }, + [EntityType.DATABASE]: { + totalRecords: 2, + failedRecords: 0, + successRecords: 2, + }, + [EntityType.GLOSSARY]: { + totalRecords: 1, + failedRecords: 0, + successRecords: 1, + }, + [EntityType.PIPELINE]: { + totalRecords: 8, + failedRecords: 0, + successRecords: 8, + }, + [EntityType.TEST_CASE]: { + totalRecords: 7, + failedRecords: 0, + successRecords: 7, + }, + [EntityType.CONTAINER]: { + totalRecords: 17, + failedRecords: 0, + successRecords: 17, + }, + [EntityType.DASHBOARD]: { + totalRecords: 14, + failedRecords: 0, + successRecords: 14, + }, + [EntityType.TEST_SUITE]: { + totalRecords: 3, + failedRecords: 0, + successRecords: 3, + }, + [EntityType.DATA_PRODUCT]: { + totalRecords: 0, + failedRecords: 0, + successRecords: 0, + }, + [EntityType.SEARCH_INDEX]: { + totalRecords: 1, + failedRecords: 0, + successRecords: 1, + }, + [EntityType.GLOSSARY_TERM]: { + totalRecords: 0, + failedRecords: 0, + successRecords: 0, + }, + [EntityType.SEARCH_SERVICE]: { + totalRecords: 1, + failedRecords: 0, + successRecords: 1, + }, + [EntityType.CLASSIFICATION]: { + totalRecords: 3, + failedRecords: 0, + successRecords: 3, + }, + [EntityType.DATABASE_SCHEMA]: { + totalRecords: 4, + failedRecords: 0, + successRecords: 4, + }, + [EntityType.MLMODEL_SERVICE]: { + totalRecords: 1, + failedRecords: 0, + successRecords: 1, + }, + [EntityType.STORAGE_SERVICE]: { + totalRecords: 1, + failedRecords: 0, + successRecords: 1, + }, + [EntityType.DATABASE_SERVICE]: { + totalRecords: 3, + failedRecords: 0, + successRecords: 3, + }, + [EntityType.METADATA_SERVICE]: { + totalRecords: 1, + failedRecords: 0, + successRecords: 1, + }, + [EntityType.PIPELINE_SERVICE]: { + totalRecords: 1, + failedRecords: 0, + successRecords: 1, + }, + [EntityType.STORED_PROCEDURE]: { + totalRecords: 12, + failedRecords: 0, + successRecords: 12, + }, + [EntityType.DASHBOARD_SERVICE]: { + totalRecords: 2, + failedRecords: 0, + successRecords: 2, + }, + [EntityType.ENTITY_REPORT_DATA]: { + totalRecords: 4, + failedRecords: 0, + successRecords: 4, + }, + [EntityType.MESSAGING_SERVICE]: { + totalRecords: 1, + failedRecords: 0, + successRecords: 1, + }, + [EntityType.INGESTION_PIPELINE]: { + totalRecords: 4, + failedRecords: 0, + successRecords: 4, + }, + [EntityType.DASHBOARD_DATA_MODEL]: { + totalRecords: 6, + failedRecords: 0, + successRecords: 6, + }, + [EntityType.WEB_ANALYTIC_ENTITY_VIEW_REPORT_DATA]: { + totalRecords: 2, + failedRecords: 0, + successRecords: 2, + }, + [EntityType.WEB_ANALYTIC_USER_ACTIVITY_REPORT_DATA]: { + totalRecords: 4, + failedRecords: 0, + successRecords: 4, + }, +}; + +export const MOCK_APPLICATION_ENTITY_STATS_DATA = [ + { + name: EntityType.TAG, + totalRecords: 10, + successRecords: 10, + failedRecords: 0, + }, + { + name: EntityType.TEAM, + totalRecords: 17, + successRecords: 17, + failedRecords: 0, + }, + { + name: EntityType.USER, + totalRecords: 105, + successRecords: 105, + failedRecords: 0, + }, + { + name: EntityType.CHART, + totalRecords: 16, + successRecords: 16, + failedRecords: 0, + }, + { + name: EntityType.QUERY, + totalRecords: 8, + successRecords: 8, + failedRecords: 0, + }, + { + name: EntityType.TABLE, + totalRecords: 180, + successRecords: 180, + failedRecords: 0, + }, + { + name: EntityType.TOPIC, + totalRecords: 10, + successRecords: 10, + failedRecords: 0, + }, + { + name: EntityType.DOMAIN, + totalRecords: 0, + successRecords: 0, + failedRecords: 0, + }, + { + name: EntityType.MLMODEL, + totalRecords: 2, + successRecords: 2, + failedRecords: 0, + }, + { + name: EntityType.DATABASE, + totalRecords: 2, + successRecords: 2, + failedRecords: 0, + }, + { + name: EntityType.GLOSSARY, + totalRecords: 1, + successRecords: 1, + failedRecords: 0, + }, + { + name: EntityType.PIPELINE, + totalRecords: 8, + successRecords: 8, + failedRecords: 0, + }, + { + name: EntityType.TEST_CASE, + totalRecords: 7, + failedRecords: 0, + successRecords: 7, + }, + { + name: EntityType.CONTAINER, + totalRecords: 17, + failedRecords: 0, + successRecords: 17, + }, + { + name: EntityType.DASHBOARD, + totalRecords: 14, + failedRecords: 0, + successRecords: 14, + }, + { + name: EntityType.TEST_SUITE, + totalRecords: 3, + failedRecords: 0, + successRecords: 3, + }, + { + name: EntityType.DATA_PRODUCT, + totalRecords: 0, + successRecords: 0, + failedRecords: 0, + }, + { + name: EntityType.SEARCH_INDEX, + totalRecords: 1, + failedRecords: 0, + successRecords: 1, + }, + { + name: EntityType.GLOSSARY_TERM, + totalRecords: 0, + successRecords: 0, + failedRecords: 0, + }, + { + name: EntityType.SEARCH_SERVICE, + totalRecords: 1, + failedRecords: 0, + successRecords: 1, + }, + { + name: EntityType.CLASSIFICATION, + totalRecords: 3, + failedRecords: 0, + successRecords: 3, + }, + { + name: EntityType.DATABASE_SCHEMA, + totalRecords: 4, + failedRecords: 0, + successRecords: 4, + }, + { + name: EntityType.MLMODEL_SERVICE, + totalRecords: 1, + failedRecords: 0, + successRecords: 1, + }, + { + name: EntityType.STORAGE_SERVICE, + totalRecords: 1, + failedRecords: 0, + successRecords: 1, + }, + { + name: EntityType.DATABASE_SERVICE, + totalRecords: 3, + failedRecords: 0, + successRecords: 3, + }, + { + name: EntityType.METADATA_SERVICE, + totalRecords: 1, + failedRecords: 0, + successRecords: 1, + }, + { + name: EntityType.PIPELINE_SERVICE, + totalRecords: 1, + failedRecords: 0, + successRecords: 1, + }, + { + name: EntityType.STORED_PROCEDURE, + totalRecords: 12, + failedRecords: 0, + successRecords: 12, + }, + { + name: EntityType.DASHBOARD_SERVICE, + totalRecords: 2, + failedRecords: 0, + successRecords: 2, + }, + { + name: EntityType.ENTITY_REPORT_DATA, + totalRecords: 4, + failedRecords: 0, + successRecords: 4, + }, + { + name: EntityType.MESSAGING_SERVICE, + totalRecords: 1, + failedRecords: 0, + successRecords: 1, + }, + { + name: EntityType.INGESTION_PIPELINE, + totalRecords: 4, + failedRecords: 0, + successRecords: 4, + }, + { + name: EntityType.DASHBOARD_DATA_MODEL, + totalRecords: 6, + failedRecords: 0, + successRecords: 6, + }, + { + name: EntityType.WEB_ANALYTIC_ENTITY_VIEW_REPORT_DATA, + totalRecords: 2, + failedRecords: 0, + successRecords: 2, + }, + { + name: EntityType.WEB_ANALYTIC_USER_ACTIVITY_REPORT_DATA, + totalRecords: 4, + failedRecords: 0, + successRecords: 4, + }, +]; diff --git a/openmetadata-ui/src/main/resources/ui/src/utils/mocks/EntitySummaryPanelUtils.mock.tsx b/openmetadata-ui/src/main/resources/ui/src/utils/mocks/EntitySummaryPanelUtils.mock.tsx index 8b3edaa9d4bf..0af725bd59bb 100644 --- a/openmetadata-ui/src/main/resources/ui/src/utils/mocks/EntitySummaryPanelUtils.mock.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/utils/mocks/EntitySummaryPanelUtils.mock.tsx @@ -11,10 +11,12 @@ * limitations under the License. */ +import Icon from '@ant-design/icons/lib/components/Icon'; import { Typography } from 'antd'; import React from 'react'; import { Link } from 'react-router-dom'; import { BasicEntityInfo } from '../../components/Explore/EntitySummaryPanel/SummaryList/SummaryList.interface'; +import { ICON_DIMENSION } from '../../constants/constants'; import { Task } from '../../generated/entity/data/pipeline'; import { Column, @@ -40,14 +42,14 @@ export const mockLinkBasedSummaryTitleResponse = ( pathname: 'http://localhost:8080/taskinstance/list/?flt1_dag_id_equals=dim_address_task', }}> -
    +
    dim_address Task - +
    ); @@ -146,14 +148,14 @@ export const mockEntityDataWithoutNestingResponse: BasicEntityInfo[] = [ pathname: 'http://localhost:8080/taskinstance/list/?flt1_dag_id_equals=assert_table_exists', }}> -
    +
    Assert Table Exists - +
    ), diff --git a/openmetadata-ui/src/main/resources/ui/yarn.lock b/openmetadata-ui/src/main/resources/ui/yarn.lock index 339cce488878..be635430ade7 100644 --- a/openmetadata-ui/src/main/resources/ui/yarn.lock +++ b/openmetadata-ui/src/main/resources/ui/yarn.lock @@ -144,22 +144,22 @@ promise-polyfill "^8.2.1" unfetch "^4.2.0" -"@azure/msal-browser@^2.37.0": - version "2.37.0" - resolved "https://registry.yarnpkg.com/@azure/msal-browser/-/msal-browser-2.37.0.tgz#32d7af74eef53f2692f8a9d6bd6818c78faf4c1b" - integrity sha512-YNGD/W/tw/5wDWlXOfmrVILaxVsorVLxYU2ovmL1PDvxkdudbQRyGk/76l4emqgDAl/kPQeqyivxjOU6w1YfvQ== +"@azure/msal-browser@^3.10.0": + version "3.10.0" + resolved "https://registry.yarnpkg.com/@azure/msal-browser/-/msal-browser-3.10.0.tgz#8925659e8d1a4bd21e389cca4683eb52658c778e" + integrity sha512-mnmi8dCXVNZI+AGRq0jKQ3YiodlIC4W9npr6FCB9WN6NQT+6rq+cIlxgUb//BjLyzKsnYo+i4LROGeMyU+6v1A== dependencies: - "@azure/msal-common" "13.0.0" + "@azure/msal-common" "14.7.1" -"@azure/msal-common@13.0.0": - version "13.0.0" - resolved "https://registry.yarnpkg.com/@azure/msal-common/-/msal-common-13.0.0.tgz#9c39184903b5d0fd6e643ccc12193fae220e912b" - integrity sha512-GqCOg5H5bouvLij9NFXFkh+asRRxsPBRwnTDsfK7o0KcxYHJbuidKw8/VXpycahGXNxgtuhqtK/n5he+5NhyEA== +"@azure/msal-common@14.7.1": + version "14.7.1" + resolved "https://registry.yarnpkg.com/@azure/msal-common/-/msal-common-14.7.1.tgz#b13443fbacc87ce2019a91e81a6582ea73847c75" + integrity sha512-v96btzjM7KrAu4NSEdOkhQSTGOuNUIIsUdB8wlyB9cdgl5KqEKnTonHUZ8+khvZ6Ap542FCErbnTyDWl8lZ2rA== -"@azure/msal-react@^1.5.11": - version "1.5.11" - resolved "https://registry.yarnpkg.com/@azure/msal-react/-/msal-react-1.5.11.tgz#d67882f28820bb0dbcae6d3988f4903d58adfcc7" - integrity sha512-ZZnbCDbSHQXA25AF0lNYVv+AdTcFFLx6jnIZdiVHUvXrrH5WJuTfl7GV337ZEAOx8x/aZa9yZfeEaQq0ZB5acg== +"@azure/msal-react@^2.0.12": + version "2.0.12" + resolved "https://registry.yarnpkg.com/@azure/msal-react/-/msal-react-2.0.12.tgz#f33c84a57663307fcf3707ca17d9ce2d7934a048" + integrity sha512-23HKdajBWQ5SSzcwwFKHAWaOCpq4UCthoOBgKpab3UoHx0OuFMQiq6CrNBzBKtBFdyxCjadBGzWshRgl0Nvk1g== "@babel/code-frame@7.12.11": version "7.12.11" @@ -7793,6 +7793,11 @@ eventemitter3@^4.0.0, eventemitter3@^4.0.1, eventemitter3@^4.0.4: resolved "https://registry.yarnpkg.com/eventemitter3/-/eventemitter3-4.0.7.tgz#2de9b68f6528d5644ef5c59526a1b4a07306169f" integrity sha512-8guHBZCwKnFhYdHr2ysuRWErTwhoN2X8XELRlrRwpmfeY2jjuUN4taQMsULKUVo1K4DvZl+0pgfyoysHxvmvEw== +eventemitter3@^5.0.1: + version "5.0.1" + resolved "https://registry.yarnpkg.com/eventemitter3/-/eventemitter3-5.0.1.tgz#53f5ffd0a492ac800721bb42c66b841de96423c4" + integrity sha512-GWkBvjiSZK87ELrYOSESUYeVIc9mvLLf/nXalMOS5dYrgZq9o5OVkbZAVM06CVxYsCwH9BDZFPlQTlPA1j4ahA== + events@^3.2.0: version "3.3.0" resolved "https://registry.yarnpkg.com/events/-/events-3.3.0.tgz#31a95ad0a924e2d2c419a813aeb2c4e878ea7400" @@ -15705,3 +15710,10 @@ zustand@^4.4.1: integrity sha512-oRy+X3ZazZvLfmv6viIaQmtLOMeij1noakIsK/Y47PWYhT8glfXzQ4j0YcP5i0P0qI1A4rIB//SGROGyZhx91A== dependencies: use-sync-external-store "1.2.0" + +zustand@^4.5.0: + version "4.5.0" + resolved "https://registry.yarnpkg.com/zustand/-/zustand-4.5.0.tgz#141354af56f91de378aa6c4b930032ab338f3ef0" + integrity sha512-zlVFqS5TQ21nwijjhJlx4f9iGrXSL0o/+Dpy4txAP22miJ8Ti6c1Ol1RLNN98BMib83lmDH/2KmLwaNXpjrO1A== + dependencies: + use-sync-external-store "1.2.0" diff --git a/pom.xml b/pom.xml index 4613f5c2a034..3ff38a484a5b 100644 --- a/pom.xml +++ b/pom.xml @@ -26,7 +26,7 @@ based on Open Metadata Standards/APIs, supporting connectors to a wide range of data services, OpenMetadata enables end-to-end metadata management, giving you the freedom to unlock the value of your data assets. - 1.4.0-SNAPSHOT + 1.3.4 https://github.com/open-metadata/OpenMetadata openmetadata-spec @@ -101,7 +101,7 @@ 2.1.0.25 2.10.1 8.0.33 - 42.7.1 + 42.7.2 1.2.1 2.6 1.18.30 @@ -113,7 +113,7 @@ 1.0.1.RELEASE 4.5.14 - 5.3.28 + 6.1.5 2.21.0 5.9.3 1.7.3 diff --git a/scripts/datamodel_generation.py b/scripts/datamodel_generation.py index cd2a357e96ad..14c84a7f1cea 100644 --- a/scripts/datamodel_generation.py +++ b/scripts/datamodel_generation.py @@ -31,6 +31,7 @@ ingestion_path = "./" if current_directory.endswith("/ingestion") else "ingestion/" directory_root = "../" if current_directory.endswith("/ingestion") else "./" +UTF_8 = "UTF-8" UNICODE_REGEX_REPLACEMENT_FILE_PATHS = [ f"{ingestion_path}src/metadata/generated/schema/entity/classification/tag.py", f"{ingestion_path}src/metadata/generated/schema/entity/events/webhook.py", @@ -44,9 +45,23 @@ main(args) for file_path in UNICODE_REGEX_REPLACEMENT_FILE_PATHS: - with open(file_path, "r", encoding="UTF-8") as file_: + with open(file_path, "r", encoding=UTF_8) as file_: content = file_.read() # Python now requires to move the global flags at the very start of the expression content = content.replace("(?U)", "(?u)") - with open(file_path, "w", encoding="UTF-8") as file_: + with open(file_path, "w", encoding=UTF_8) as file_: file_.write(content) + + +# Until https://github.com/koxudaxi/datamodel-code-generator/issues/1895 +MISSING_IMPORTS = [f"{ingestion_path}src/metadata/generated/schema/entity/applications/app.py",] +WRITE_AFTER = "from __future__ import annotations" + +for file_path in MISSING_IMPORTS: + with open(file_path, "r", encoding=UTF_8) as file_: + lines = file_.readlines() + with open(file_path, "w", encoding=UTF_8) as file_: + for line in lines: + file_.write(line) + if line.strip() == WRITE_AFTER: + file_.write("from typing import Union # custom generate import\n\n")