diff --git a/bootstrap/sql/migrations/native/1.5.15/mysql/postDataMigrationSQLScript.sql b/bootstrap/sql/migrations/native/1.5.15/mysql/postDataMigrationSQLScript.sql new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/bootstrap/sql/migrations/native/1.5.15/mysql/schemaChanges.sql b/bootstrap/sql/migrations/native/1.5.15/mysql/schemaChanges.sql new file mode 100644 index 000000000000..19762625a9d8 --- /dev/null +++ b/bootstrap/sql/migrations/native/1.5.15/mysql/schemaChanges.sql @@ -0,0 +1,5 @@ +-- Make domain policy and role non-system +UPDATE policy_entity SET json = JSON_SET(json, '$.provider', 'user') where name = 'DomainOnlyAccessPolicy'; +UPDATE policy_entity SET json = JSON_SET(json, '$.allowDelete', true) where name = 'DomainOnlyAccessPolicy'; +UPDATE role_entity SET json = JSON_SET(json, '$.provider', 'user') where name = 'DomainOnlyAccessRole'; +UPDATE role_entity SET json = JSON_SET(json, '$.allowDelete', true) where name = 'DomainOnlyAccessRole'; \ No newline at end of file diff --git a/bootstrap/sql/migrations/native/1.5.15/postgres/postDataMigrationSQLScript.sql b/bootstrap/sql/migrations/native/1.5.15/postgres/postDataMigrationSQLScript.sql new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/bootstrap/sql/migrations/native/1.5.15/postgres/schemaChanges.sql b/bootstrap/sql/migrations/native/1.5.15/postgres/schemaChanges.sql new file mode 100644 index 000000000000..6f92fbea754c --- /dev/null +++ b/bootstrap/sql/migrations/native/1.5.15/postgres/schemaChanges.sql @@ -0,0 +1,5 @@ +-- Make domain policy and role non-system +UPDATE policy_entity SET json = JSONB_SET(json::jsonb, '{provider}', '"user"', true) where name = 'DomainOnlyAccessPolicy'; +UPDATE policy_entity SET json = JSONB_SET(json::jsonb, '{allowDelete}', 'true', true) WHERE name = 'DomainOnlyAccessPolicy'; +UPDATE role_entity SET json = JSONB_SET(json::jsonb, '{provider}', '"user"', true) where name = 'DomainOnlyAccessRole'; +UPDATE role_entity SET json = JSONB_SET(json::jsonb, '{allowDelete}', 'true', true) WHERE name = 'DomainOnlyAccessRole'; diff --git a/conf/openmetadata.yaml b/conf/openmetadata.yaml index dedceb705dd9..cf7f07b7f286 100644 --- a/conf/openmetadata.yaml +++ b/conf/openmetadata.yaml @@ -180,6 +180,7 @@ authenticationConfiguration: # This will only be valid when provider type specified is customOidc providerName: ${CUSTOM_OIDC_AUTHENTICATION_PROVIDER_NAME:-""} publicKeyUrls: ${AUTHENTICATION_PUBLIC_KEYS:-[http://localhost:8585/api/v1/system/config/jwks]} + tokenValidationAlgorithm: ${AUTHENTICATION_TOKEN_VALIDATION_ALGORITHM:-"RS256"} authority: ${AUTHENTICATION_AUTHORITY:-https://accounts.google.com} clientId: ${AUTHENTICATION_CLIENT_ID:-""} callbackUrl: ${AUTHENTICATION_CALLBACK_URL:-""} diff --git a/ingestion/src/metadata/ingestion/source/storage/s3/metadata.py b/ingestion/src/metadata/ingestion/source/storage/s3/metadata.py index 777944bc204b..e6bca41938f5 100644 --- a/ingestion/src/metadata/ingestion/source/storage/s3/metadata.py +++ b/ingestion/src/metadata/ingestion/source/storage/s3/metadata.py @@ -293,15 +293,26 @@ def _generate_container_details( ) # if we have a sample file to fetch a schema from if sample_key: - columns = self._get_columns( - container_name=bucket_name, - sample_key=sample_key, - metadata_entry=metadata_entry, - config_source=S3Config( - securityConfig=self.service_connection.awsConfig - ), - client=self.s3_client, - ) + try: + columns = self._get_columns( + container_name=bucket_name, + sample_key=sample_key, + metadata_entry=metadata_entry, + config_source=S3Config( + securityConfig=self.service_connection.awsConfig + ), + client=self.s3_client, + ) + except Exception as err: + logger.warning() + self.status.failed( + error=StackTraceError( + name=f"{bucket_name}/{sample_key}", + error=f"Error extracting columns from [{bucket_name}/{sample_key}] due to: [{err}]", + stackTrace=traceback.format_exc(), + ) + ) + return None if columns: prefix = ( f"{KEY_SEPARATOR}{metadata_entry.dataPath.strip(KEY_SEPARATOR)}" @@ -413,7 +424,7 @@ def _yield_nested_unstructured_containers( candidate_keys = [ entry["Key"] for entry in response[S3_CLIENT_ROOT_RESPONSE] - if entry and entry.get("Key") + if entry and entry.get("Key") and not entry.get("Key").endswith("/") ] for key in candidate_keys: if self.is_valid_unstructured_file(metadata_entry.unstructuredFormats, key): @@ -622,7 +633,7 @@ def _get_sample_file_path( candidate_keys = [ entry["Key"] for entry in response[S3_CLIENT_ROOT_RESPONSE] - if entry and entry.get("Key") + if entry and entry.get("Key") and not entry.get("Key").endswith("/") ] # pick a random key out of the candidates if any were returned if candidate_keys: diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/OpenMetadataApplication.java b/openmetadata-service/src/main/java/org/openmetadata/service/OpenMetadataApplication.java index 9fedc0c89534..8efa36c4bc47 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/OpenMetadataApplication.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/OpenMetadataApplication.java @@ -79,6 +79,7 @@ import org.openmetadata.service.exception.JsonMappingExceptionMapper; import org.openmetadata.service.exception.OMErrorPageHandler; import org.openmetadata.service.fernet.Fernet; +import org.openmetadata.service.governance.workflows.WorkflowHandler; import org.openmetadata.service.jdbi3.CollectionDAO; import org.openmetadata.service.jdbi3.EntityRepository; import org.openmetadata.service.jdbi3.MigrationDAO; @@ -173,6 +174,9 @@ public void run(OpenMetadataApplicationConfig catalogConfig, Environment environ // Configure the Fernet instance Fernet.getInstance().setFernetKey(catalogConfig); + // Initialize Workflow Handler + WorkflowHandler.initialize(catalogConfig); + // Init Settings Cache after repositories SettingsCache.initialize(catalogConfig); @@ -186,7 +190,10 @@ public void run(OpenMetadataApplicationConfig catalogConfig, Environment environ EntityMaskerFactory.createEntityMasker(); // Instantiate JWT Token Generator - JWTTokenGenerator.getInstance().init(catalogConfig.getJwtTokenConfiguration()); + JWTTokenGenerator.getInstance() + .init( + catalogConfig.getAuthenticationConfiguration().getTokenValidationAlgorithm(), + catalogConfig.getJwtTokenConfiguration()); // Set the Database type for choosing correct queries from annotations jdbi.getConfig(SqlObjects.class) diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/WorkflowHandler.java b/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/WorkflowHandler.java index 899c4b7f9c10..3e28d2d39885 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/WorkflowHandler.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/WorkflowHandler.java @@ -64,7 +64,7 @@ public void initializeNewProcessEngine( ProcessEngineConfiguration currentProcessEngineConfiguration) { ProcessEngines.destroy(); SystemRepository systemRepository = Entity.getSystemRepository(); - WorkflowSettings workflowSettings = systemRepository.getWorkflowSettings(); + WorkflowSettings workflowSettings = systemRepository.getWorkflowSettingsOrDefault(); StandaloneProcessEngineConfiguration processEngineConfiguration = new StandaloneProcessEngineConfiguration(); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java index 809a53e5d6b4..0d9609a71361 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java @@ -4152,7 +4152,7 @@ List listWithoutEntityFilter( @Bind("eventType") String eventType, @Bind("timestamp") long timestamp); @SqlQuery( - "SELECT json FROM change_event ce where ce.offset > :offset ORDER BY ce.eventTime ASC LIMIT :limit") + "SELECT json FROM change_event WHERE offset > :offset ORDER BY offset ASC LIMIT :limit") List list(@Bind("limit") long limit, @Bind("offset") long offset); @ConnectionAwareSqlQuery(value = "SELECT MAX(offset) FROM change_event", connectionType = MYSQL) diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityRepository.java index 221da3625c89..e86f736d444c 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityRepository.java @@ -131,6 +131,7 @@ import org.openmetadata.schema.entity.feed.Suggestion; import org.openmetadata.schema.entity.teams.Team; import org.openmetadata.schema.entity.teams.User; +import org.openmetadata.schema.entity.type.Style; import org.openmetadata.schema.system.EntityError; import org.openmetadata.schema.type.ApiStatus; import org.openmetadata.schema.type.AssetCertification; @@ -2969,6 +2970,14 @@ private static List getEntityReferences(List r private void updateStyle() { if (supportsStyle) { + Style originalStyle = original.getStyle(); + Style updatedStyle = updated.getStyle(); + + if (originalStyle == updatedStyle) return; + if (operation == Operation.PUT && updatedStyle == null) { + updatedStyle = originalStyle; + updated.setStyle(updatedStyle); + } recordChange(FIELD_STYLE, original.getStyle(), updated.getStyle(), true); } } @@ -3023,7 +3032,7 @@ private void updateCertification() { SystemRepository systemRepository = Entity.getSystemRepository(); AssetCertificationSettings assetCertificationSettings = - systemRepository.getAssetCertificationSettings(); + systemRepository.getAssetCertificationSettingOrDefault(); String certificationLabel = updatedCertification.getTagLabel().getTagFQN(); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/SystemRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/SystemRepository.java index b854c39a6ee5..302882eb0c7d 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/SystemRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/SystemRepository.java @@ -16,6 +16,8 @@ import org.jdbi.v3.sqlobject.transaction.Transaction; import org.openmetadata.api.configuration.UiThemePreference; import org.openmetadata.schema.configuration.AssetCertificationSettings; +import org.openmetadata.schema.configuration.ExecutorConfiguration; +import org.openmetadata.schema.configuration.HistoryCleanUpConfiguration; import org.openmetadata.schema.configuration.WorkflowSettings; import org.openmetadata.schema.email.SmtpSettings; import org.openmetadata.schema.entity.services.ingestionPipelines.PipelineServiceClientResponse; @@ -121,6 +123,17 @@ public AssetCertificationSettings getAssetCertificationSettings() { .orElse(null); } + public AssetCertificationSettings getAssetCertificationSettingOrDefault() { + AssetCertificationSettings assetCertificationSettings = getAssetCertificationSettings(); + if (assetCertificationSettings == null) { + assetCertificationSettings = + new AssetCertificationSettings() + .withAllowedClassification("Certification") + .withValidityPeriod("P30D"); + } + return assetCertificationSettings; + } + public WorkflowSettings getWorkflowSettings() { Optional oWorkflowSettings = Optional.ofNullable(getConfigWithKey(SettingsType.WORKFLOW_SETTINGS.value())); @@ -130,6 +143,17 @@ public WorkflowSettings getWorkflowSettings() { .orElse(null); } + public WorkflowSettings getWorkflowSettingsOrDefault() { + WorkflowSettings workflowSettings = getWorkflowSettings(); + if (workflowSettings == null) { + workflowSettings = + new WorkflowSettings() + .withExecutorConfiguration(new ExecutorConfiguration()) + .withHistoryCleanUpConfiguration(new HistoryCleanUpConfiguration()); + } + return workflowSettings; + } + public Settings getEmailConfigInternal() { try { Settings setting = dao.getConfigWithKey(SettingsType.EMAIL_CONFIGURATION.value()); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/governance/WorkflowDefinitionResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/governance/WorkflowDefinitionResource.java index 7cbc8e066a97..f76306509325 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/governance/WorkflowDefinitionResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/governance/WorkflowDefinitionResource.java @@ -70,7 +70,6 @@ public static class WorkflowDefinitionList extends ResultList DEFAULT_PUBLIC_KEY_URLS = Arrays.asList( @@ -123,6 +125,7 @@ public JwtFilter( this.principalDomain = authorizerConfiguration.getPrincipalDomain(); this.enforcePrincipalDomain = authorizerConfiguration.getEnforcePrincipalDomain(); this.useRolesFromProvider = authorizerConfiguration.getUseRolesFromProvider(); + this.tokenValidationAlgorithm = authenticationConfiguration.getTokenValidationAlgorithm(); } @VisibleForTesting @@ -224,7 +227,8 @@ public Map validateJwtAndGetClaims(String token) { // Validate JWT with public key Jwk jwk = jwkProvider.get(jwt.getKeyId()); - Algorithm algorithm = Algorithm.RSA256((RSAPublicKey) jwk.getPublicKey(), null); + Algorithm algorithm = + getAlgorithm(tokenValidationAlgorithm, (RSAPublicKey) jwk.getPublicKey(), null); try { algorithm.verify(jwt); } catch (RuntimeException runtimeException) { diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/security/jwt/JWTTokenGenerator.java b/openmetadata-service/src/main/java/org/openmetadata/service/security/jwt/JWTTokenGenerator.java index 868175326469..21aaeeeef9b2 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/security/jwt/JWTTokenGenerator.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/security/jwt/JWTTokenGenerator.java @@ -37,6 +37,7 @@ import java.util.Set; import lombok.Getter; import lombok.extern.slf4j.Slf4j; +import org.openmetadata.schema.api.security.AuthenticationConfiguration; import org.openmetadata.schema.api.security.jwt.JWTTokenConfiguration; import org.openmetadata.schema.auth.JWTAuthMechanism; import org.openmetadata.schema.auth.JWTTokenExpiry; @@ -56,6 +57,7 @@ public class JWTTokenGenerator { @Getter private RSAPublicKey publicKey; private String issuer; private String kid; + private AuthenticationConfiguration.TokenValidationAlgorithm tokenValidationAlgorithm; private JWTTokenGenerator() { /* Private constructor for singleton */ @@ -66,7 +68,9 @@ public static JWTTokenGenerator getInstance() { } /** Expected to be initialized only once during application start */ - public void init(JWTTokenConfiguration jwtTokenConfiguration) { + public void init( + AuthenticationConfiguration.TokenValidationAlgorithm algorithm, + JWTTokenConfiguration jwtTokenConfiguration) { try { if (jwtTokenConfiguration.getRsaprivateKeyFilePath() != null && !jwtTokenConfiguration.getRsaprivateKeyFilePath().isEmpty() @@ -84,6 +88,7 @@ public void init(JWTTokenConfiguration jwtTokenConfiguration) { publicKey = (RSAPublicKey) kf.generatePublic(spec); issuer = jwtTokenConfiguration.getJwtissuer(); kid = jwtTokenConfiguration.getKeyId(); + tokenValidationAlgorithm = algorithm; } } catch (Exception ex) { LOG.error("Failed to initialize JWTTokenGenerator ", ex); @@ -141,7 +146,7 @@ public JWTAuthMechanism getJwtAuthMechanism( } } JWTAuthMechanism jwtAuthMechanism = new JWTAuthMechanism().withJWTTokenExpiry(expiry); - Algorithm algorithm = Algorithm.RSA256(null, privateKey); + Algorithm algorithm = getAlgorithm(tokenValidationAlgorithm, null, privateKey); String token = JWT.create() .withIssuer(issuer) @@ -214,4 +219,15 @@ public Date getTokenExpiryFromJWT(String token) { return jwt.getExpiresAt(); } + + public static Algorithm getAlgorithm( + AuthenticationConfiguration.TokenValidationAlgorithm algorithm, + RSAPublicKey publicKey, + RSAPrivateKey privateKey) { + return switch (algorithm) { + case RS_256 -> Algorithm.RSA256(publicKey, privateKey); + case RS_384 -> Algorithm.RSA384(publicKey, privateKey); + case RS_512 -> Algorithm.RSA512(publicKey, privateKey); + }; + } } diff --git a/openmetadata-service/src/main/resources/json/data/policy/DomainAccessPolicy.json b/openmetadata-service/src/main/resources/json/data/policy/DomainAccessPolicy.json index 572760b5ef01..d103fff85265 100644 --- a/openmetadata-service/src/main/resources/json/data/policy/DomainAccessPolicy.json +++ b/openmetadata-service/src/main/resources/json/data/policy/DomainAccessPolicy.json @@ -4,8 +4,8 @@ "fullyQualifiedName": "DomainOnlyAccessPolicy", "description": "This Policy adds restrictions so that users will have access to domain related data. If the user has some domain, then he will be able to access data only for that domain. If the user does not have any domain assigned , he will be able to access only assets which also does not have any domain.", "enabled": true, - "allowDelete": false, - "provider": "system", + "allowDelete": true, + "provider": "user", "rules": [ { "name": "DomainOnlyAccessRule", diff --git a/openmetadata-service/src/main/resources/json/data/role/DomainOnlyAccessRole.json b/openmetadata-service/src/main/resources/json/data/role/DomainOnlyAccessRole.json index b18aeae18424..ec770210e4da 100644 --- a/openmetadata-service/src/main/resources/json/data/role/DomainOnlyAccessRole.json +++ b/openmetadata-service/src/main/resources/json/data/role/DomainOnlyAccessRole.json @@ -2,8 +2,8 @@ "name": "DomainOnlyAccessRole", "displayName": "Domain Only Access Role", "description": "Role Corresponding to Domain Access Restriction.", - "allowDelete": false, - "provider": "system", + "allowDelete": true, + "provider": "user", "policies" : [ { "type" : "policy", diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/resources/system/SystemResourceTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/resources/system/SystemResourceTest.java index c0bd093aa5cc..1cf64c31e5ea 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/resources/system/SystemResourceTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/resources/system/SystemResourceTest.java @@ -47,6 +47,7 @@ import org.openmetadata.schema.auth.JWTAuthMechanism; import org.openmetadata.schema.auth.JWTTokenExpiry; import org.openmetadata.schema.configuration.AssetCertificationSettings; +import org.openmetadata.schema.configuration.WorkflowSettings; import org.openmetadata.schema.email.SmtpSettings; import org.openmetadata.schema.entity.data.Table; import org.openmetadata.schema.entity.teams.AuthenticationMechanism; @@ -551,6 +552,48 @@ void testLineageSettings() throws HttpResponseException { assertEquals(4, updatedLineageConfig.getDownstreamDepth()); } + @Test + void testWorkflowSettings() throws HttpResponseException { + // Retrieve the default workflow settings + Settings setting = getSystemConfig(SettingsType.WORKFLOW_SETTINGS); + WorkflowSettings workflowSettings = + JsonUtils.convertValue(setting.getConfigValue(), WorkflowSettings.class); + + // Assert default values + assertEquals(50, workflowSettings.getExecutorConfiguration().getCorePoolSize()); + assertEquals(1000, workflowSettings.getExecutorConfiguration().getQueueSize()); + assertEquals(100, workflowSettings.getExecutorConfiguration().getMaxPoolSize()); + assertEquals(20, workflowSettings.getExecutorConfiguration().getTasksDuePerAcquisition()); + assertEquals(7, workflowSettings.getHistoryCleanUpConfiguration().getCleanAfterNumberOfDays()); + + // Update workflow settings + workflowSettings.getExecutorConfiguration().setCorePoolSize(100); + workflowSettings.getExecutorConfiguration().setQueueSize(2000); + workflowSettings.getExecutorConfiguration().setMaxPoolSize(200); + workflowSettings.getExecutorConfiguration().setTasksDuePerAcquisition(40); + workflowSettings.getHistoryCleanUpConfiguration().setCleanAfterNumberOfDays(10); + + Settings updatedSetting = + new Settings() + .withConfigType(SettingsType.WORKFLOW_SETTINGS) + .withConfigValue(workflowSettings); + + updateSystemConfig(updatedSetting); + + // Retrieve the updated settings + Settings updatedSettings = getSystemConfig(SettingsType.WORKFLOW_SETTINGS); + WorkflowSettings updateWorkflowSettings = + JsonUtils.convertValue(updatedSettings.getConfigValue(), WorkflowSettings.class); + + // Assert updated values + assertEquals(100, updateWorkflowSettings.getExecutorConfiguration().getCorePoolSize()); + assertEquals(2000, updateWorkflowSettings.getExecutorConfiguration().getQueueSize()); + assertEquals(200, updateWorkflowSettings.getExecutorConfiguration().getMaxPoolSize()); + assertEquals(40, updateWorkflowSettings.getExecutorConfiguration().getTasksDuePerAcquisition()); + assertEquals( + 10, updateWorkflowSettings.getHistoryCleanUpConfiguration().getCleanAfterNumberOfDays()); + } + @Test void globalProfilerConfig(TestInfo test) throws HttpResponseException { // Create a profiler config diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/security/JWTTokenGeneratorTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/security/JWTTokenGeneratorTest.java index b7d935f5c624..555a079ddd30 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/security/JWTTokenGeneratorTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/security/JWTTokenGeneratorTest.java @@ -15,6 +15,7 @@ import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestInstance; +import org.openmetadata.schema.api.security.AuthenticationConfiguration; import org.openmetadata.schema.api.security.jwt.JWTTokenConfiguration; import org.openmetadata.schema.auth.JWTAuthMechanism; import org.openmetadata.schema.auth.JWTTokenExpiry; @@ -38,7 +39,8 @@ public void setup() { jwtTokenConfiguration.setRsaprivateKeyFilePath(rsaPrivateKeyPath); jwtTokenConfiguration.setRsapublicKeyFilePath(rsaPublicKeyPath); jwtTokenGenerator = JWTTokenGenerator.getInstance(); - jwtTokenGenerator.init(jwtTokenConfiguration); + jwtTokenGenerator.init( + AuthenticationConfiguration.TokenValidationAlgorithm.RS_256, jwtTokenConfiguration); } @Test diff --git a/openmetadata-spec/src/main/resources/json/schema/configuration/authenticationConfiguration.json b/openmetadata-spec/src/main/resources/json/schema/configuration/authenticationConfiguration.json index 670401107ca8..ac7d5075ecae 100644 --- a/openmetadata-spec/src/main/resources/json/schema/configuration/authenticationConfiguration.json +++ b/openmetadata-spec/src/main/resources/json/schema/configuration/authenticationConfiguration.json @@ -46,6 +46,12 @@ "type": "string" } }, + "tokenValidationAlgorithm": { + "description": "Token Validation Algorithm to use.", + "type": "string", + "enum": ["RS256", "RS384", "RS512"], + "default": "RS256" + }, "authority": { "description": "Authentication Authority", "type": "string" diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/database/mysqlConnection.json b/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/database/mysqlConnection.json index 5d269f58a6f0..663c3b70b85f 100644 --- a/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/database/mysqlConnection.json +++ b/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/database/mysqlConnection.json @@ -44,6 +44,7 @@ "authType": { "title": "Auth Configuration Type", "description": "Choose Auth Config Type.", + "mask": true, "oneOf": [ { "$ref": "./common/basicAuth.json" diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/database/postgresConnection.json b/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/database/postgresConnection.json index 71defb2941b8..019e6816e743 100644 --- a/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/database/postgresConnection.json +++ b/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/database/postgresConnection.json @@ -45,6 +45,7 @@ "authType": { "title": "Auth Configuration Type", "description": "Choose Auth Config Type.", + "mask": true, "oneOf": [ { "$ref": "./common/basicAuth.json" diff --git a/openmetadata-spec/src/main/resources/json/schema/governance/workflows/elements/nodes/userTask/userApprovalTask.json b/openmetadata-spec/src/main/resources/json/schema/governance/workflows/elements/nodes/userTask/userApprovalTask.json index fe4ae4636a7c..b175fba1d6e8 100644 --- a/openmetadata-spec/src/main/resources/json/schema/governance/workflows/elements/nodes/userTask/userApprovalTask.json +++ b/openmetadata-spec/src/main/resources/json/schema/governance/workflows/elements/nodes/userTask/userApprovalTask.json @@ -40,13 +40,6 @@ "description": "Add the Reviewers to the assignees List.", "type": "boolean", "default": false - }, - "extraAssignees": { - "description": "Manually add Specific Assignees.", - "type": "array", - "items": { - "$ref": "../../../../../type/entityReference.json" - } } } } diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/configuration/authenticationConfiguration.ts b/openmetadata-ui/src/main/resources/ui/src/generated/configuration/authenticationConfiguration.ts index c21521c1562a..41f559faee8a 100644 --- a/openmetadata-ui/src/main/resources/ui/src/generated/configuration/authenticationConfiguration.ts +++ b/openmetadata-ui/src/main/resources/ui/src/generated/configuration/authenticationConfiguration.ts @@ -10,9 +10,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - - - /** +/** * This schema defines the Authentication Configuration. */ export interface AuthenticationConfiguration { @@ -69,6 +67,10 @@ export interface AuthenticationConfiguration { * Saml Configuration that is applicable only when the provider is Saml */ samlConfiguration?: SamlSSOClientConfig; + /** + * Token Validation Algorithm to use. + */ + tokenValidationAlgorithm?: TokenValidationAlgorithm; } /** @@ -492,3 +494,12 @@ export interface SP { */ spX509Certificate?: string; } + +/** + * Token Validation Algorithm to use. + */ +export enum TokenValidationAlgorithm { + Rs256 = "RS256", + Rs384 = "RS384", + Rs512 = "RS512", +} diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/entity/services/connections/database/mysqlConnection.ts b/openmetadata-ui/src/main/resources/ui/src/generated/entity/services/connections/database/mysqlConnection.ts index 4aab7d80f0fa..035b8b79deba 100644 --- a/openmetadata-ui/src/main/resources/ui/src/generated/entity/services/connections/database/mysqlConnection.ts +++ b/openmetadata-ui/src/main/resources/ui/src/generated/entity/services/connections/database/mysqlConnection.ts @@ -10,9 +10,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - - - /** +/** * Mysql Database Connection Config */ export interface MysqlConnection { diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/entity/services/connections/database/postgresConnection.ts b/openmetadata-ui/src/main/resources/ui/src/generated/entity/services/connections/database/postgresConnection.ts index c681b859806b..4294e5c637ac 100644 --- a/openmetadata-ui/src/main/resources/ui/src/generated/entity/services/connections/database/postgresConnection.ts +++ b/openmetadata-ui/src/main/resources/ui/src/generated/entity/services/connections/database/postgresConnection.ts @@ -10,9 +10,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - - - /** +/** * Postgres Database Connection Config */ export interface PostgresConnection { diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/governance/workflows/elements/nodes/userTask/userApprovalTask.ts b/openmetadata-ui/src/main/resources/ui/src/generated/governance/workflows/elements/nodes/userTask/userApprovalTask.ts index 5cab66f590ec..6d523d5b7427 100644 --- a/openmetadata-ui/src/main/resources/ui/src/generated/governance/workflows/elements/nodes/userTask/userApprovalTask.ts +++ b/openmetadata-ui/src/main/resources/ui/src/generated/governance/workflows/elements/nodes/userTask/userApprovalTask.ts @@ -10,9 +10,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - - - /** +/** * Defines a Task for a given User to approve. */ export interface UserApprovalTask { @@ -51,58 +49,5 @@ export interface Assignees { * Add the Reviewers to the assignees List. */ addReviewers?: boolean; - /** - * Manually add Specific Assignees. - */ - extraAssignees?: EntityReference[]; [property: string]: any; } - -/** - * This schema defines the EntityReference type used for referencing an entity. - * EntityReference is used for capturing relationships from one entity to another. For - * example, a table has an attribute called database of type EntityReference that captures - * the relationship of a table `belongs to a` database. - */ -export interface EntityReference { - /** - * If true the entity referred to has been soft-deleted. - */ - deleted?: boolean; - /** - * Optional description of entity. - */ - description?: string; - /** - * Display Name that identifies this entity. - */ - displayName?: string; - /** - * Fully qualified name of the entity instance. For entities such as tables, databases - * fullyQualifiedName is returned in this field. For entities that don't have name hierarchy - * such as `user` and `team` this will be same as the `name` field. - */ - fullyQualifiedName?: string; - /** - * Link to the entity resource. - */ - href?: string; - /** - * Unique identifier that identifies an entity instance. - */ - id: string; - /** - * If true the relationship indicated by this entity reference is inherited from the parent - * entity. - */ - inherited?: boolean; - /** - * Name of the entity instance. - */ - name?: string; - /** - * Entity type/class name - Examples: `database`, `table`, `metrics`, `databaseService`, - * `dashboardService`... - */ - type: string; -} diff --git a/openmetadata-ui/src/main/resources/ui/src/pages/LogsViewerPage/LogsViewerPage.tsx b/openmetadata-ui/src/main/resources/ui/src/pages/LogsViewerPage/LogsViewerPage.tsx index 530ce44a3587..906934722adf 100644 --- a/openmetadata-ui/src/main/resources/ui/src/pages/LogsViewerPage/LogsViewerPage.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/pages/LogsViewerPage/LogsViewerPage.tsx @@ -54,7 +54,10 @@ import { } from '../../rest/ingestionPipelineAPI'; import { getEpochMillisForPastDays } from '../../utils/date-time/DateTimeUtils'; import { getEntityName } from '../../utils/EntityUtils'; -import { downloadIngestionLog } from '../../utils/IngestionLogs/LogsUtils'; +import { + downloadAppLogs, + downloadIngestionLog, +} from '../../utils/IngestionLogs/LogsUtils'; import logsClassBase from '../../utils/LogsClassBase'; import { showErrorToast } from '../../utils/ToastUtils'; import './logs-viewer-page.style.less'; @@ -307,18 +310,24 @@ const LogsViewerPage = () => { ); updateProgress(paging?.after ? progress : 1); - - const logs = await downloadIngestionLog( - ingestionDetails?.id, + let logs = ''; + let fileName = `${getEntityName(ingestionDetails)}-${ ingestionDetails?.pipelineType - ); + }.log`; + if (isApplicationType) { + logs = await downloadAppLogs(ingestionName); + fileName = `${ingestionName}.log`; + } else { + logs = await downloadIngestionLog( + ingestionDetails?.id, + ingestionDetails?.pipelineType + ); + } const element = document.createElement('a'); const file = new Blob([logs || ''], { type: 'text/plain' }); element.href = URL.createObjectURL(file); - element.download = `${getEntityName(ingestionDetails)}-${ - ingestionDetails?.pipelineType - }.log`; + element.download = fileName; document.body.appendChild(element); element.click(); document.body.removeChild(element); diff --git a/openmetadata-ui/src/main/resources/ui/src/rest/applicationAPI.ts b/openmetadata-ui/src/main/resources/ui/src/rest/applicationAPI.ts index 1a2f5d636edb..b34632fb346c 100644 --- a/openmetadata-ui/src/main/resources/ui/src/rest/applicationAPI.ts +++ b/openmetadata-ui/src/main/resources/ui/src/rest/applicationAPI.ts @@ -139,3 +139,11 @@ export const restoreApp = async (id: string) => { export const stopApp = async (name: string) => { return await APIClient.post(`${BASE_URL}/stop/${getEncodedFqn(name)}`); }; + +export const getApplicationLogs = (appName: string, after?: string) => { + return APIClient.get(`${BASE_URL}/name/${appName}/logs`, { + params: { + after, + }, + }); +}; diff --git a/openmetadata-ui/src/main/resources/ui/src/utils/IngestionLogs/LogsUtils.ts b/openmetadata-ui/src/main/resources/ui/src/utils/IngestionLogs/LogsUtils.ts index 7fc27e901f26..d0f794dad2f5 100644 --- a/openmetadata-ui/src/main/resources/ui/src/utils/IngestionLogs/LogsUtils.ts +++ b/openmetadata-ui/src/main/resources/ui/src/utils/IngestionLogs/LogsUtils.ts @@ -15,6 +15,7 @@ import { round } from 'lodash'; import { PipelineType } from '../../generated/entity/services/ingestionPipelines/ingestionPipeline'; import { useDownloadProgressStore } from '../../hooks/useDownloadProgressStore'; import { IngestionPipelineLogByIdInterface } from '../../pages/LogsViewerPage/LogsViewerPage.interfaces'; +import { getApplicationLogs } from '../../rest/applicationAPI'; import { getIngestionPipelineLogById } from '../../rest/ingestionPipelineAPI'; import { showErrorToast } from '../ToastUtils'; @@ -61,10 +62,13 @@ export const fetchLogsRecursively = async ( after?: string ) => { let logs = ''; + const { data: { total, after: afterCursor, ...rest }, - } = await getIngestionPipelineLogById(ingestionId, after); - + } = + pipelineType === PipelineType.Application + ? await getApplicationLogs(ingestionId, after) + : await getIngestionPipelineLogById(ingestionId, after); logs = logs.concat(getLogsFromResponse(rest, pipelineType)); if (afterCursor && total) { const progress = round((Number(afterCursor) * 100) / Number(total)); @@ -94,3 +98,17 @@ export const downloadIngestionLog = async ( return ''; } }; + +export const downloadAppLogs = async (appName?: string) => { + if (!appName) { + return ''; + } + + try { + return await fetchLogsRecursively(appName, PipelineType.Application); + } catch (err) { + showErrorToast(err as AxiosError); + + return ''; + } +};