From e8eb2bda3c4a5264c4ac3f381a2ce63c2008b4dc Mon Sep 17 00:00:00 2001 From: Pere Miquel Brull Date: Mon, 8 Jan 2024 14:57:07 +0100 Subject: [PATCH] MINOR - Hide AutoTagger, add MetaPilot search, and fix apps status calls (#14201) * MINOR - Hide AutoTagger & MetaPilot search & Fix apps status calls * Fix method call * Fix method call * Fix method call * "additionalProperties": true * Use EntityRef * Use ES sink * Update UI schema type --- .../src/metadata/workflow/application.py | 2 +- .../workflow/workflow_output_handler.py | 18 +++-------- .../test_workflow_creation.py | 3 +- .../bundles/insights/DataInsightsApp.java | 5 +-- .../AutoTaggerApplication.json | 24 -------------- .../external/metaPilotAppConfig.json | 14 ++++++--- .../schema/metadataIngestion/application.json | 1 + .../AutoTaggerApplication.json | 31 ------------------- .../MetaPilotApplication.json | 17 +++++++--- .../ui/src/utils/i18next/i18nextUtil.ts | 2 +- 10 files changed, 35 insertions(+), 82 deletions(-) delete mode 100644 openmetadata-service/src/main/resources/json/data/appMarketPlaceDefinition/AutoTaggerApplication.json delete mode 100644 openmetadata-ui/src/main/resources/ui/src/utils/ApplicationSchemas/AutoTaggerApplication.json diff --git a/ingestion/src/metadata/workflow/application.py b/ingestion/src/metadata/workflow/application.py index fd43d0a51bb1..44aa6898181f 100644 --- a/ingestion/src/metadata/workflow/application.py +++ b/ingestion/src/metadata/workflow/application.py @@ -133,7 +133,7 @@ def calculate_success(self) -> float: return self.runner.get_status().calculate_success() def get_failures(self) -> List[StackTraceError]: - return self.source.get_status().failures + return self.workflow_steps()[0].get_status().failures def workflow_steps(self) -> List[Step]: return [self.runner] diff --git a/ingestion/src/metadata/workflow/workflow_output_handler.py b/ingestion/src/metadata/workflow/workflow_output_handler.py index 7461054b0e9c..fde37905314b 100644 --- a/ingestion/src/metadata/workflow/workflow_output_handler.py +++ b/ingestion/src/metadata/workflow/workflow_output_handler.py @@ -28,8 +28,6 @@ DEFAULT_EXAMPLE_FILE, EXAMPLES_WORKFLOW_PATH, WORKFLOW_FAILURE_MESSAGE, - WORKFLOW_SUCCESS_MESSAGE, - WORKFLOW_WARNING_MESSAGE, WorkflowType, print_error_msg, print_more_info, @@ -122,12 +120,14 @@ def print_status(workflow: "IngestionWorkflow") -> None: print_workflow_summary(workflow) - if workflow.source.get_status().source_start_time: + # Get the time to execute the first step + first_step = workflow.workflow_steps()[0] + if first_step.get_status().source_start_time: log_ansi_encoded_string( color=ANSI.BRIGHT_CYAN, bold=True, message="Workflow finished in time: " - f"{pretty_print_time_duration(time.time()-workflow.source.get_status().source_start_time)}", + f"{pretty_print_time_duration(time.time()-first_step.get_status().source_start_time)}", ) if workflow.result_status() == 1: @@ -136,13 +136,3 @@ def print_status(workflow: "IngestionWorkflow") -> None: bold=True, message=WORKFLOW_FAILURE_MESSAGE, ) - elif workflow.source.get_status().warnings or ( - hasattr(workflow, "sink") and workflow.sink.get_status().warnings - ): - log_ansi_encoded_string( - color=ANSI.YELLOW, bold=True, message=WORKFLOW_WARNING_MESSAGE - ) - else: - log_ansi_encoded_string( - color=ANSI.GREEN, bold=True, message=WORKFLOW_SUCCESS_MESSAGE - ) diff --git a/openmetadata-airflow-apis/tests/unit/ingestion_pipeline/test_workflow_creation.py b/openmetadata-airflow-apis/tests/unit/ingestion_pipeline/test_workflow_creation.py index ac946a4e2ab9..d5f95fada27d 100644 --- a/openmetadata-airflow-apis/tests/unit/ingestion_pipeline/test_workflow_creation.py +++ b/openmetadata-airflow-apis/tests/unit/ingestion_pipeline/test_workflow_creation.py @@ -52,6 +52,7 @@ PipelineType, ) from metadata.generated.schema.metadataIngestion.applicationPipeline import ( + ApplicationConfigType, ApplicationPipeline, ) from metadata.generated.schema.metadataIngestion.databaseServiceMetadataPipeline import ( @@ -376,7 +377,7 @@ def test_application_workflow(self): fullyQualifiedName="OpenMetadata.test_auto_tagger_application", sourceConfig=SourceConfig( config=ApplicationPipeline( - type="Application", + type=ApplicationConfigType.Application, appConfig=AppConfig( __root__=AutoTaggerAppConfig(confidenceLevel=80) ), diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/insights/DataInsightsApp.java b/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/insights/DataInsightsApp.java index 0751f9bfb51a..a05f64f31f8b 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/insights/DataInsightsApp.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/insights/DataInsightsApp.java @@ -9,7 +9,7 @@ import org.openmetadata.schema.entity.services.ingestionPipelines.AirflowConfig; import org.openmetadata.schema.entity.services.ingestionPipelines.IngestionPipeline; import org.openmetadata.schema.entity.services.ingestionPipelines.PipelineType; -import org.openmetadata.schema.metadataIngestion.DataInsightPipeline; +import org.openmetadata.schema.metadataIngestion.MetadataToElasticSearchPipeline; import org.openmetadata.schema.metadataIngestion.SourceConfig; import org.openmetadata.schema.type.EntityReference; import org.openmetadata.schema.type.ProviderType; @@ -98,7 +98,8 @@ private void createAndBindIngestionPipeline( .withDisplayName(INGESTION_PIPELINE_NAME) .withDescription(PIPELINE_DESCRIPTION) .withPipelineType(PipelineType.DATA_INSIGHT) - .withSourceConfig(new SourceConfig().withConfig(new DataInsightPipeline())) + // We're configuring Data Insights to use ES as a sink, so we apply ES in the config + .withSourceConfig(new SourceConfig().withConfig(new MetadataToElasticSearchPipeline())) .withAirflowConfig( new AirflowConfig() .withScheduleInterval(getCronMapper().map(quartzCron).asString())) diff --git a/openmetadata-service/src/main/resources/json/data/appMarketPlaceDefinition/AutoTaggerApplication.json b/openmetadata-service/src/main/resources/json/data/appMarketPlaceDefinition/AutoTaggerApplication.json deleted file mode 100644 index 66c8f9dbf4b1..000000000000 --- a/openmetadata-service/src/main/resources/json/data/appMarketPlaceDefinition/AutoTaggerApplication.json +++ /dev/null @@ -1,24 +0,0 @@ -{ - "name": "AutoTaggerApplication", - "displayName": "AutoTagger", - "description": "Use NLP models to automatically tag PII information and add your own business rules for automatic data tagging.", - "features": "Auto Tag information in your tables & columns", - "appType": "external", - "appScreenshots": [ - "AutoPIIApplication.png" - ], - "developer": "Collate Inc.", - "developerUrl": "https://www.getcollate.io", - "privacyPolicyUrl": "https://www.getcollate.io", - "supportEmail": "support@getcollate.io", - "scheduleType": "Scheduled", - "permission": "All", - "className": "org.openmetadata.service.apps.AbstractNativeApplication", - "sourcePythonClass": "metadata.applications.auto_tagger.AutoTaggerApp", - "runtime": { - "enabled": "true" - }, - "appConfiguration": { - "confidenceLevel": 80 - } -} \ No newline at end of file diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/applications/configuration/external/metaPilotAppConfig.json b/openmetadata-spec/src/main/resources/json/schema/entity/applications/configuration/external/metaPilotAppConfig.json index a67359d5ece7..c9f085d19a4d 100644 --- a/openmetadata-spec/src/main/resources/json/schema/entity/applications/configuration/external/metaPilotAppConfig.json +++ b/openmetadata-spec/src/main/resources/json/schema/entity/applications/configuration/external/metaPilotAppConfig.json @@ -17,22 +17,28 @@ "description": "Choose the service and its databases you want to generate descriptions from.", "type": "object", "properties": { - "serviceName": { + "service": { "title": "Service Name", + "placeholder": "Search Service", "description": "Service Name to get descriptions from.", - "type": "string" + "$ref": "../../../../type/entityReference.json", + "format": "autoComplete", + "autoCompleteType": "database_service_search_index" }, "databases": { "title": "Databases", "description": "List of database names from the Service to get descriptions from.", "type": "array", "items": { - "type": "string" + "placeholder": "Search Databases", + "$ref": "../../../../type/entityReference.json", + "format": "autoComplete", + "autoCompleteType": "database_search_index" } } }, "additionalProperties": false, - "required": ["serviceName", "databases"] + "required": ["service", "databases"] } }, "properties": { diff --git a/openmetadata-spec/src/main/resources/json/schema/metadataIngestion/application.json b/openmetadata-spec/src/main/resources/json/schema/metadataIngestion/application.json index b0dada118cc4..2ee79748248c 100644 --- a/openmetadata-spec/src/main/resources/json/schema/metadataIngestion/application.json +++ b/openmetadata-spec/src/main/resources/json/schema/metadataIngestion/application.json @@ -3,6 +3,7 @@ "$schema": "http://json-schema.org/draft-07/schema#", "title": "OpenMetadataApplicationConfig", "description": "OpenMetadata Ingestion Framework definition for Applications, i.e., the YAML shape we require.", + "javaType": "org.openmetadata.schema.metadataIngestion.OpenMetadataAppConfig", "type": "object", "properties": { "workflowConfig": { diff --git a/openmetadata-ui/src/main/resources/ui/src/utils/ApplicationSchemas/AutoTaggerApplication.json b/openmetadata-ui/src/main/resources/ui/src/utils/ApplicationSchemas/AutoTaggerApplication.json deleted file mode 100644 index 64e015ce4961..000000000000 --- a/openmetadata-ui/src/main/resources/ui/src/utils/ApplicationSchemas/AutoTaggerApplication.json +++ /dev/null @@ -1,31 +0,0 @@ -{ - "$id": "https://open-metadata.org/schema/entity/applications/configuration/external/autoTaggerAppConfig.json", - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "AutoTaggerAppConfig", - "description": "Configuration for the Auto Tagger External Application.", - "type": "object", - "javaType": "org.openmetadata.schema.entity.app.external.AutoTaggerAppConfig", - "definitions": { - "autoTaggerApp": { - "description": "Application type.", - "type": "string", - "enum": ["AutoTagger"], - "default": "AutoTagger" - } - }, - "properties": { - "type": { - "title": "Service Type", - "description": "Service Type", - "$ref": "#/definitions/autoTaggerApp", - "default": "AutoTagger" - }, - "confidenceLevel": { - "title": "Confidence Level", - "type": "integer", - "description": "Confidence level for the ML models to apply the PII tagging.", - "default": 80 - } - }, - "additionalProperties": false -} diff --git a/openmetadata-ui/src/main/resources/ui/src/utils/ApplicationSchemas/MetaPilotApplication.json b/openmetadata-ui/src/main/resources/ui/src/utils/ApplicationSchemas/MetaPilotApplication.json index a67359d5ece7..099d382bf49d 100644 --- a/openmetadata-ui/src/main/resources/ui/src/utils/ApplicationSchemas/MetaPilotApplication.json +++ b/openmetadata-ui/src/main/resources/ui/src/utils/ApplicationSchemas/MetaPilotApplication.json @@ -17,22 +17,31 @@ "description": "Choose the service and its databases you want to generate descriptions from.", "type": "object", "properties": { - "serviceName": { + "service": { "title": "Service Name", + "placeholder": "Search Service", "description": "Service Name to get descriptions from.", - "type": "string" + "type": "string", + "format": "autoComplete", + "autoCompleteType": "database_service_search_index", + "additionalProperties": true }, "databases": { "title": "Databases", "description": "List of database names from the Service to get descriptions from.", "type": "array", + "additionalProperties": true, "items": { - "type": "string" + "placeholder": "Search Databases", + "type": "string", + "format": "autoComplete", + "autoCompleteType": "database_search_index", + "additionalProperties": true } } }, "additionalProperties": false, - "required": ["serviceName", "databases"] + "required": ["service", "databases"] } }, "properties": { diff --git a/openmetadata-ui/src/main/resources/ui/src/utils/i18next/i18nextUtil.ts b/openmetadata-ui/src/main/resources/ui/src/utils/i18next/i18nextUtil.ts index 731993382a85..c0710f5e8f58 100644 --- a/openmetadata-ui/src/main/resources/ui/src/utils/i18next/i18nextUtil.ts +++ b/openmetadata-ui/src/main/resources/ui/src/utils/i18next/i18nextUtil.ts @@ -18,10 +18,10 @@ import enUS from '../../locale/languages/en-us.json'; import esES from '../../locale/languages/es-es.json'; import frFR from '../../locale/languages/fr-fr.json'; import jaJP from '../../locale/languages/ja-jp.json'; +import nlNL from '../../locale/languages/nl-nl.json'; import ptBR from '../../locale/languages/pt-br.json'; import ruRU from '../../locale/languages/ru-ru.json'; import zhCN from '../../locale/languages/zh-cn.json'; -import nlNL from '../../locale/languages/nl-nl.json'; export enum SupportedLocales { English = 'en-US',