MINOR - Hide AutoTagger, add MetaPilot search, and fix apps status ca…

…lls (#14201)

* MINOR - Hide AutoTagger & MetaPilot search & Fix apps status calls

* Fix method call

* Fix method call

* Fix method call

* "additionalProperties": true

* Use EntityRef

* Use ES sink

* Update UI schema type

ingestion/src/metadata/workflow/application.py

@@ -133,7 +133,7 @@ def calculate_success(self) -> float:
  return self.runner.get_status().calculate_success()
 
  def get_failures(self) -> List[StackTraceError]:
- return self.source.get_status().failures
+ return self.workflow_steps()[0].get_status().failures
 
  def workflow_steps(self) -> List[Step]:
  return [self.runner]

ingestion/src/metadata/workflow/workflow_output_handler.py

@@ -28,8 +28,6 @@
  DEFAULT_EXAMPLE_FILE,
  EXAMPLES_WORKFLOW_PATH,
  WORKFLOW_FAILURE_MESSAGE,
- WORKFLOW_SUCCESS_MESSAGE,
- WORKFLOW_WARNING_MESSAGE,
  WorkflowType,
  print_error_msg,
  print_more_info,
@@ -122,12 +120,14 @@ def print_status(workflow: "IngestionWorkflow") -> None:
 
  print_workflow_summary(workflow)
 
- if workflow.source.get_status().source_start_time:
+ # Get the time to execute the first step
+ first_step = workflow.workflow_steps()[0]
+ if first_step.get_status().source_start_time:
  log_ansi_encoded_string(
  color=ANSI.BRIGHT_CYAN,
  bold=True,
  message="Workflow finished in time: "
- f"{pretty_print_time_duration(time.time()-workflow.source.get_status().source_start_time)}",
+ f"{pretty_print_time_duration(time.time()-first_step.get_status().source_start_time)}",
  )
 
  if workflow.result_status() == 1:
@@ -136,13 +136,3 @@ def print_status(workflow: "IngestionWorkflow") -> None:
  bold=True,
  message=WORKFLOW_FAILURE_MESSAGE,
  )
- elif workflow.source.get_status().warnings or (
- hasattr(workflow, "sink") and workflow.sink.get_status().warnings
- ):
- log_ansi_encoded_string(
- color=ANSI.YELLOW, bold=True, message=WORKFLOW_WARNING_MESSAGE
- )
- else:
- log_ansi_encoded_string(
- color=ANSI.GREEN, bold=True, message=WORKFLOW_SUCCESS_MESSAGE
- )

openmetadata-airflow-apis/tests/unit/ingestion_pipeline/test_workflow_creation.py

@@ -52,6 +52,7 @@
  PipelineType,
 )
 from metadata.generated.schema.metadataIngestion.applicationPipeline import (
+ ApplicationConfigType,
  ApplicationPipeline,
 )
 from metadata.generated.schema.metadataIngestion.databaseServiceMetadataPipeline import (
@@ -376,7 +377,7 @@ def test_application_workflow(self):
  fullyQualifiedName="OpenMetadata.test_auto_tagger_application",
  sourceConfig=SourceConfig(
  config=ApplicationPipeline(
- type="Application",
+ type=ApplicationConfigType.Application,
  appConfig=AppConfig(
  __root__=AutoTaggerAppConfig(confidenceLevel=80)
  ),

openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/insights/DataInsightsApp.java

@@ -9,7 +9,7 @@
 import org.openmetadata.schema.entity.services.ingestionPipelines.AirflowConfig;
 import org.openmetadata.schema.entity.services.ingestionPipelines.IngestionPipeline;
 import org.openmetadata.schema.entity.services.ingestionPipelines.PipelineType;
-import org.openmetadata.schema.metadataIngestion.DataInsightPipeline;
+import org.openmetadata.schema.metadataIngestion.MetadataToElasticSearchPipeline;
 import org.openmetadata.schema.metadataIngestion.SourceConfig;
 import org.openmetadata.schema.type.EntityReference;
 import org.openmetadata.schema.type.ProviderType;
@@ -98,7 +98,8 @@ private void createAndBindIngestionPipeline(
  .withDisplayName(INGESTION_PIPELINE_NAME)
  .withDescription(PIPELINE_DESCRIPTION)
  .withPipelineType(PipelineType.DATA_INSIGHT)
- .withSourceConfig(new SourceConfig().withConfig(new DataInsightPipeline()))
+ // We're configuring Data Insights to use ES as a sink, so we apply ES in the config
+ .withSourceConfig(new SourceConfig().withConfig(new MetadataToElasticSearchPipeline()))
  .withAirflowConfig(
  new AirflowConfig()
  .withScheduleInterval(getCronMapper().map(quartzCron).asString()))

openmetadata-spec/src/main/resources/json/schema/entity/applications/configuration/external/metaPilotAppConfig.json

@@ -17,22 +17,28 @@
  "description": "Choose the service and its databases you want to generate descriptions from.",
  "type": "object",
  "properties": {
- "serviceName": {
+ "service": {
  "title": "Service Name",
+ "placeholder": "Search Service",
  "description": "Service Name to get descriptions from.",
- "type": "string"
+ "$ref": "../../../../type/entityReference.json",
+ "format": "autoComplete",
+ "autoCompleteType": "database_service_search_index"
  },
  "databases": {
  "title": "Databases",
  "description": "List of database names from the Service to get descriptions from.",
  "type": "array",
  "items": {
- "type": "string"
+ "placeholder": "Search Databases",
+ "$ref": "../../../../type/entityReference.json",
+ "format": "autoComplete",
+ "autoCompleteType": "database_search_index"
  }
  }
  },
  "additionalProperties": false,
- "required": ["serviceName", "databases"]
+ "required": ["service", "databases"]
  }
  },
  "properties": {

openmetadata-spec/src/main/resources/json/schema/metadataIngestion/application.json

@@ -3,6 +3,7 @@
  "$schema": "http://json-schema.org/draft-07/schema#",
  "title": "OpenMetadataApplicationConfig",
  "description": "OpenMetadata Ingestion Framework definition for Applications, i.e., the YAML shape we require.",
+ "javaType": "org.openmetadata.schema.metadataIngestion.OpenMetadataAppConfig",
  "type": "object",
  "properties": {
  "workflowConfig": {

openmetadata-ui/src/main/resources/ui/src/utils/ApplicationSchemas/MetaPilotApplication.json

@@ -17,22 +17,31 @@
  "description": "Choose the service and its databases you want to generate descriptions from.",
  "type": "object",
  "properties": {
- "serviceName": {
+ "service": {
  "title": "Service Name",
+ "placeholder": "Search Service",
  "description": "Service Name to get descriptions from.",
- "type": "string"
+ "type": "string",
+ "format": "autoComplete",
+ "autoCompleteType": "database_service_search_index",
+ "additionalProperties": true
  },
  "databases": {
  "title": "Databases",
  "description": "List of database names from the Service to get descriptions from.",
  "type": "array",
+ "additionalProperties": true,
  "items": {
- "type": "string"
+ "placeholder": "Search Databases",
+ "type": "string",
+ "format": "autoComplete",
+ "autoCompleteType": "database_search_index",
+ "additionalProperties": true
  }
  }
  },
  "additionalProperties": false,
- "required": ["serviceName", "databases"]
+ "required": ["service", "databases"]
  }
  },
  "properties": {

openmetadata-ui/src/main/resources/ui/src/utils/i18next/i18nextUtil.ts

@@ -18,10 +18,10 @@ import enUS from '../../locale/languages/en-us.json';
 import esES from '../../locale/languages/es-es.json';
 import frFR from '../../locale/languages/fr-fr.json';
 import jaJP from '../../locale/languages/ja-jp.json';
+import nlNL from '../../locale/languages/nl-nl.json';
 import ptBR from '../../locale/languages/pt-br.json';
 import ruRU from '../../locale/languages/ru-ru.json';
 import zhCN from '../../locale/languages/zh-cn.json';
-import nlNL from '../../locale/languages/nl-nl.json';
 
 export enum SupportedLocales {
  English = 'en-US',