es sink fixes for ingestion

open-metadata · Jun 24, 2023 · c1df6f9 · c1df6f9
1 parent b7804be
commit c1df6f9
Show file tree

Hide file tree

Showing 2 changed files with 9 additions and 2 deletions.
diff --git a/ingestion/src/metadata/utils/constants.py b/ingestion/src/metadata/utils/constants.py
@@ -31,6 +31,13 @@
  "verifyCerts": "verify_certs",
 }
 
+ES_SOURCE_IGNORE_KEYS = {
+ "searchIndexMappingLanguage",
+ "batchSize",
+ "recreateIndex",
+ "type",
+}
+
 QUERY_WITH_OM_VERSION = '/* {"app": "OpenMetadata"'
 
 QUERY_WITH_DBT = '/* {"app": "dbt"'

diff --git a/...metadata-airflow-apis/openmetadata_managed_apis/workflows/ingestion/elasticsearch_sink.py b/...metadata-airflow-apis/openmetadata_managed_apis/workflows/ingestion/elasticsearch_sink.py
@@ -19,7 +19,7 @@
 )
 from metadata.generated.schema.metadataIngestion.workflow import Sink
 from metadata.generated.schema.type.basic import ComponentConfig
-from metadata.utils.constants import ES_SOURCE_TO_ES_OBJ_ARGS
+from metadata.utils.constants import ES_SOURCE_IGNORE_KEYS, ES_SOURCE_TO_ES_OBJ_ARGS
 
 
 def build_elasticsearch_sink(
@@ -42,7 +42,7 @@ def build_elasticsearch_sink(
  elasticsearch_source_config_dict = {
  ES_SOURCE_TO_ES_OBJ_ARGS[key]: value
  for key, value in ingestion_pipeline.sourceConfig.config.dict().items()
- if value and key != "type"
+ if value and key not in ES_SOURCE_IGNORE_KEYS
  }
 
  return Sink(