diff --git a/ingestion/src/metadata/utils/constants.py b/ingestion/src/metadata/utils/constants.py index 721e3c9a40c0..8b3c8d4fa03d 100644 --- a/ingestion/src/metadata/utils/constants.py +++ b/ingestion/src/metadata/utils/constants.py @@ -31,6 +31,13 @@ "verifyCerts": "verify_certs", } +ES_SOURCE_IGNORE_KEYS = { + "searchIndexMappingLanguage", + "batchSize", + "recreateIndex", + "type", +} + QUERY_WITH_OM_VERSION = '/* {"app": "OpenMetadata"' QUERY_WITH_DBT = '/* {"app": "dbt"' diff --git a/openmetadata-airflow-apis/openmetadata_managed_apis/workflows/ingestion/elasticsearch_sink.py b/openmetadata-airflow-apis/openmetadata_managed_apis/workflows/ingestion/elasticsearch_sink.py index 9acaa4ae84e9..2586235a0166 100644 --- a/openmetadata-airflow-apis/openmetadata_managed_apis/workflows/ingestion/elasticsearch_sink.py +++ b/openmetadata-airflow-apis/openmetadata_managed_apis/workflows/ingestion/elasticsearch_sink.py @@ -19,7 +19,7 @@ ) from metadata.generated.schema.metadataIngestion.workflow import Sink from metadata.generated.schema.type.basic import ComponentConfig -from metadata.utils.constants import ES_SOURCE_TO_ES_OBJ_ARGS +from metadata.utils.constants import ES_SOURCE_IGNORE_KEYS, ES_SOURCE_TO_ES_OBJ_ARGS def build_elasticsearch_sink( @@ -42,7 +42,7 @@ def build_elasticsearch_sink( elasticsearch_source_config_dict = { ES_SOURCE_TO_ES_OBJ_ARGS[key]: value for key, value in ingestion_pipeline.sourceConfig.config.dict().items() - if value and key != "type" + if value and key not in ES_SOURCE_IGNORE_KEYS } return Sink(