diff --git a/ingestion/src/metadata/utils/constants.py b/ingestion/src/metadata/utils/constants.py index 721e3c9a40c0..8b3c8d4fa03d 100644 --- a/ingestion/src/metadata/utils/constants.py +++ b/ingestion/src/metadata/utils/constants.py @@ -31,6 +31,13 @@ "verifyCerts": "verify_certs", } +ES_SOURCE_IGNORE_KEYS = { + "searchIndexMappingLanguage", + "batchSize", + "recreateIndex", + "type", +} + QUERY_WITH_OM_VERSION = '/* {"app": "OpenMetadata"' QUERY_WITH_DBT = '/* {"app": "dbt"' diff --git a/openmetadata-airflow-apis/openmetadata_managed_apis/workflows/ingestion/elasticsearch_sink.py b/openmetadata-airflow-apis/openmetadata_managed_apis/workflows/ingestion/elasticsearch_sink.py index 9acaa4ae84e9..2586235a0166 100644 --- a/openmetadata-airflow-apis/openmetadata_managed_apis/workflows/ingestion/elasticsearch_sink.py +++ b/openmetadata-airflow-apis/openmetadata_managed_apis/workflows/ingestion/elasticsearch_sink.py @@ -19,7 +19,7 @@ ) from metadata.generated.schema.metadataIngestion.workflow import Sink from metadata.generated.schema.type.basic import ComponentConfig -from metadata.utils.constants import ES_SOURCE_TO_ES_OBJ_ARGS +from metadata.utils.constants import ES_SOURCE_IGNORE_KEYS, ES_SOURCE_TO_ES_OBJ_ARGS def build_elasticsearch_sink( @@ -42,7 +42,7 @@ def build_elasticsearch_sink( elasticsearch_source_config_dict = { ES_SOURCE_TO_ES_OBJ_ARGS[key]: value for key, value in ingestion_pipeline.sourceConfig.config.dict().items() - if value and key != "type" + if value and key not in ES_SOURCE_IGNORE_KEYS } return Sink( diff --git a/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/AddIngestion.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/AddIngestion.component.tsx index 5102e3d58c94..135b14df2328 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/AddIngestion.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/AddIngestion.component.tsx @@ -67,6 +67,7 @@ import { ModifiedDbtConfig, } from './addIngestion.interface'; import ConfigureIngestion from './Steps/ConfigureIngestion'; +import DataInsightMetadataToESConfigForm from './Steps/DataInsightMetadataToESConfigForm/DataInsightMetadataToESConfigForm'; import MetadataToESConfigForm from './Steps/MetadataToESConfigForm/MetadataToESConfigForm'; import ScheduleInterval from './Steps/ScheduleInterval'; @@ -242,6 +243,9 @@ const AddIngestion = ({ useAwsCredentials: Boolean(sourceConfig?.useAwsCredentials), useSSL: Boolean(sourceConfig?.useSSL), verifyCerts: Boolean(sourceConfig?.verifyCerts), + batchSize: sourceConfig?.batchSize, + searchIndexMappingLanguage: sourceConfig?.searchIndexMappingLanguage, + recreateIndex: sourceConfig?.recreateIndex, }, dbtUpdateDescriptions: sourceConfig?.dbtUpdateDescriptions ?? false, confidence: sourceConfig?.confidence, @@ -774,15 +778,27 @@ const AddIngestion = ({ /> )} - {activeIngestionStep === 3 && isServiceTypeOpenMetadata && ( - - )} + {activeIngestionStep === 3 && + pipelineType === PipelineType.ElasticSearchReindex && ( + + )} + + {activeIngestionStep === 3 && + pipelineType === PipelineType.DataInsight && ( + + )} {activeIngestionStep === 4 && ( void; + handlePrev: () => void; + handleNext: () => void; + onFocus: (fieldId: string) => void; +} + +const { Item } = Form; + +const DataInsightMetadataToESConfigForm = ({ + handleMetadataToESConfig, + handlePrev, + handleNext, + onFocus, + data, +}: Props) => { + const { t } = useTranslation(); + const [form] = Form.useForm(); + + const handleSubmit = (values: ConfigClass) => { + handleMetadataToESConfig({ + ...values, + }); + handleNext(); + }; + + const initialValues = useMemo( + () => ({ + caCerts: data.metadataToESConfig?.caCerts, + regionName: data.metadataToESConfig?.regionName, + timeout: data.metadataToESConfig?.timeout, + useAwsCredentials: data.metadataToESConfig?.useAwsCredentials, + useSSL: data.metadataToESConfig?.useSSL, + verifyCerts: data.metadataToESConfig?.verifyCerts, + }), + [data] + ); + + return ( +
onFocus(e.target.id)}> + + + + + + + + + + + + + {t('label.use-aws-credential-plural')} + + + form.setFieldsValue({ useAwsCredentials: value }) + } + /> + + + + + + + {t('label.use-ssl-uppercase')} + + form.setFieldsValue({ useSSL: value })} + /> + + + + + + + {t('label.verify-cert-plural')} + + form.setFieldsValue({ verifyCerts: value })} + /> + + + + + + + + + + + + + + ); +}; + +export default DataInsightMetadataToESConfigForm;