From b6752cd801286d4f4fd3624e0a97bf51b5478c3e Mon Sep 17 00:00:00 2001
From: Aniket Katkar <aniketkatkar97@gmail.com>
Date: Fri, 23 Jun 2023 16:24:09 +0530
Subject: [PATCH 1/2] Reverted the ES config step for Data Insights pipeline

---
 .../AddIngestion/AddIngestion.component.tsx   |  34 +++--
 .../DataInsightMetadataToESConfigForm.tsx     | 135 ++++++++++++++++++
 2 files changed, 160 insertions(+), 9 deletions(-)
 create mode 100644 openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/Steps/DataInsightMetadataToESConfigForm/DataInsightMetadataToESConfigForm.tsx

diff --git a/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/AddIngestion.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/AddIngestion.component.tsx
index 5102e3d58c94..135b14df2328 100644
--- a/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/AddIngestion.component.tsx
+++ b/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/AddIngestion.component.tsx
@@ -67,6 +67,7 @@ import {
   ModifiedDbtConfig,
 } from './addIngestion.interface';
 import ConfigureIngestion from './Steps/ConfigureIngestion';
+import DataInsightMetadataToESConfigForm from './Steps/DataInsightMetadataToESConfigForm/DataInsightMetadataToESConfigForm';
 import MetadataToESConfigForm from './Steps/MetadataToESConfigForm/MetadataToESConfigForm';
 import ScheduleInterval from './Steps/ScheduleInterval';
 
@@ -242,6 +243,9 @@ const AddIngestion = ({
         useAwsCredentials: Boolean(sourceConfig?.useAwsCredentials),
         useSSL: Boolean(sourceConfig?.useSSL),
         verifyCerts: Boolean(sourceConfig?.verifyCerts),
+        batchSize: sourceConfig?.batchSize,
+        searchIndexMappingLanguage: sourceConfig?.searchIndexMappingLanguage,
+        recreateIndex: sourceConfig?.recreateIndex,
       },
       dbtUpdateDescriptions: sourceConfig?.dbtUpdateDescriptions ?? false,
       confidence: sourceConfig?.confidence,
@@ -774,15 +778,27 @@ const AddIngestion = ({
           />
         )}
 
-        {activeIngestionStep === 3 && isServiceTypeOpenMetadata && (
-          <MetadataToESConfigForm
-            data={state}
-            handleMetadataToESConfig={handleMetadataToESConfig}
-            handleNext={handleNext}
-            handlePrev={handlePrev}
-            onFocus={onFocus}
-          />
-        )}
+        {activeIngestionStep === 3 &&
+          pipelineType === PipelineType.ElasticSearchReindex && (
+            <MetadataToESConfigForm
+              data={state}
+              handleMetadataToESConfig={handleMetadataToESConfig}
+              handleNext={handleNext}
+              handlePrev={handlePrev}
+              onFocus={onFocus}
+            />
+          )}
+
+        {activeIngestionStep === 3 &&
+          pipelineType === PipelineType.DataInsight && (
+            <DataInsightMetadataToESConfigForm
+              data={state}
+              handleMetadataToESConfig={handleMetadataToESConfig}
+              handleNext={handleNext}
+              handlePrev={handlePrev}
+              onFocus={onFocus}
+            />
+          )}
 
         {activeIngestionStep === 4 && (
           <ScheduleInterval
diff --git a/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/Steps/DataInsightMetadataToESConfigForm/DataInsightMetadataToESConfigForm.tsx b/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/Steps/DataInsightMetadataToESConfigForm/DataInsightMetadataToESConfigForm.tsx
new file mode 100644
index 000000000000..389b0f1e8f13
--- /dev/null
+++ b/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/Steps/DataInsightMetadataToESConfigForm/DataInsightMetadataToESConfigForm.tsx
@@ -0,0 +1,135 @@
+/*
+ *  Copyright 2022 Collate.
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *  http://www.apache.org/licenses/LICENSE-2.0
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+import { Button, Col, Divider, Form, Input, Row, Switch } from 'antd';
+import { AddIngestionState } from 'components/AddIngestion/addIngestion.interface';
+import React, { useMemo } from 'react';
+import { useTranslation } from 'react-i18next';
+import { ConfigClass } from '../../../../generated/entity/services/ingestionPipelines/ingestionPipeline';
+import '../MetadataToESConfigForm/MetadataToESConfigForm.less';
+
+interface Props {
+  data: AddIngestionState;
+  handleMetadataToESConfig: (data: ConfigClass) => void;
+  handlePrev: () => void;
+  handleNext: () => void;
+  onFocus: (fieldId: string) => void;
+}
+
+const { Item } = Form;
+
+const DataInsightMetadataToESConfigForm = ({
+  handleMetadataToESConfig,
+  handlePrev,
+  handleNext,
+  onFocus,
+  data,
+}: Props) => {
+  const { t } = useTranslation();
+  const [form] = Form.useForm();
+
+  const handleSubmit = (values: ConfigClass) => {
+    handleMetadataToESConfig({
+      ...values,
+    });
+    handleNext();
+  };
+
+  const initialValues = useMemo(
+    () => ({
+      caCerts: data.metadataToESConfig?.caCerts,
+      regionName: data.metadataToESConfig?.regionName,
+      timeout: data.metadataToESConfig?.timeout,
+      useAwsCredentials: data.metadataToESConfig?.useAwsCredentials,
+      useSSL: data.metadataToESConfig?.useSSL,
+      verifyCerts: data.metadataToESConfig?.verifyCerts,
+    }),
+    [data]
+  );
+
+  return (
+    <Form
+      className="metadata-to-es-config-form"
+      form={form}
+      initialValues={initialValues}
+      layout="vertical"
+      onFinish={handleSubmit}
+      onFocus={(e) => onFocus(e.target.id)}>
+      <Item label={t('label.ca-certs')} name="caCerts">
+        <Input id="root/caCerts" />
+      </Item>
+      <Item label={t('label.region-name')} name="regionName">
+        <Input id="root/regionName" />
+      </Item>
+      <Item label={t('label.timeout')} name="timeout">
+        <Input id="root/timeout" type="number" />
+      </Item>
+      <Divider />
+      <Item name="useAwsCredentials">
+        <Row>
+          <Col span={8}>{t('label.use-aws-credential-plural')}</Col>
+          <Col span={16}>
+            <Switch
+              defaultChecked={initialValues.useAwsCredentials}
+              id="root/useAwsCredentials"
+              onChange={(value) =>
+                form.setFieldsValue({ useAwsCredentials: value })
+              }
+            />
+          </Col>
+        </Row>
+      </Item>
+      <Divider />
+      <Item name="useSSL">
+        <Row>
+          <Col span={8}>{t('label.use-ssl-uppercase')}</Col>
+          <Col span={16}>
+            <Switch
+              defaultChecked={initialValues.useSSL}
+              id="root/useSSL"
+              onChange={(value) => form.setFieldsValue({ useSSL: value })}
+            />
+          </Col>
+        </Row>
+      </Item>
+      <Divider />
+      <Item name="verifyCerts">
+        <Row>
+          <Col span={8}>{t('label.verify-cert-plural')}</Col>
+          <Col span={16}>
+            <Switch
+              defaultChecked={initialValues.verifyCerts}
+              id="root/verifyCerts"
+              onChange={(value) => form.setFieldsValue({ verifyCerts: value })}
+            />
+          </Col>
+        </Row>
+      </Item>
+      <Divider />
+      <Row justify="end">
+        <Col>
+          <Button type="link" onClick={handlePrev}>
+            {t('label.back')}
+          </Button>
+        </Col>
+        <Col>
+          <Button htmlType="submit" type="primary">
+            {t('label.next')}
+          </Button>
+        </Col>
+      </Row>
+    </Form>
+  );
+};
+
+export default DataInsightMetadataToESConfigForm;

From c1df6f9fc3dfdde0a9751cff6f9242423e42820d Mon Sep 17 00:00:00 2001
From: ulixius9 <mayursingal9@gmail.com>
Date: Sat, 24 Jun 2023 15:24:18 +0530
Subject: [PATCH 2/2] es sink fixes for ingestion

---
 ingestion/src/metadata/utils/constants.py                  | 7 +++++++
 .../workflows/ingestion/elasticsearch_sink.py              | 4 ++--
 2 files changed, 9 insertions(+), 2 deletions(-)

diff --git a/ingestion/src/metadata/utils/constants.py b/ingestion/src/metadata/utils/constants.py
index 721e3c9a40c0..8b3c8d4fa03d 100644
--- a/ingestion/src/metadata/utils/constants.py
+++ b/ingestion/src/metadata/utils/constants.py
@@ -31,6 +31,13 @@
     "verifyCerts": "verify_certs",
 }
 
+ES_SOURCE_IGNORE_KEYS = {
+    "searchIndexMappingLanguage",
+    "batchSize",
+    "recreateIndex",
+    "type",
+}
+
 QUERY_WITH_OM_VERSION = '/* {"app": "OpenMetadata"'
 
 QUERY_WITH_DBT = '/* {"app": "dbt"'
diff --git a/openmetadata-airflow-apis/openmetadata_managed_apis/workflows/ingestion/elasticsearch_sink.py b/openmetadata-airflow-apis/openmetadata_managed_apis/workflows/ingestion/elasticsearch_sink.py
index 9acaa4ae84e9..2586235a0166 100644
--- a/openmetadata-airflow-apis/openmetadata_managed_apis/workflows/ingestion/elasticsearch_sink.py
+++ b/openmetadata-airflow-apis/openmetadata_managed_apis/workflows/ingestion/elasticsearch_sink.py
@@ -19,7 +19,7 @@
 )
 from metadata.generated.schema.metadataIngestion.workflow import Sink
 from metadata.generated.schema.type.basic import ComponentConfig
-from metadata.utils.constants import ES_SOURCE_TO_ES_OBJ_ARGS
+from metadata.utils.constants import ES_SOURCE_IGNORE_KEYS, ES_SOURCE_TO_ES_OBJ_ARGS
 
 
 def build_elasticsearch_sink(
@@ -42,7 +42,7 @@ def build_elasticsearch_sink(
     elasticsearch_source_config_dict = {
         ES_SOURCE_TO_ES_OBJ_ARGS[key]: value
         for key, value in ingestion_pipeline.sourceConfig.config.dict().items()
-        if value and key != "type"
+        if value and key not in ES_SOURCE_IGNORE_KEYS
     }
 
     return Sink(