Skip to content

Commit

Permalink
fix(UI): reverted the ES config step for 'Data Insights Pipeline' (#1…
Browse files Browse the repository at this point in the history
…2121)

* Reverted the ES config step for Data Insights pipeline

* es sink fixes for ingestion

---------

Co-authored-by: Chirag Madlani <12962843+chirag-madlani@users.noreply.github.com>
Co-authored-by: ulixius9 <mayursingal9@gmail.com>
  • Loading branch information
3 people authored Jun 24, 2023
1 parent 9fcaaaf commit 67ffa1c
Show file tree
Hide file tree
Showing 4 changed files with 169 additions and 11 deletions.
7 changes: 7 additions & 0 deletions ingestion/src/metadata/utils/constants.py
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,13 @@
"verifyCerts": "verify_certs",
}

ES_SOURCE_IGNORE_KEYS = {
"searchIndexMappingLanguage",
"batchSize",
"recreateIndex",
"type",
}

QUERY_WITH_OM_VERSION = '/* {"app": "OpenMetadata"'

QUERY_WITH_DBT = '/* {"app": "dbt"'
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@
)
from metadata.generated.schema.metadataIngestion.workflow import Sink
from metadata.generated.schema.type.basic import ComponentConfig
from metadata.utils.constants import ES_SOURCE_TO_ES_OBJ_ARGS
from metadata.utils.constants import ES_SOURCE_IGNORE_KEYS, ES_SOURCE_TO_ES_OBJ_ARGS


def build_elasticsearch_sink(
Expand All @@ -42,7 +42,7 @@ def build_elasticsearch_sink(
elasticsearch_source_config_dict = {
ES_SOURCE_TO_ES_OBJ_ARGS[key]: value
for key, value in ingestion_pipeline.sourceConfig.config.dict().items()
if value and key != "type"
if value and key not in ES_SOURCE_IGNORE_KEYS
}

return Sink(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,7 @@ import {
ModifiedDbtConfig,
} from './addIngestion.interface';
import ConfigureIngestion from './Steps/ConfigureIngestion';
import DataInsightMetadataToESConfigForm from './Steps/DataInsightMetadataToESConfigForm/DataInsightMetadataToESConfigForm';
import MetadataToESConfigForm from './Steps/MetadataToESConfigForm/MetadataToESConfigForm';
import ScheduleInterval from './Steps/ScheduleInterval';

Expand Down Expand Up @@ -242,6 +243,9 @@ const AddIngestion = ({
useAwsCredentials: Boolean(sourceConfig?.useAwsCredentials),
useSSL: Boolean(sourceConfig?.useSSL),
verifyCerts: Boolean(sourceConfig?.verifyCerts),
batchSize: sourceConfig?.batchSize,
searchIndexMappingLanguage: sourceConfig?.searchIndexMappingLanguage,
recreateIndex: sourceConfig?.recreateIndex,
},
dbtUpdateDescriptions: sourceConfig?.dbtUpdateDescriptions ?? false,
confidence: sourceConfig?.confidence,
Expand Down Expand Up @@ -774,15 +778,27 @@ const AddIngestion = ({
/>
)}

{activeIngestionStep === 3 && isServiceTypeOpenMetadata && (
<MetadataToESConfigForm
data={state}
handleMetadataToESConfig={handleMetadataToESConfig}
handleNext={handleNext}
handlePrev={handlePrev}
onFocus={onFocus}
/>
)}
{activeIngestionStep === 3 &&
pipelineType === PipelineType.ElasticSearchReindex && (
<MetadataToESConfigForm
data={state}
handleMetadataToESConfig={handleMetadataToESConfig}
handleNext={handleNext}
handlePrev={handlePrev}
onFocus={onFocus}
/>
)}

{activeIngestionStep === 3 &&
pipelineType === PipelineType.DataInsight && (
<DataInsightMetadataToESConfigForm
data={state}
handleMetadataToESConfig={handleMetadataToESConfig}
handleNext={handleNext}
handlePrev={handlePrev}
onFocus={onFocus}
/>
)}

{activeIngestionStep === 4 && (
<ScheduleInterval
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,135 @@
/*
* Copyright 2022 Collate.
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* http://www.apache.org/licenses/LICENSE-2.0
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

import { Button, Col, Divider, Form, Input, Row, Switch } from 'antd';
import { AddIngestionState } from 'components/AddIngestion/addIngestion.interface';
import React, { useMemo } from 'react';
import { useTranslation } from 'react-i18next';
import { ConfigClass } from '../../../../generated/entity/services/ingestionPipelines/ingestionPipeline';
import '../MetadataToESConfigForm/MetadataToESConfigForm.less';

interface Props {
data: AddIngestionState;
handleMetadataToESConfig: (data: ConfigClass) => void;
handlePrev: () => void;
handleNext: () => void;
onFocus: (fieldId: string) => void;
}

const { Item } = Form;

const DataInsightMetadataToESConfigForm = ({
handleMetadataToESConfig,
handlePrev,
handleNext,
onFocus,
data,
}: Props) => {
const { t } = useTranslation();
const [form] = Form.useForm();

const handleSubmit = (values: ConfigClass) => {
handleMetadataToESConfig({
...values,
});
handleNext();
};

const initialValues = useMemo(
() => ({
caCerts: data.metadataToESConfig?.caCerts,
regionName: data.metadataToESConfig?.regionName,
timeout: data.metadataToESConfig?.timeout,
useAwsCredentials: data.metadataToESConfig?.useAwsCredentials,
useSSL: data.metadataToESConfig?.useSSL,
verifyCerts: data.metadataToESConfig?.verifyCerts,
}),
[data]
);

return (
<Form
className="metadata-to-es-config-form"
form={form}
initialValues={initialValues}
layout="vertical"
onFinish={handleSubmit}
onFocus={(e) => onFocus(e.target.id)}>
<Item label={t('label.ca-certs')} name="caCerts">
<Input id="root/caCerts" />
</Item>
<Item label={t('label.region-name')} name="regionName">
<Input id="root/regionName" />
</Item>
<Item label={t('label.timeout')} name="timeout">
<Input id="root/timeout" type="number" />
</Item>
<Divider />
<Item name="useAwsCredentials">
<Row>
<Col span={8}>{t('label.use-aws-credential-plural')}</Col>
<Col span={16}>
<Switch
defaultChecked={initialValues.useAwsCredentials}
id="root/useAwsCredentials"
onChange={(value) =>
form.setFieldsValue({ useAwsCredentials: value })
}
/>
</Col>
</Row>
</Item>
<Divider />
<Item name="useSSL">
<Row>
<Col span={8}>{t('label.use-ssl-uppercase')}</Col>
<Col span={16}>
<Switch
defaultChecked={initialValues.useSSL}
id="root/useSSL"
onChange={(value) => form.setFieldsValue({ useSSL: value })}
/>
</Col>
</Row>
</Item>
<Divider />
<Item name="verifyCerts">
<Row>
<Col span={8}>{t('label.verify-cert-plural')}</Col>
<Col span={16}>
<Switch
defaultChecked={initialValues.verifyCerts}
id="root/verifyCerts"
onChange={(value) => form.setFieldsValue({ verifyCerts: value })}
/>
</Col>
</Row>
</Item>
<Divider />
<Row justify="end">
<Col>
<Button type="link" onClick={handlePrev}>
{t('label.back')}
</Button>
</Col>
<Col>
<Button htmlType="submit" type="primary">
{t('label.next')}
</Button>
</Col>
</Row>
</Form>
);
};

export default DataInsightMetadataToESConfigForm;

0 comments on commit 67ffa1c

Please sign in to comment.