From 654a5be2a49808c355161e956e54eccf480775d1 Mon Sep 17 00:00:00 2001 From: karanh37 Date: Mon, 20 Nov 2023 12:44:56 +0530 Subject: [PATCH 01/15] fix: allow config update for external app --- .../AppDetails/AppDetails.component.tsx | 5 +---- .../src/pages/AppInstall/AppInstall.component.tsx | 13 ++----------- 2 files changed, 3 insertions(+), 15 deletions(-) diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Applications/AppDetails/AppDetails.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Applications/AppDetails/AppDetails.component.tsx index 5b13f11e287d..7637b7d18647 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Applications/AppDetails/AppDetails.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Applications/AppDetails/AppDetails.component.tsx @@ -51,7 +51,6 @@ import { import { ServiceCategory } from '../../../enums/service.enum'; import { App, - AppType, ScheduleTimeline, } from '../../../generated/entity/applications/app'; import { Include } from '../../../generated/type/include'; @@ -310,9 +309,7 @@ const AppDetails = () => { const tabs = useMemo(() => { const tabConfiguration = - appData?.appConfiguration && - appData.appType === AppType.Internal && - jsonSchema + appData?.appConfiguration && jsonSchema ? [ { label: ( diff --git a/openmetadata-ui/src/main/resources/ui/src/pages/AppInstall/AppInstall.component.tsx b/openmetadata-ui/src/main/resources/ui/src/pages/AppInstall/AppInstall.component.tsx index 7cd96933ce59..a7058f3cdc49 100644 --- a/openmetadata-ui/src/main/resources/ui/src/pages/AppInstall/AppInstall.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/pages/AppInstall/AppInstall.component.tsx @@ -67,15 +67,6 @@ const AppInstall = () => { [appData] ); - const stepperList = useMemo( - () => - isExternalApp - ? STEPS_FOR_APP_INSTALL.filter((item) => item.step !== 2) - : STEPS_FOR_APP_INSTALL, - - [isExternalApp] - ); - const fetchAppDetails = useCallback(async () => { setIsLoading(true); try { @@ -146,7 +137,7 @@ const AppInstall = () => { isExternalApp ? t('label.schedule') : t('label.configure') } onCancel={onCancel} - onSave={() => setActiveServiceStep(isExternalApp ? 3 : 2)} + onSave={() => setActiveServiceStep(2)} /> ); @@ -211,7 +202,7 @@ const AppInstall = () => { From 220981689345a3bda2041d7c30f24047ab020411 Mon Sep 17 00:00:00 2001 From: Pere Miquel Brull Date: Mon, 20 Nov 2023 10:03:38 +0100 Subject: [PATCH 02/15] Prep config --- ...cation.json => AutoTaggerApplication.json} | 0 .../DataInsightsApplication.json | 13 +--- .../MetaPilotApplication.json | 69 +++++++++++++++++++ 3 files changed, 70 insertions(+), 12 deletions(-) rename openmetadata-ui/src/main/resources/ui/src/utils/ApplicationSchemas/{AutoPIIApplication.json => AutoTaggerApplication.json} (100%) create mode 100644 openmetadata-ui/src/main/resources/ui/src/utils/ApplicationSchemas/MetaPilotApplication.json diff --git a/openmetadata-ui/src/main/resources/ui/src/utils/ApplicationSchemas/AutoPIIApplication.json b/openmetadata-ui/src/main/resources/ui/src/utils/ApplicationSchemas/AutoTaggerApplication.json similarity index 100% rename from openmetadata-ui/src/main/resources/ui/src/utils/ApplicationSchemas/AutoPIIApplication.json rename to openmetadata-ui/src/main/resources/ui/src/utils/ApplicationSchemas/AutoTaggerApplication.json diff --git a/openmetadata-ui/src/main/resources/ui/src/utils/ApplicationSchemas/DataInsightsApplication.json b/openmetadata-ui/src/main/resources/ui/src/utils/ApplicationSchemas/DataInsightsApplication.json index 420032ad9912..338b7124ce27 100644 --- a/openmetadata-ui/src/main/resources/ui/src/utils/ApplicationSchemas/DataInsightsApplication.json +++ b/openmetadata-ui/src/main/resources/ui/src/utils/ApplicationSchemas/DataInsightsApplication.json @@ -3,17 +3,6 @@ "$schema": "http://json-schema.org/draft-07/schema#", "title": "Data Insights Application", "description": "This schema defines configuration for Search Reindexing Application.", - "properties": { - "sendToAdmins": { - "title": "Send To Admins", - "type": "boolean", - "default": true - }, - "sendToTeams": { - "title": "Send To Teams", - "type": "boolean", - "default": true - } - }, + "properties": {}, "additionalProperties": false } diff --git a/openmetadata-ui/src/main/resources/ui/src/utils/ApplicationSchemas/MetaPilotApplication.json b/openmetadata-ui/src/main/resources/ui/src/utils/ApplicationSchemas/MetaPilotApplication.json new file mode 100644 index 000000000000..ca6edd2023e3 --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/utils/ApplicationSchemas/MetaPilotApplication.json @@ -0,0 +1,69 @@ +{ + "$id": "https://open-metadata.org/schema/entity/applications/configuration/external/metaPilotAppConfig.json", + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "MetaPilotAppConfig", + "description": "Configuration for the MetaPilot External Application.", + "type": "object", + "javaType": "org.openmetadata.schema.entity.app.external.MetaPilotAppConfig", + "definitions": { + "metaPilotApp": { + "description": "Application type.", + "type": "string", + "enum": ["MetaPilot"], + "default": "MetaPilot" + }, + "serviceDatabases": { + "title": "Service Databases", + "description": "Choose the service and its databases you want to generate descriptions from.", + "type": "object", + "properties": { + "serviceName": { + "title": "Service Name", + "description": "Service Name to get descriptions from.", + "type": "string" + }, + "databases": { + "title": "Databases", + "description": "List of database names from the Service to get descriptions from.", + "type": "array", + "items": { + "type": "string" + } + } + }, + "additionalProperties": false, + "required": ["serviceName", "databases"] + } + }, + "properties": { + "type": { + "title": "Service Type", + "description": "Service Type", + "$ref": "#/definitions/metaPilotApp", + "default": "MetaPilot" + }, + "waiiInstance": { + "title": "WAII Instance", + "description": "WAII API host URL", + "type": "string", + "format": "URI", + "default": "https://tweakit.waii.ai/api/" + }, + "token": { + "title": "WAII API Token", + "description": "WAII API Token", + "type": "string", + "format": "password" + }, + "serviceDatabases": { + "title": "Service Databases", + "description": "Services and Databases configured to get the descriptions from.", + "type": "array", + "items": { + "$ref": "#/definitions/serviceDatabases" + } + } + }, + "additionalProperties": false, + "required": ["token"] +} From 1ae4b57bade98bc02f0f23988b17d7ae73bf90e8 Mon Sep 17 00:00:00 2001 From: Pere Miquel Brull Date: Mon, 20 Nov 2023 15:12:10 +0100 Subject: [PATCH 03/15] Add configs --- .../AutoTaggerApplication.json | 34 +++++++++++++------ .../DataInsightsApplication.json | 2 +- 2 files changed, 24 insertions(+), 12 deletions(-) diff --git a/openmetadata-ui/src/main/resources/ui/src/utils/ApplicationSchemas/AutoTaggerApplication.json b/openmetadata-ui/src/main/resources/ui/src/utils/ApplicationSchemas/AutoTaggerApplication.json index 47bbe6a38f8a..64e015ce4961 100644 --- a/openmetadata-ui/src/main/resources/ui/src/utils/ApplicationSchemas/AutoTaggerApplication.json +++ b/openmetadata-ui/src/main/resources/ui/src/utils/ApplicationSchemas/AutoTaggerApplication.json @@ -1,19 +1,31 @@ { - "$id": "https://open-metadata.org/schema/entity/applications/configuration/externalApplicationConfig.json", + "$id": "https://open-metadata.org/schema/entity/applications/configuration/external/autoTaggerAppConfig.json", "$schema": "http://json-schema.org/draft-07/schema#", - "title": "ExternalApplicationConfig", - "description": "External Application Pipeline Configuration.", + "title": "AutoTaggerAppConfig", + "description": "Configuration for the Auto Tagger External Application.", "type": "object", + "javaType": "org.openmetadata.schema.entity.app.external.AutoTaggerAppConfig", + "definitions": { + "autoTaggerApp": { + "description": "Application type.", + "type": "string", + "enum": ["AutoTagger"], + "default": "AutoTagger" + } + }, "properties": { - "config": { - "$ref": "../../../type/basic.json#/definitions/componentConfig", - "description": "Application-specific configuration" + "type": { + "title": "Service Type", + "description": "Service Type", + "$ref": "#/definitions/autoTaggerApp", + "default": "AutoTagger" }, - "sourcePythonClass": { - "description": "Source Python Class Name to run the application", - "type": "string" + "confidenceLevel": { + "title": "Confidence Level", + "type": "integer", + "description": "Confidence level for the ML models to apply the PII tagging.", + "default": 80 } }, - "additionalProperties": false, - "required": ["sourcePythonClass"] + "additionalProperties": false } diff --git a/openmetadata-ui/src/main/resources/ui/src/utils/ApplicationSchemas/DataInsightsApplication.json b/openmetadata-ui/src/main/resources/ui/src/utils/ApplicationSchemas/DataInsightsApplication.json index 338b7124ce27..40b011c8144d 100644 --- a/openmetadata-ui/src/main/resources/ui/src/utils/ApplicationSchemas/DataInsightsApplication.json +++ b/openmetadata-ui/src/main/resources/ui/src/utils/ApplicationSchemas/DataInsightsApplication.json @@ -1,7 +1,7 @@ { "$id": "DataInsightsApplication.json", "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Data Insights Application", + "title": "Data Insights Application Config", "description": "This schema defines configuration for Search Reindexing Application.", "properties": {}, "additionalProperties": false From 33d6ce66e34080647c098f0faf38047917cd1c0c Mon Sep 17 00:00:00 2001 From: karanh37 Date: Tue, 21 Nov 2023 14:00:46 +0530 Subject: [PATCH 04/15] fix: call configure app on saving of configurations --- .../Applications/AppDetails/AppDetails.component.tsx | 3 +++ .../src/main/resources/ui/src/rest/applicationAPI.ts | 7 +++++++ 2 files changed, 10 insertions(+) diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Applications/AppDetails/AppDetails.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Applications/AppDetails/AppDetails.component.tsx index 7637b7d18647..758ec4103d71 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Applications/AppDetails/AppDetails.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Applications/AppDetails/AppDetails.component.tsx @@ -55,6 +55,7 @@ import { } from '../../../generated/entity/applications/app'; import { Include } from '../../../generated/type/include'; import { + configureApp, deployApp, getApplicationByName, patchApplication, @@ -242,6 +243,8 @@ const AppDetails = () => { try { const response = await patchApplication(appData.id, jsonPatch); + // call configure endpoint also to update configuration + await configureApp(appData.fullyQualifiedName ?? '', updatedFormData); setAppData(response); showSuccessToast( t('message.entity-saved-successfully', { diff --git a/openmetadata-ui/src/main/resources/ui/src/rest/applicationAPI.ts b/openmetadata-ui/src/main/resources/ui/src/rest/applicationAPI.ts index e818dea89353..67d673bce900 100644 --- a/openmetadata-ui/src/main/resources/ui/src/rest/applicationAPI.ts +++ b/openmetadata-ui/src/main/resources/ui/src/rest/applicationAPI.ts @@ -103,6 +103,13 @@ export const deployApp = (appName: string): Promise => { return APIClient.post(`${BASE_URL}/deploy/${appName}`); }; +export const configureApp = ( + appName: string, + data: Record +): Promise => { + return APIClient.post(`${BASE_URL}/configure/${appName}`, data); +}; + export const restoreApp = async (id: string) => { const response = await APIClient.put>( `${BASE_URL}/restore`, From d5d47e4a419770a996b9984e2d247c79b30c6f3f Mon Sep 17 00:00:00 2001 From: karanh37 Date: Tue, 21 Nov 2023 14:04:16 +0530 Subject: [PATCH 05/15] fix: add autocomplete widget for json schema forms --- .../JsonSchemaWidgets/AsyncSelectWidget.tsx | 63 +++++++++++++++++++ .../common/FormBuilder/FormBuilder.tsx | 2 + .../ui/src/constants/Applications.constant.ts | 16 +++++ .../pages/AppInstall/AppInstall.component.tsx | 6 +- 4 files changed, 86 insertions(+), 1 deletion(-) create mode 100644 openmetadata-ui/src/main/resources/ui/src/components/JsonSchemaWidgets/AsyncSelectWidget.tsx diff --git a/openmetadata-ui/src/main/resources/ui/src/components/JsonSchemaWidgets/AsyncSelectWidget.tsx b/openmetadata-ui/src/main/resources/ui/src/components/JsonSchemaWidgets/AsyncSelectWidget.tsx new file mode 100644 index 000000000000..9f2adf55994c --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/components/JsonSchemaWidgets/AsyncSelectWidget.tsx @@ -0,0 +1,63 @@ +/* + * Copyright 2023 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { WidgetProps } from '@rjsf/utils'; +import React, { useCallback } from 'react'; +import { PAGE_SIZE_MEDIUM } from '../../constants/constants'; +import { SearchIndex } from '../../enums/search.enum'; +import { searchQuery } from '../../rest/searchAPI'; +import { getEntityName } from '../../utils/EntityUtils'; +import { AsyncSelect } from '../AsyncSelect/AsyncSelect'; + +const AsyncSelectWidget = ({ + onFocus, + onBlur, + onChange, + schema, + ...rest +}: WidgetProps) => { + const type = rest?.uiSchema?.['ui:options']?.autoCompleteType as SearchIndex; + + const fetchEntities = useCallback(async (searchText: string) => { + try { + const res = await searchQuery({ + pageNumber: 1, + pageSize: PAGE_SIZE_MEDIUM, + searchIndex: type ?? SearchIndex.TABLE, + query: searchText, + }); + + return res.hits.hits.map((value) => ({ + label: getEntityName(value._source), + value: value._source.id, + })); + } catch (_) { + return []; + } + }, []); + + return ( + { + onChange(value); + }} + /> + ); +}; + +export default AsyncSelectWidget; diff --git a/openmetadata-ui/src/main/resources/ui/src/components/common/FormBuilder/FormBuilder.tsx b/openmetadata-ui/src/main/resources/ui/src/components/common/FormBuilder/FormBuilder.tsx index e9c4453729b2..27b6ed76e8a8 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/common/FormBuilder/FormBuilder.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/common/FormBuilder/FormBuilder.tsx @@ -29,6 +29,7 @@ import { ArrayFieldTemplate } from '../../JSONSchemaTemplate/ArrayFieldTemplate' import DescriptionFieldTemplate from '../../JSONSchemaTemplate/DescriptionFieldTemplate'; import { FieldErrorTemplate } from '../../JSONSchemaTemplate/FieldErrorTemplate/FieldErrorTemplate'; import { ObjectFieldTemplate } from '../../JSONSchemaTemplate/ObjectFieldTemplate'; +import AsyncSelectWidget from '../../JsonSchemaWidgets/AsyncSelectWidget'; import MultiSelectWidget from '../../JsonSchemaWidgets/MultiSelectWidget'; import PasswordWidget from '../../JsonSchemaWidgets/PasswordWidget'; import Loader from '../../Loader/Loader'; @@ -78,6 +79,7 @@ const FormBuilder: FunctionComponent = ({ const widgets = { PasswordWidget: PasswordWidget, + autoComplete: AsyncSelectWidget, ...(useSelectWidget && { SelectWidget: MultiSelectWidget }), }; diff --git a/openmetadata-ui/src/main/resources/ui/src/constants/Applications.constant.ts b/openmetadata-ui/src/main/resources/ui/src/constants/Applications.constant.ts index 136ca3f460bf..f42f01ee5fa8 100644 --- a/openmetadata-ui/src/main/resources/ui/src/constants/Applications.constant.ts +++ b/openmetadata-ui/src/main/resources/ui/src/constants/Applications.constant.ts @@ -12,6 +12,7 @@ */ import { t } from 'i18next'; import { StepperStepType } from 'Models'; +import { SearchIndex } from '../enums/search.enum'; export const STEPS_FOR_APP_INSTALL: Array = [ { @@ -21,3 +22,18 @@ export const STEPS_FOR_APP_INSTALL: Array = [ { name: t('label.configure'), step: 2 }, { name: t('label.schedule'), step: 3 }, ]; + +export const APPLICATION_UI_SCHEMA = { + databases: { + 'ui:widget': 'autoComplete', + 'ui:options': { + autoCompleteType: SearchIndex.DATABASE, + }, + }, + owner: { + 'ui:widget': 'autoComplete', + 'ui:options': { + autoCompleteType: SearchIndex.USER, + }, + }, +}; diff --git a/openmetadata-ui/src/main/resources/ui/src/pages/AppInstall/AppInstall.component.tsx b/openmetadata-ui/src/main/resources/ui/src/pages/AppInstall/AppInstall.component.tsx index a7058f3cdc49..11ec9ff081d9 100644 --- a/openmetadata-ui/src/main/resources/ui/src/pages/AppInstall/AppInstall.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/pages/AppInstall/AppInstall.component.tsx @@ -25,7 +25,10 @@ import FormBuilder from '../../components/common/FormBuilder/FormBuilder'; import IngestionStepper from '../../components/IngestionStepper/IngestionStepper.component'; import Loader from '../../components/Loader/Loader'; import PageLayoutV1 from '../../components/PageLayoutV1/PageLayoutV1'; -import { STEPS_FOR_APP_INSTALL } from '../../constants/Applications.constant'; +import { + APPLICATION_UI_SCHEMA, + STEPS_FOR_APP_INSTALL, +} from '../../constants/Applications.constant'; import { GlobalSettingOptions, GlobalSettingsMenuCategory, @@ -154,6 +157,7 @@ const AppInstall = () => { serviceCategory={ServiceCategory.DASHBOARD_SERVICES} serviceType="" showTestConnection={false} + uiSchema={APPLICATION_UI_SCHEMA} validator={validator} onCancel={() => setActiveServiceStep(1)} onSubmit={onSaveConfiguration} From c2f25477ab16545e60e025b976b0f51763a22e71 Mon Sep 17 00:00:00 2001 From: Pere Miquel Brull Date: Tue, 21 Nov 2023 14:48:07 +0100 Subject: [PATCH 06/15] ORganize configuration handling --- .../src/metadata/applications/auto_tagger.py | 3 +- .../src/metadata/workflow/application.py | 5 +- .../workflows/ingestion/application.py | 5 +- .../apps/AbstractNativeApplication.java | 82 +--------------- .../apps/ExternalApplicationHandler.java | 7 +- .../bundles/insights/DataInsightsApp.java | 95 +++++++++++++++++++ .../insights/DataInsightsReportApp.java | 2 +- .../service/resources/apps/AppResource.java | 3 +- .../data/app/DataInsightsApplication.json | 36 +------ .../DataInsightsApplication.json | 30 +----- .../json/schema/entity/applications/app.json | 8 +- .../schema/entity/applications/appConfig.json | 21 ---- ...tionConfig.json => applicationConfig.json} | 21 ++-- .../configuration/dataInsightsApp.json | 24 ----- .../external/autoTaggerAppConfig.json | 4 +- .../external/metaPilotAppConfig.json | 8 +- .../externalAppIngestionConfig.json | 55 ----------- .../internal/dataInsightsAppConfig.json | 9 ++ .../internal/dataInsightsReportAppConfig.json | 33 +++++++ .../searchIndexingAppConfig.json} | 16 +++- .../schema/entity/applications/jobStatus.json | 2 +- .../marketplace/appMarketPlaceDefinition.json | 8 +- .../schema/metadataIngestion/application.json | 2 +- .../applicationPipeline.json | 4 +- .../DataInsightsApplication.json | 2 +- 25 files changed, 203 insertions(+), 282 deletions(-) delete mode 100644 openmetadata-spec/src/main/resources/json/schema/entity/applications/appConfig.json rename openmetadata-spec/src/main/resources/json/schema/entity/applications/configuration/{externalApplicationConfig.json => applicationConfig.json} (54%) delete mode 100644 openmetadata-spec/src/main/resources/json/schema/entity/applications/configuration/dataInsightsApp.json delete mode 100644 openmetadata-spec/src/main/resources/json/schema/entity/applications/configuration/externalAppIngestionConfig.json create mode 100644 openmetadata-spec/src/main/resources/json/schema/entity/applications/configuration/internal/dataInsightsAppConfig.json create mode 100644 openmetadata-spec/src/main/resources/json/schema/entity/applications/configuration/internal/dataInsightsReportAppConfig.json rename openmetadata-spec/src/main/resources/json/schema/entity/applications/configuration/{searchIndexingApp.json => internal/searchIndexingAppConfig.json} (64%) diff --git a/ingestion/src/metadata/applications/auto_tagger.py b/ingestion/src/metadata/applications/auto_tagger.py index b9636f432d0c..c9b11428a895 100644 --- a/ingestion/src/metadata/applications/auto_tagger.py +++ b/ingestion/src/metadata/applications/auto_tagger.py @@ -46,7 +46,8 @@ class AutoTaggerApp(AppRunner): with a YAML file like: sourcePythonClass: metadata.applications.auto_tagger.AutoTaggerApp - config: + appConfig: + type: AutoTagger confidenceLevel: 80 workflowConfig: loggerLevel: INFO diff --git a/ingestion/src/metadata/workflow/application.py b/ingestion/src/metadata/workflow/application.py index 7a191792eebd..26747b868c90 100644 --- a/ingestion/src/metadata/workflow/application.py +++ b/ingestion/src/metadata/workflow/application.py @@ -14,10 +14,9 @@ from abc import ABC, abstractmethod from typing import List, Optional +from metadata.generated.schema.entity.applications.configuration.applicationConfig import AppConfig + from metadata.config.common import WorkflowExecutionError -from metadata.generated.schema.entity.applications.configuration.externalApplicationConfig import ( - AppConfig, -) from metadata.generated.schema.entity.services.connections.metadata.openMetadataConnection import ( OpenMetadataConnection, ) diff --git a/openmetadata-airflow-apis/openmetadata_managed_apis/workflows/ingestion/application.py b/openmetadata-airflow-apis/openmetadata_managed_apis/workflows/ingestion/application.py index e05fd3388227..055368333b84 100644 --- a/openmetadata-airflow-apis/openmetadata_managed_apis/workflows/ingestion/application.py +++ b/openmetadata-airflow-apis/openmetadata_managed_apis/workflows/ingestion/application.py @@ -14,15 +14,14 @@ import json from airflow import DAG +from metadata.generated.schema.entity.applications.configuration.applicationConfig import AppConfig + from openmetadata_managed_apis.utils.logger import set_operator_logger from openmetadata_managed_apis.workflows.ingestion.common import ( build_dag, build_workflow_config_property, ) -from metadata.generated.schema.entity.applications.configuration.externalApplicationConfig import ( - AppConfig, -) from metadata.generated.schema.entity.services.ingestionPipelines.ingestionPipeline import ( IngestionPipeline, ) diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/apps/AbstractNativeApplication.java b/openmetadata-service/src/main/java/org/openmetadata/service/apps/AbstractNativeApplication.java index 4c744a7e794d..e460de0cbffb 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/apps/AbstractNativeApplication.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/apps/AbstractNativeApplication.java @@ -9,10 +9,8 @@ import static org.openmetadata.service.exception.CatalogExceptionMessage.LIVE_APP_SCHEDULE_ERR; import com.cronutils.mapper.CronMapper; -import com.cronutils.model.Cron; import com.cronutils.model.definition.CronDefinitionBuilder; import com.cronutils.parser.CronParser; -import java.util.List; import lombok.Getter; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; @@ -21,24 +19,16 @@ import org.openmetadata.schema.entity.app.App; import org.openmetadata.schema.entity.app.AppRunRecord; import org.openmetadata.schema.entity.app.AppType; -import org.openmetadata.schema.entity.app.ExternalAppIngestionConfig; import org.openmetadata.schema.entity.app.ScheduleType; import org.openmetadata.schema.entity.app.ScheduledExecutionContext; -import org.openmetadata.schema.entity.services.ServiceType; import org.openmetadata.schema.entity.services.ingestionPipelines.IngestionPipeline; import org.openmetadata.schema.services.connections.metadata.OpenMetadataConnection; -import org.openmetadata.schema.type.EntityReference; -import org.openmetadata.schema.type.ProviderType; -import org.openmetadata.schema.type.Relationship; import org.openmetadata.service.Entity; import org.openmetadata.service.apps.scheduler.AppScheduler; import org.openmetadata.service.apps.scheduler.OmAppJobListener; -import org.openmetadata.service.exception.EntityNotFoundException; import org.openmetadata.service.jdbi3.CollectionDAO; -import org.openmetadata.service.jdbi3.EntityRepository; import org.openmetadata.service.jdbi3.IngestionPipelineRepository; import org.openmetadata.service.search.SearchRepository; -import org.openmetadata.service.util.FullyQualifiedName; import org.openmetadata.service.util.JsonUtils; import org.openmetadata.service.util.OpenMetadataConnectionBuilder; import org.quartz.JobExecutionContext; @@ -87,77 +77,7 @@ public void scheduleInternal() { @Override public void initializeExternalApp() { - if (app.getAppType() == AppType.External && app.getScheduleType().equals(ScheduleType.Scheduled)) { - IngestionPipelineRepository ingestionPipelineRepository = - (IngestionPipelineRepository) Entity.getEntityRepository(Entity.INGESTION_PIPELINE); - ExternalAppIngestionConfig ingestionConfig = - JsonUtils.convertValue(app.getAppConfiguration(), ExternalAppIngestionConfig.class); - - try { - // Check if the Pipeline Already Exists - String fqn = FullyQualifiedName.add(ingestionConfig.getService().getName(), ingestionConfig.getName()); - IngestionPipeline storedPipeline = - ingestionPipelineRepository.getByName(null, fqn, ingestionPipelineRepository.getFields("id")); - - // Init Application Code for Some Initialization - List records = - collectionDAO - .relationshipDAO() - .findTo(app.getId(), Entity.APPLICATION, Relationship.HAS.ordinal(), Entity.INGESTION_PIPELINE); - - if (records.isEmpty()) { - // Add Ingestion Pipeline to Application - collectionDAO - .relationshipDAO() - .insert( - app.getId(), - storedPipeline.getId(), - Entity.APPLICATION, - Entity.INGESTION_PIPELINE, - Relationship.HAS.ordinal()); - } - } catch (EntityNotFoundException ex) { - // Pipeline needs to be created - EntityRepository serviceRepository = - Entity.getServiceEntityRepository(ServiceType.fromValue(ingestionConfig.getService().getType())); - EntityReference service = - serviceRepository - .getByName(null, ingestionConfig.getService().getName(), serviceRepository.getFields("id")) - .getEntityReference(); - - Cron quartzCron = cronParser.parse(app.getAppSchedule().getCronExpression()); - - CreateIngestionPipeline createPipelineRequest = - new CreateIngestionPipeline() - .withName(ingestionConfig.getName()) - .withDisplayName(ingestionConfig.getDisplayName()) - .withDescription(ingestionConfig.getDescription()) - .withPipelineType(ingestionConfig.getPipelineType()) - .withSourceConfig(ingestionConfig.getSourceConfig()) - .withAirflowConfig( - ingestionConfig.getAirflowConfig().withScheduleInterval(cronMapper.map(quartzCron).asString())) - .withService(service); - - // Get Pipeline - IngestionPipeline dataInsightPipeline = - getIngestionPipeline(createPipelineRequest, String.format("%sBot", app.getName()), "admin") - .withProvider(ProviderType.USER); - ingestionPipelineRepository.setFullyQualifiedName(dataInsightPipeline); - ingestionPipelineRepository.initializeEntity(dataInsightPipeline); - - // Add Ingestion Pipeline to Application - collectionDAO - .relationshipDAO() - .insert( - app.getId(), - dataInsightPipeline.getId(), - Entity.APPLICATION, - Entity.INGESTION_PIPELINE, - Relationship.HAS.ordinal()); - } - } else { - throw new IllegalArgumentException(INVALID_APP_TYPE); - } + // External apps should either use or extend `ExternalApplicationHandler` } protected void validateServerExecutableApp(AppRuntime context) { diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/apps/ExternalApplicationHandler.java b/openmetadata-service/src/main/java/org/openmetadata/service/apps/ExternalApplicationHandler.java index 69f14ad689dc..aa958701e572 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/apps/ExternalApplicationHandler.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/apps/ExternalApplicationHandler.java @@ -4,7 +4,7 @@ import java.util.List; import lombok.extern.slf4j.Slf4j; import org.openmetadata.schema.api.services.ingestionPipelines.CreateIngestionPipeline; -import org.openmetadata.schema.entity.applications.configuration.ExternalApplicationConfig; +import org.openmetadata.schema.entity.applications.configuration.ApplicationConfig; import org.openmetadata.schema.entity.services.ingestionPipelines.AirflowConfig; import org.openmetadata.schema.entity.services.ingestionPipelines.IngestionPipeline; import org.openmetadata.schema.entity.services.ingestionPipelines.PipelineType; @@ -34,8 +34,7 @@ public class ExternalApplicationHandler extends AbstractNativeApplication { @Override public void initializeExternalApp() { - ExternalApplicationConfig config = - JsonUtils.convertValue(this.getApp().getAppConfiguration(), ExternalApplicationConfig.class); + ApplicationConfig config = JsonUtils.convertValue(this.getApp().getAppConfiguration(), ApplicationConfig.class); IngestionPipelineRepository ingestionPipelineRepository = (IngestionPipelineRepository) Entity.getEntityRepository(Entity.INGESTION_PIPELINE); @@ -85,7 +84,7 @@ public void initializeExternalApp() { .withConfig( new ApplicationPipeline() .withSourcePythonClass(this.getApp().getSourcePythonClass()) - .withAppConfig(config.getConfig()))) + .withAppConfig(config))) .withAirflowConfig( new AirflowConfig().withScheduleInterval(this.getCronMapper().map(quartzCron).asString())) .withService(service); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/insights/DataInsightsApp.java b/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/insights/DataInsightsApp.java index 39330905bdca..a6e29f38d19c 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/insights/DataInsightsApp.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/insights/DataInsightsApp.java @@ -1,17 +1,112 @@ package org.openmetadata.service.apps.bundles.insights; +import static org.openmetadata.service.exception.CatalogExceptionMessage.INVALID_APP_TYPE; + +import com.cronutils.model.Cron; +import java.util.List; import lombok.extern.slf4j.Slf4j; +import org.openmetadata.schema.api.services.ingestionPipelines.CreateIngestionPipeline; import org.openmetadata.schema.entity.app.App; +import org.openmetadata.schema.entity.app.AppType; +import org.openmetadata.schema.entity.app.ScheduleType; +import org.openmetadata.schema.entity.services.ServiceType; +import org.openmetadata.schema.entity.services.ingestionPipelines.AirflowConfig; +import org.openmetadata.schema.entity.services.ingestionPipelines.IngestionPipeline; +import org.openmetadata.schema.entity.services.ingestionPipelines.PipelineType; +import org.openmetadata.schema.metadataIngestion.SourceConfig; +import org.openmetadata.schema.services.connections.metadata.ComponentConfig; +import org.openmetadata.schema.type.EntityReference; +import org.openmetadata.schema.type.ProviderType; +import org.openmetadata.schema.type.Relationship; +import org.openmetadata.service.Entity; import org.openmetadata.service.apps.AbstractNativeApplication; +import org.openmetadata.service.exception.EntityNotFoundException; import org.openmetadata.service.jdbi3.CollectionDAO; +import org.openmetadata.service.jdbi3.EntityRepository; +import org.openmetadata.service.jdbi3.IngestionPipelineRepository; import org.openmetadata.service.search.SearchRepository; +import org.openmetadata.service.util.FullyQualifiedName; @Slf4j public class DataInsightsApp extends AbstractNativeApplication { + private static final String INGESTION_PIPELINE_NAME = "OpenMetadata_dataInsight"; + private static final String SERVICE_NAME = "OpenMetadata"; + private static final String SERVICE_TYPE = "Metadata"; + private static final String PIPELINE_DESCRIPTION = "OpenMetadata DataInsight Pipeline"; + @Override public void init(App app, CollectionDAO dao, SearchRepository searchRepository) { super.init(app, dao, searchRepository); LOG.info("Data Insights App is initialized"); } + + @Override + public void initializeExternalApp() { + if (getApp().getAppType() == AppType.External && getApp().getScheduleType().equals(ScheduleType.Scheduled)) { + IngestionPipelineRepository ingestionPipelineRepository = + (IngestionPipelineRepository) Entity.getEntityRepository(Entity.INGESTION_PIPELINE); + + try { + // Check if the Pipeline Already Exists + String fqn = FullyQualifiedName.add(SERVICE_NAME, INGESTION_PIPELINE_NAME); + IngestionPipeline storedPipeline = + ingestionPipelineRepository.getByName(null, fqn, ingestionPipelineRepository.getFields("id")); + + // Init Application Code for Some Initialization + List records = + collectionDAO + .relationshipDAO() + .findTo(getApp().getId(), Entity.APPLICATION, Relationship.HAS.ordinal(), Entity.INGESTION_PIPELINE); + + if (records.isEmpty()) { + // Add Ingestion Pipeline to Application + collectionDAO + .relationshipDAO() + .insert( + getApp().getId(), + storedPipeline.getId(), + Entity.APPLICATION, + Entity.INGESTION_PIPELINE, + Relationship.HAS.ordinal()); + } + } catch (EntityNotFoundException ex) { + // Pipeline needs to be created + EntityRepository serviceRepository = Entity.getServiceEntityRepository(ServiceType.fromValue(SERVICE_TYPE)); + EntityReference service = + serviceRepository.getByName(null, SERVICE_NAME, serviceRepository.getFields("id")).getEntityReference(); + + Cron quartzCron = getCronParser().parse(getApp().getAppSchedule().getCronExpression()); + + CreateIngestionPipeline createPipelineRequest = + new CreateIngestionPipeline() + .withName(INGESTION_PIPELINE_NAME) + .withDisplayName(INGESTION_PIPELINE_NAME) + .withDescription(PIPELINE_DESCRIPTION) + .withPipelineType(PipelineType.DATA_INSIGHT) + .withSourceConfig(new SourceConfig().withConfig(new ComponentConfig())) + .withAirflowConfig(new AirflowConfig().withScheduleInterval(getCronMapper().map(quartzCron).asString())) + .withService(service); + + // Get Pipeline + IngestionPipeline dataInsightPipeline = + getIngestionPipeline(createPipelineRequest, String.format("%sBot", getApp().getName()), "admin") + .withProvider(ProviderType.USER); + ingestionPipelineRepository.setFullyQualifiedName(dataInsightPipeline); + ingestionPipelineRepository.initializeEntity(dataInsightPipeline); + + // Add Ingestion Pipeline to Application + collectionDAO + .relationshipDAO() + .insert( + getApp().getId(), + dataInsightPipeline.getId(), + Entity.APPLICATION, + Entity.INGESTION_PIPELINE, + Relationship.HAS.ordinal()); + } + } else { + throw new IllegalArgumentException(INVALID_APP_TYPE); + } + } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/insights/DataInsightsReportApp.java b/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/insights/DataInsightsReportApp.java index a51334b63e03..8956a760efe7 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/insights/DataInsightsReportApp.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/insights/DataInsightsReportApp.java @@ -37,7 +37,7 @@ import org.openmetadata.schema.dataInsight.type.TotalEntitiesByType; import org.openmetadata.schema.entity.app.App; import org.openmetadata.schema.entity.app.AppSchedule; -import org.openmetadata.schema.entity.app.DataInsightsReportAppConfig; +import org.openmetadata.schema.entity.applications.configuration.internal.DataInsightsReportAppConfig; import org.openmetadata.schema.entity.teams.Team; import org.openmetadata.schema.entity.teams.User; import org.openmetadata.schema.type.EntityReference; diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/apps/AppResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/apps/AppResource.java index 72a41ec2d3cb..da4d01043f0d 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/apps/AppResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/apps/AppResource.java @@ -804,7 +804,8 @@ private App getApplication( .withAppLogoUrl(marketPlaceDefinition.getAppLogoUrl()) .withAppScreenshots(marketPlaceDefinition.getAppScreenshots()) .withFeatures(marketPlaceDefinition.getFeatures()) - .withSourcePythonClass(marketPlaceDefinition.getSourcePythonClass()); + .withSourcePythonClass(marketPlaceDefinition.getSourcePythonClass()) + .withAllowConfiguration(marketPlaceDefinition.getAllowConfiguration()); // validate Bot if provided validateAndAddBot(app, createAppRequest.getBot()); diff --git a/openmetadata-service/src/main/resources/json/data/app/DataInsightsApplication.json b/openmetadata-service/src/main/resources/json/data/app/DataInsightsApplication.json index 05a4acdf7fb3..9e26dfeeb6e6 100644 --- a/openmetadata-service/src/main/resources/json/data/app/DataInsightsApplication.json +++ b/openmetadata-service/src/main/resources/json/data/app/DataInsightsApplication.json @@ -1,35 +1 @@ -{ - "name": "DataInsightsApplication", - "appConfiguration": { - "name": "OpenMetadata_dataInsight", - "displayName": "OpenMetadata_dataInsight", - "description": "OpenMetadata DataInsight Pipeline", - "pipelineType": "dataInsight", - "sourceConfig": { - "config": { - "type": "MetadataToElasticSearch" - } - }, - "airflowConfig": { - "pausePipeline": false, - "concurrency": 1, - "pipelineTimezone": "UTC", - "retries": 3, - "retryDelay": 300, - "pipelineCatchup": false, - "scheduleInterval": "0 0 * * *", - "maxActiveRuns": 1, - "workflowDefaultView": "tree", - "workflowDefaultViewOrientation": "LR" - }, - "loggerLevel": "INFO", - "service": { - "name": "OpenMetadata", - "type": "Metadata" - } - }, - "appSchedule": { - "scheduleType": "Custom", - "cronExpression": "0 0 0 * * ?" - } -} \ No newline at end of file +{} \ No newline at end of file diff --git a/openmetadata-service/src/main/resources/json/data/appMarketPlaceDefinition/DataInsightsApplication.json b/openmetadata-service/src/main/resources/json/data/appMarketPlaceDefinition/DataInsightsApplication.json index e3f415423f6a..452391f940da 100644 --- a/openmetadata-service/src/main/resources/json/data/appMarketPlaceDefinition/DataInsightsApplication.json +++ b/openmetadata-service/src/main/resources/json/data/appMarketPlaceDefinition/DataInsightsApplication.json @@ -15,32 +15,6 @@ "runtime": { "enabled": "true" }, - "appConfiguration": { - "name": "OpenMetadata_dataInsight", - "displayName": "OpenMetadata_dataInsight", - "description": "OpenMetadata DataInsight Pipeline", - "pipelineType": "dataInsight", - "sourceConfig": { - "config": { - "type": "MetadataToElasticSearch" - } - }, - "airflowConfig": { - "pausePipeline": false, - "concurrency": 1, - "pipelineTimezone": "UTC", - "retries": 3, - "retryDelay": 300, - "pipelineCatchup": false, - "scheduleInterval": "0 0 * * *", - "maxActiveRuns": 1, - "workflowDefaultView": "tree", - "workflowDefaultViewOrientation": "LR" - }, - "loggerLevel": "INFO", - "service": { - "name": "OpenMetadata", - "type": "Metadata" - } - } + "allowConfiguration": false, + "appConfiguration": {} } \ No newline at end of file diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/applications/app.json b/openmetadata-spec/src/main/resources/json/schema/entity/applications/app.json index f7e6feca872a..13651561eeae 100644 --- a/openmetadata-spec/src/main/resources/json/schema/entity/applications/app.json +++ b/openmetadata-spec/src/main/resources/json/schema/entity/applications/app.json @@ -190,8 +190,14 @@ "description": "Execution Configuration.", "$ref": "#/definitions/executionContext" }, + "allowConfiguration": { + "description": "Allow users to configure the app from the UI. If `false`, the `configure` step will be hidden.", + "type": "boolean", + "default": true + }, "appConfiguration": { - "description": "Application Configuration object." + "description": "Application Configuration object.", + "$ref": "./configuration/applicationConfig.json#/definitions/appConfig" }, "pipelines": { "description": "References to pipelines deployed for this database service to extract metadata, usage, lineage etc..", diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/applications/appConfig.json b/openmetadata-spec/src/main/resources/json/schema/entity/applications/appConfig.json deleted file mode 100644 index 397d5557d459..000000000000 --- a/openmetadata-spec/src/main/resources/json/schema/entity/applications/appConfig.json +++ /dev/null @@ -1,21 +0,0 @@ -{ - "$id": "https://open-metadata.org/schema/entity/applications/appConfig.json", - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "AppConfiguration", - "javaType": "org.openmetadata.schema.entity.app.AppConfiguration", - "description": "Configuration object.", - "type": "object", - "definitions": { - "configuration": { - "description": "Configuration Object.", - "oneOf": [ - { - "$ref": "./configuration/searchIndexingApp.json" - }, - { - "$ref": "./configuration/dataInsightsApp.json" - } - ] - } - } -} \ No newline at end of file diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/applications/configuration/externalApplicationConfig.json b/openmetadata-spec/src/main/resources/json/schema/entity/applications/configuration/applicationConfig.json similarity index 54% rename from openmetadata-spec/src/main/resources/json/schema/entity/applications/configuration/externalApplicationConfig.json rename to openmetadata-spec/src/main/resources/json/schema/entity/applications/configuration/applicationConfig.json index 5f076d06d57c..2a85c58f4d1c 100644 --- a/openmetadata-spec/src/main/resources/json/schema/entity/applications/configuration/externalApplicationConfig.json +++ b/openmetadata-spec/src/main/resources/json/schema/entity/applications/configuration/applicationConfig.json @@ -1,8 +1,8 @@ { - "$id": "https://open-metadata.org/schema/entity/applications/configuration/externalApplicationConfig.json", + "$id": "https://open-metadata.org/schema/entity/applications/configuration/applicationConfig.json", "$schema": "http://json-schema.org/draft-07/schema#", - "title": "ExternalApplicationConfig", - "description": "External Application Pipeline Configuration.", + "title": "ApplicationConfigModel", + "description": "Application Pipeline Configuration.", "type": "object", "definitions": { "appConfig": { @@ -12,15 +12,14 @@ }, { "$ref": "./external/metaPilotAppConfig.json" + }, + { + "$ref": "internal/dataInsightsReportAppConfig.json" + }, + { + "$ref": "internal/searchIndexingAppConfig.json" } ] } - }, - "properties": { - "config": { - "$ref": "#/definitions/appConfig" - } - }, - "additionalProperties": false, - "required": ["config"] + } } diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/applications/configuration/dataInsightsApp.json b/openmetadata-spec/src/main/resources/json/schema/entity/applications/configuration/dataInsightsApp.json deleted file mode 100644 index 9e9d608dc21b..000000000000 --- a/openmetadata-spec/src/main/resources/json/schema/entity/applications/configuration/dataInsightsApp.json +++ /dev/null @@ -1,24 +0,0 @@ -{ - "$id": "https://open-metadata.org/schema/entity/applications/configuration/dataInsightsApp.json", - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "DataInsightsReportAppConfig", - "description": "This schema defines Data Insight config for receiving reports from OpenMetadata.", - "type": "object", - "javaType": "org.openmetadata.schema.entity.app.DataInsightsReportAppConfig", - "javaInterfaces": [ - "org.openmetadata.schema.SubscriptionAction" - ], - "properties": { - "sendToAdmins": { - "description": "Send the Mails to Admins", - "type": "boolean", - "default": false - }, - "sendToTeams": { - "description": "Send the Mails to Teams", - "type": "boolean", - "default": false - } - }, - "additionalProperties": false -} diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/applications/configuration/external/autoTaggerAppConfig.json b/openmetadata-spec/src/main/resources/json/schema/entity/applications/configuration/external/autoTaggerAppConfig.json index 64e015ce4961..763f9a1dbfab 100644 --- a/openmetadata-spec/src/main/resources/json/schema/entity/applications/configuration/external/autoTaggerAppConfig.json +++ b/openmetadata-spec/src/main/resources/json/schema/entity/applications/configuration/external/autoTaggerAppConfig.json @@ -6,7 +6,7 @@ "type": "object", "javaType": "org.openmetadata.schema.entity.app.external.AutoTaggerAppConfig", "definitions": { - "autoTaggerApp": { + "autoTaggerAppType": { "description": "Application type.", "type": "string", "enum": ["AutoTagger"], @@ -17,7 +17,7 @@ "type": { "title": "Service Type", "description": "Service Type", - "$ref": "#/definitions/autoTaggerApp", + "$ref": "#/definitions/autoTaggerAppType", "default": "AutoTagger" }, "confidenceLevel": { diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/applications/configuration/external/metaPilotAppConfig.json b/openmetadata-spec/src/main/resources/json/schema/entity/applications/configuration/external/metaPilotAppConfig.json index ca6edd2023e3..e17d7dfe0d64 100644 --- a/openmetadata-spec/src/main/resources/json/schema/entity/applications/configuration/external/metaPilotAppConfig.json +++ b/openmetadata-spec/src/main/resources/json/schema/entity/applications/configuration/external/metaPilotAppConfig.json @@ -6,7 +6,7 @@ "type": "object", "javaType": "org.openmetadata.schema.entity.app.external.MetaPilotAppConfig", "definitions": { - "metaPilotApp": { + "metaPilotAppType": { "description": "Application type.", "type": "string", "enum": ["MetaPilot"], @@ -37,9 +37,9 @@ }, "properties": { "type": { - "title": "Service Type", - "description": "Service Type", - "$ref": "#/definitions/metaPilotApp", + "title": "Application Type", + "description": "Application Type", + "$ref": "#/definitions/metaPilotAppType", "default": "MetaPilot" }, "waiiInstance": { diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/applications/configuration/externalAppIngestionConfig.json b/openmetadata-spec/src/main/resources/json/schema/entity/applications/configuration/externalAppIngestionConfig.json deleted file mode 100644 index 8f4282e713a2..000000000000 --- a/openmetadata-spec/src/main/resources/json/schema/entity/applications/configuration/externalAppIngestionConfig.json +++ /dev/null @@ -1,55 +0,0 @@ -{ - "$id": "https://open-metadata.org/schema/entity/applications/configuration/externalAppIngestionConfig.json", - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "ExternalAppIngestionConfig", - "description": "This schema defines External App Ingestion Config used for app working with Ingestion. Only used for Data Insights.", - "type": "object", - "javaType": "org.openmetadata.schema.entity.app.ExternalAppIngestionConfig", - "definitions": { - "service": { - "description": "Link to the service for which ingestion pipeline is ingesting the metadata.", - "properties": { - "name": { - "description": "Name of the Service.", - "type": "string" - }, - "type": { - "description": "Type of Service", - "type": "string" - } - } - } - }, - "properties": { - "name": { - "description": "Name of the ingestion Pipeline.", - "$ref": "../../../type/basic.json#/definitions/entityName" - }, - "displayName": { - "description": "Display Name that identifies this ingestion pipeline.", - "type": "string" - }, - "description": { - "description": "Description of the pipeline.", - "$ref": "../../../type/basic.json#/definitions/markdown" - }, - "pipelineType": { - "$ref": "../../../entity/services/ingestionPipelines/ingestionPipeline.json#/definitions/pipelineType" - }, - "sourceConfig": { - "$ref": "../../../metadataIngestion/workflow.json#/definitions/sourceConfig" - }, - "airflowConfig": { - "$ref": "../../../entity/services/ingestionPipelines/ingestionPipeline.json#/definitions/airflowConfig" - }, - "loggerLevel": { - "description": "Set the logging level for the workflow.", - "$ref": "../../../metadataIngestion/workflow.json#/definitions/logLevels" - }, - "service": { - "$ref": "#/definitions/service" - } - }, - "required": ["name", "pipelineType", "service"], - "additionalProperties": false -} diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/applications/configuration/internal/dataInsightsAppConfig.json b/openmetadata-spec/src/main/resources/json/schema/entity/applications/configuration/internal/dataInsightsAppConfig.json new file mode 100644 index 000000000000..1c3908784553 --- /dev/null +++ b/openmetadata-spec/src/main/resources/json/schema/entity/applications/configuration/internal/dataInsightsAppConfig.json @@ -0,0 +1,9 @@ +{ + "$id": "https://open-metadata.org/schema/entity/applications/configuration/dataInsightsApp.json", + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "DataInsightsAppConfig", + "description": "No configuration needed to instantiate the Data Insights Pipeline. The logic is handled in the backend.", + "type": "object", + "properties": {}, + "additionalProperties": false +} diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/applications/configuration/internal/dataInsightsReportAppConfig.json b/openmetadata-spec/src/main/resources/json/schema/entity/applications/configuration/internal/dataInsightsReportAppConfig.json new file mode 100644 index 000000000000..31e00b7ef43d --- /dev/null +++ b/openmetadata-spec/src/main/resources/json/schema/entity/applications/configuration/internal/dataInsightsReportAppConfig.json @@ -0,0 +1,33 @@ +{ + "$id": "DataInsightsReportApplication.json", + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "DataInsightsReportAppConfig", + "description": "This schema defines configuration for Data Insights Report Application.", + "definitions": { + "dataInsightsReportAppType": { + "description": "Application type.", + "type": "string", + "enum": ["DataInsightsReport"], + "default": "DataInsightsReport" + } + }, + "properties": { + "type": { + "title": "Application Type", + "description": "Application Type", + "$ref": "#/definitions/dataInsightsReportAppType", + "default": "DataInsightsReport" + }, + "sendToAdmins": { + "title": "Send To Admins", + "type": "boolean", + "default": true + }, + "sendToTeams": { + "title": "Send To Teams", + "type": "boolean", + "default": true + } + }, + "additionalProperties": false +} diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/applications/configuration/searchIndexingApp.json b/openmetadata-spec/src/main/resources/json/schema/entity/applications/configuration/internal/searchIndexingAppConfig.json similarity index 64% rename from openmetadata-spec/src/main/resources/json/schema/entity/applications/configuration/searchIndexingApp.json rename to openmetadata-spec/src/main/resources/json/schema/entity/applications/configuration/internal/searchIndexingAppConfig.json index 1314536692ee..d7bb62b90591 100644 --- a/openmetadata-spec/src/main/resources/json/schema/entity/applications/configuration/searchIndexingApp.json +++ b/openmetadata-spec/src/main/resources/json/schema/entity/applications/configuration/internal/searchIndexingAppConfig.json @@ -4,7 +4,21 @@ "title": "SearchIndexingApp", "type": "object", "description": "Search Indexing App.", + "definitions": { + "searchIndexingType": { + "description": "Application type.", + "type": "string", + "enum": ["SearchIndexing"], + "default": "SearchIndexing" + } + }, "properties" : { + "type": { + "title": "Application Type", + "description": "Application Type", + "$ref": "#/definitions/searchIndexingType", + "default": "SearchIndexing" + }, "entities": { "description": "List of Entities to Reindex", "type": "array", @@ -26,7 +40,7 @@ }, "searchIndexMappingLanguage": { "description": "Recreate Indexes with updated Language", - "$ref": "../../../configuration/elasticSearchConfiguration.json#/definitions/searchIndexMappingLanguage" + "$ref": "../../../../configuration/elasticSearchConfiguration.json#/definitions/searchIndexMappingLanguage" } }, "additionalProperties": false diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/applications/jobStatus.json b/openmetadata-spec/src/main/resources/json/schema/entity/applications/jobStatus.json index 025c7d406bcc..f8ba929466fd 100644 --- a/openmetadata-spec/src/main/resources/json/schema/entity/applications/jobStatus.json +++ b/openmetadata-spec/src/main/resources/json/schema/entity/applications/jobStatus.json @@ -10,7 +10,7 @@ "description": "Configuration Object.", "oneOf": [ { - "$ref": "./configuration/searchIndexingApp.json" + "$ref": "configuration/internal/searchIndexingAppConfig.json" } ] } diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/applications/marketplace/appMarketPlaceDefinition.json b/openmetadata-spec/src/main/resources/json/schema/entity/applications/marketplace/appMarketPlaceDefinition.json index 16cfa0841fa4..649e72b2a2b6 100644 --- a/openmetadata-spec/src/main/resources/json/schema/entity/applications/marketplace/appMarketPlaceDefinition.json +++ b/openmetadata-spec/src/main/resources/json/schema/entity/applications/marketplace/appMarketPlaceDefinition.json @@ -109,8 +109,14 @@ "description": "If app type is live, user can provide additional runtime context.", "$ref": "../app.json#/definitions/executionContext" }, + "allowConfiguration": { + "description": "Allow users to configure the app from the UI. If `false`, the `configure` step will be hidden.", + "type": "boolean", + "default": true + }, "appConfiguration": { - "description": "Application Configuration object." + "description": "Application Configuration object.", + "$ref": "../configuration/applicationConfig.json#/definitions/appConfig" }, "appLogoUrl": { "description": "Application Logo Url.", diff --git a/openmetadata-spec/src/main/resources/json/schema/metadataIngestion/application.json b/openmetadata-spec/src/main/resources/json/schema/metadataIngestion/application.json index b27f974b25f9..b0dada118cc4 100644 --- a/openmetadata-spec/src/main/resources/json/schema/metadataIngestion/application.json +++ b/openmetadata-spec/src/main/resources/json/schema/metadataIngestion/application.json @@ -14,7 +14,7 @@ "type": "string" }, "appConfig": { - "$ref": "../entity/applications/configuration/externalApplicationConfig.json#/definitions/appConfig", + "$ref": "../entity/applications/configuration/applicationConfig.json#/definitions/appConfig", "description": "External Application configuration" }, "ingestionPipelineFQN": { diff --git a/openmetadata-spec/src/main/resources/json/schema/metadataIngestion/applicationPipeline.json b/openmetadata-spec/src/main/resources/json/schema/metadataIngestion/applicationPipeline.json index 783ae5e22583..72248e41ce4c 100644 --- a/openmetadata-spec/src/main/resources/json/schema/metadataIngestion/applicationPipeline.json +++ b/openmetadata-spec/src/main/resources/json/schema/metadataIngestion/applicationPipeline.json @@ -23,8 +23,8 @@ "type": "string" }, "appConfig": { - "$ref": "../entity/applications/configuration/externalApplicationConfig.json#/definitions/appConfig", - "description": "External Application configuration" + "$ref": "../entity/applications/configuration/applicationConfig.json#/definitions/appConfig", + "description": "Application configuration" } }, "additionalProperties": false diff --git a/openmetadata-ui/src/main/resources/ui/src/utils/ApplicationSchemas/DataInsightsApplication.json b/openmetadata-ui/src/main/resources/ui/src/utils/ApplicationSchemas/DataInsightsApplication.json index 40b011c8144d..85b59d14d994 100644 --- a/openmetadata-ui/src/main/resources/ui/src/utils/ApplicationSchemas/DataInsightsApplication.json +++ b/openmetadata-ui/src/main/resources/ui/src/utils/ApplicationSchemas/DataInsightsApplication.json @@ -1,7 +1,7 @@ { "$id": "DataInsightsApplication.json", "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Data Insights Application Config", + "title": "DataInsightsAppConfig", "description": "This schema defines configuration for Search Reindexing Application.", "properties": {}, "additionalProperties": false From 4788c3fdf6a39805c75b5d1943bcb6a8060a9fb1 Mon Sep 17 00:00:00 2001 From: Pere Miquel Brull Date: Tue, 21 Nov 2023 16:35:46 +0100 Subject: [PATCH 07/15] Allow config --- .../service/apps/bundles/insights/DataInsightsApp.java | 4 ++-- .../marketplace/createAppMarketPlaceDefinitionReq.json | 8 +++++++- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/insights/DataInsightsApp.java b/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/insights/DataInsightsApp.java index a6e29f38d19c..cca449c5316b 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/insights/DataInsightsApp.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/insights/DataInsightsApp.java @@ -13,8 +13,8 @@ import org.openmetadata.schema.entity.services.ingestionPipelines.AirflowConfig; import org.openmetadata.schema.entity.services.ingestionPipelines.IngestionPipeline; import org.openmetadata.schema.entity.services.ingestionPipelines.PipelineType; +import org.openmetadata.schema.metadataIngestion.DataInsightPipeline; import org.openmetadata.schema.metadataIngestion.SourceConfig; -import org.openmetadata.schema.services.connections.metadata.ComponentConfig; import org.openmetadata.schema.type.EntityReference; import org.openmetadata.schema.type.ProviderType; import org.openmetadata.schema.type.Relationship; @@ -84,7 +84,7 @@ public void initializeExternalApp() { .withDisplayName(INGESTION_PIPELINE_NAME) .withDescription(PIPELINE_DESCRIPTION) .withPipelineType(PipelineType.DATA_INSIGHT) - .withSourceConfig(new SourceConfig().withConfig(new ComponentConfig())) + .withSourceConfig(new SourceConfig().withConfig(new DataInsightPipeline())) .withAirflowConfig(new AirflowConfig().withScheduleInterval(getCronMapper().map(quartzCron).asString())) .withService(service); diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/applications/marketplace/createAppMarketPlaceDefinitionReq.json b/openmetadata-spec/src/main/resources/json/schema/entity/applications/marketplace/createAppMarketPlaceDefinitionReq.json index f708735680bb..f94d1ca89df6 100644 --- a/openmetadata-spec/src/main/resources/json/schema/entity/applications/marketplace/createAppMarketPlaceDefinitionReq.json +++ b/openmetadata-spec/src/main/resources/json/schema/entity/applications/marketplace/createAppMarketPlaceDefinitionReq.json @@ -76,8 +76,14 @@ "description": "If app type is live, user can provide additional runtime context.", "$ref": "../app.json#/definitions/executionContext" }, + "allowConfiguration": { + "description": "Allow users to configure the app from the UI. If `false`, the `configure` step will be hidden.", + "type": "boolean", + "default": true + }, "appConfiguration": { - "description": "Application Configuration object." + "description": "Application Configuration object.", + "$ref": "../configuration/applicationConfig.json#/definitions/appConfig" }, "appLogoUrl": { "description": "Application Logo Url.", From eb59d0fbb8faebaec0e00c47de8b557f1ecbeb74 Mon Sep 17 00:00:00 2001 From: Pere Miquel Brull Date: Wed, 22 Nov 2023 06:31:30 +0100 Subject: [PATCH 08/15] Format --- ingestion/src/metadata/workflow/application.py | 5 +++-- .../workflows/ingestion/application.py | 5 +++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/ingestion/src/metadata/workflow/application.py b/ingestion/src/metadata/workflow/application.py index 26747b868c90..0abdfd03ca34 100644 --- a/ingestion/src/metadata/workflow/application.py +++ b/ingestion/src/metadata/workflow/application.py @@ -14,9 +14,10 @@ from abc import ABC, abstractmethod from typing import List, Optional -from metadata.generated.schema.entity.applications.configuration.applicationConfig import AppConfig - from metadata.config.common import WorkflowExecutionError +from metadata.generated.schema.entity.applications.configuration.applicationConfig import ( + AppConfig, +) from metadata.generated.schema.entity.services.connections.metadata.openMetadataConnection import ( OpenMetadataConnection, ) diff --git a/openmetadata-airflow-apis/openmetadata_managed_apis/workflows/ingestion/application.py b/openmetadata-airflow-apis/openmetadata_managed_apis/workflows/ingestion/application.py index 055368333b84..3b8a770ce209 100644 --- a/openmetadata-airflow-apis/openmetadata_managed_apis/workflows/ingestion/application.py +++ b/openmetadata-airflow-apis/openmetadata_managed_apis/workflows/ingestion/application.py @@ -14,14 +14,15 @@ import json from airflow import DAG -from metadata.generated.schema.entity.applications.configuration.applicationConfig import AppConfig - from openmetadata_managed_apis.utils.logger import set_operator_logger from openmetadata_managed_apis.workflows.ingestion.common import ( build_dag, build_workflow_config_property, ) +from metadata.generated.schema.entity.applications.configuration.applicationConfig import ( + AppConfig, +) from metadata.generated.schema.entity.services.ingestionPipelines.ingestionPipeline import ( IngestionPipeline, ) From aa390fc8eb4590d47070444230108fd0526f553f Mon Sep 17 00:00:00 2001 From: Pere Miquel Brull Date: Wed, 22 Nov 2023 06:32:46 +0100 Subject: [PATCH 09/15] Fix test --- .../tests/unit/ingestion_pipeline/test_workflow_creation.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/openmetadata-airflow-apis/tests/unit/ingestion_pipeline/test_workflow_creation.py b/openmetadata-airflow-apis/tests/unit/ingestion_pipeline/test_workflow_creation.py index 2fb423ccd515..ac946a4e2ab9 100644 --- a/openmetadata-airflow-apis/tests/unit/ingestion_pipeline/test_workflow_creation.py +++ b/openmetadata-airflow-apis/tests/unit/ingestion_pipeline/test_workflow_creation.py @@ -36,12 +36,12 @@ build_usage_workflow_config, ) +from metadata.generated.schema.entity.applications.configuration.applicationConfig import ( + AppConfig, +) from metadata.generated.schema.entity.applications.configuration.external.autoTaggerAppConfig import ( AutoTaggerAppConfig, ) -from metadata.generated.schema.entity.applications.configuration.externalApplicationConfig import ( - AppConfig, -) from metadata.generated.schema.entity.services.connections.metadata.openMetadataConnection import ( OpenMetadataConnection, ) From 0100b928832573ddf10e43e81c62f7caa9eb1b00 Mon Sep 17 00:00:00 2001 From: Pere Miquel Brull Date: Wed, 22 Nov 2023 07:42:24 +0100 Subject: [PATCH 10/15] Prep migrations for allowConfiguration --- .../1.3.0/mysql/postDataMigrationSQLScript.sql | 0 .../native/1.3.0/mysql/schemaChanges.sql | 16 ++++++++++++++++ .../postgres/postDataMigrationSQLScript.sql | 0 .../native/1.3.0/postgres/schemaChanges.sql | 0 .../resources/apps/AppMarketPlaceResource.java | 3 ++- 5 files changed, 18 insertions(+), 1 deletion(-) create mode 100644 bootstrap/sql/migrations/native/1.3.0/mysql/postDataMigrationSQLScript.sql create mode 100644 bootstrap/sql/migrations/native/1.3.0/mysql/schemaChanges.sql create mode 100644 bootstrap/sql/migrations/native/1.3.0/postgres/postDataMigrationSQLScript.sql create mode 100644 bootstrap/sql/migrations/native/1.3.0/postgres/schemaChanges.sql diff --git a/bootstrap/sql/migrations/native/1.3.0/mysql/postDataMigrationSQLScript.sql b/bootstrap/sql/migrations/native/1.3.0/mysql/postDataMigrationSQLScript.sql new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/bootstrap/sql/migrations/native/1.3.0/mysql/schemaChanges.sql b/bootstrap/sql/migrations/native/1.3.0/mysql/schemaChanges.sql new file mode 100644 index 000000000000..17d89b9be399 --- /dev/null +++ b/bootstrap/sql/migrations/native/1.3.0/mysql/schemaChanges.sql @@ -0,0 +1,16 @@ +-- DataInsightsApplication should not allow configuration +update apps_marketplace +set json = JSON_INSERT( + JSON_REMOVE(json, '$.allowConfiguration'), + '$.allowConfiguration', + false +) +where name = 'DataInsightsApplication'; + +update installed_apps +set json = JSON_INSERT( + JSON_REMOVE(json, '$.allowConfiguration'), + '$.allowConfiguration', + false +) +where name = 'DataInsightsApplication'; diff --git a/bootstrap/sql/migrations/native/1.3.0/postgres/postDataMigrationSQLScript.sql b/bootstrap/sql/migrations/native/1.3.0/postgres/postDataMigrationSQLScript.sql new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/bootstrap/sql/migrations/native/1.3.0/postgres/schemaChanges.sql b/bootstrap/sql/migrations/native/1.3.0/postgres/schemaChanges.sql new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/apps/AppMarketPlaceResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/apps/AppMarketPlaceResource.java index 89499a18fddb..eedfcb1725df 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/apps/AppMarketPlaceResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/apps/AppMarketPlaceResource.java @@ -405,7 +405,8 @@ private AppMarketPlaceDefinition getApplicationDefinition( .withAppLogoUrl(create.getAppLogoUrl()) .withAppScreenshots(create.getAppScreenshots()) .withFeatures(create.getFeatures()) - .withSourcePythonClass(create.getSourcePythonClass()); + .withSourcePythonClass(create.getSourcePythonClass()) + .withAllowConfiguration(create.getAllowConfiguration()); // Validate App validateApplication(app); From 4127f63a07635002d55c056bdd631b50aeea741a Mon Sep 17 00:00:00 2001 From: Pere Miquel Brull Date: Wed, 22 Nov 2023 07:42:48 +0100 Subject: [PATCH 11/15] TODO --- .../sql/migrations/native/1.3.0/postgres/schemaChanges.sql | 2 ++ 1 file changed, 2 insertions(+) diff --git a/bootstrap/sql/migrations/native/1.3.0/postgres/schemaChanges.sql b/bootstrap/sql/migrations/native/1.3.0/postgres/schemaChanges.sql index e69de29bb2d1..751f4f548946 100644 --- a/bootstrap/sql/migrations/native/1.3.0/postgres/schemaChanges.sql +++ b/bootstrap/sql/migrations/native/1.3.0/postgres/schemaChanges.sql @@ -0,0 +1,2 @@ +-- DataInsightsApplication should not allow configuration +-- TODO MIGRATIONS FOR POSTGRES \ No newline at end of file From fae63045844b820f04a405fb05e2b18d077926e6 Mon Sep 17 00:00:00 2001 From: Pere Miquel Brull Date: Wed, 22 Nov 2023 08:50:45 +0100 Subject: [PATCH 12/15] Add pg migrations --- .../native/1.3.0/postgres/schemaChanges.sql | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/bootstrap/sql/migrations/native/1.3.0/postgres/schemaChanges.sql b/bootstrap/sql/migrations/native/1.3.0/postgres/schemaChanges.sql index 751f4f548946..b29cc7cbb3f2 100644 --- a/bootstrap/sql/migrations/native/1.3.0/postgres/schemaChanges.sql +++ b/bootstrap/sql/migrations/native/1.3.0/postgres/schemaChanges.sql @@ -1,2 +1,16 @@ -- DataInsightsApplication should not allow configuration --- TODO MIGRATIONS FOR POSTGRES \ No newline at end of file +UPDATE apps_marketplace +SET json = jsonb_set( + json::jsonb, + '{allowConfiguration}', + to_jsonb(false) +) +where name = 'DataInsightsApplication'; + +UPDATE installed_apps +SET json = jsonb_set( + json::jsonb, + '{allowConfiguration}', + to_jsonb(false) +) +where name = 'DataInsightsApplication'; From 51f438bdfbe7becb4bad296afd3c30540e5bfeac Mon Sep 17 00:00:00 2001 From: karanh37 Date: Wed, 22 Nov 2023 14:54:00 +0530 Subject: [PATCH 13/15] fix: add allowConfiguration support --- .../AppDetails/AppDetails.component.tsx | 4 +++- .../ui/src/constants/Applications.constant.ts | 1 + .../pages/AppInstall/AppInstall.component.tsx | 22 +++++++++++++++---- 3 files changed, 22 insertions(+), 5 deletions(-) diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Applications/AppDetails/AppDetails.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Applications/AppDetails/AppDetails.component.tsx index 758ec4103d71..d45ca169876c 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Applications/AppDetails/AppDetails.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Applications/AppDetails/AppDetails.component.tsx @@ -43,6 +43,7 @@ import { ReactComponent as IconDropdown } from '../../../assets/svg/menu.svg'; import Loader from '../../../components/Loader/Loader'; import PageLayoutV1 from '../../../components/PageLayoutV1/PageLayoutV1'; import TabsLabel from '../../../components/TabsLabel/TabsLabel.component'; +import { APPLICATION_UI_SCHEMA } from '../../../constants/Applications.constant'; import { DE_ACTIVE_COLOR } from '../../../constants/constants'; import { GlobalSettingOptions, @@ -312,7 +313,7 @@ const AppDetails = () => { const tabs = useMemo(() => { const tabConfiguration = - appData?.appConfiguration && jsonSchema + appData?.appConfiguration && appData.allowConfiguration && jsonSchema ? [ { label: ( @@ -334,6 +335,7 @@ const AppDetails = () => { serviceCategory={ServiceCategory.DASHBOARD_SERVICES} serviceType="" showTestConnection={false} + uiSchema={APPLICATION_UI_SCHEMA} validator={validator} onCancel={noop} onSubmit={onConfigSave} diff --git a/openmetadata-ui/src/main/resources/ui/src/constants/Applications.constant.ts b/openmetadata-ui/src/main/resources/ui/src/constants/Applications.constant.ts index f42f01ee5fa8..6bff623c6559 100644 --- a/openmetadata-ui/src/main/resources/ui/src/constants/Applications.constant.ts +++ b/openmetadata-ui/src/main/resources/ui/src/constants/Applications.constant.ts @@ -36,4 +36,5 @@ export const APPLICATION_UI_SCHEMA = { autoCompleteType: SearchIndex.USER, }, }, + type: { 'ui:widget': 'hidden' }, }; diff --git a/openmetadata-ui/src/main/resources/ui/src/pages/AppInstall/AppInstall.component.tsx b/openmetadata-ui/src/main/resources/ui/src/pages/AppInstall/AppInstall.component.tsx index 11ec9ff081d9..c168ca3f7ff4 100644 --- a/openmetadata-ui/src/main/resources/ui/src/pages/AppInstall/AppInstall.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/pages/AppInstall/AppInstall.component.tsx @@ -70,6 +70,14 @@ const AppInstall = () => { [appData] ); + const stepperList = useMemo( + () => + !appData?.allowConfiguration + ? STEPS_FOR_APP_INSTALL.filter((item) => item.step !== 2) + : STEPS_FOR_APP_INSTALL, + [appData] + ); + const fetchAppDetails = useCallback(async () => { setIsLoading(true); try { @@ -137,10 +145,14 @@ const AppInstall = () => { setActiveServiceStep(2)} + onSave={() => + setActiveServiceStep(appData?.allowConfiguration ? 2 : 3) + } /> ); @@ -172,7 +184,9 @@ const AppInstall = () => { isQuartzCron includePeriodOptions={isExternalApp ? ['Day'] : undefined} initialData={getIngestionFrequency(PipelineType.Application)} - onCancel={() => setActiveServiceStep(isExternalApp ? 1 : 2)} + onCancel={() => + setActiveServiceStep(appData.allowConfiguration ? 2 : 1) + } onSubmit={onSubmit} /> @@ -206,7 +220,7 @@ const AppInstall = () => { From 92641d9ff883060488bdcd070a9635c08759161a Mon Sep 17 00:00:00 2001 From: Pere Miquel Brull Date: Wed, 22 Nov 2023 17:10:10 +0100 Subject: [PATCH 14/15] Autotagger --- .../data/appMarketPlaceDefinition/AutoTaggerApplication.json | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/openmetadata-service/src/main/resources/json/data/appMarketPlaceDefinition/AutoTaggerApplication.json b/openmetadata-service/src/main/resources/json/data/appMarketPlaceDefinition/AutoTaggerApplication.json index e658e5e4eaa7..96dd049a2c22 100644 --- a/openmetadata-service/src/main/resources/json/data/appMarketPlaceDefinition/AutoTaggerApplication.json +++ b/openmetadata-service/src/main/resources/json/data/appMarketPlaceDefinition/AutoTaggerApplication.json @@ -19,8 +19,6 @@ "enabled": "true" }, "appConfiguration": { - "config": { - "confidenceLevel": 80 - } + "confidenceLevel": 80 } } \ No newline at end of file From 63e0dc2c729049d7e1daa80495580b3fe35a35ba Mon Sep 17 00:00:00 2001 From: Pere Miquel Brull Date: Mon, 27 Nov 2023 07:48:10 +0100 Subject: [PATCH 15/15] Update install flow --- .../apps/AbstractNativeApplication.java | 119 +++++++++++++++-- .../service/apps/ApplicationHandler.java | 27 ++-- .../apps/ExternalApplicationHandler.java | 110 --------------- .../service/apps/NativeApplication.java | 6 +- .../bundles/insights/DataInsightsApp.java | 126 +++++++++--------- .../service/resources/apps/AppResource.java | 12 +- .../AutoTaggerApplication.json | 2 +- 7 files changed, 188 insertions(+), 214 deletions(-) delete mode 100644 openmetadata-service/src/main/java/org/openmetadata/service/apps/ExternalApplicationHandler.java diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/apps/AbstractNativeApplication.java b/openmetadata-service/src/main/java/org/openmetadata/service/apps/AbstractNativeApplication.java index e460de0cbffb..c6e55ee90cc3 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/apps/AbstractNativeApplication.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/apps/AbstractNativeApplication.java @@ -5,12 +5,13 @@ import static org.openmetadata.service.apps.scheduler.AppScheduler.APP_INFO_KEY; import static org.openmetadata.service.apps.scheduler.AppScheduler.COLLECTION_DAO_KEY; import static org.openmetadata.service.apps.scheduler.AppScheduler.SEARCH_CLIENT_KEY; -import static org.openmetadata.service.exception.CatalogExceptionMessage.INVALID_APP_TYPE; import static org.openmetadata.service.exception.CatalogExceptionMessage.LIVE_APP_SCHEDULE_ERR; import com.cronutils.mapper.CronMapper; +import com.cronutils.model.Cron; import com.cronutils.model.definition.CronDefinitionBuilder; import com.cronutils.parser.CronParser; +import java.util.List; import lombok.Getter; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; @@ -21,14 +22,25 @@ import org.openmetadata.schema.entity.app.AppType; import org.openmetadata.schema.entity.app.ScheduleType; import org.openmetadata.schema.entity.app.ScheduledExecutionContext; +import org.openmetadata.schema.entity.applications.configuration.ApplicationConfig; +import org.openmetadata.schema.entity.services.ingestionPipelines.AirflowConfig; import org.openmetadata.schema.entity.services.ingestionPipelines.IngestionPipeline; +import org.openmetadata.schema.entity.services.ingestionPipelines.PipelineType; +import org.openmetadata.schema.metadataIngestion.ApplicationPipeline; +import org.openmetadata.schema.metadataIngestion.SourceConfig; import org.openmetadata.schema.services.connections.metadata.OpenMetadataConnection; +import org.openmetadata.schema.type.EntityReference; +import org.openmetadata.schema.type.ProviderType; +import org.openmetadata.schema.type.Relationship; import org.openmetadata.service.Entity; import org.openmetadata.service.apps.scheduler.AppScheduler; import org.openmetadata.service.apps.scheduler.OmAppJobListener; +import org.openmetadata.service.exception.EntityNotFoundException; import org.openmetadata.service.jdbi3.CollectionDAO; import org.openmetadata.service.jdbi3.IngestionPipelineRepository; +import org.openmetadata.service.jdbi3.MetadataServiceRepository; import org.openmetadata.service.search.SearchRepository; +import org.openmetadata.service.util.FullyQualifiedName; import org.openmetadata.service.util.JsonUtils; import org.openmetadata.service.util.OpenMetadataConnectionBuilder; import org.quartz.JobExecutionContext; @@ -42,6 +54,9 @@ public class AbstractNativeApplication implements NativeApplication { private final @Getter CronMapper cronMapper = CronMapper.fromQuartzToUnix(); private final @Getter CronParser cronParser = new CronParser(CronDefinitionBuilder.instanceDefinitionFor(QUARTZ)); + // Default service that contains external apps' Ingestion Pipelines + private static final String SERVICE_NAME = "OpenMetadata"; + @Override public void init(App app, CollectionDAO dao, SearchRepository searchRepository) { this.collectionDAO = dao; @@ -49,6 +64,15 @@ public void init(App app, CollectionDAO dao, SearchRepository searchRepository) this.app = app; } + @Override + public void install() { + if (app.getAppType() == AppType.Internal && app.getScheduleType().equals(ScheduleType.Scheduled)) { + scheduleInternal(); + } else if (app.getAppType() == AppType.External && app.getScheduleType().equals(ScheduleType.Scheduled)) { + scheduleExternal(); + } + } + @Override public void triggerOnDemand() { // Validate Native Application @@ -62,22 +86,93 @@ public void triggerOnDemand() { } } - @Override public void scheduleInternal() { // Validate Native Application - if (app.getAppType() == AppType.Internal && app.getScheduleType().equals(ScheduleType.Scheduled)) { - AppRuntime runtime = JsonUtils.convertValue(app.getRuntime(), ScheduledExecutionContext.class); - validateServerExecutableApp(runtime); - // Schedule New Application Run - AppScheduler.getInstance().addApplicationSchedule(app); - return; + AppRuntime runtime = JsonUtils.convertValue(app.getRuntime(), ScheduledExecutionContext.class); + validateServerExecutableApp(runtime); + // Schedule New Application Run + AppScheduler.getInstance().addApplicationSchedule(app); + } + + public void scheduleExternal() { + IngestionPipelineRepository ingestionPipelineRepository = + (IngestionPipelineRepository) Entity.getEntityRepository(Entity.INGESTION_PIPELINE); + + try { + bindExistingIngestionToApplication(ingestionPipelineRepository); + } catch (EntityNotFoundException ex) { + ApplicationConfig config = JsonUtils.convertValue(this.getApp().getAppConfiguration(), ApplicationConfig.class); + createAndBindIngestionPipeline(ingestionPipelineRepository, config); } - throw new IllegalArgumentException(INVALID_APP_TYPE); } - @Override - public void initializeExternalApp() { - // External apps should either use or extend `ExternalApplicationHandler` + private void bindExistingIngestionToApplication(IngestionPipelineRepository ingestionPipelineRepository) { + String fqn = FullyQualifiedName.add(SERVICE_NAME, this.getApp().getName()); + IngestionPipeline storedPipeline = + ingestionPipelineRepository.getByName(null, fqn, ingestionPipelineRepository.getFields("id")); + + // Init Application Code for Some Initialization + List records = + collectionDAO + .relationshipDAO() + .findTo(this.getApp().getId(), Entity.APPLICATION, Relationship.HAS.ordinal(), Entity.INGESTION_PIPELINE); + + if (records.isEmpty()) { + // Add Ingestion Pipeline to Application + collectionDAO + .relationshipDAO() + .insert( + this.getApp().getId(), + storedPipeline.getId(), + Entity.APPLICATION, + Entity.INGESTION_PIPELINE, + Relationship.HAS.ordinal()); + } + } + + private void createAndBindIngestionPipeline( + IngestionPipelineRepository ingestionPipelineRepository, ApplicationConfig config) { + MetadataServiceRepository serviceEntityRepository = + (MetadataServiceRepository) Entity.getEntityRepository(Entity.METADATA_SERVICE); + EntityReference service = + serviceEntityRepository + .getByName(null, SERVICE_NAME, serviceEntityRepository.getFields("id")) + .getEntityReference(); + + Cron quartzCron = this.getCronParser().parse(this.getApp().getAppSchedule().getCronExpression()); + + CreateIngestionPipeline createPipelineRequest = + new CreateIngestionPipeline() + .withName(this.getApp().getName()) + .withDisplayName(this.getApp().getDisplayName()) + .withDescription(this.getApp().getDescription()) + .withPipelineType(PipelineType.APPLICATION) + .withSourceConfig( + new SourceConfig() + .withConfig( + new ApplicationPipeline() + .withSourcePythonClass(this.getApp().getSourcePythonClass()) + .withAppConfig(config))) + .withAirflowConfig( + new AirflowConfig().withScheduleInterval(this.getCronMapper().map(quartzCron).asString())) + .withService(service); + + // Get Pipeline + IngestionPipeline ingestionPipeline = + getIngestionPipeline(createPipelineRequest, String.format("%sBot", this.getApp().getName()), "admin") + .withProvider(ProviderType.USER); + ingestionPipelineRepository.setFullyQualifiedName(ingestionPipeline); + ingestionPipelineRepository.initializeEntity(ingestionPipeline); + + // Add Ingestion Pipeline to Application + collectionDAO + .relationshipDAO() + .insert( + this.getApp().getId(), + ingestionPipeline.getId(), + Entity.APPLICATION, + Entity.INGESTION_PIPELINE, + Relationship.HAS.ordinal()); } protected void validateServerExecutableApp(AppRuntime context) { diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/apps/ApplicationHandler.java b/openmetadata-service/src/main/java/org/openmetadata/service/apps/ApplicationHandler.java index ffe9bb9c836d..e13ceceab396 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/apps/ApplicationHandler.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/apps/ApplicationHandler.java @@ -4,7 +4,6 @@ import java.lang.reflect.Method; import lombok.extern.slf4j.Slf4j; import org.openmetadata.schema.entity.app.App; -import org.openmetadata.schema.entity.app.AppType; import org.openmetadata.service.jdbi3.CollectionDAO; import org.openmetadata.service.search.SearchRepository; @@ -17,24 +16,20 @@ private ApplicationHandler() { public static void triggerApplicationOnDemand( App app, CollectionDAO daoCollection, SearchRepository searchRepository) { - runMethodFromApplication(app, daoCollection, searchRepository, "triggerOnDemand", "triggerOnDemand"); + runMethodFromApplication(app, daoCollection, searchRepository, "triggerOnDemand"); } - public static void scheduleApplication(App app, CollectionDAO daoCollection, SearchRepository searchRepository) { - runMethodFromApplication(app, daoCollection, searchRepository, "scheduleInternal", "initializeExternalApp"); + public static void installApplication(App app, CollectionDAO daoCollection, SearchRepository searchRepository) { + runMethodFromApplication(app, daoCollection, searchRepository, "install"); } public static void configureApplication(App app, CollectionDAO daoCollection, SearchRepository searchRepository) { - runMethodFromApplication(app, daoCollection, searchRepository, "configure", "configure"); + runMethodFromApplication(app, daoCollection, searchRepository, "configure"); } /** Load an App from its className and call its methods dynamically */ public static void runMethodFromApplication( - App app, - CollectionDAO daoCollection, - SearchRepository searchRepository, - String internalMethodName, - String externalMethodName) { + App app, CollectionDAO daoCollection, SearchRepository searchRepository, String methodName) { // Native Application try { Class clz = Class.forName(app.getClassName()); @@ -44,14 +39,10 @@ public static void runMethodFromApplication( Method initMethod = resource.getClass().getMethod("init", App.class, CollectionDAO.class, SearchRepository.class); initMethod.invoke(resource, app, daoCollection, searchRepository); - // Call Trigger On Demand Method - if (app.getAppType() == AppType.Internal) { - Method scheduleMethod = resource.getClass().getMethod(internalMethodName); - scheduleMethod.invoke(resource); - } else if (app.getAppType() == AppType.External) { - Method scheduleMethod = resource.getClass().getMethod(externalMethodName); - scheduleMethod.invoke(resource); - } + // Call method on demand + Method scheduleMethod = resource.getClass().getMethod(methodName); + scheduleMethod.invoke(resource); + } catch (NoSuchMethodException | InstantiationException | IllegalAccessException | InvocationTargetException e) { LOG.error("Exception encountered", e); throw new RuntimeException(e); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/apps/ExternalApplicationHandler.java b/openmetadata-service/src/main/java/org/openmetadata/service/apps/ExternalApplicationHandler.java deleted file mode 100644 index aa958701e572..000000000000 --- a/openmetadata-service/src/main/java/org/openmetadata/service/apps/ExternalApplicationHandler.java +++ /dev/null @@ -1,110 +0,0 @@ -package org.openmetadata.service.apps; - -import com.cronutils.model.Cron; -import java.util.List; -import lombok.extern.slf4j.Slf4j; -import org.openmetadata.schema.api.services.ingestionPipelines.CreateIngestionPipeline; -import org.openmetadata.schema.entity.applications.configuration.ApplicationConfig; -import org.openmetadata.schema.entity.services.ingestionPipelines.AirflowConfig; -import org.openmetadata.schema.entity.services.ingestionPipelines.IngestionPipeline; -import org.openmetadata.schema.entity.services.ingestionPipelines.PipelineType; -import org.openmetadata.schema.metadataIngestion.ApplicationPipeline; -import org.openmetadata.schema.metadataIngestion.SourceConfig; -import org.openmetadata.schema.type.EntityReference; -import org.openmetadata.schema.type.ProviderType; -import org.openmetadata.schema.type.Relationship; -import org.openmetadata.service.Entity; -import org.openmetadata.service.exception.EntityNotFoundException; -import org.openmetadata.service.jdbi3.CollectionDAO; -import org.openmetadata.service.jdbi3.IngestionPipelineRepository; -import org.openmetadata.service.jdbi3.MetadataServiceRepository; -import org.openmetadata.service.util.FullyQualifiedName; -import org.openmetadata.service.util.JsonUtils; - -@Slf4j -public class ExternalApplicationHandler extends AbstractNativeApplication { - private static final String SERVICE_NAME = "OpenMetadata"; - - /** - * MetaPilot is an external App that accepts one ApiKey parameter and runs a workflow based on it. - * - *

The App will register an IngestionPipeline against the OpenMetadata service with default daily scheduled - * configurations. - */ - @Override - public void initializeExternalApp() { - - ApplicationConfig config = JsonUtils.convertValue(this.getApp().getAppConfiguration(), ApplicationConfig.class); - IngestionPipelineRepository ingestionPipelineRepository = - (IngestionPipelineRepository) Entity.getEntityRepository(Entity.INGESTION_PIPELINE); - - // Check if the Ingestion Pipeline has already been created - try { - String fqn = FullyQualifiedName.add(SERVICE_NAME, this.getApp().getName()); - IngestionPipeline storedPipeline = - ingestionPipelineRepository.getByName(null, fqn, ingestionPipelineRepository.getFields("id")); - - // Init Application Code for Some Initialization - List records = - collectionDAO - .relationshipDAO() - .findTo(this.getApp().getId(), Entity.APPLICATION, Relationship.HAS.ordinal(), Entity.INGESTION_PIPELINE); - - if (records.isEmpty()) { - // Add Ingestion Pipeline to Application - collectionDAO - .relationshipDAO() - .insert( - this.getApp().getId(), - storedPipeline.getId(), - Entity.APPLICATION, - Entity.INGESTION_PIPELINE, - Relationship.HAS.ordinal()); - } - - // Otherwise, create it - } catch (EntityNotFoundException ex) { - MetadataServiceRepository serviceEntityRepository = - (MetadataServiceRepository) Entity.getEntityRepository(Entity.METADATA_SERVICE); - EntityReference service = - serviceEntityRepository - .getByName(null, SERVICE_NAME, serviceEntityRepository.getFields("id")) - .getEntityReference(); - - Cron quartzCron = this.getCronParser().parse(this.getApp().getAppSchedule().getCronExpression()); - - CreateIngestionPipeline createPipelineRequest = - new CreateIngestionPipeline() - .withName(this.getApp().getName()) - .withDisplayName(this.getApp().getDisplayName()) - .withDescription(this.getApp().getDescription()) - .withPipelineType(PipelineType.APPLICATION) - .withSourceConfig( - new SourceConfig() - .withConfig( - new ApplicationPipeline() - .withSourcePythonClass(this.getApp().getSourcePythonClass()) - .withAppConfig(config))) - .withAirflowConfig( - new AirflowConfig().withScheduleInterval(this.getCronMapper().map(quartzCron).asString())) - .withService(service); - - // Get Pipeline - IngestionPipeline dataInsightPipeline = - getIngestionPipeline(createPipelineRequest, String.format("%sBot", this.getApp().getName()), "admin") - .withProvider(ProviderType.USER); - ingestionPipelineRepository.setFullyQualifiedName(dataInsightPipeline); - ingestionPipelineRepository.initializeEntity(dataInsightPipeline); - - // Add Ingestion Pipeline to Application - collectionDAO - .relationshipDAO() - .insert( - this.getApp().getId(), - dataInsightPipeline.getId(), - Entity.APPLICATION, - Entity.INGESTION_PIPELINE, - Relationship.HAS.ordinal()); - } - } -} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/apps/NativeApplication.java b/openmetadata-service/src/main/java/org/openmetadata/service/apps/NativeApplication.java index c8f90ae2dfb5..e0ed3e228746 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/apps/NativeApplication.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/apps/NativeApplication.java @@ -9,11 +9,9 @@ public interface NativeApplication extends Job { void init(App app, CollectionDAO dao, SearchRepository searchRepository); - void triggerOnDemand(); - - void scheduleInternal(); + void install(); - void initializeExternalApp(); + void triggerOnDemand(); void configure(); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/insights/DataInsightsApp.java b/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/insights/DataInsightsApp.java index cca449c5316b..936800b5feaa 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/insights/DataInsightsApp.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/insights/DataInsightsApp.java @@ -1,14 +1,10 @@ package org.openmetadata.service.apps.bundles.insights; -import static org.openmetadata.service.exception.CatalogExceptionMessage.INVALID_APP_TYPE; - import com.cronutils.model.Cron; import java.util.List; import lombok.extern.slf4j.Slf4j; import org.openmetadata.schema.api.services.ingestionPipelines.CreateIngestionPipeline; import org.openmetadata.schema.entity.app.App; -import org.openmetadata.schema.entity.app.AppType; -import org.openmetadata.schema.entity.app.ScheduleType; import org.openmetadata.schema.entity.services.ServiceType; import org.openmetadata.schema.entity.services.ingestionPipelines.AirflowConfig; import org.openmetadata.schema.entity.services.ingestionPipelines.IngestionPipeline; @@ -42,71 +38,75 @@ public void init(App app, CollectionDAO dao, SearchRepository searchRepository) } @Override - public void initializeExternalApp() { - if (getApp().getAppType() == AppType.External && getApp().getScheduleType().equals(ScheduleType.Scheduled)) { - IngestionPipelineRepository ingestionPipelineRepository = - (IngestionPipelineRepository) Entity.getEntityRepository(Entity.INGESTION_PIPELINE); + public void install() { + IngestionPipelineRepository ingestionPipelineRepository = + (IngestionPipelineRepository) Entity.getEntityRepository(Entity.INGESTION_PIPELINE); + + try { + bindExistingIngestionToApplication(ingestionPipelineRepository); + } catch (EntityNotFoundException ex) { + createAndBindIngestionPipeline(ingestionPipelineRepository); + } + } + + private void bindExistingIngestionToApplication(IngestionPipelineRepository ingestionPipelineRepository) { + // Check if the Pipeline Already Exists + String fqn = FullyQualifiedName.add(SERVICE_NAME, INGESTION_PIPELINE_NAME); + IngestionPipeline storedPipeline = + ingestionPipelineRepository.getByName(null, fqn, ingestionPipelineRepository.getFields("id")); - try { - // Check if the Pipeline Already Exists - String fqn = FullyQualifiedName.add(SERVICE_NAME, INGESTION_PIPELINE_NAME); - IngestionPipeline storedPipeline = - ingestionPipelineRepository.getByName(null, fqn, ingestionPipelineRepository.getFields("id")); + // Init Application Code for Some Initialization + List records = + collectionDAO + .relationshipDAO() + .findTo(getApp().getId(), Entity.APPLICATION, Relationship.HAS.ordinal(), Entity.INGESTION_PIPELINE); - // Init Application Code for Some Initialization - List records = - collectionDAO - .relationshipDAO() - .findTo(getApp().getId(), Entity.APPLICATION, Relationship.HAS.ordinal(), Entity.INGESTION_PIPELINE); + if (records.isEmpty()) { + // Add Ingestion Pipeline to Application + collectionDAO + .relationshipDAO() + .insert( + getApp().getId(), + storedPipeline.getId(), + Entity.APPLICATION, + Entity.INGESTION_PIPELINE, + Relationship.HAS.ordinal()); + } + } - if (records.isEmpty()) { - // Add Ingestion Pipeline to Application - collectionDAO - .relationshipDAO() - .insert( - getApp().getId(), - storedPipeline.getId(), - Entity.APPLICATION, - Entity.INGESTION_PIPELINE, - Relationship.HAS.ordinal()); - } - } catch (EntityNotFoundException ex) { - // Pipeline needs to be created - EntityRepository serviceRepository = Entity.getServiceEntityRepository(ServiceType.fromValue(SERVICE_TYPE)); - EntityReference service = - serviceRepository.getByName(null, SERVICE_NAME, serviceRepository.getFields("id")).getEntityReference(); + private void createAndBindIngestionPipeline(IngestionPipelineRepository ingestionPipelineRepository) { + // Pipeline needs to be created + EntityRepository serviceRepository = Entity.getServiceEntityRepository(ServiceType.fromValue(SERVICE_TYPE)); + EntityReference service = + serviceRepository.getByName(null, SERVICE_NAME, serviceRepository.getFields("id")).getEntityReference(); - Cron quartzCron = getCronParser().parse(getApp().getAppSchedule().getCronExpression()); + Cron quartzCron = getCronParser().parse(getApp().getAppSchedule().getCronExpression()); - CreateIngestionPipeline createPipelineRequest = - new CreateIngestionPipeline() - .withName(INGESTION_PIPELINE_NAME) - .withDisplayName(INGESTION_PIPELINE_NAME) - .withDescription(PIPELINE_DESCRIPTION) - .withPipelineType(PipelineType.DATA_INSIGHT) - .withSourceConfig(new SourceConfig().withConfig(new DataInsightPipeline())) - .withAirflowConfig(new AirflowConfig().withScheduleInterval(getCronMapper().map(quartzCron).asString())) - .withService(service); + CreateIngestionPipeline createPipelineRequest = + new CreateIngestionPipeline() + .withName(INGESTION_PIPELINE_NAME) + .withDisplayName(INGESTION_PIPELINE_NAME) + .withDescription(PIPELINE_DESCRIPTION) + .withPipelineType(PipelineType.DATA_INSIGHT) + .withSourceConfig(new SourceConfig().withConfig(new DataInsightPipeline())) + .withAirflowConfig(new AirflowConfig().withScheduleInterval(getCronMapper().map(quartzCron).asString())) + .withService(service); - // Get Pipeline - IngestionPipeline dataInsightPipeline = - getIngestionPipeline(createPipelineRequest, String.format("%sBot", getApp().getName()), "admin") - .withProvider(ProviderType.USER); - ingestionPipelineRepository.setFullyQualifiedName(dataInsightPipeline); - ingestionPipelineRepository.initializeEntity(dataInsightPipeline); + // Get Pipeline + IngestionPipeline dataInsightPipeline = + getIngestionPipeline(createPipelineRequest, String.format("%sBot", getApp().getName()), "admin") + .withProvider(ProviderType.USER); + ingestionPipelineRepository.setFullyQualifiedName(dataInsightPipeline); + ingestionPipelineRepository.initializeEntity(dataInsightPipeline); - // Add Ingestion Pipeline to Application - collectionDAO - .relationshipDAO() - .insert( - getApp().getId(), - dataInsightPipeline.getId(), - Entity.APPLICATION, - Entity.INGESTION_PIPELINE, - Relationship.HAS.ordinal()); - } - } else { - throw new IllegalArgumentException(INVALID_APP_TYPE); - } + // Add Ingestion Pipeline to Application + collectionDAO + .relationshipDAO() + .insert( + getApp().getId(), + dataInsightPipeline.getId(), + Entity.APPLICATION, + Entity.INGESTION_PIPELINE, + Relationship.HAS.ordinal()); } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/apps/AppResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/apps/AppResource.java index ce1c39b6fb91..048d0cc0c78f 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/apps/AppResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/apps/AppResource.java @@ -131,7 +131,7 @@ public void initialize(OpenMetadataApplicationConfig config) { // Schedule if (app != null && app.getScheduleType().equals(ScheduleType.Scheduled)) { - ApplicationHandler.scheduleApplication(app, Entity.getCollectionDAO(), searchRepository); + ApplicationHandler.installApplication(app, Entity.getCollectionDAO(), searchRepository); } } catch (Exception ex) { @@ -473,7 +473,7 @@ public Response create(@Context UriInfo uriInfo, @Context SecurityContext securi app.setOpenMetadataServerConnection( new OpenMetadataConnectionBuilder(openMetadataApplicationConfig, app.getBot().getName()).build()); if (app.getScheduleType().equals(ScheduleType.Scheduled)) { - ApplicationHandler.scheduleApplication(app, Entity.getCollectionDAO(), searchRepository); + ApplicationHandler.installApplication(app, Entity.getCollectionDAO(), searchRepository); ApplicationHandler.configureApplication(app, Entity.getCollectionDAO(), searchRepository); } // We don't want to store this information @@ -525,7 +525,7 @@ public Response createOrUpdate( App app = getApplication(definition, create, securityContext.getUserPrincipal().getName()); AppScheduler.getInstance().deleteScheduledApplication(app); if (app.getScheduleType().equals(ScheduleType.Scheduled)) { - ApplicationHandler.scheduleApplication(app, Entity.getCollectionDAO(), searchRepository); + ApplicationHandler.installApplication(app, Entity.getCollectionDAO(), searchRepository); } return createOrUpdate(uriInfo, securityContext, app); } @@ -597,7 +597,7 @@ public Response restoreApp( if (response.getStatus() == Response.Status.OK.getStatusCode()) { App app = (App) response.getEntity(); if (app.getScheduleType().equals(ScheduleType.Scheduled)) { - ApplicationHandler.scheduleApplication(app, Entity.getCollectionDAO(), searchRepository); + ApplicationHandler.installApplication(app, Entity.getCollectionDAO(), searchRepository); } } return response; @@ -622,7 +622,7 @@ public Response scheduleApplication( @Context SecurityContext securityContext) { App app = repository.getByName(uriInfo, name, new EntityUtil.Fields(repository.getAllowedFields())); if (app.getScheduleType().equals(ScheduleType.Scheduled)) { - ApplicationHandler.scheduleApplication(app, repository.getDaoCollection(), searchRepository); + ApplicationHandler.installApplication(app, repository.getDaoCollection(), searchRepository); return Response.status(Response.Status.OK).entity("App is Scheduled.").build(); } throw new IllegalArgumentException("App is not of schedule type Scheduled."); @@ -721,7 +721,7 @@ public Response deployApplicationFlow( EntityUtil.Fields fields = getFields(String.format("%s,bot,pipelines", FIELD_OWNER)); App app = repository.getByName(uriInfo, name, fields); if (app.getAppType().equals(AppType.Internal)) { - ApplicationHandler.scheduleApplication(app, Entity.getCollectionDAO(), searchRepository); + ApplicationHandler.installApplication(app, Entity.getCollectionDAO(), searchRepository); return Response.status(Response.Status.OK).entity("Application Deployed").build(); } else { if (!app.getPipelines().isEmpty()) { diff --git a/openmetadata-service/src/main/resources/json/data/appMarketPlaceDefinition/AutoTaggerApplication.json b/openmetadata-service/src/main/resources/json/data/appMarketPlaceDefinition/AutoTaggerApplication.json index 96dd049a2c22..66c8f9dbf4b1 100644 --- a/openmetadata-service/src/main/resources/json/data/appMarketPlaceDefinition/AutoTaggerApplication.json +++ b/openmetadata-service/src/main/resources/json/data/appMarketPlaceDefinition/AutoTaggerApplication.json @@ -13,7 +13,7 @@ "supportEmail": "support@getcollate.io", "scheduleType": "Scheduled", "permission": "All", - "className": "org.openmetadata.service.apps.ExternalApplicationHandler", + "className": "org.openmetadata.service.apps.AbstractNativeApplication", "sourcePythonClass": "metadata.applications.auto_tagger.AutoTaggerApp", "runtime": { "enabled": "true"