diff --git a/ingestion/src/metadata/ingestion/api/parser.py b/ingestion/src/metadata/ingestion/api/parser.py index 639c26492bf2..84aa898844cd 100644 --- a/ingestion/src/metadata/ingestion/api/parser.py +++ b/ingestion/src/metadata/ingestion/api/parser.py @@ -18,6 +18,10 @@ from metadata.generated.schema.entity.automations.workflow import ( Workflow as AutomationWorkflow, ) +from metadata.generated.schema.entity.services.apiService import ( + ApiServiceConnection, + APIServiceType, +) from metadata.generated.schema.entity.services.dashboardService import ( DashboardConnection, DashboardServiceType, @@ -53,6 +57,10 @@ StorageConnection, StorageServiceType, ) +from metadata.generated.schema.metadataIngestion.apiServiceMetadataPipeline import ( + ApiMetadataConfigType, + ApiServiceMetadataPipeline, +) from metadata.generated.schema.metadataIngestion.dashboardServiceMetadataPipeline import ( DashboardMetadataConfigType, DashboardServiceMetadataPipeline, @@ -127,6 +135,7 @@ # Build a service type map dynamically from JSON Schema covered types SERVICE_TYPE_MAP = { "Backend": PipelineConnection, # For Airflow backend + **{service: ApiServiceConnection for service in APIServiceType.__members__}, **{service: DatabaseConnection for service in DatabaseServiceType.__members__}, **{service: DashboardConnection for service in DashboardServiceType.__members__}, **{service: MessagingConnection for service in MessagingServiceType.__members__}, @@ -138,6 +147,7 @@ } SOURCE_CONFIG_CLASS_MAP = { + ApiMetadataConfigType.ApiMetadata.value: ApiServiceMetadataPipeline, DashboardMetadataConfigType.DashboardMetadata.value: DashboardServiceMetadataPipeline, ProfilerConfigType.Profiler.value: DatabaseServiceProfilerPipeline, DatabaseUsageConfigType.DatabaseUsage.value: DatabaseServiceQueryUsagePipeline, @@ -173,6 +183,7 @@ class InvalidWorkflowException(Exception): def get_service_type( source_type: str, ) -> Union[ + Type[ApiServiceConnection], Type[DashboardConnection], Type[DatabaseConnection], Type[MessagingConnection], @@ -196,6 +207,7 @@ def get_service_type( def get_source_config_class( source_config_type: str, ) -> Union[ + Type[ApiServiceMetadataPipeline], Type[DashboardServiceMetadataPipeline], Type[DatabaseServiceProfilerPipeline], Type[DatabaseServiceQueryUsagePipeline], @@ -221,6 +233,7 @@ def get_source_config_class( def get_connection_class( source_type: str, service_type: Union[ + Type[ApiServiceConnection], Type[DashboardConnection], Type[DatabaseConnection], Type[MessagingConnection], diff --git a/ingestion/src/metadata/ingestion/models/custom_types.py b/ingestion/src/metadata/ingestion/models/custom_types.py index 6de0ee04e15f..11da3c1e123a 100644 --- a/ingestion/src/metadata/ingestion/models/custom_types.py +++ b/ingestion/src/metadata/ingestion/models/custom_types.py @@ -14,6 +14,7 @@ from typing import NewType, Union +from metadata.generated.schema.entity.services.apiService import ApiService from metadata.generated.schema.entity.services.dashboardService import DashboardService from metadata.generated.schema.entity.services.databaseService import DatabaseService from metadata.generated.schema.entity.services.messagingService import MessagingService @@ -25,6 +26,7 @@ ServiceWithConnectionType = NewType( "ServiceWithConnectionType", Union[ + ApiService, DashboardService, DatabaseService, MessagingService, diff --git a/ingestion/src/metadata/utils/class_helper.py b/ingestion/src/metadata/utils/class_helper.py index 8157e252d89a..281afb854db6 100644 --- a/ingestion/src/metadata/utils/class_helper.py +++ b/ingestion/src/metadata/utils/class_helper.py @@ -22,6 +22,9 @@ PipelineType, ) from metadata.generated.schema.entity.services.serviceType import ServiceType +from metadata.generated.schema.metadataIngestion.apiServiceMetadataPipeline import ( + ApiServiceMetadataPipeline, +) from metadata.generated.schema.metadataIngestion.dashboardServiceMetadataPipeline import ( DashboardServiceMetadataPipeline, ) @@ -65,6 +68,7 @@ from metadata.generated.schema.metadataIngestion.workflow import SourceConfig SERVICE_TYPE_REF = { + ServiceType.API.value: "apiService", ServiceType.Database.value: "databaseService", ServiceType.Dashboard.value: "dashboardService", ServiceType.Pipeline.value: "pipelineService", @@ -76,6 +80,7 @@ } SOURCE_CONFIG_TYPE_INGESTION = { + ApiServiceMetadataPipeline.__name__: PipelineType.metadata, DatabaseServiceMetadataPipeline.__name__: PipelineType.metadata, DatabaseServiceQueryUsagePipeline.__name__: PipelineType.usage, DatabaseServiceQueryLineagePipeline.__name__: PipelineType.lineage, diff --git a/ingestion/src/metadata/utils/constants.py b/ingestion/src/metadata/utils/constants.py index ebc15825276d..fcf212d04f8e 100644 --- a/ingestion/src/metadata/utils/constants.py +++ b/ingestion/src/metadata/utils/constants.py @@ -26,6 +26,7 @@ from metadata.generated.schema.entity.data.topic import Topic from metadata.generated.schema.entity.domains.dataProduct import DataProduct from metadata.generated.schema.entity.domains.domain import Domain +from metadata.generated.schema.entity.services.apiService import ApiService from metadata.generated.schema.entity.services.dashboardService import DashboardService from metadata.generated.schema.entity.services.databaseService import DatabaseService from metadata.generated.schema.entity.services.messagingService import MessagingService @@ -76,6 +77,7 @@ # Service Entities "databaseService": DatabaseService, "messagingService": MessagingService, + "apiService": ApiService, "dashboardService": DashboardService, "pipelineService": PipelineService, "storageService": StorageService, diff --git a/ingestion/src/metadata/utils/fqn.py b/ingestion/src/metadata/utils/fqn.py index 25ebf1e33029..05fa11e08bd3 100644 --- a/ingestion/src/metadata/utils/fqn.py +++ b/ingestion/src/metadata/utils/fqn.py @@ -27,6 +27,7 @@ from metadata.generated.antlr.FqnLexer import FqnLexer from metadata.generated.antlr.FqnParser import FqnParser from metadata.generated.schema.entity.classification.tag import Tag +from metadata.generated.schema.entity.data.apiCollection import APICollection from metadata.generated.schema.entity.data.chart import Chart from metadata.generated.schema.entity.data.container import Container from metadata.generated.schema.entity.data.dashboard import Dashboard @@ -250,6 +251,20 @@ def _( return _build(service_name, dashboard_name) +@fqn_build_registry.add(APICollection) +def _( + _: Optional[OpenMetadata], # ES Index not necessary for dashboard FQN building + *, + service_name: str, + api_collection_name: str, +) -> str: + if not service_name or not api_collection_name: + raise FQNBuildingException( + f"Args should be informed, but got service=`{service_name}`, collection=`{api_collection_name}``" + ) + return _build(service_name, api_collection_name) + + @fqn_build_registry.add(Chart) def _( _: Optional[OpenMetadata], # ES Index not necessary for dashboard FQN building diff --git a/ingestion/tests/unit/test_workflow_parse.py b/ingestion/tests/unit/test_workflow_parse.py index 5803fd84f4df..375dfb717e46 100644 --- a/ingestion/tests/unit/test_workflow_parse.py +++ b/ingestion/tests/unit/test_workflow_parse.py @@ -42,6 +42,9 @@ MessagingConnection, ) from metadata.generated.schema.entity.services.metadataService import MetadataConnection +from metadata.generated.schema.metadataIngestion.apiServiceMetadataPipeline import ( + ApiServiceMetadataPipeline, +) from metadata.generated.schema.metadataIngestion.dashboardServiceMetadataPipeline import ( DashboardServiceMetadataPipeline, ) @@ -132,6 +135,10 @@ def test_get_source_config_class(self): connection = get_source_config_class(source_config_type) self.assertEqual(connection, DashboardServiceMetadataPipeline) + source_config_type = "ApiMetadata" + connection = get_source_config_class(source_config_type) + self.assertEqual(connection, ApiServiceMetadataPipeline) + def test_parsing_ok(self): """ Test MSSQL JSON Config parsing OK diff --git a/openmetadata-airflow-apis/openmetadata_managed_apis/workflows/ingestion/common.py b/openmetadata-airflow-apis/openmetadata_managed_apis/workflows/ingestion/common.py index f9f1814f8a36..c3c07433e5a2 100644 --- a/openmetadata-airflow-apis/openmetadata_managed_apis/workflows/ingestion/common.py +++ b/openmetadata-airflow-apis/openmetadata_managed_apis/workflows/ingestion/common.py @@ -22,6 +22,7 @@ from pydantic import ValidationError from requests.utils import quote +from metadata.generated.schema.entity.services.apiService import ApiService from metadata.generated.schema.entity.services.dashboardService import DashboardService from metadata.generated.schema.entity.services.databaseService import DatabaseService from metadata.generated.schema.entity.services.messagingService import MessagingService @@ -71,6 +72,7 @@ logger = workflow_logger() ENTITY_CLASS_MAP = { + "apiService": ApiService, "databaseService": DatabaseService, "pipelineService": PipelineService, "dashboardService": DashboardService, diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/resources/services/APIServiceResourceTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/resources/services/APIServiceResourceTest.java index a021eb97cd89..a1efa41da3af 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/resources/services/APIServiceResourceTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/resources/services/APIServiceResourceTest.java @@ -51,7 +51,7 @@ public void setupAPIService(TestInfo test) throws HttpResponseException { apiServiceResourceTest .createRequest(test) .withName("openmetadata") - .withServiceType(CreateAPIService.ApiServiceType.REST) + .withServiceType(CreateAPIService.APIServiceType.REST) .withConnection(TestUtils.API_SERVICE_CONNECTION); APIService omAPIService = @@ -62,7 +62,7 @@ public void setupAPIService(TestInfo test) throws HttpResponseException { sampleAPIServiceResourceTest .createRequest(test) .withName("sampleAPI") - .withServiceType(CreateAPIService.ApiServiceType.REST) + .withServiceType(CreateAPIService.APIServiceType.REST) .withConnection(TestUtils.API_SERVICE_CONNECTION); APIService sampleAPIService = new APIServiceResourceTest().createEntity(createAPIService, ADMIN_AUTH_HEADERS); @@ -153,7 +153,7 @@ public APIService putTestConnectionResult( public CreateAPIService createRequest(String name) { return new CreateAPIService() .withName(name) - .withServiceType(CreateAPIService.ApiServiceType.REST) + .withServiceType(CreateAPIService.APIServiceType.REST) .withConnection( new APIServiceConnection() .withConfig( @@ -211,7 +211,7 @@ public void assertFieldChange(String fieldName, Object expected, Object actual) private void validateConnection( APIServiceConnection expectedConnection, APIServiceConnection actualConnection, - CreateAPIService.ApiServiceType serviceType) { + CreateAPIService.APIServiceType serviceType) { if (expectedConnection != null && actualConnection != null) { RESTConnection restConnection = (RESTConnection) expectedConnection.getConfig(); RESTConnection actualESConnection = diff --git a/openmetadata-spec/src/main/resources/json/schema/api/services/createApiService.json b/openmetadata-spec/src/main/resources/json/schema/api/services/createApiService.json index 9a0400c7a071..db1af1eb6e07 100644 --- a/openmetadata-spec/src/main/resources/json/schema/api/services/createApiService.json +++ b/openmetadata-spec/src/main/resources/json/schema/api/services/createApiService.json @@ -21,7 +21,7 @@ "$ref": "../../type/basic.json#/definitions/markdown" }, "serviceType": { - "$ref": "../../entity/services/apiService.json#/definitions/apiServiceType" + "$ref": "../../entity/services/apiService.json#/definitions/APIServiceType" }, "connection": { "$ref": "../../entity/services/apiService.json#/definitions/apiServiceConnection" diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/data/apiCollection.json b/openmetadata-spec/src/main/resources/json/schema/entity/data/apiCollection.json index f39bf6481ad9..370f9ad0ce13 100644 --- a/openmetadata-spec/src/main/resources/json/schema/entity/data/apiCollection.json +++ b/openmetadata-spec/src/main/resources/json/schema/entity/data/apiCollection.json @@ -74,7 +74,7 @@ }, "serviceType": { "description": "Service type where this API Collection is hosted in.", - "$ref": "../services/apiService.json#/definitions/apiServiceType" + "$ref": "../services/apiService.json#/definitions/APIServiceType" }, "changeDescription": { "description": "Change that lead to this version of the entity.", diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/data/apiEndpoint.json b/openmetadata-spec/src/main/resources/json/schema/entity/data/apiEndpoint.json index dcbb280247ff..8c028fafaf28 100644 --- a/openmetadata-spec/src/main/resources/json/schema/entity/data/apiEndpoint.json +++ b/openmetadata-spec/src/main/resources/json/schema/entity/data/apiEndpoint.json @@ -136,7 +136,7 @@ }, "serviceType": { "description": "Service type where this API Collection is hosted in.", - "$ref": "../services/apiService.json#/definitions/apiServiceType" + "$ref": "../services/apiService.json#/definitions/APIServiceType" }, "changeDescription": { "description": "Change that lead to this version of the entity.", diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/services/apiService.json b/openmetadata-spec/src/main/resources/json/schema/entity/services/apiService.json index 875794f96ae5..4619f680a1b8 100644 --- a/openmetadata-spec/src/main/resources/json/schema/entity/services/apiService.json +++ b/openmetadata-spec/src/main/resources/json/schema/entity/services/apiService.json @@ -10,7 +10,7 @@ "org.openmetadata.schema.ServiceEntityInterface" ], "definitions": { - "apiServiceType": { + "APIServiceType": { "description": "Type of api service such as REST, Webhook,...", "javaInterfaces": [ "org.openmetadata.schema.EnumInterface" @@ -41,7 +41,7 @@ "mask": true, "oneOf": [ { - "$ref": "connections/apiService/restConnection.json" + "$ref": "./connections/apiService/restConnection.json" } ] } @@ -68,7 +68,7 @@ }, "serviceType": { "description": "Type of API service such as REST, WEBHOOK..", - "$ref": "#/definitions/apiServiceType" + "$ref": "#/definitions/APIServiceType" }, "description": { "description": "Description of a API service instance.", diff --git a/openmetadata-spec/src/main/resources/json/schema/metadataIngestion/workflow.json b/openmetadata-spec/src/main/resources/json/schema/metadataIngestion/workflow.json index 78b67926eee7..41c512e0d748 100644 --- a/openmetadata-spec/src/main/resources/json/schema/metadataIngestion/workflow.json +++ b/openmetadata-spec/src/main/resources/json/schema/metadataIngestion/workflow.json @@ -12,6 +12,9 @@ "properties": { "config": { "oneOf": [ + { + "$ref": "apiServiceMetadataPipeline.json" + }, { "$ref": "databaseServiceMetadataPipeline.json" }, diff --git a/openmetadata-ui/src/main/resources/ui/src/utils/IngestionWorkflowUtils.ts b/openmetadata-ui/src/main/resources/ui/src/utils/IngestionWorkflowUtils.ts index b2e1f8be33d5..ec38b82d1dfa 100644 --- a/openmetadata-ui/src/main/resources/ui/src/utils/IngestionWorkflowUtils.ts +++ b/openmetadata-ui/src/main/resources/ui/src/utils/IngestionWorkflowUtils.ts @@ -17,6 +17,7 @@ import { Pipeline, PipelineType as WorkflowType, } from '../generated/api/services/ingestionPipelines/createIngestionPipeline'; +import apiServiceMetadataPipeline from '../jsons/ingestionSchemas/apiServiceMetadataPipeline.json'; import dashboardMetadataPipeline from '../jsons/ingestionSchemas/dashboardServiceMetadataPipeline.json'; import databaseMetadataPipeline from '../jsons/ingestionSchemas/databaseServiceMetadataPipeline.json'; import databaseProfilerPipeline from '../jsons/ingestionSchemas/databaseServiceProfilerPipeline.json'; @@ -39,6 +40,8 @@ export const getMetadataSchemaByServiceCategory = ( case ServiceCategory.METADATA_SERVICES: case ServiceCategory.DATABASE_SERVICES: return databaseMetadataPipeline; + case ServiceCategory.API_SERVICES: + return apiServiceMetadataPipeline; case ServiceCategory.DASHBOARD_SERVICES: return dashboardMetadataPipeline; case ServiceCategory.MESSAGING_SERVICES: