configure api service metadata

ingestion/src/metadata/ingestion/api/parser.py

@@ -18,6 +18,10 @@
 from metadata.generated.schema.entity.automations.workflow import (
  Workflow as AutomationWorkflow,
 )
+from metadata.generated.schema.entity.services.apiService import (
+ ApiServiceConnection,
+ APIServiceType,
+)
 from metadata.generated.schema.entity.services.dashboardService import (
  DashboardConnection,
  DashboardServiceType,
@@ -53,6 +57,10 @@
  StorageConnection,
  StorageServiceType,
 )
+from metadata.generated.schema.metadataIngestion.apiServiceMetadataPipeline import (
+ ApiMetadataConfigType,
+ ApiServiceMetadataPipeline,
+)
 from metadata.generated.schema.metadataIngestion.dashboardServiceMetadataPipeline import (
  DashboardMetadataConfigType,
  DashboardServiceMetadataPipeline,
@@ -127,6 +135,7 @@
 # Build a service type map dynamically from JSON Schema covered types
 SERVICE_TYPE_MAP = {
  "Backend": PipelineConnection, # For Airflow backend
+ **{service: ApiServiceConnection for service in APIServiceType.__members__},
  **{service: DatabaseConnection for service in DatabaseServiceType.__members__},
  **{service: DashboardConnection for service in DashboardServiceType.__members__},
  **{service: MessagingConnection for service in MessagingServiceType.__members__},
@@ -138,6 +147,7 @@
 }
 
 SOURCE_CONFIG_CLASS_MAP = {
+ ApiMetadataConfigType.ApiMetadata.value: ApiServiceMetadataPipeline,
  DashboardMetadataConfigType.DashboardMetadata.value: DashboardServiceMetadataPipeline,
  ProfilerConfigType.Profiler.value: DatabaseServiceProfilerPipeline,
  DatabaseUsageConfigType.DatabaseUsage.value: DatabaseServiceQueryUsagePipeline,
@@ -173,6 +183,7 @@ class InvalidWorkflowException(Exception):
 def get_service_type(
  source_type: str,
 ) -> Union[
+ Type[ApiServiceConnection],
  Type[DashboardConnection],
  Type[DatabaseConnection],
  Type[MessagingConnection],
@@ -196,6 +207,7 @@ def get_service_type(
 def get_source_config_class(
  source_config_type: str,
 ) -> Union[
+ Type[ApiServiceMetadataPipeline],
  Type[DashboardServiceMetadataPipeline],
  Type[DatabaseServiceProfilerPipeline],
  Type[DatabaseServiceQueryUsagePipeline],
@@ -221,6 +233,7 @@ def get_source_config_class(
 def get_connection_class(
  source_type: str,
  service_type: Union[
+ Type[ApiServiceConnection],
  Type[DashboardConnection],
  Type[DatabaseConnection],
  Type[MessagingConnection],

ingestion/src/metadata/ingestion/models/custom_types.py

@@ -14,6 +14,7 @@
 
 from typing import NewType, Union
 
+from metadata.generated.schema.entity.services.apiService import ApiService
 from metadata.generated.schema.entity.services.dashboardService import DashboardService
 from metadata.generated.schema.entity.services.databaseService import DatabaseService
 from metadata.generated.schema.entity.services.messagingService import MessagingService
@@ -25,6 +26,7 @@
 ServiceWithConnectionType = NewType(
  "ServiceWithConnectionType",
  Union[
+ ApiService,
  DashboardService,
  DatabaseService,
  MessagingService,

ingestion/src/metadata/utils/class_helper.py

@@ -22,6 +22,9 @@
  PipelineType,
 )
 from metadata.generated.schema.entity.services.serviceType import ServiceType
+from metadata.generated.schema.metadataIngestion.apiServiceMetadataPipeline import (
+ ApiServiceMetadataPipeline,
+)
 from metadata.generated.schema.metadataIngestion.dashboardServiceMetadataPipeline import (
  DashboardServiceMetadataPipeline,
 )
@@ -65,6 +68,7 @@
 from metadata.generated.schema.metadataIngestion.workflow import SourceConfig
 
 SERVICE_TYPE_REF = {
+ ServiceType.API.value: "apiService",
  ServiceType.Database.value: "databaseService",
  ServiceType.Dashboard.value: "dashboardService",
  ServiceType.Pipeline.value: "pipelineService",
@@ -76,6 +80,7 @@
 }
 
 SOURCE_CONFIG_TYPE_INGESTION = {
+ ApiServiceMetadataPipeline.__name__: PipelineType.metadata,
  DatabaseServiceMetadataPipeline.__name__: PipelineType.metadata,
  DatabaseServiceQueryUsagePipeline.__name__: PipelineType.usage,
  DatabaseServiceQueryLineagePipeline.__name__: PipelineType.lineage,

ingestion/src/metadata/utils/constants.py

@@ -26,6 +26,7 @@
 from metadata.generated.schema.entity.data.topic import Topic
 from metadata.generated.schema.entity.domains.dataProduct import DataProduct
 from metadata.generated.schema.entity.domains.domain import Domain
+from metadata.generated.schema.entity.services.apiService import ApiService
 from metadata.generated.schema.entity.services.dashboardService import DashboardService
 from metadata.generated.schema.entity.services.databaseService import DatabaseService
 from metadata.generated.schema.entity.services.messagingService import MessagingService
@@ -76,6 +77,7 @@
  # Service Entities
  "databaseService": DatabaseService,
  "messagingService": MessagingService,
+ "apiService": ApiService,
  "dashboardService": DashboardService,
  "pipelineService": PipelineService,
  "storageService": StorageService,

ingestion/src/metadata/utils/fqn.py

@@ -27,6 +27,7 @@
 from metadata.generated.antlr.FqnLexer import FqnLexer
 from metadata.generated.antlr.FqnParser import FqnParser
 from metadata.generated.schema.entity.classification.tag import Tag
+from metadata.generated.schema.entity.data.apiCollection import APICollection
 from metadata.generated.schema.entity.data.chart import Chart
 from metadata.generated.schema.entity.data.container import Container
 from metadata.generated.schema.entity.data.dashboard import Dashboard
@@ -250,6 +251,20 @@ def _(
  return _build(service_name, dashboard_name)
 
 
+@fqn_build_registry.add(APICollection)
+def _(
+ _: Optional[OpenMetadata], # ES Index not necessary for dashboard FQN building
+ *,
+ service_name: str,
+ api_collection_name: str,
+) -> str:
+ if not service_name or not api_collection_name:
+ raise FQNBuildingException(
+ f"Args should be informed, but got service=`{service_name}`, collection=`{api_collection_name}``"
+ )
+ return _build(service_name, api_collection_name)
+
+
 @fqn_build_registry.add(Chart)
 def _(
  _: Optional[OpenMetadata], # ES Index not necessary for dashboard FQN building

ingestion/tests/unit/test_workflow_parse.py

@@ -42,6 +42,9 @@
  MessagingConnection,
 )
 from metadata.generated.schema.entity.services.metadataService import MetadataConnection
+from metadata.generated.schema.metadataIngestion.apiServiceMetadataPipeline import (
+ ApiServiceMetadataPipeline,
+)
 from metadata.generated.schema.metadataIngestion.dashboardServiceMetadataPipeline import (
  DashboardServiceMetadataPipeline,
 )
@@ -132,6 +135,10 @@ def test_get_source_config_class(self):
  connection = get_source_config_class(source_config_type)
  self.assertEqual(connection, DashboardServiceMetadataPipeline)
 
+ source_config_type = "ApiMetadata"
+ connection = get_source_config_class(source_config_type)
+ self.assertEqual(connection, ApiServiceMetadataPipeline)
+
  def test_parsing_ok(self):
  """
  Test MSSQL JSON Config parsing OK

openmetadata-airflow-apis/openmetadata_managed_apis/workflows/ingestion/common.py

@@ -22,6 +22,7 @@
 from pydantic import ValidationError
 from requests.utils import quote
 
+from metadata.generated.schema.entity.services.apiService import ApiService
 from metadata.generated.schema.entity.services.dashboardService import DashboardService
 from metadata.generated.schema.entity.services.databaseService import DatabaseService
 from metadata.generated.schema.entity.services.messagingService import MessagingService
@@ -71,6 +72,7 @@
 logger = workflow_logger()
 
 ENTITY_CLASS_MAP = {
+ "apiService": ApiService,
  "databaseService": DatabaseService,
  "pipelineService": PipelineService,
  "dashboardService": DashboardService,

openmetadata-service/src/test/java/org/openmetadata/service/resources/services/APIServiceResourceTest.java

@@ -51,7 +51,7 @@ public void setupAPIService(TestInfo test) throws HttpResponseException {
  apiServiceResourceTest
  .createRequest(test)
  .withName("openmetadata")
- .withServiceType(CreateAPIService.ApiServiceType.REST)
+ .withServiceType(CreateAPIService.APIServiceType.REST)
  .withConnection(TestUtils.API_SERVICE_CONNECTION);
 
  APIService omAPIService =
@@ -62,7 +62,7 @@ public void setupAPIService(TestInfo test) throws HttpResponseException {
  sampleAPIServiceResourceTest
  .createRequest(test)
  .withName("sampleAPI")
- .withServiceType(CreateAPIService.ApiServiceType.REST)
+ .withServiceType(CreateAPIService.APIServiceType.REST)
  .withConnection(TestUtils.API_SERVICE_CONNECTION);
  APIService sampleAPIService =
  new APIServiceResourceTest().createEntity(createAPIService, ADMIN_AUTH_HEADERS);
@@ -153,7 +153,7 @@ public APIService putTestConnectionResult(
  public CreateAPIService createRequest(String name) {
  return new CreateAPIService()
  .withName(name)
- .withServiceType(CreateAPIService.ApiServiceType.REST)
+ .withServiceType(CreateAPIService.APIServiceType.REST)
  .withConnection(
  new APIServiceConnection()
  .withConfig(
@@ -211,7 +211,7 @@ public void assertFieldChange(String fieldName, Object expected, Object actual)
  private void validateConnection(
  APIServiceConnection expectedConnection,
  APIServiceConnection actualConnection,
- CreateAPIService.ApiServiceType serviceType) {
+ CreateAPIService.APIServiceType serviceType) {
  if (expectedConnection != null && actualConnection != null) {
  RESTConnection restConnection = (RESTConnection) expectedConnection.getConfig();
  RESTConnection actualESConnection =

openmetadata-spec/src/main/resources/json/schema/api/services/createApiService.json

@@ -21,7 +21,7 @@
  "$ref": "../../type/basic.json#/definitions/markdown"
  },
  "serviceType": {
- "$ref": "../../entity/services/apiService.json#/definitions/apiServiceType"
+ "$ref": "../../entity/services/apiService.json#/definitions/APIServiceType"
  },
  "connection": {
  "$ref": "../../entity/services/apiService.json#/definitions/apiServiceConnection"

openmetadata-spec/src/main/resources/json/schema/entity/data/apiCollection.json

@@ -74,7 +74,7 @@
  },
  "serviceType": {
  "description": "Service type where this API Collection is hosted in.",
- "$ref": "../services/apiService.json#/definitions/apiServiceType"
+ "$ref": "../services/apiService.json#/definitions/APIServiceType"
  },
  "changeDescription": {
  "description": "Change that lead to this version of the entity.",

openmetadata-spec/src/main/resources/json/schema/entity/data/apiEndpoint.json

@@ -136,7 +136,7 @@
  },
  "serviceType": {
  "description": "Service type where this API Collection is hosted in.",
- "$ref": "../services/apiService.json#/definitions/apiServiceType"
+ "$ref": "../services/apiService.json#/definitions/APIServiceType"
  },
  "changeDescription": {
  "description": "Change that lead to this version of the entity.",

openmetadata-spec/src/main/resources/json/schema/entity/services/apiService.json

@@ -10,7 +10,7 @@
  "org.openmetadata.schema.ServiceEntityInterface"
  ],
  "definitions": {
- "apiServiceType": {
+ "APIServiceType": {
  "description": "Type of api service such as REST, Webhook,...",
  "javaInterfaces": [
  "org.openmetadata.schema.EnumInterface"
@@ -41,7 +41,7 @@
  "mask": true,
  "oneOf": [
  {
- "$ref": "connections/apiService/restConnection.json"
+ "$ref": "./connections/apiService/restConnection.json"
  }
  ]
  }
@@ -68,7 +68,7 @@
  },
  "serviceType": {
  "description": "Type of API service such as REST, WEBHOOK..",
- "$ref": "#/definitions/apiServiceType"
+ "$ref": "#/definitions/APIServiceType"
  },
  "description": {
  "description": "Description of a API service instance.",

openmetadata-spec/src/main/resources/json/schema/metadataIngestion/workflow.json

@@ -12,6 +12,9 @@
  "properties": {
  "config": {
  "oneOf": [
+ {
+ "$ref": "apiServiceMetadataPipeline.json"
+ },
  {
  "$ref": "databaseServiceMetadataPipeline.json"
  },

openmetadata-ui/src/main/resources/ui/src/utils/IngestionWorkflowUtils.ts

@@ -17,6 +17,7 @@ import {
  Pipeline,
  PipelineType as WorkflowType,
 } from '../generated/api/services/ingestionPipelines/createIngestionPipeline';
+import apiServiceMetadataPipeline from '../jsons/ingestionSchemas/apiServiceMetadataPipeline.json';
 import dashboardMetadataPipeline from '../jsons/ingestionSchemas/dashboardServiceMetadataPipeline.json';
 import databaseMetadataPipeline from '../jsons/ingestionSchemas/databaseServiceMetadataPipeline.json';
 import databaseProfilerPipeline from '../jsons/ingestionSchemas/databaseServiceProfilerPipeline.json';
@@ -39,6 +40,8 @@ export const getMetadataSchemaByServiceCategory = (
  case ServiceCategory.METADATA_SERVICES:
  case ServiceCategory.DATABASE_SERVICES:
  return databaseMetadataPipeline;
+ case ServiceCategory.API_SERVICES:
+ return apiServiceMetadataPipeline;
  case ServiceCategory.DASHBOARD_SERVICES:
  return dashboardMetadataPipeline;
  case ServiceCategory.MESSAGING_SERVICES: