diff --git a/ingestion/src/metadata/ingestion/source/api/rest/connection.py b/ingestion/src/metadata/ingestion/source/api/rest/connection.py index 4f51011ca047..bdfad1db9869 100644 --- a/ingestion/src/metadata/ingestion/source/api/rest/connection.py +++ b/ingestion/src/metadata/ingestion/source/api/rest/connection.py @@ -52,10 +52,10 @@ def test_connection( """ def custom_url_exec(): - if client.status_code == 200: + if client.headers.get("content-type") == "application/json": return [] raise SchemaURLError( - f"Failed to get access to provided schema url. Please check with url and its permissions" + f"Failed to parse JSON schema url. Please check if provided url is valid JSON schema." ) test_fn = {"CheckURL": custom_url_exec} diff --git a/ingestion/src/metadata/ingestion/source/api/rest/metadata.py b/ingestion/src/metadata/ingestion/source/api/rest/metadata.py index 5c4d7f07f030..f0f22bad894b 100644 --- a/ingestion/src/metadata/ingestion/source/api/rest/metadata.py +++ b/ingestion/src/metadata/ingestion/source/api/rest/metadata.py @@ -43,6 +43,7 @@ from metadata.ingestion.api.steps import InvalidSourceException from metadata.ingestion.ometa.ometa_api import OpenMetadata from metadata.ingestion.source.api.api_service import ApiServiceSource +from metadata.ingestion.source.api.rest.models import RESTCollection, RESTEndpoint from metadata.utils import fqn from metadata.utils.logger import ingestion_logger @@ -93,24 +94,40 @@ def yield_api_collection( ) -> Iterable[Either[CreateAPICollectionRequest]]: """Method to return api collection Entities""" try: - collection_request = CreateAPICollectionRequest( + rest_collection = RESTCollection( name=EntityName(collection.get("name")), - displayName=collection.get("name"), + display_name=collection.get("name"), description=Markdown(collection.get("description")) if collection.get("description") else None, service=FullyQualifiedEntityName(self.context.get().api_service), - endpointURL=AnyUrl( + endpoint_url=AnyUrl( f"{self.config.serviceConnection.root.config.openAPISchemaURL}#tag/{collection.get('name')}" ), ) + except Exception as err: + yield Either( + left=StackTraceError( + name=collection.get("name"), + error=f"Error parsing collection: {err}", + stackTrace=traceback.format_exc(), + ) + ) + try: + collection_request = CreateAPICollectionRequest( + name=rest_collection.name, + displayName=rest_collection.display_name, + description=rest_collection.description, + service=rest_collection.service, + endpointURL=rest_collection.endpoint_url, + ) yield Either(right=collection_request) self.register_record(collection_request=collection_request) except Exception as exc: yield Either( left=StackTraceError( name=collection.get("name"), - error=f"Error creating collection: {exc}", + error=f"Error creating api collection request: {exc}", stackTrace=traceback.format_exc(), ) ) @@ -122,34 +139,52 @@ def yield_api_endpoint( filtered_endpoints = self._filter_collection_endpoints(collection) or {} for path, methods in filtered_endpoints.items(): for method_type, info in methods.items(): + try: + api_endpoint = RESTEndpoint( + name=EntityName(info.get("operationId")), + description=Markdown(info.get("description")) + if info.get("description") + else None, + api_collection=FullyQualifiedEntityName( + fqn.build( + self.metadata, + entity_type=APICollection, + service_name=self.context.get().api_service, + api_collection_name=collection.get("name"), + ) + ), + endpoint_url=AnyUrl( + f"{self.config.serviceConnection.root.config.openAPISchemaURL}#operation/{info.get('operationId')}", + ), + request_method=self._get_api_request_method(method_type), + request_schema=self._get_request_schema(info), + response_schema=self._get_response_schema(info), + ) + except Exception as err: + yield Either( + left=StackTraceError( + name=collection.get("name"), + error=f"Error parsing api endpoint: {err}", + stackTrace=traceback.format_exc(), + ) + ) try: yield Either( right=CreateAPIEndpointRequest( - name=EntityName(info.get("operationId")), - description=Markdown(info.get("description")) - if info.get("description") - else None, - apiCollection=FullyQualifiedEntityName( - fqn.build( - self.metadata, - entity_type=APICollection, - service_name=self.context.get().api_service, - api_collection_name=collection.get("name"), - ) - ), - endpointURL=AnyUrl( - f"{self.config.serviceConnection.root.config.openAPISchemaURL}#operation/{info.get('operationId')}", - ), - requestMethod=self._get_api_request_method(method_type), - requestSchema=self._get_request_schema(info), - responseSchema=self._get_response_schema(info), + name=api_endpoint.name, + description=api_endpoint.description, + apiCollection=api_endpoint.api_collection, + endpointURL=api_endpoint.endpoint_url, + requestMethod=api_endpoint.request_method, + requestSchema=api_endpoint.request_schema, + responseSchema=api_endpoint.response_schema, ) ) except Exception as exc: # pylint: disable=broad-except yield Either( left=StackTraceError( name=collection.get("name"), - error=f"Error creating API Endpoint [{info.get('operationId')}]: {exc}", + error=f"Error creating API Endpoint request [{info.get('operationId')}]: {exc}", stackTrace=traceback.format_exc(), ) ) diff --git a/ingestion/src/metadata/ingestion/source/api/rest/models.py b/ingestion/src/metadata/ingestion/source/api/rest/models.py new file mode 100644 index 000000000000..9d1ee8f1a276 --- /dev/null +++ b/ingestion/src/metadata/ingestion/source/api/rest/models.py @@ -0,0 +1,43 @@ +# Copyright 2024 Collate +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +OpenAPI REST API Models +""" +from typing import Optional + +from pydantic import AnyUrl, BaseModel + +from metadata.generated.schema.entity.data.apiEndpoint import ApiRequestMethod +from metadata.generated.schema.type import basic +from metadata.generated.schema.type.apiSchema import APISchema + + +class RESTCollection(BaseModel): + """REST colleciton model""" + + name: basic.EntityName + display_name: Optional[str] = None + description: Optional[basic.Markdown] = None + endpoint_url: Optional[AnyUrl] = None + service: basic.FullyQualifiedEntityName + + +class RESTEndpoint(BaseModel): + """REST endpoint model""" + + name: basic.EntityName + display_name: Optional[str] = None + description: Optional[basic.Markdown] = None + api_collection: basic.FullyQualifiedEntityName + endpoint_url: AnyUrl + request_method: Optional[ApiRequestMethod] = None + request_schema: Optional[APISchema] = None + response_schema: Optional[APISchema] = None