add models, fix test con.

ingestion/src/metadata/ingestion/source/api/rest/connection.py

@@ -52,10 +52,10 @@ def test_connection(
  """
 
  def custom_url_exec():
- if client.status_code == 200:
+ if client.headers.get("content-type") == "application/json":
  return []
  raise SchemaURLError(
- f"Failed to get access to provided schema url. Please check with url and its permissions"
+ f"Failed to parse JSON schema url. Please check if provided url is valid JSON schema."
  )
 
  test_fn = {"CheckURL": custom_url_exec}

ingestion/src/metadata/ingestion/source/api/rest/metadata.py

@@ -43,6 +43,7 @@
 from metadata.ingestion.api.steps import InvalidSourceException
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
 from metadata.ingestion.source.api.api_service import ApiServiceSource
+from metadata.ingestion.source.api.rest.models import RESTCollection, RESTEndpoint
 from metadata.utils import fqn
 from metadata.utils.logger import ingestion_logger
 
@@ -93,24 +94,40 @@ def yield_api_collection(
  ) -> Iterable[Either[CreateAPICollectionRequest]]:
  """Method to return api collection Entities"""
  try:
- collection_request = CreateAPICollectionRequest(
+ rest_collection = RESTCollection(
  name=EntityName(collection.get("name")),
- displayName=collection.get("name"),
+ display_name=collection.get("name"),
  description=Markdown(collection.get("description"))
  if collection.get("description")
  else None,
  service=FullyQualifiedEntityName(self.context.get().api_service),
- endpointURL=AnyUrl(
+ endpoint_url=AnyUrl(
  f"{self.config.serviceConnection.root.config.openAPISchemaURL}#tag/{collection.get('name')}"
  ),
  )
+ except Exception as err:
+ yield Either(
+ left=StackTraceError(
+ name=collection.get("name"),
+ error=f"Error parsing collection: {err}",
+ stackTrace=traceback.format_exc(),
+ )
+ )
+ try:
+ collection_request = CreateAPICollectionRequest(
+ name=rest_collection.name,
+ displayName=rest_collection.display_name,
+ description=rest_collection.description,
+ service=rest_collection.service,
+ endpointURL=rest_collection.endpoint_url,
+ )
  yield Either(right=collection_request)
  self.register_record(collection_request=collection_request)
  except Exception as exc:
  yield Either(
  left=StackTraceError(
  name=collection.get("name"),
- error=f"Error creating collection: {exc}",
+ error=f"Error creating api collection request: {exc}",
  stackTrace=traceback.format_exc(),
  )
  )
@@ -122,34 +139,52 @@ def yield_api_endpoint(
  filtered_endpoints = self._filter_collection_endpoints(collection) or {}
  for path, methods in filtered_endpoints.items():
  for method_type, info in methods.items():
+ try:
+ api_endpoint = RESTEndpoint(
+ name=EntityName(info.get("operationId")),
+ description=Markdown(info.get("description"))
+ if info.get("description")
+ else None,
+ api_collection=FullyQualifiedEntityName(
+ fqn.build(
+ self.metadata,
+ entity_type=APICollection,
+ service_name=self.context.get().api_service,
+ api_collection_name=collection.get("name"),
+ )
+ ),
+ endpoint_url=AnyUrl(
+ f"{self.config.serviceConnection.root.config.openAPISchemaURL}#operation/{info.get('operationId')}",
+ ),
+ request_method=self._get_api_request_method(method_type),
+ request_schema=self._get_request_schema(info),
+ response_schema=self._get_response_schema(info),
+ )
+ except Exception as err:
+ yield Either(
+ left=StackTraceError(
+ name=collection.get("name"),
+ error=f"Error parsing api endpoint: {err}",
+ stackTrace=traceback.format_exc(),
+ )
+ )
  try:
  yield Either(
  right=CreateAPIEndpointRequest(
- name=EntityName(info.get("operationId")),
- description=Markdown(info.get("description"))
- if info.get("description")
- else None,
- apiCollection=FullyQualifiedEntityName(
- fqn.build(
- self.metadata,
- entity_type=APICollection,
- service_name=self.context.get().api_service,
- api_collection_name=collection.get("name"),
- )
- ),
- endpointURL=AnyUrl(
- f"{self.config.serviceConnection.root.config.openAPISchemaURL}#operation/{info.get('operationId')}",
- ),
- requestMethod=self._get_api_request_method(method_type),
- requestSchema=self._get_request_schema(info),
- responseSchema=self._get_response_schema(info),
+ name=api_endpoint.name,
+ description=api_endpoint.description,
+ apiCollection=api_endpoint.api_collection,
+ endpointURL=api_endpoint.endpoint_url,
+ requestMethod=api_endpoint.request_method,
+ requestSchema=api_endpoint.request_schema,
+ responseSchema=api_endpoint.response_schema,
  )
  )
  except Exception as exc: # pylint: disable=broad-except
  yield Either(
  left=StackTraceError(
  name=collection.get("name"),
- error=f"Error creating API Endpoint [{info.get('operationId')}]: {exc}",
+ error=f"Error creating API Endpoint request [{info.get('operationId')}]: {exc}",
  stackTrace=traceback.format_exc(),
  )
  )

ingestion/src/metadata/ingestion/source/api/rest/models.py

@@ -0,0 +1,43 @@
+# Copyright 2024 Collate
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+OpenAPI REST API Models
+"""
+from typing import Optional
+
+from pydantic import AnyUrl, BaseModel
+
+from metadata.generated.schema.entity.data.apiEndpoint import ApiRequestMethod
+from metadata.generated.schema.type import basic
+from metadata.generated.schema.type.apiSchema import APISchema
+
+
+class RESTCollection(BaseModel):
+ """REST colleciton model"""
+
+ name: basic.EntityName
+ display_name: Optional[str] = None
+ description: Optional[basic.Markdown] = None
+ endpoint_url: Optional[AnyUrl] = None
+ service: basic.FullyQualifiedEntityName
+
+
+class RESTEndpoint(BaseModel):
+ """REST endpoint model"""
+
+ name: basic.EntityName
+ display_name: Optional[str] = None
+ description: Optional[basic.Markdown] = None
+ api_collection: basic.FullyQualifiedEntityName
+ endpoint_url: AnyUrl
+ request_method: Optional[ApiRequestMethod] = None
+ request_schema: Optional[APISchema] = None
+ response_schema: Optional[APISchema] = None