feature/gra-808-server-update-not-correctly-handling-sourced-metadata (…

…#623)

* Standardized JSON encoders

* Standardized YAML encoders

* Client library fixes for subscripted generics checks in older version versions

* Fixed server metadata update logic

* Standardized schema -> server model adapters

* Fixed server YAML adapter

grai-client/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "grai-client"
-version = "0.3.1"
+version = "0.3.2"
 description = ""
 authors = ["Ian Eaves <ian@grai.io>"]
 license = "Elastic-2.0"
@@ -18,7 +18,7 @@ pydantic = "^1.9.1"
 requests = "^2.28.1"
 multimethod = "^1.9"
 pyyaml = "^6.0"
-grai-schemas = "^0.2.0"
+grai-schemas = "^0.2.4"
 httpx = {extras = ["http2"], version = "^0.24.0"}
 brotli = "^1.0.9"
 tqdm = "^4.65.0"

grai-client/src/grai_client/endpoints/client.py

@@ -21,7 +21,7 @@
 
 import httpx
 from furl import furl
-from grai_schemas.base import Edge, Node
+from grai_schemas.base import Edge, Node, SourcedEdge, SourcedNode
 from httpx import Auth, BasicAuth, QueryParams, Response
 from multimethod import multimethod
 from pydantic import BaseModel, SecretStr
@@ -650,8 +650,8 @@ def type_segmentation(
 
 
 PRIORITY_ORDER_MAP = {
-    "post": (Node, Edge),
-    "delete": (Edge, Node),
+    "post": (Node, SourcedNode, Edge, SourcedEdge),
+    "delete": (Edge, SourcedEdge, Node, SourcedNode),
     "patch": (),
     "get": (),
 }

grai-client/src/grai_client/endpoints/utilities.py

@@ -21,10 +21,10 @@
 )
 from uuid import UUID
 
-import orjson
-from grai_schemas.generics import GraiBaseModel, MalformedMetadata
+from grai_schemas.generics import MalformedMetadata
+from grai_schemas.serializers import dump_json, load_json
 from httpx import Response
-from pydantic import BaseModel, ValidationError
+from pydantic import ValidationError
 from requests import RequestException
 
 from grai_client.errors import InvalidResponseError, ObjectNotFoundError
@@ -110,56 +110,6 @@ def response_status_check(resp: Response) -> Response:
     raise RequestException(message)
 
 
-def orjson_defaults(obj: Any) -> Any:
-    """
-
-    Args:
-        obj (Any):
-
-    Returns:
-
-    Raises:
-
-    """
-    if isinstance(obj, set):
-        return list(obj)
-    elif isinstance(obj, (pathlib.PosixPath, pathlib.WindowsPath)):
-        return str(obj)
-    elif isinstance(obj, GraiBaseModel):
-        return obj.json()
-    elif isinstance(obj, BaseModel):
-        return obj.dict()
-    else:
-        raise Exception(f"No supported JSON serialization format for objects of type {type(obj)}")
-
-
-class GraiEncoder(json.JSONEncoder):
-    """Needed for the base python json implementation"""
-
-    def default(self, obj: Any) -> Any:
-        """
-
-        Args:
-            obj (Any):
-
-        Returns:
-
-        Raises:
-
-        """
-        if isinstance(obj, (UUID, pathlib.PosixPath, pathlib.WindowsPath)):
-            return str(obj)
-        elif isinstance(obj, (GraiBaseModel, BaseModel)):
-            return obj.dict()
-        elif isinstance(obj, datetime.date):
-            # datetime is a date but date is not a datetime
-            # TODO: TZ management
-            return obj.isoformat()
-        elif isinstance(obj, set):
-            return list(obj)
-        return json.JSONEncoder.default(self, obj)
-
-
 def serialize_obj(obj: Dict) -> bytes:
     """
 
@@ -171,23 +121,7 @@ def serialize_obj(obj: Dict) -> bytes:
     Raises:
 
     """
-    json_obj = orjson.dumps(obj, default=orjson_defaults)
-    return json_obj
-
-
-def serialize_obj_fallback(obj: Dict) -> str:
-    """
-
-    Args:
-        obj (Dict):
-
-    Returns:
-
-    Raises:
-
-    """
-    json_obj = json.dumps(obj, cls=GraiEncoder)
-    return json_obj
+    return dump_json(obj)
 
 
 def add_query_params(url: str, params: dict) -> str:

grai-client/src/grai_client/endpoints/v1/get/utils.py

@@ -1,4 +1,4 @@
-from typing import Any, Dict, Tuple, TypeVar, Union
+from typing import Any, Dict, Tuple, TypeVar, Union, get_args
 
 from grai_schemas.v1 import EdgeV1, NodeV1, SourcedEdgeV1, SourcedNodeV1
 from grai_schemas.v1.edge import EdgeSpec, SourcedEdgeSpec
@@ -37,16 +37,20 @@ def source_edge_builder(resp: Dict[str, Any]) -> SourcedEdgeV1:
     return SourcedEdgeV1.from_spec(resp)
 
 
+source_edge_spec_args = get_args(SourcedEdgeSpec)
+source_node_spec_args = get_args(SourcedNodeSpec)
+
+
 def get_source_and_spec(
     client: ClientV1, grai_type: Union[SourcedNodeSpec, SourcedEdgeSpec]
 ) -> Tuple[SourceSpec, Union[NodeSpec, EdgeSpec]]:
     source = grai_type.data_source
     if source.id is None:
         source = get_is_unique(client, source).spec
 
-    if isinstance(grai_type, SourcedEdgeSpec):
+    if isinstance(grai_type, source_edge_spec_args):
         label = "Edge"
-    elif isinstance(grai_type, SourcedNodeSpec):
+    elif isinstance(grai_type, source_node_spec_args):
         label = "Node"
     else:
         raise ValueError(f"Unexpected type: {type(grai_type)}")

grai-client/src/grai_client/update.py

@@ -1,5 +1,6 @@
 from typing import Dict, List, Optional, Tuple, TypeVar, Union
 
+from grai_schemas.utilities import compute_graph_changes
 from grai_schemas.v1.edge import EdgeV1, SourcedEdgeV1
 from grai_schemas.v1.node import NodeV1, SourcedNodeV1
 from grai_schemas.v1.source import SourceSpec, SourceV1
@@ -9,33 +10,6 @@
 T = TypeVar("T", SourcedNodeV1, SourcedEdgeV1)
 
 
-def compute_graph_changes(items: List[T], active_items: List[T]) -> Tuple[List[T], List[T], List[T]]:
-    """
-    Args:
-        items:
-        active_items:
-
-    Returns:
-
-    Raises:
-
-    """
-
-    active_item_map = {hash(item.spec): item for item in active_items}
-    item_map: Dict[int, T] = {hash(item.spec): item for item in items}
-
-    new_item_keys = item_map.keys() - active_item_map.keys()
-    deleted_source_item_keys = active_item_map.keys() - item_map.keys()
-    updated_item_keys = item_map.keys() - new_item_keys
-
-    deleted_from_sources = [active_item_map[k] for k in deleted_source_item_keys]
-
-    new_items: List[T] = [item_map[k] for k in new_item_keys]
-    updated_items = [item_map[k] for k in updated_item_keys if item_map[k] != active_item_map[k]]
-
-    return new_items, updated_items, deleted_from_sources
-
-
 def update(
     client: BaseClient,
     items: List[Union[SourcedNodeV1, SourcedEdgeV1]],
@@ -63,7 +37,7 @@ def update(
             raise ValueError(
                 f"All items provided to `update` must be of the same type. Instead got a mix of types:" f" {item_types}"
             )
-    if any(isinstance(item, Union[NodeV1, EdgeV1]) for item in items):
+    if any(isinstance(item, (NodeV1, EdgeV1)) for item in items):
         raise NotImplementedError(
             f"Update is not supported for NodeV1 or EdgeV1. Please use SourcedNodeV1 or SourcedEdgeV1 instead."
         )

grai-client/tests/test_v1.py

@@ -1,4 +1,5 @@
 from functools import cache
+from typing import get_args
 from uuid import UUID, uuid4
 
 import pytest
@@ -234,7 +235,7 @@ class TestNodesV1:
     def test_get_nodes(client):
         nodes = client.get("node")
         assert all(isinstance(node, NodeV1) for node in nodes)
-        assert all(isinstance(node.spec.metadata, MetadataV1) for node in nodes)
+        assert all(isinstance(node.spec.metadata, get_args(MetadataV1)) for node in nodes)
 
     @classmethod
     def test_post_node(cls, client, mock_v1):
@@ -315,7 +316,7 @@ class TestEdgesV1:
     def test_get_edges(client):
         result = client.get("edge")
         assert all(isinstance(edge, EdgeV1) for edge in result)
-        assert all(isinstance(edge.spec.metadata, MetadataV1) for edge in result)
+        assert all(isinstance(edge.spec.metadata, get_args(MetadataV1)) for edge in result)
 
     @staticmethod
     def test_post_edge(client, mock_v1):