From 268285b0b33fbb39e0cef052512995cad378ddb1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Leon=20M=C3=B6ller?= Date: Wed, 13 Mar 2024 21:33:58 +0100 Subject: [PATCH 1/6] adapter.{json,xml}: make (de-)serialization interfaces coherent lxml supports paths already, no modification is necessary there. However, the `lxml.etree.ElementTree.write()` function requires `BinaryIO`, i.e. files opened with the 'b' mode. While it would be possible to access the underlying binary buffer of files opened in text mode via `open()`, this isn't possible for `io.StringIO()`, as it doesn't have the `buffer` property. Thus, even if we could support files opened via `open()` in text mode, we couldn't annotate the XML serialization functions with `TextIO`, as `io.StringIO()` remains unsupported. Because of that, I decided to not support `TextIO` for the XML serialization. The builtin JSON module only supports file handles, with the `json.dump()` method only supporting `TextIO` and `json.load()` supporting `TextIO` and `BinaryIO`. Thus, the JSON adapter is modified to `open()` given paths, while the JSON serialization is additionally modified to wrap `BinaryIO` with `io.TextIOWrapper`. Fix #42 --- basyx/aas/adapter/_generic.py | 9 ++++- .../aas/adapter/json/json_deserialization.py | 24 +++++++++---- basyx/aas/adapter/json/json_serialization.py | 34 ++++++++++++++++--- basyx/aas/adapter/xml/xml_deserialization.py | 12 +++---- basyx/aas/adapter/xml/xml_serialization.py | 17 ++++++++-- 5 files changed, 76 insertions(+), 20 deletions(-) diff --git a/basyx/aas/adapter/_generic.py b/basyx/aas/adapter/_generic.py index 34c3412b1..3ca90cc28 100644 --- a/basyx/aas/adapter/_generic.py +++ b/basyx/aas/adapter/_generic.py @@ -8,10 +8,17 @@ The dicts defined in this module are used in the json and xml modules to translate enum members of our implementation to the respective string and vice versa. """ -from typing import Dict, Type +import os +from typing import BinaryIO, Dict, IO, Type, Union from basyx.aas import model +# type aliases for path-like objects and IO +# used by write_aas_xml_file, read_aas_xml_file, write_aas_json_file, read_aas_json_file +Path = Union[str, bytes, os.PathLike] +PathOrBinaryIO = Union[Path, BinaryIO] +PathOrIO = Union[Path, IO] # IO is TextIO or BinaryIO + # XML Namespace definition XML_NS_MAP = {"aas": "https://admin-shell.io/aas/3/0"} XML_NS_AAS = "{" + XML_NS_MAP["aas"] + "}" diff --git a/basyx/aas/adapter/json/json_deserialization.py b/basyx/aas/adapter/json/json_deserialization.py index 1e3aecbd1..df8a7f244 100644 --- a/basyx/aas/adapter/json/json_deserialization.py +++ b/basyx/aas/adapter/json/json_deserialization.py @@ -30,15 +30,16 @@ Other embedded objects are converted using a number of helper constructor methods. """ import base64 +import contextlib import json import logging import pprint -from typing import Dict, Callable, TypeVar, Type, List, IO, Optional, Set +from typing import Dict, Callable, ContextManager, TypeVar, Type, List, IO, Optional, Set, get_args from basyx.aas import model from .._generic import MODELLING_KIND_INVERSE, ASSET_KIND_INVERSE, KEY_TYPES_INVERSE, ENTITY_TYPES_INVERSE, \ IEC61360_DATA_TYPES_INVERSE, IEC61360_LEVEL_TYPES_INVERSE, KEY_TYPES_CLASSES_INVERSE, REFERENCE_TYPES_INVERSE, \ - DIRECTION_INVERSE, STATE_OF_EVENT_INVERSE, QUALIFIER_KIND_INVERSE + DIRECTION_INVERSE, STATE_OF_EVENT_INVERSE, QUALIFIER_KIND_INVERSE, PathOrIO, Path logger = logging.getLogger(__name__) @@ -794,7 +795,7 @@ def _select_decoder(failsafe: bool, stripped: bool, decoder: Optional[Type[AASFr return StrictAASFromJsonDecoder -def read_aas_json_file_into(object_store: model.AbstractObjectStore, file: IO, replace_existing: bool = False, +def read_aas_json_file_into(object_store: model.AbstractObjectStore, file: PathOrIO, replace_existing: bool = False, ignore_existing: bool = False, failsafe: bool = True, stripped: bool = False, decoder: Optional[Type[AASFromJsonDecoder]] = None) -> Set[model.Identifier]: """ @@ -803,7 +804,7 @@ def read_aas_json_file_into(object_store: model.AbstractObjectStore, file: IO, r :param object_store: The :class:`ObjectStore ` in which the identifiable objects should be stored - :param file: A file-like object to read the JSON-serialized data from + :param file: A filename or file-like object to read the JSON-serialized data from :param replace_existing: Whether to replace existing objects with the same identifier in the object store or not :param ignore_existing: Whether to ignore existing objects (e.g. log a message) or raise an error. This parameter is ignored if replace_existing is ``True``. @@ -819,8 +820,19 @@ def read_aas_json_file_into(object_store: model.AbstractObjectStore, file: IO, r ret: Set[model.Identifier] = set() decoder_ = _select_decoder(failsafe, stripped, decoder) + # json.load() accepts TextIO and BinaryIO + cm: ContextManager[IO] + if isinstance(file, get_args(Path)): + # 'file' is a path, needs to be opened first + cm = open(file, "r", encoding="utf-8-sig") + else: + # 'file' is not a path, thus it must already be IO + # mypy seems to have issues narrowing the type due to get_args() + cm = contextlib.nullcontext(file) # type: ignore[arg-type] + # read, parse and convert JSON file - data = json.load(file, cls=decoder_) + with cm as fp: + data = json.load(fp, cls=decoder_) for name, expected_type in (('assetAdministrationShells', model.AssetAdministrationShell), ('submodels', model.Submodel), @@ -864,7 +876,7 @@ def read_aas_json_file_into(object_store: model.AbstractObjectStore, file: IO, r return ret -def read_aas_json_file(file: IO, **kwargs) -> model.DictObjectStore[model.Identifiable]: +def read_aas_json_file(file: PathOrIO, **kwargs) -> model.DictObjectStore[model.Identifiable]: """ A wrapper of :meth:`~basyx.aas.adapter.json.json_deserialization.read_aas_json_file_into`, that reads all objects in an empty :class:`~basyx.aas.model.provider.DictObjectStore`. This function supports the same keyword arguments as diff --git a/basyx/aas/adapter/json/json_serialization.py b/basyx/aas/adapter/json/json_serialization.py index 2faadb82b..5891f1be3 100644 --- a/basyx/aas/adapter/json/json_serialization.py +++ b/basyx/aas/adapter/json/json_serialization.py @@ -27,8 +27,10 @@ conversion functions to handle all the attributes of abstract base classes. """ import base64 +import contextlib import inspect -from typing import List, Dict, IO, Optional, Type, Callable +import io +from typing import ContextManager, List, Dict, Optional, TextIO, Type, Callable, get_args import json from basyx.aas import model @@ -732,13 +734,21 @@ def object_store_to_json(data: model.AbstractObjectStore, stripped: bool = False return json.dumps(_create_dict(data), cls=encoder_, **kwargs) -def write_aas_json_file(file: IO, data: model.AbstractObjectStore, stripped: bool = False, +class _DetachingTextIOWrapper(io.TextIOWrapper): + """ + Like :class:`io.TextIOWrapper`, but detaches on context exit instead of closing the wrapped buffer. + """ + def __exit__(self, exc_type, exc_val, exc_tb): + self.detach() + + +def write_aas_json_file(file: _generic.PathOrIO, data: model.AbstractObjectStore, stripped: bool = False, encoder: Optional[Type[AASToJsonEncoder]] = None, **kwargs) -> None: """ Write a set of AAS objects to an Asset Administration Shell JSON file according to 'Details of the Asset Administration Shell', chapter 5.5 - :param file: A file-like object to write the JSON-serialized data to + :param file: A filename or file-like object to write the JSON-serialized data to :param data: :class:`ObjectStore ` which contains different objects of the AAS meta model which should be serialized to a JSON file :param stripped: If `True`, objects are serialized to stripped json objects. @@ -748,5 +758,21 @@ def write_aas_json_file(file: IO, data: model.AbstractObjectStore, stripped: boo :param kwargs: Additional keyword arguments to be passed to `json.dump()` """ encoder_ = _select_encoder(stripped, encoder) + + # json.dump() only accepts TextIO + cm: ContextManager[TextIO] + if isinstance(file, get_args(_generic.Path)): + # 'file' is a path, needs to be opened first + cm = open(file, "w", encoding="utf-8") + elif not hasattr(file, "encoding"): + # only TextIO has this attribute, so this must be BinaryIO, which needs to be wrapped + # mypy seems to have issues narrowing the type due to get_args() + cm = _DetachingTextIOWrapper(file, "utf-8", write_through=True) # type: ignore[arg-type] + else: + # we already got TextIO, nothing needs to be done + # mypy seems to have issues narrowing the type due to get_args() + cm = contextlib.nullcontext(file) # type: ignore[arg-type] + # serialize object to json - json.dump(_create_dict(data), file, cls=encoder_, **kwargs) + with cm as fp: + json.dump(_create_dict(data), fp, cls=encoder_, **kwargs) diff --git a/basyx/aas/adapter/xml/xml_deserialization.py b/basyx/aas/adapter/xml/xml_deserialization.py index 31b5012fb..255cfb539 100644 --- a/basyx/aas/adapter/xml/xml_deserialization.py +++ b/basyx/aas/adapter/xml/xml_deserialization.py @@ -47,10 +47,10 @@ import base64 import enum -from typing import Any, Callable, Dict, IO, Iterable, Optional, Set, Tuple, Type, TypeVar +from typing import Any, Callable, Dict, Iterable, Optional, Set, Tuple, Type, TypeVar from .._generic import XML_NS_MAP, XML_NS_AAS, MODELLING_KIND_INVERSE, ASSET_KIND_INVERSE, KEY_TYPES_INVERSE, \ ENTITY_TYPES_INVERSE, IEC61360_DATA_TYPES_INVERSE, IEC61360_LEVEL_TYPES_INVERSE, KEY_TYPES_CLASSES_INVERSE, \ - REFERENCE_TYPES_INVERSE, DIRECTION_INVERSE, STATE_OF_EVENT_INVERSE, QUALIFIER_KIND_INVERSE + REFERENCE_TYPES_INVERSE, DIRECTION_INVERSE, STATE_OF_EVENT_INVERSE, QUALIFIER_KIND_INVERSE, PathOrIO NS_AAS = XML_NS_AAS REQUIRED_NAMESPACES: Set[str] = {XML_NS_MAP["aas"]} @@ -1186,7 +1186,7 @@ class StrictStrippedAASFromXmlDecoder(StrictAASFromXmlDecoder, StrippedAASFromXm pass -def _parse_xml_document(file: IO, failsafe: bool = True, **parser_kwargs: Any) -> Optional[etree.Element]: +def _parse_xml_document(file: PathOrIO, failsafe: bool = True, **parser_kwargs: Any) -> Optional[etree.Element]: """ Parse an XML document into an element tree @@ -1289,7 +1289,7 @@ class XMLConstructables(enum.Enum): DATA_SPECIFICATION_IEC61360 = enum.auto() -def read_aas_xml_element(file: IO, construct: XMLConstructables, failsafe: bool = True, stripped: bool = False, +def read_aas_xml_element(file: PathOrIO, construct: XMLConstructables, failsafe: bool = True, stripped: bool = False, decoder: Optional[Type[AASFromXmlDecoder]] = None, **constructor_kwargs) -> Optional[object]: """ Construct a single object from an XML string. The namespaces have to be declared on the object itself, since there @@ -1397,7 +1397,7 @@ def read_aas_xml_element(file: IO, construct: XMLConstructables, failsafe: bool return _failsafe_construct(element, constructor, decoder_.failsafe, **constructor_kwargs) -def read_aas_xml_file_into(object_store: model.AbstractObjectStore[model.Identifiable], file: IO, +def read_aas_xml_file_into(object_store: model.AbstractObjectStore[model.Identifiable], file: PathOrIO, replace_existing: bool = False, ignore_existing: bool = False, failsafe: bool = True, stripped: bool = False, decoder: Optional[Type[AASFromXmlDecoder]] = None, **parser_kwargs: Any) -> Set[model.Identifier]: @@ -1470,7 +1470,7 @@ def read_aas_xml_file_into(object_store: model.AbstractObjectStore[model.Identif return ret -def read_aas_xml_file(file: IO, **kwargs: Any) -> model.DictObjectStore[model.Identifiable]: +def read_aas_xml_file(file: PathOrIO, **kwargs: Any) -> model.DictObjectStore[model.Identifiable]: """ A wrapper of :meth:`~basyx.aas.adapter.xml.xml_deserialization.read_aas_xml_file_into`, that reads all objects in an empty :class:`~basyx.aas.model.provider.DictObjectStore`. This function supports diff --git a/basyx/aas/adapter/xml/xml_serialization.py b/basyx/aas/adapter/xml/xml_serialization.py index c6eb2be1c..bb882a952 100644 --- a/basyx/aas/adapter/xml/xml_serialization.py +++ b/basyx/aas/adapter/xml/xml_serialization.py @@ -16,10 +16,21 @@ - For serializing any object to an XML fragment, that fits the XML specification from 'Details of the Asset Administration Shell', chapter 5.4, check out ``_to_xml()``. These functions return an :class:`~lxml.etree.Element` object to be serialized into XML. + +.. attention:: + Unlike the XML deserialization and the JSON (de-)serialization, the XML serialization only supports + :class:`~typing.BinaryIO` and not :class:`~typing.TextIO`. Thus, if you open files by yourself, you have to open + them in binary mode, see the mode table of :func:`open`. + + .. code:: python + + # wb = open for writing + binary mode + with open("example.xml", "wb") as fp: + write_aas_xml_file(fp, object_store) """ from lxml import etree # type: ignore -from typing import Dict, IO, Optional, Type +from typing import Dict, Optional, Type import base64 from basyx.aas import model @@ -840,14 +851,14 @@ def basic_event_element_to_xml(obj: model.BasicEventElement, tag: str = NS_AAS+" # ############################################################## -def write_aas_xml_file(file: IO, +def write_aas_xml_file(file: _generic.PathOrBinaryIO, data: model.AbstractObjectStore, **kwargs) -> None: """ Write a set of AAS objects to an Asset Administration Shell XML file according to 'Details of the Asset Administration Shell', chapter 5.4 - :param file: A file-like object to write the XML-serialized data to + :param file: A filename or file-like object to write the XML-serialized data to :param data: :class:`ObjectStore ` which contains different objects of the AAS meta model which should be serialized to an XML file :param kwargs: Additional keyword arguments to be passed to :meth:`~lxml.etree.ElementTree.write` From 790bc7b5896e6bfe95f8bd943d12c80cc0983a94 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Leon=20M=C3=B6ller?= Date: Wed, 13 Mar 2024 22:31:01 +0100 Subject: [PATCH 2/6] readme, examples: simplify by passing filenames instead of file handles --- README.md | 3 +-- .../tutorial_serialization_deserialization.py | 19 ++++--------------- 2 files changed, 5 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index 63b5de26b..6932cc055 100644 --- a/README.md +++ b/README.md @@ -130,8 +130,7 @@ from basyx.aas.adapter.xml import write_aas_xml_file data: model.DictObjectStore[model.Identifiable] = model.DictObjectStore() data.add(submodel) -with open('Simple_Submodel.xml', 'wb') as f: - write_aas_xml_file(file=f, data=data) +write_aas_xml_file(file='Simple_Submodel.xml', data=data) ``` diff --git a/basyx/aas/examples/tutorial_serialization_deserialization.py b/basyx/aas/examples/tutorial_serialization_deserialization.py index a163dd9f0..4200cfc11 100755 --- a/basyx/aas/examples/tutorial_serialization_deserialization.py +++ b/basyx/aas/examples/tutorial_serialization_deserialization.py @@ -108,18 +108,13 @@ aashell.update() # step 4.3: writing the contents of the ObjectStore to a JSON file -# Heads up! It is important to open the file in text-mode with utf-8 encoding! -with open('data.json', 'w', encoding='utf-8') as json_file: - basyx.aas.adapter.json.write_aas_json_file(json_file, obj_store) +basyx.aas.adapter.json.write_aas_json_file('data.json', obj_store) # We can pass the additional keyword argument `indent=4` to `write_aas_json_file()` to format the JSON file in a more # human-readable (but much more space-consuming) manner. # step 4.4: writing the contents of the ObjectStore to an XML file -# Heads up! For writing XML files -- in contrast to writing JSON --, the file must be opened in binary mode! The XML -# writer will handle character encoding internally. -with open('data.xml', 'wb') as xml_file: - basyx.aas.adapter.xml.write_aas_xml_file(xml_file, obj_store) +basyx.aas.adapter.xml.write_aas_xml_file('data.xml', obj_store) ################################################################## @@ -127,19 +122,13 @@ ################################################################## # step 5.1: reading contents of the JSON file as an ObjectStore -# Heads up! It is important to open the file in text-mode with utf-8 encoding! Using 'utf-8-sig' is recommended to -# handle unicode Byte Order Marks (BOM) correctly. -with open('data.json', encoding='utf-8-sig') as json_file: - json_file_data = basyx.aas.adapter.json.read_aas_json_file(json_file) +json_file_data = basyx.aas.adapter.json.read_aas_json_file('data.json') # By passing the `failsafe=False` argument to `read_aas_json_file()`, we can switch to the `StrictAASFromJsonDecoder` # (see step 3) for a stricter error reporting. # step 5.2: reading contents of the XML file as an ObjectStore -# Heads up! For reading XML files -- in contrast to reading JSON --, the file must be opened in binary mode! The XML -# writer will handle character encoding internally. -with open('data.xml', 'rb') as xml_file: - xml_file_data = basyx.aas.adapter.xml.read_aas_xml_file(xml_file) +xml_file_data = basyx.aas.adapter.xml.read_aas_xml_file('data.xml') # Again, we can use `failsafe=False` for switching on stricter error reporting in the parser. From 40b420c1ccd93e370cb6c39ed7cb0d24c1236fb4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Leon=20M=C3=B6ller?= Date: Wed, 13 Mar 2024 22:40:17 +0100 Subject: [PATCH 3/6] test.adater.xml.test_xml_deserialization: simplify ... by using `StringIO` instead of `BytesIO`. --- test/adapter/xml/test_xml_deserialization.py | 45 +++++++++----------- 1 file changed, 21 insertions(+), 24 deletions(-) diff --git a/test/adapter/xml/test_xml_deserialization.py b/test/adapter/xml/test_xml_deserialization.py index 4ff06aa60..3db7d123c 100644 --- a/test/adapter/xml/test_xml_deserialization.py +++ b/test/adapter/xml/test_xml_deserialization.py @@ -18,10 +18,7 @@ def _xml_wrap(xml: str) -> str: - return \ - """""" \ - f""" """ \ - + xml + """""" + return f'{xml}' def _root_cause(exception: BaseException) -> BaseException: @@ -44,11 +41,11 @@ def _assertInExceptionAndLog(self, xml: str, strings: Union[Iterable[str], str], """ if isinstance(strings, str): strings = [strings] - bytes_io = io.BytesIO(xml.encode("utf-8")) + string_io = io.StringIO(xml) with self.assertLogs(logging.getLogger(), level=log_level) as log_ctx: - read_aas_xml_file(bytes_io, failsafe=True) + read_aas_xml_file(string_io, failsafe=True) with self.assertRaises(error_type) as err_ctx: - read_aas_xml_file(bytes_io, failsafe=False) + read_aas_xml_file(string_io, failsafe=False) cause = _root_cause(err_ctx.exception) for s in strings: self.assertIn(s, log_ctx.output[0]) @@ -142,7 +139,7 @@ def test_no_modelling_kind(self) -> None: """) # should get parsed successfully - object_store = read_aas_xml_file(io.BytesIO(xml.encode("utf-8")), failsafe=False) + object_store = read_aas_xml_file(io.StringIO(xml), failsafe=False) # modelling kind should default to INSTANCE submodel = object_store.pop() self.assertIsInstance(submodel, model.Submodel) @@ -171,7 +168,7 @@ def test_reference_kind_mismatch(self) -> None: """) with self.assertLogs(logging.getLogger(), level=logging.WARNING) as context: - read_aas_xml_file(io.BytesIO(xml.encode("utf-8")), failsafe=False) + read_aas_xml_file(io.StringIO(xml), failsafe=False) for s in ("SUBMODEL", "http://acplt.org/test_ref", "AssetAdministrationShell"): self.assertIn(s, context.output[0]) @@ -254,7 +251,7 @@ def test_operation_variable_too_many_submodel_elements(self) -> None: """) with self.assertLogs(logging.getLogger(), level=logging.WARNING) as context: - read_aas_xml_file(io.BytesIO(xml.encode("utf-8")), failsafe=False) + read_aas_xml_file(io.StringIO(xml), failsafe=False) self.assertIn("aas:value", context.output[0]) self.assertIn("more than one submodel element", context.output[0]) @@ -294,10 +291,10 @@ def get_clean_store() -> model.DictObjectStore: """) - bytes_io = io.BytesIO(xml.encode("utf-8")) + string_io = io.StringIO(xml) object_store = get_clean_store() - identifiers = read_aas_xml_file_into(object_store, bytes_io, replace_existing=True, ignore_existing=False) + identifiers = read_aas_xml_file_into(object_store, string_io, replace_existing=True, ignore_existing=False) self.assertEqual(identifiers.pop(), sm_id) submodel = object_store.pop() self.assertIsInstance(submodel, model.Submodel) @@ -305,7 +302,7 @@ def get_clean_store() -> model.DictObjectStore: object_store = get_clean_store() with self.assertLogs(logging.getLogger(), level=logging.INFO) as log_ctx: - identifiers = read_aas_xml_file_into(object_store, bytes_io, replace_existing=False, ignore_existing=True) + identifiers = read_aas_xml_file_into(object_store, string_io, replace_existing=False, ignore_existing=True) self.assertEqual(len(identifiers), 0) self.assertIn("already exists in the object store", log_ctx.output[0]) submodel = object_store.pop() @@ -314,7 +311,7 @@ def get_clean_store() -> model.DictObjectStore: object_store = get_clean_store() with self.assertRaises(KeyError) as err_ctx: - identifiers = read_aas_xml_file_into(object_store, bytes_io, replace_existing=False, ignore_existing=False) + identifiers = read_aas_xml_file_into(object_store, string_io, replace_existing=False, ignore_existing=False) self.assertEqual(len(identifiers), 0) cause = _root_cause(err_ctx.exception) self.assertIn("already exists in the object store", str(cause)) @@ -328,9 +325,9 @@ def test_read_aas_xml_element(self) -> None: http://acplt.org/test_submodel """ - bytes_io = io.BytesIO(xml.encode("utf-8")) + string_io = io.StringIO(xml) - submodel = read_aas_xml_element(bytes_io, XMLConstructables.SUBMODEL) + submodel = read_aas_xml_element(string_io, XMLConstructables.SUBMODEL) self.assertIsInstance(submodel, model.Submodel) @@ -358,10 +355,10 @@ def test_stripped_qualifiable(self) -> None: """ - bytes_io = io.BytesIO(xml.encode("utf-8")) + string_io = io.StringIO(xml) # check if XML with qualifiers can be parsed successfully - submodel = read_aas_xml_element(bytes_io, XMLConstructables.SUBMODEL, failsafe=False) + submodel = read_aas_xml_element(string_io, XMLConstructables.SUBMODEL, failsafe=False) self.assertIsInstance(submodel, model.Submodel) assert isinstance(submodel, model.Submodel) self.assertEqual(len(submodel.qualifier), 1) @@ -369,7 +366,7 @@ def test_stripped_qualifiable(self) -> None: self.assertEqual(len(operation.qualifier), 1) # check if qualifiers are ignored in stripped mode - submodel = read_aas_xml_element(bytes_io, XMLConstructables.SUBMODEL, failsafe=False, stripped=True) + submodel = read_aas_xml_element(string_io, XMLConstructables.SUBMODEL, failsafe=False, stripped=True) self.assertIsInstance(submodel, model.Submodel) assert isinstance(submodel, model.Submodel) self.assertEqual(len(submodel.qualifier), 0) @@ -396,16 +393,16 @@ def test_stripped_asset_administration_shell(self) -> None: """ - bytes_io = io.BytesIO(xml.encode("utf-8")) + string_io = io.StringIO(xml) # check if XML with submodels can be parsed successfully - aas = read_aas_xml_element(bytes_io, XMLConstructables.ASSET_ADMINISTRATION_SHELL, failsafe=False) + aas = read_aas_xml_element(string_io, XMLConstructables.ASSET_ADMINISTRATION_SHELL, failsafe=False) self.assertIsInstance(aas, model.AssetAdministrationShell) assert isinstance(aas, model.AssetAdministrationShell) self.assertEqual(len(aas.submodel), 1) # check if submodels are ignored in stripped mode - aas = read_aas_xml_element(bytes_io, XMLConstructables.ASSET_ADMINISTRATION_SHELL, failsafe=False, + aas = read_aas_xml_element(string_io, XMLConstructables.ASSET_ADMINISTRATION_SHELL, failsafe=False, stripped=True) self.assertIsInstance(aas, model.AssetAdministrationShell) assert isinstance(aas, model.AssetAdministrationShell) @@ -430,9 +427,9 @@ def construct_submodel(cls, element: etree.Element, object_class=EnhancedSubmode http://acplt.org/test_stripped_submodel """ - bytes_io = io.BytesIO(xml.encode("utf-8")) + string_io = io.StringIO(xml) - submodel = read_aas_xml_element(bytes_io, XMLConstructables.SUBMODEL, decoder=EnhancedAASDecoder) + submodel = read_aas_xml_element(string_io, XMLConstructables.SUBMODEL, decoder=EnhancedAASDecoder) self.assertIsInstance(submodel, EnhancedSubmodel) assert isinstance(submodel, EnhancedSubmodel) self.assertEqual(submodel.enhanced_attribute, "fancy!") From 77846ff3cf0944cf3ffa749a4d234572ead906c4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Leon=20M=C3=B6ller?= Date: Wed, 13 Mar 2024 22:48:37 +0100 Subject: [PATCH 4/6] test.adapter.json: add `BytesIO` test --- ...test_json_serialization_deserialization.py | 22 +++++++++++-------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/test/adapter/json/test_json_serialization_deserialization.py b/test/adapter/json/test_json_serialization_deserialization.py index 2d64af353..351b71845 100644 --- a/test/adapter/json/test_json_serialization_deserialization.py +++ b/test/adapter/json/test_json_serialization_deserialization.py @@ -16,6 +16,8 @@ example_aas_mandatory_attributes, example_submodel_template, create_example from basyx.aas.examples.data._helper import AASDataChecker +from typing import Iterable, IO + class JsonSerializationDeserializationTest(unittest.TestCase): def test_random_object_serialization_deserialization(self) -> None: @@ -41,15 +43,17 @@ def test_random_object_serialization_deserialization(self) -> None: json_object_store = read_aas_json_file(io.StringIO(json_data), failsafe=False) def test_example_serialization_deserialization(self) -> None: - data = example_aas.create_full_example() - file = io.StringIO() - write_aas_json_file(file=file, data=data) - - # try deserializing the json string into a DictObjectStore of AAS objects with help of the json module - file.seek(0) - json_object_store = read_aas_json_file(file, failsafe=False) - checker = AASDataChecker(raise_immediately=True) - example_aas.check_full_example(checker, json_object_store) + # test with TextIO and BinaryIO, which should both be supported + t: Iterable[IO] = (io.StringIO(), io.BytesIO()) + for file in t: + data = example_aas.create_full_example() + write_aas_json_file(file=file, data=data) + + # try deserializing the json string into a DictObjectStore of AAS objects with help of the json module + file.seek(0) + json_object_store = read_aas_json_file(file, failsafe=False) + checker = AASDataChecker(raise_immediately=True) + example_aas.check_full_example(checker, json_object_store) class JsonSerializationDeserializationTest2(unittest.TestCase): From d4486674938ac73e182f17d3e0d8d7c188e23669 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Leon=20M=C3=B6ller?= Date: Wed, 13 Mar 2024 23:12:32 +0100 Subject: [PATCH 5/6] adapter.xml: add function for serializing single objects --- basyx/aas/adapter/xml/__init__.py | 3 +- basyx/aas/adapter/xml/xml_serialization.py | 153 ++++++++++++++++-- .../test_xml_serialization_deserialization.py | 15 +- 3 files changed, 154 insertions(+), 17 deletions(-) diff --git a/basyx/aas/adapter/xml/__init__.py b/basyx/aas/adapter/xml/__init__.py index 714c80663..af58ad0be 100644 --- a/basyx/aas/adapter/xml/__init__.py +++ b/basyx/aas/adapter/xml/__init__.py @@ -11,7 +11,8 @@ """ import os.path -from .xml_serialization import write_aas_xml_file +from .xml_serialization import object_store_to_xml_element, write_aas_xml_file, object_to_xml_element, \ + write_aas_xml_element from .xml_deserialization import AASFromXmlDecoder, StrictAASFromXmlDecoder, StrippedAASFromXmlDecoder, \ StrictStrippedAASFromXmlDecoder, XMLConstructables, read_aas_xml_file, read_aas_xml_file_into, read_aas_xml_element diff --git a/basyx/aas/adapter/xml/xml_serialization.py b/basyx/aas/adapter/xml/xml_serialization.py index bb882a952..cd02ead14 100644 --- a/basyx/aas/adapter/xml/xml_serialization.py +++ b/basyx/aas/adapter/xml/xml_serialization.py @@ -14,8 +14,10 @@ - For generating an XML-File from a :class:`~basyx.aas.model.provider.AbstractObjectStore`, check out the function :func:`write_aas_xml_file`. - For serializing any object to an XML fragment, that fits the XML specification from 'Details of the - Asset Administration Shell', chapter 5.4, check out ``_to_xml()``. These functions return - an :class:`~lxml.etree.Element` object to be serialized into XML. + Asset Administration Shell', chapter 5.4, you can either use :func:`object_to_xml_element`, which serializes a given + object and returns it as :class:`~lxml.etree.Element`, **or** :func:`write_aas_xml_element`, which does the same + thing, but writes the :class:`~lxml.etree.Element` to a file instead of returning it. + As a third alternative, you can also use the functions ``_to_xml()`` directly. .. attention:: Unlike the XML deserialization and the JSON (de-)serialization, the XML serialization only supports @@ -30,7 +32,7 @@ """ from lxml import etree # type: ignore -from typing import Dict, Optional, Type +from typing import Callable, Dict, Optional, Type import base64 from basyx.aas import model @@ -231,6 +233,20 @@ def data_element_to_xml(obj: model.DataElement) -> etree.Element: return reference_element_to_xml(obj) +def key_to_xml(obj: model.Key, tag: str = NS_AAS+"key") -> etree.Element: + """ + Serialization of objects of class :class:`~basyx.aas.model.base.Key` to XML + + :param obj: Object of class :class:`~basyx.aas.model.base.Key` + :param tag: Namespace+Tag of the returned element. Default is ``aas:key`` + :return: Serialized :class:`~lxml.etree.Element` object + """ + et_key = _generate_element(tag) + et_key.append(_generate_element(name=NS_AAS + "type", text=_generic.KEY_TYPES[obj.type])) + et_key.append(_generate_element(name=NS_AAS + "value", text=obj.value)) + return et_key + + def reference_to_xml(obj: model.Reference, tag: str = NS_AAS+"reference") -> etree.Element: """ Serialization of objects of class :class:`~basyx.aas.model.base.Reference` to XML @@ -245,10 +261,7 @@ def reference_to_xml(obj: model.Reference, tag: str = NS_AAS+"reference") -> etr et_reference.append(reference_to_xml(obj.referred_semantic_id, NS_AAS + "referredSemanticId")) et_keys = _generate_element(name=NS_AAS + "keys") for aas_key in obj.key: - et_key = _generate_element(name=NS_AAS + "key") - et_key.append(_generate_element(name=NS_AAS + "type", text=_generic.KEY_TYPES[aas_key.type])) - et_key.append(_generate_element(name=NS_AAS + "value", text=aas_key.value)) - et_keys.append(et_key) + et_keys.append(key_to_xml(aas_key)) et_reference.append(et_keys) return et_reference @@ -850,18 +863,114 @@ def basic_event_element_to_xml(obj: model.BasicEventElement, tag: str = NS_AAS+" # general functions # ############################################################## +def _write_element(file: _generic.PathOrBinaryIO, element: etree.Element, **kwargs) -> None: + etree.ElementTree(element).write(file, encoding="UTF-8", xml_declaration=True, method="xml", **kwargs) + + +def object_to_xml_element(obj: object) -> etree.Element: + """ + Serialize a single object to an :class:`~lxml.etree.Element`. + + :param obj: The object to serialize + """ + serialization_func: Callable[..., etree.Element] + + if isinstance(obj, model.Key): + serialization_func = key_to_xml + elif isinstance(obj, model.Reference): + serialization_func = reference_to_xml + elif isinstance(obj, model.Reference): + serialization_func = reference_to_xml + elif isinstance(obj, model.AdministrativeInformation): + serialization_func = administrative_information_to_xml + elif isinstance(obj, model.Qualifier): + serialization_func = qualifier_to_xml + elif isinstance(obj, model.AnnotatedRelationshipElement): + serialization_func = annotated_relationship_element_to_xml + elif isinstance(obj, model.BasicEventElement): + serialization_func = basic_event_element_to_xml + elif isinstance(obj, model.Blob): + serialization_func = blob_to_xml + elif isinstance(obj, model.Capability): + serialization_func = capability_to_xml + elif isinstance(obj, model.Entity): + serialization_func = entity_to_xml + elif isinstance(obj, model.Extension): + serialization_func = extension_to_xml + elif isinstance(obj, model.File): + serialization_func = file_to_xml + elif isinstance(obj, model.Resource): + serialization_func = resource_to_xml + elif isinstance(obj, model.MultiLanguageProperty): + serialization_func = multi_language_property_to_xml + elif isinstance(obj, model.Operation): + serialization_func = operation_to_xml + elif isinstance(obj, model.Property): + serialization_func = property_to_xml + elif isinstance(obj, model.Range): + serialization_func = range_to_xml + elif isinstance(obj, model.ReferenceElement): + serialization_func = reference_element_to_xml + elif isinstance(obj, model.RelationshipElement): + serialization_func = relationship_element_to_xml + elif isinstance(obj, model.SubmodelElementCollection): + serialization_func = submodel_element_collection_to_xml + elif isinstance(obj, model.SubmodelElementList): + serialization_func = submodel_element_list_to_xml + elif isinstance(obj, model.AssetAdministrationShell): + serialization_func = asset_administration_shell_to_xml + elif isinstance(obj, model.AssetInformation): + serialization_func = asset_information_to_xml + elif isinstance(obj, model.SpecificAssetId): + serialization_func = specific_asset_id_to_xml + elif isinstance(obj, model.Submodel): + serialization_func = submodel_to_xml + elif isinstance(obj, model.ValueReferencePair): + serialization_func = value_reference_pair_to_xml + elif isinstance(obj, model.ConceptDescription): + serialization_func = concept_description_to_xml + elif isinstance(obj, model.LangStringSet): + serialization_func = lang_string_set_to_xml + elif isinstance(obj, model.EmbeddedDataSpecification): + serialization_func = embedded_data_specification_to_xml + elif isinstance(obj, model.DataSpecificationIEC61360): + serialization_func = data_specification_iec61360_to_xml + # generic serialization using the functions for abstract classes + elif isinstance(obj, model.DataElement): + serialization_func = data_element_to_xml + elif isinstance(obj, model.SubmodelElement): + serialization_func = submodel_to_xml + elif isinstance(obj, model.DataSpecificationContent): + serialization_func = data_specification_content_to_xml + # type aliases + elif isinstance(obj, model.ValueList): + serialization_func = value_list_to_xml + else: + raise ValueError(f"{obj!r} cannot be serialized!") -def write_aas_xml_file(file: _generic.PathOrBinaryIO, - data: model.AbstractObjectStore, - **kwargs) -> None: + return serialization_func(obj) + + +def write_aas_xml_element(file: _generic.PathOrBinaryIO, obj: object, **kwargs) -> None: """ - Write a set of AAS objects to an Asset Administration Shell XML file according to 'Details of the Asset - Administration Shell', chapter 5.4 + Serialize a single object to XML. Namespace declarations are added to the object itself, as there is no surrounding + environment element. :param file: A filename or file-like object to write the XML-serialized data to + :param obj: The object to serialize + :param kwargs: Additional keyword arguments to be passed to :meth:`~lxml.etree.ElementTree.write` + """ + return _write_element(file, object_to_xml_element(obj), **kwargs) + + +def object_store_to_xml_element(data: model.AbstractObjectStore) -> etree.Element: + """ + Serialize a set of AAS objects to an Asset Administration Shell as :class:`~lxml.etree.Element`. + This function is used internally by :meth:`write_aas_xml_file` and shouldn't be + called directly for most use-cases. + :param data: :class:`ObjectStore ` which contains different objects of the AAS meta model which should be serialized to an XML file - :param kwargs: Additional keyword arguments to be passed to :meth:`~lxml.etree.ElementTree.write` """ # separate different kind of objects asset_administration_shells = [] @@ -893,5 +1002,19 @@ def write_aas_xml_file(file: _generic.PathOrBinaryIO, et_concept_descriptions.append(concept_description_to_xml(con_obj)) root.append(et_concept_descriptions) - tree = etree.ElementTree(root) - tree.write(file, encoding="UTF-8", xml_declaration=True, method="xml", **kwargs) + return root + + +def write_aas_xml_file(file: _generic.PathOrBinaryIO, + data: model.AbstractObjectStore, + **kwargs) -> None: + """ + Write a set of AAS objects to an Asset Administration Shell XML file according to 'Details of the Asset + Administration Shell', chapter 5.4 + + :param file: A filename or file-like object to write the XML-serialized data to + :param data: :class:`ObjectStore ` which contains different objects of + the AAS meta model which should be serialized to an XML file + :param kwargs: Additional keyword arguments to be passed to :meth:`~lxml.etree.ElementTree.write` + """ + return _write_element(file, object_store_to_xml_element(data), **kwargs) diff --git a/test/adapter/xml/test_xml_serialization_deserialization.py b/test/adapter/xml/test_xml_serialization_deserialization.py index c32653914..919092b2b 100644 --- a/test/adapter/xml/test_xml_serialization_deserialization.py +++ b/test/adapter/xml/test_xml_serialization_deserialization.py @@ -9,7 +9,8 @@ import unittest from basyx.aas import model -from basyx.aas.adapter.xml import write_aas_xml_file, read_aas_xml_file +from basyx.aas.adapter.xml import write_aas_xml_file, read_aas_xml_file, write_aas_xml_element, read_aas_xml_element, \ + XMLConstructables from basyx.aas.examples.data import example_aas_missing_attributes, example_aas, \ example_aas_mandatory_attributes, example_submodel_template, create_example @@ -53,3 +54,15 @@ def test_example_all_examples_serialization_deserialization(self) -> None: object_store = _serialize_and_deserialize(data) checker = AASDataChecker(raise_immediately=True) checker.check_object_store(object_store, data) + + +class XMLSerializationDeserializationSingleObjectTest(unittest.TestCase): + def test_submodel_serialization_deserialization(self) -> None: + submodel: model.Submodel = example_submodel_template.create_example_submodel_template() + bytes_io = io.BytesIO() + write_aas_xml_element(bytes_io, submodel) + bytes_io.seek(0) + submodel2: model.Submodel = read_aas_xml_element(bytes_io, # type: ignore[assignment] + XMLConstructables.SUBMODEL, failsafe=False) + checker = AASDataChecker(raise_immediately=True) + checker.check_submodel_equal(submodel2, submodel) From 81e090367fbf5bec46436dbbdb80ba3d5230b928 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Leon=20M=C3=B6ller?= Date: Wed, 13 Mar 2024 23:41:39 +0100 Subject: [PATCH 6/6] adapter.{json,xml}: improve docstrings --- .../aas/adapter/json/json_deserialization.py | 12 ++++++++++ basyx/aas/adapter/xml/xml_deserialization.py | 22 ++++++++++++++++++- 2 files changed, 33 insertions(+), 1 deletion(-) diff --git a/basyx/aas/adapter/json/json_deserialization.py b/basyx/aas/adapter/json/json_deserialization.py index df8a7f244..f6c7c4183 100644 --- a/basyx/aas/adapter/json/json_deserialization.py +++ b/basyx/aas/adapter/json/json_deserialization.py @@ -815,6 +815,13 @@ def read_aas_json_file_into(object_store: model.AbstractObjectStore, file: PathO See https://git.rwth-aachen.de/acplt/pyi40aas/-/issues/91 This parameter is ignored if a decoder class is specified. :param decoder: The decoder class used to decode the JSON objects + :raises KeyError: **Non-failsafe**: Encountered a duplicate identifier + :raises KeyError: Encountered an identifier that already exists in the given ``object_store`` with both + ``replace_existing`` and ``ignore_existing`` set to ``False`` + :raises (~basyx.aas.model.base.AASConstraintViolation, KeyError, ValueError, TypeError): **Non-failsafe**: + Errors during construction of the objects + :raises TypeError: **Non-failsafe**: Encountered an element in the wrong list + (e.g. an AssetAdministrationShell in ``submodels``) :return: A set of :class:`Identifiers ` that were added to object_store """ ret: Set[model.Identifier] = set() @@ -884,6 +891,11 @@ def read_aas_json_file(file: PathOrIO, **kwargs) -> model.DictObjectStore[model. :param file: A filename or file-like object to read the JSON-serialized data from :param kwargs: Keyword arguments passed to :meth:`read_aas_json_file_into` + :raises KeyError: **Non-failsafe**: Encountered a duplicate identifier + :raises (~basyx.aas.model.base.AASConstraintViolation, KeyError, ValueError, TypeError): **Non-failsafe**: + Errors during construction of the objects + :raises TypeError: **Non-failsafe**: Encountered an element in the wrong list + (e.g. an AssetAdministrationShell in ``submodels``) :return: A :class:`~basyx.aas.model.provider.DictObjectStore` containing all AAS objects from the JSON file """ object_store: model.DictObjectStore[model.Identifiable] = model.DictObjectStore() diff --git a/basyx/aas/adapter/xml/xml_deserialization.py b/basyx/aas/adapter/xml/xml_deserialization.py index 255cfb539..14c0c9dce 100644 --- a/basyx/aas/adapter/xml/xml_deserialization.py +++ b/basyx/aas/adapter/xml/xml_deserialization.py @@ -1194,6 +1194,8 @@ def _parse_xml_document(file: PathOrIO, failsafe: bool = True, **parser_kwargs: :param failsafe: If True, the file is parsed in a failsafe way: Instead of raising an Exception if the document is malformed, parsing is aborted, an error is logged and None is returned :param parser_kwargs: Keyword arguments passed to the XMLParser constructor + :raises ~lxml.etree.XMLSyntaxError: **Non-failsafe**: If the given file(-handle) has invalid XML + :raises KeyError: **Non-failsafe**: If a required namespace has not been declared on the XML document :return: The root element of the element tree """ @@ -1293,7 +1295,7 @@ def read_aas_xml_element(file: PathOrIO, construct: XMLConstructables, failsafe: decoder: Optional[Type[AASFromXmlDecoder]] = None, **constructor_kwargs) -> Optional[object]: """ Construct a single object from an XML string. The namespaces have to be declared on the object itself, since there - is no surrounding aasenv element. + is no surrounding environment element. :param file: A filename or file-like object to read the XML-serialized data from :param construct: A member of the enum :class:`~.XMLConstructables`, specifying which type to construct. @@ -1305,6 +1307,10 @@ def read_aas_xml_element(file: PathOrIO, construct: XMLConstructables, failsafe: This parameter is ignored if a decoder class is specified. :param decoder: The decoder class used to decode the XML elements :param constructor_kwargs: Keyword arguments passed to the constructor function + :raises ~lxml.etree.XMLSyntaxError: **Non-failsafe**: If the given file(-handle) has invalid XML + :raises KeyError: **Non-failsafe**: If a required namespace has not been declared on the XML document + :raises (~basyx.aas.model.base.AASConstraintViolation, KeyError, ValueError): **Non-failsafe**: Errors during + construction of the objects :return: The constructed object or None, if an error occurred in failsafe mode. """ decoder_ = _select_decoder(failsafe, stripped, decoder) @@ -1419,6 +1425,14 @@ def read_aas_xml_file_into(object_store: model.AbstractObjectStore[model.Identif This parameter is ignored if a decoder class is specified. :param decoder: The decoder class used to decode the XML elements :param parser_kwargs: Keyword arguments passed to the XMLParser constructor + :raises ~lxml.etree.XMLSyntaxError: **Non-failsafe**: If the given file(-handle) has invalid XML + :raises KeyError: **Non-failsafe**: If a required namespace has not been declared on the XML document + :raises KeyError: **Non-failsafe**: Encountered a duplicate identifier + :raises KeyError: Encountered an identifier that already exists in the given ``object_store`` with both + ``replace_existing`` and ``ignore_existing`` set to ``False`` + :raises (~basyx.aas.model.base.AASConstraintViolation, KeyError, ValueError): **Non-failsafe**: Errors during + construction of the objects + :raises TypeError: **Non-failsafe**: Encountered an undefined top-level list (e.g. ````) :return: A set of :class:`Identifiers ` that were added to object_store """ ret: Set[model.Identifier] = set() @@ -1478,6 +1492,12 @@ def read_aas_xml_file(file: PathOrIO, **kwargs: Any) -> model.DictObjectStore[mo :param file: A filename or file-like object to read the XML-serialized data from :param kwargs: Keyword arguments passed to :meth:`~basyx.aas.adapter.xml.xml_deserialization.read_aas_xml_file_into` + :raises ~lxml.etree.XMLSyntaxError: **Non-failsafe**: If the given file(-handle) has invalid XML + :raises KeyError: **Non-failsafe**: If a required namespace has not been declared on the XML document + :raises KeyError: **Non-failsafe**: Encountered a duplicate identifier + :raises (~basyx.aas.model.base.AASConstraintViolation, KeyError, ValueError): **Non-failsafe**: Errors during + construction of the objects + :raises TypeError: **Non-failsafe**: Encountered an undefined top-level list (e.g. ````) :return: A :class:`~basyx.aas.model.provider.DictObjectStore` containing all AAS objects from the XML file """ object_store: model.DictObjectStore[model.Identifiable] = model.DictObjectStore()