From ff29f380259e94f2f9038dd44cd638f9147ee2a9 Mon Sep 17 00:00:00 2001 From: Steven Masfaraud Date: Fri, 28 Oct 2022 20:06:42 +0200 Subject: [PATCH 01/11] feat: checks --- dessia_common/checks.py | 87 ++++++++++++++++++++ dessia_common/core.py | 175 +++++++++++++++++++++++----------------- tests/models_test.py | 2 + 3 files changed, 188 insertions(+), 76 deletions(-) create mode 100644 dessia_common/checks.py diff --git a/dessia_common/checks.py b/dessia_common/checks.py new file mode 100644 index 000000000..be1c0b095 --- /dev/null +++ b/dessia_common/checks.py @@ -0,0 +1,87 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +""" +General checks & checklists +""" + +from dessia_common.core import SerializableObject + + +class PassedCheck(SerializableObject): + level = 'info' + + def __init__(self, message: str): + self.message = message + + def __repr__(self): + return f'[{self.level}]: {self.__class__.__name__} {self.message}' + + +class CheckWarning(PassedCheck): + level = 'warning' + + +class FailedCheck(PassedCheck): + level = 'error' + + +class BadType(FailedCheck): + pass + + +class GeometricInconsistance(FailedCheck): + pass + + +class CheckList(SerializableObject): + def __init__(self, checks): + self.checks = checks + + def __repr__(self): + rep = '' + for check in self.checks: + rep += str(check) + return rep + + def __add__(self, other_checklist): + return self.__class__(self.checks + other_checklist.checks) + + +def is_int(value, level='error'): + if not isinstance(value, int): + return CheckList([FailedCheck(f'Value {value} is not an int')]) + if level == 'info': + return CheckList([PassedCheck(f'value {value} is an int')]) + return CheckList([]) + + +def is_float(value, level='error'): + if not isinstance(value, float): + return CheckList([FailedCheck(f'Value {value} is not an float')]) + if level == 'info': + return CheckList([PassedCheck(f'value {value} is an float')]) + return CheckList([]) + + +def is_str(value, level='error'): + if not isinstance(value, int): + return CheckList([FailedCheck(f'Value {value} is not an str')]) + if level == 'info': + return CheckList([PassedCheck(f'value {value} is an str')]) + return CheckList([]) + + +def type_check(value, expected_type, level='error'): + if expected_type is int: + return is_int(value) + if expected_type is float: + return is_float(value) + if expected_type is str: + return is_str(value) + + if not issubclass(value.__class__, expected_type): + return CheckList([FailedCheck(f'Value {value} is not of type {expected_type}')]) + if level == 'info': + return CheckList([PassedCheck(f'value {value} is of expected type {expected_type}')]) + + return CheckList([]) diff --git a/dessia_common/core.py b/dessia_common/core.py index f5530a111..6831e3b53 100755 --- a/dessia_common/core.py +++ b/dessia_common/core.py @@ -38,6 +38,7 @@ from dessia_common.exports import XLSXWriter from dessia_common.typings import JsonSerializable from dessia_common import templates +from dessia_common.checks import type_check, CheckList from dessia_common.displays import DisplayObject, DisplaySetting from dessia_common.breakdown import attrmethod_getter, get_in_object_from_path @@ -67,7 +68,79 @@ def deprecation_warning(name, object_type, use_instead=None): return msg -class DessiaObject: +class SerializableObject: + """ + Serialization capabilities of Dessia Object + """ + _non_serializable_attributes = [] + + def base_dict(self): + """ + A base dict for to_dict: put name, object class and version in a dict + """ + package_name = self.__module__.split('.', maxsplit=1)[0] + if package_name in sys.modules: + package = sys.modules[package_name] + if hasattr(package, '__version__'): + package_version = package.__version__ + else: + package_version = None + else: + package_version = None + + object_class = self.__module__ + '.' + self.__class__.__name__ + dict_ = {'object_class': object_class} + if package_version: + dict_['package_version'] = package_version + return dict_ + + def _serializable_dict(self): + """ + Returns a dict of attribute_name, values (still python, not serialized) + Keys are filtered with non serializable attributes controls + """ + + dict_ = {k: v for k, v in self.__dict__.items() + if k not in self._non_serializable_attributes and not k.startswith('_')} + return dict_ + + def to_dict(self, use_pointers: bool = True, memo=None, path: str = '#') -> JsonSerializable: + """ + Generic to_dict method + """ + if memo is None: + memo = {} + + # Default to dict + serialized_dict = self.base_dict() + dict_ = self._serializable_dict() + if use_pointers: + serialized_dict.update(serialize_dict_with_pointers(dict_, memo, path)[0]) + else: + serialized_dict.update(serialize_dict(dict_)) + + return serialized_dict + + @classmethod + def dict_to_object(cls, dict_: JsonSerializable, force_generic: bool = False, global_dict=None, + pointers_memo: Dict[str, Any] = None, path: str = '#') -> 'DessiaObject': + """ + Generic dict_to_object method + """ + if cls is not DessiaObject: + obj = dict_to_object(dict_=dict_, class_=cls, force_generic=force_generic, global_dict=global_dict, + pointers_memo=pointers_memo, path=path) + return obj + + if 'object_class' in dict_: + obj = dict_to_object(dict_=dict_, force_generic=force_generic, global_dict=global_dict, + pointers_memo=pointers_memo, path=path) + return obj + + raise NotImplementedError('No object_class in dict') + + +class DessiaObject(SerializableObject): """ Base class for Dessia's platform compatible objects. Gathers generic methods and attributes @@ -106,7 +179,6 @@ class DessiaObject: :ivar Any kwargs: Additionnal user metadata """ _standalone_in_db = False - _non_serializable_attributes = [] _non_editable_attributes = [] _non_data_eq_attributes = ['name'] _non_data_hash_attributes = ['name'] @@ -127,6 +199,11 @@ def __init__(self, name: str = '', **kwargs): for property_name, property_value in kwargs.items(): setattr(self, property_name, property_value) + def base_dict(self): + dict_ = SerializableObject.base_dict(self) + dict_['name'] = self.name + return dict_ + def __hash__(self): """ Compute a int from object @@ -184,71 +261,6 @@ def full_classname(self): """ return full_classname(self) - def base_dict(self): - """ - A base dict for to_dict: put name, object class and version in a dict - """ - package_name = self.__module__.split('.', maxsplit=1)[0] - if package_name in sys.modules: - package = sys.modules[package_name] - if hasattr(package, '__version__'): - package_version = package.__version__ - else: - package_version = None - else: - package_version = None - - object_class = self.__module__ + '.' + self.__class__.__name__ - dict_ = {'name': self.name, 'object_class': object_class} - if package_version: - dict_['package_version'] = package_version - return dict_ - - def _serializable_dict(self): - """ - Returns a dict of attribute_name, values (still python, not serialized) - Keys are filtered with non serializable attributes controls - """ - - dict_ = {k: v for k, v in self.__dict__.items() - if k not in self._non_serializable_attributes and not k.startswith('_')} - return dict_ - - def to_dict(self, use_pointers: bool = True, memo=None, path: str = '#') -> JsonSerializable: - """ - Generic to_dict method - """ - if memo is None: - memo = {} - - # Default to dict - serialized_dict = self.base_dict() - dict_ = self._serializable_dict() - if use_pointers: - serialized_dict.update(serialize_dict_with_pointers(dict_, memo, path)[0]) - else: - serialized_dict.update(serialize_dict(dict_)) - - return serialized_dict - - @classmethod - def dict_to_object(cls, dict_: JsonSerializable, force_generic: bool = False, global_dict=None, - pointers_memo: Dict[str, Any] = None, path: str = '#') -> 'DessiaObject': - """ - Generic dict_to_object method - """ - if cls is not DessiaObject: - obj = dict_to_object(dict_=dict_, class_=cls, force_generic=force_generic, global_dict=global_dict, - pointers_memo=pointers_memo, path=path) - return obj - - if 'object_class' in dict_: - obj = dict_to_object(dict_=dict_, force_generic=force_generic, global_dict=global_dict, - pointers_memo=pointers_memo, path=path) - return obj - - raise NotImplementedError('No object_class in dict') - @classmethod def base_jsonschema(cls): jsonschema = deepcopy(JSONSCHEMA_HEADER) @@ -444,8 +456,19 @@ def load_from_file(cls, filepath: str): return cls.dict_to_object(dict_) - def is_valid(self): - return True + def checks(self, level='error'): + class_argspec = inspect.getfullargspec(self.__class__) + annotations = inspect.signature(self.__init__).parameters + check_list = CheckList([]) + + for arg in class_argspec.args: + if arg != 'self': + if arg in annotations: + value = self.__dict__[arg] + # print(annotations[arg], type(annotations[arg])) + print(annotations[arg].annotation) + check_list += type_check(value, annotations[arg].annotation.__class__, level=level) + return check_list def copy(self, deep=True, memo=None): if deep: @@ -1198,13 +1221,13 @@ def enhanced_deep_attr(obj, sequence): path = f"#/{'/'.join(sequence)}" return get_in_object_from_path(object_=obj, path=path) - # # Sequence is a string and not a sequence of deep attributes - # if '/' in sequence: - # # Is deep attribute reference - # sequence = sequence.split('/') - # return enhanced_deep_attr(obj=obj, sequence=sequence) - # # Is direct attribute - # return enhanced_get_attr(obj=obj, attr=sequence) + # # Sequence is a string and not a sequence of deep attributes + # if '/' in sequence: + # # Is deep attribute reference + # sequence = sequence.split('/') + # return enhanced_deep_attr(obj=obj, sequence=sequence) + # # Is direct attribute + # return enhanced_get_attr(obj=obj, attr=sequence) # # # Get direct attribute # subobj = enhanced_get_attr(obj=obj, attr=sequence[0]) diff --git a/tests/models_test.py b/tests/models_test.py index 936cf92c2..d4c1222d4 100644 --- a/tests/models_test.py +++ b/tests/models_test.py @@ -32,6 +32,8 @@ system1._check_platform() system1.jsonschema() +checks = system1.checks() + system1.save_to_file('system1') system1_lff = dc.DessiaObject.load_from_file('system1.json') assert system1_lff == system1 From a3d6f2479187bab375fd8e042eef233b30389e61 Mon Sep 17 00:00:00 2001 From: Steven Masfaraud Date: Tue, 8 Nov 2022 20:55:28 +0100 Subject: [PATCH 02/11] fix: move to base Serializable object --- code_pylint.py | 4 +- dessia_common/base.py | 84 +++++++++++++++++++++++ dessia_common/checks.py | 3 + dessia_common/core.py | 122 +++++++++------------------------ dessia_common/workflow/core.py | 14 ++-- 5 files changed, 127 insertions(+), 100 deletions(-) create mode 100644 dessia_common/base.py diff --git a/code_pylint.py b/code_pylint.py index 7d46bda65..64d629c46 100644 --- a/code_pylint.py +++ b/code_pylint.py @@ -28,8 +28,8 @@ 'too-many-branches': 12, 'wrong-import-order': 1, 'too-many-branches': 12, - 'unused-argument': 5, - 'cyclic-import': 12, + 'unused-argument': 6, + 'cyclic-import': 14, 'no-self-use': 6, 'trailing-whitespace': 11, 'empty-docstring': 1, diff --git a/dessia_common/base.py b/dessia_common/base.py new file mode 100644 index 000000000..c0a0dfd9e --- /dev/null +++ b/dessia_common/base.py @@ -0,0 +1,84 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +""" +Created on Tue Nov 8 19:39:07 2022 + +@author: steven +""" + +import sys +from typing import Dict, Any +from dessia_common.typings import JsonSerializable +from dessia_common.utils.serialization import dict_to_object, serialize_dict_with_pointers, serialize_dict + + +class SerializableObject: + """ + Serialization capabilities of Dessia Object + """ + _non_serializable_attributes = [] + + def base_dict(self): + """ + A base dict for to_dict: put name, object class and version in a dict + """ + package_name = self.__module__.split('.', maxsplit=1)[0] + if package_name in sys.modules: + package = sys.modules[package_name] + if hasattr(package, '__version__'): + package_version = package.__version__ + else: + package_version = None + else: + package_version = None + + object_class = self.__module__ + '.' + self.__class__.__name__ + dict_ = {'object_class': object_class} + if package_version: + dict_['package_version'] = package_version + return dict_ + + def _serializable_dict(self): + """ + Returns a dict of attribute_name, values (still python, not serialized) + Keys are filtered with non serializable attributes controls + """ + + dict_ = {k: v for k, v in self.__dict__.items() + if k not in self._non_serializable_attributes and not k.startswith('_')} + return dict_ + + def to_dict(self, use_pointers: bool = True, memo=None, path: str = '#') -> JsonSerializable: + """ + Generic to_dict method + """ + if memo is None: + memo = {} + + # Default to dict + serialized_dict = self.base_dict() + dict_ = self._serializable_dict() + if use_pointers: + serialized_dict.update(serialize_dict_with_pointers(dict_, memo, path)[0]) + else: + serialized_dict.update(serialize_dict(dict_)) + + return serialized_dict + + @classmethod + def dict_to_object(cls, dict_: JsonSerializable, force_generic: bool = False, global_dict=None, + pointers_memo: Dict[str, Any] = None, path: str = '#') -> 'SerializableObject': + """ + Generic dict_to_object method + """ + if 'object_class' in dict_: + obj = dict_to_object(dict_=dict_, force_generic=force_generic, global_dict=global_dict, + pointers_memo=pointers_memo, path=path) + return obj + + if cls is not SerializableObject: + obj = dict_to_object(dict_=dict_, class_=cls, force_generic=force_generic, global_dict=global_dict, + pointers_memo=pointers_memo, path=path) + return obj + + raise NotImplementedError('No object_class in dict') diff --git a/dessia_common/checks.py b/dessia_common/checks.py index be1c0b095..8e6f2f69e 100644 --- a/dessia_common/checks.py +++ b/dessia_common/checks.py @@ -72,6 +72,9 @@ def is_str(value, level='error'): def type_check(value, expected_type, level='error'): + """ + This is experimental! + """ if expected_type is int: return is_int(value) if expected_type is float: diff --git a/dessia_common/core.py b/dessia_common/core.py index 6831e3b53..60a4ce023 100755 --- a/dessia_common/core.py +++ b/dessia_common/core.py @@ -6,7 +6,6 @@ """ import time -import sys import warnings import operator import math @@ -20,7 +19,7 @@ import inspect import json -from typing import List, Dict, Any, Tuple, get_type_hints +from typing import List, Tuple, get_type_hints import traceback as tb from importlib import import_module @@ -28,17 +27,18 @@ import dessia_common.errors from dessia_common.utils.diff import data_eq, diff, dict_hash, list_hash -from dessia_common.utils.serialization import dict_to_object, serialize_dict_with_pointers, serialize_dict,\ - deserialize_argument, serialize +from dessia_common.utils.serialization import deserialize_argument, serialize from dessia_common.utils.types import full_classname, is_sequence, is_bson_valid, TYPES_FROM_STRING from dessia_common.utils.copy import deepcopy_value from dessia_common.utils.jsonschema import default_dict, jsonschema_from_annotation, JSONSCHEMA_HEADER,\ set_default_value from dessia_common.utils.docstrings import parse_docstring, FAILED_DOCSTRING_PARSING + +from dessia_common.base import SerializableObject from dessia_common.exports import XLSXWriter from dessia_common.typings import JsonSerializable from dessia_common import templates -from dessia_common.checks import type_check, CheckList +from dessia_common.checks import CheckList, FailedCheck from dessia_common.displays import DisplayObject, DisplaySetting from dessia_common.breakdown import attrmethod_getter, get_in_object_from_path @@ -68,78 +68,6 @@ def deprecation_warning(name, object_type, use_instead=None): return msg -class SerializableObject: - """ - Serialization capabilities of Dessia Object - """ - _non_serializable_attributes = [] - - def base_dict(self): - """ - A base dict for to_dict: put name, object class and version in a dict - """ - package_name = self.__module__.split('.', maxsplit=1)[0] - if package_name in sys.modules: - package = sys.modules[package_name] - if hasattr(package, '__version__'): - package_version = package.__version__ - else: - package_version = None - else: - package_version = None - - object_class = self.__module__ + '.' + self.__class__.__name__ - dict_ = {'object_class': object_class} - if package_version: - dict_['package_version'] = package_version - return dict_ - - def _serializable_dict(self): - """ - Returns a dict of attribute_name, values (still python, not serialized) - Keys are filtered with non serializable attributes controls - """ - - dict_ = {k: v for k, v in self.__dict__.items() - if k not in self._non_serializable_attributes and not k.startswith('_')} - return dict_ - - def to_dict(self, use_pointers: bool = True, memo=None, path: str = '#') -> JsonSerializable: - """ - Generic to_dict method - """ - if memo is None: - memo = {} - - # Default to dict - serialized_dict = self.base_dict() - dict_ = self._serializable_dict() - if use_pointers: - serialized_dict.update(serialize_dict_with_pointers(dict_, memo, path)[0]) - else: - serialized_dict.update(serialize_dict(dict_)) - - return serialized_dict - - @classmethod - def dict_to_object(cls, dict_: JsonSerializable, force_generic: bool = False, global_dict=None, - pointers_memo: Dict[str, Any] = None, path: str = '#') -> 'DessiaObject': - """ - Generic dict_to_object method - """ - if cls is not DessiaObject: - obj = dict_to_object(dict_=dict_, class_=cls, force_generic=force_generic, global_dict=global_dict, - pointers_memo=pointers_memo, path=path) - return obj - - if 'object_class' in dict_: - obj = dict_to_object(dict_=dict_, force_generic=force_generic, global_dict=global_dict, - pointers_memo=pointers_memo, path=path) - return obj - - raise NotImplementedError('No object_class in dict') - - class DessiaObject(SerializableObject): """ Base class for Dessia's platform compatible objects. @@ -457,17 +385,22 @@ def load_from_file(cls, filepath: str): return cls.dict_to_object(dict_) def checks(self, level='error'): - class_argspec = inspect.getfullargspec(self.__class__) - annotations = inspect.signature(self.__init__).parameters check_list = CheckList([]) - for arg in class_argspec.args: - if arg != 'self': - if arg in annotations: - value = self.__dict__[arg] - # print(annotations[arg], type(annotations[arg])) - print(annotations[arg].annotation) - check_list += type_check(value, annotations[arg].annotation.__class__, level=level) + check_list += self._check_platform(level=level) + + # Type checking: not ready yet + # class_argspec = inspect.getfullargspec(self.__class__) + # annotations = inspect.signature(self.__init__).parameters + + # for arg in class_argspec.args: + # if arg != 'self': + # if arg in annotations: + # value = self.__dict__[arg] + # # print(annotations[arg], type(annotations[arg])) + # print(annotations[arg].annotation) + # check_list += type_check(value, annotations[arg].annotation.__class__, level=level) + return check_list def copy(self, deep=True, memo=None): @@ -616,11 +549,12 @@ def _performance_analysis(self): display_time = time.time() - display_time print(f'Generation of display {display_setting.selector} in: {round(display_time, 6)} seconds') - def _check_platform(self): + def _check_platform(self, level='error'): """ Reproduce lifecycle on platform (serialization, display) raise an error if something is wrong """ + checks = [] try: dict_ = self.to_dict(use_pointers=True) except TypeError: @@ -630,22 +564,28 @@ def _check_platform(self): deserialized_object = self.dict_to_object(decoded_json) if not deserialized_object._data_eq(self): print('data diff: ', self._data_diff(deserialized_object)) - raise dessia_common.errors.DeserializationError('Object is not equal to itself' - ' after serialization/deserialization') + # raise dessia_common.errors.DeserializationError('Object is not equal to itself' + # ' after serialization/deserialization') + checks.append(FailedCheck('Object is not equal to itself after serialization/deserialization')) copied_object = self.copy() if not copied_object._data_eq(self): try: print('data diff: ', self._data_diff(copied_object)) except: pass - raise dessia_common.errors.CopyError('Object is not equal to itself after copy') + checks.append(FailedCheck('Object is not equal to itself after serialization/deserialization')) + # raise dessia_common.errors.CopyError('Object is not equal to itself after copy') valid, hint = is_bson_valid(stringify_dict_keys(dict_)) if not valid: - raise ValueError(hint) + # raise ValueError(hint) + checks.append(FailedCheck(f'Object is not bson valid {hint}')) + json.dumps(self._displays()) json.dumps(self._method_jsonschemas) + return CheckList(checks) + def to_xlsx(self, filepath: str): """ Exports the object to an XLSX file given by the filepath diff --git a/dessia_common/workflow/core.py b/dessia_common/workflow/core.py index f3a80a22f..1fb5cc381 100644 --- a/dessia_common/workflow/core.py +++ b/dessia_common/workflow/core.py @@ -21,9 +21,10 @@ from dessia_common.graph import get_column_by_node from dessia_common.templates import workflow_template from dessia_common import DessiaObject, is_sequence, JSONSCHEMA_HEADER, jsonschema_from_annotation, \ - deserialize_argument, set_default_value, prettyname, serialize_dict, DisplaySetting + deserialize_argument, set_default_value, prettyname, DisplaySetting -from dessia_common.utils.serialization import deserialize, serialize_with_pointers, serialize, update_pointers_data +from dessia_common.utils.serialization import deserialize, serialize_with_pointers, serialize, update_pointers_data, \ + serialize_dict from dessia_common.utils.types import serialize_typing, deserialize_typing, recursive_type, typematch, is_serializable from dessia_common.utils.copy import deepcopy_value from dessia_common.utils.docstrings import FAILED_ATTRIBUTE_PARSING, EMPTY_PARSED_ATTRIBUTE @@ -44,7 +45,7 @@ class Variable(DessiaObject): _eq_is_data_eq = False has_default_value: bool = False - def __init__(self, name: str = '', position = None): + def __init__(self, name: str = '', position=None): """ Variable for workflow """ @@ -60,7 +61,6 @@ def to_dict(self, use_pointers=True, memo=None, path: str = '#'): 'position': self.position}) return dict_ - def _to_script(self) -> ToScriptElement: script = self._get_to_script_elements() script.declaration = f"{self.__class__.__name__}({script.declaration})" @@ -76,7 +76,7 @@ def _get_to_script_elements(self): class TypedVariable(Variable): has_default_value: bool = False - def __init__(self, type_: Type, name: str = '', position = None): + def __init__(self, type_: Type, name: str = '', position=None): """ Variable for workflow with a typing """ @@ -121,7 +121,7 @@ def __init__(self, default_value: Any, name: str = '', position=None): class TypedVariableWithDefaultValue(TypedVariable): has_default_value: bool = True - def __init__(self, type_: Type, default_value: Any, name: str = '', position = None): + def __init__(self, type_: Type, default_value: Any, name: str = '', position=None): """ Workflow variables wit a type and a default value """ @@ -1152,7 +1152,7 @@ def variable_compatibility(self, variable: Variable, other_variable: Variable) - @property def layout_graph(self) -> nx.DiGraph: graph = nx.DiGraph() - graph.add_nodes_from(self.nodes) #does not handle detached_variable + graph.add_nodes_from(self.nodes) # does not handle detached_variable for pipe in self.pipes: if pipe.input_variable in self.nonblock_variables: From 4e48b926bba55dea6b11efb59c4961e65bdc89bd Mon Sep 17 00:00:00 2001 From: jezequel Date: Mon, 14 Nov 2022 17:32:31 +0100 Subject: [PATCH 03/11] fix(pylint): last merge choices --- code_pylint.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/code_pylint.py b/code_pylint.py index d389266df..ee5ee1cd9 100644 --- a/code_pylint.py +++ b/code_pylint.py @@ -26,7 +26,7 @@ 'no-member': 3, 'too-many-locals': 12, # Reduce by dropping vectored objects 'wrong-import-order': 1, - 'too-many-branches': 12, + 'too-many-branches': 13, 'unused-argument': 6, 'cyclic-import': 14, 'no-self-use': 6, From b45e514ba4889dc63638ddf72327804f1e84713a Mon Sep 17 00:00:00 2001 From: unknown Date: Tue, 15 Nov 2022 14:58:48 -0500 Subject: [PATCH 04/11] grammar correction and usecase --- dessia_common/checks.py | 8 ++++---- tests/checks.py | 35 +++++++++++++++++++++++++++++++++++ 2 files changed, 39 insertions(+), 4 deletions(-) create mode 100644 tests/checks.py diff --git a/dessia_common/checks.py b/dessia_common/checks.py index 8e6f2f69e..8927f0f5e 100644 --- a/dessia_common/checks.py +++ b/dessia_common/checks.py @@ -57,17 +57,17 @@ def is_int(value, level='error'): def is_float(value, level='error'): if not isinstance(value, float): - return CheckList([FailedCheck(f'Value {value} is not an float')]) + return CheckList([FailedCheck(f'Value {value} is not a float')]) if level == 'info': - return CheckList([PassedCheck(f'value {value} is an float')]) + return CheckList([PassedCheck(f'value {value} is a float')]) return CheckList([]) def is_str(value, level='error'): if not isinstance(value, int): - return CheckList([FailedCheck(f'Value {value} is not an str')]) + return CheckList([FailedCheck(f'Value {value} is not a str')]) if level == 'info': - return CheckList([PassedCheck(f'value {value} is an str')]) + return CheckList([PassedCheck(f'value {value} is a str')]) return CheckList([]) diff --git a/tests/checks.py b/tests/checks.py new file mode 100644 index 000000000..861856c68 --- /dev/null +++ b/tests/checks.py @@ -0,0 +1,35 @@ +from dessia_common import DessiaObject +# from dessia_common.datatools.dataset import Dataset +import dessia_common.checks as checks + + +class Battery(DessiaObject): + + def __init__(self, capacity: int, max_dc: int, + name: str = ''): + + DessiaObject.__init__(self, name=name) + self.capacity = capacity + self.max_dc = max_dc + + # def is_valid(self): + # if self.max_dc < 50: + # return False + # return True + + def is_valid(self): + if checks.is_float(self.max_dc) or checks.is_float(self.max_dc): + return False + return True + +class ElectricChar2(DessiaObject): + + def __init__(self, battery: Battery, brand: str, model: str, price: int, autonomy: int, + name: str = ''): + + DessiaObject.__init__(self, name=name) + self.battery = battery + self.brand = brand + self.model = model + self.price = price + self.autonomy = autonomy \ No newline at end of file From 07d6765ed11daa3ca538136a2e8afb07229fd708 Mon Sep 17 00:00:00 2001 From: Steven Masfaraud Date: Thu, 17 Nov 2022 14:59:26 +0100 Subject: [PATCH 05/11] fix: little change --- .drone.yml | 2 +- code_pylint.py | 2 +- dessia_common/checks.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.drone.yml b/.drone.yml index a8aa8ada5..fbce5506d 100644 --- a/.drone.yml +++ b/.drone.yml @@ -54,8 +54,8 @@ steps: - cd ../tests - coverage run --source dessia_common ci_tests.py - coverage json - - python coverage.py - coverage report + - python coverage.py - name: develop image: python:3.8 diff --git a/code_pylint.py b/code_pylint.py index ee5ee1cd9..8f0ca76d5 100644 --- a/code_pylint.py +++ b/code_pylint.py @@ -28,7 +28,7 @@ 'wrong-import-order': 1, 'too-many-branches': 13, 'unused-argument': 6, - 'cyclic-import': 14, + 'cyclic-import': 15, 'no-self-use': 6, 'trailing-whitespace': 11, 'empty-docstring': 1, diff --git a/dessia_common/checks.py b/dessia_common/checks.py index 8927f0f5e..625d90061 100644 --- a/dessia_common/checks.py +++ b/dessia_common/checks.py @@ -82,7 +82,7 @@ def type_check(value, expected_type, level='error'): if expected_type is str: return is_str(value) - if not issubclass(value.__class__, expected_type): + if not isinstance(value, expected_type): return CheckList([FailedCheck(f'Value {value} is not of type {expected_type}')]) if level == 'info': return CheckList([PassedCheck(f'value {value} is of expected type {expected_type}')]) From 3ed84801b42fc2b5634f20a7e0a47c2f2fbffbba Mon Sep 17 00:00:00 2001 From: Steven Masfaraud Date: Thu, 17 Nov 2022 15:07:31 +0100 Subject: [PATCH 06/11] tests: adding tests --- tests/checks.py | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/tests/checks.py b/tests/checks.py index 861856c68..795064cdf 100644 --- a/tests/checks.py +++ b/tests/checks.py @@ -4,32 +4,38 @@ class Battery(DessiaObject): - + def __init__(self, capacity: int, max_dc: int, - name: str = ''): - + name: str = ''): + DessiaObject.__init__(self, name=name) self.capacity = capacity self.max_dc = max_dc - + # def is_valid(self): # if self.max_dc < 50: # return False # return True - + def is_valid(self): if checks.is_float(self.max_dc) or checks.is_float(self.max_dc): return False return True + +battery = Battery(3, 2) +battery.checks() +battery.is_valid() + + class ElectricChar2(DessiaObject): - + def __init__(self, battery: Battery, brand: str, model: str, price: int, autonomy: int, - name: str = ''): - + name: str = ''): + DessiaObject.__init__(self, name=name) self.battery = battery self.brand = brand self.model = model self.price = price - self.autonomy = autonomy \ No newline at end of file + self.autonomy = autonomy From f818635fa460a66ae3ce9f3ca27571a79aacded4 Mon Sep 17 00:00:00 2001 From: Steven Masfaraud Date: Fri, 18 Nov 2022 19:32:24 +0100 Subject: [PATCH 07/11] fix: checks feature adapted --- dessia_common/checks.py | 26 ++++++++++++++++++++++---- tests/checks.py | 35 ++++++++++++++++++++++++++--------- 2 files changed, 48 insertions(+), 13 deletions(-) diff --git a/dessia_common/checks.py b/dessia_common/checks.py index 625d90061..bfc9c0c3d 100644 --- a/dessia_common/checks.py +++ b/dessia_common/checks.py @@ -7,6 +7,9 @@ from dessia_common.core import SerializableObject +LEVEL_TO_INT = {'debug': 0, 'info': 1, 'warning': 2, 'error': 3} + + class PassedCheck(SerializableObject): level = 'info' @@ -38,14 +41,19 @@ def __init__(self, checks): self.checks = checks def __repr__(self): - rep = '' - for check in self.checks: - rep += str(check) + rep = f'Check list containing {len(self.checks)} checks:\n' + for check_idx, check in enumerate(self.checks): + rep += f'Check {check_idx+1}: {check}\n' return rep def __add__(self, other_checklist): return self.__class__(self.checks + other_checklist.checks) + def raise_if_above_level(self, level='error'): + for check in self.checks: + if LEVEL_TO_INT[check.level] >= LEVEL_TO_INT[level]: + raise ValueError(f'Check: {check} is above level "{level}"') + def is_int(value, level='error'): if not isinstance(value, int): @@ -56,6 +64,16 @@ def is_int(value, level='error'): def is_float(value, level='error'): + """ + + :param value: DESCRIPTION + :type value: TYPE + :param level: DESCRIPTION, defaults to 'error' + :type level: TYPE, optional + :return: DESCRIPTION + :rtype: TYPE + + """ if not isinstance(value, float): return CheckList([FailedCheck(f'Value {value} is not a float')]) if level == 'info': @@ -64,7 +82,7 @@ def is_float(value, level='error'): def is_str(value, level='error'): - if not isinstance(value, int): + if not isinstance(value, str): return CheckList([FailedCheck(f'Value {value} is not a str')]) if level == 'info': return CheckList([PassedCheck(f'value {value} is a str')]) diff --git a/tests/checks.py b/tests/checks.py index 795064cdf..a77ee55c7 100644 --- a/tests/checks.py +++ b/tests/checks.py @@ -5,27 +5,44 @@ class Battery(DessiaObject): - def __init__(self, capacity: int, max_dc: int, + def __init__(self, capacity: float, number_cells: int, name: str = ''): DessiaObject.__init__(self, name=name) self.capacity = capacity - self.max_dc = max_dc + self.number_cells = number_cells # def is_valid(self): # if self.max_dc < 50: # return False # return True - def is_valid(self): - if checks.is_float(self.max_dc) or checks.is_float(self.max_dc): - return False - return True + def checks(self, level='info'): + check_list = DessiaObject.checks(self, level=level) + check_list += checks.is_float(self.capacity, level=level) + check_list += checks.is_int(self.number_cells, level=level) + check_list += checks.is_str(self.name, level=level) -battery = Battery(3, 2) -battery.checks() -battery.is_valid() + return check_list + + +battery = Battery(3., 2, 'Good name') +check_list = battery.checks() +print(check_list) +check_list.raise_if_above_level('error') + +battery = Battery(None, 22.2, 1) +check_list2 = battery.checks() +print(check_list2) + +raised = False +try: + check_list2.raise_if_above_level('error') +except: + raised = True + +assert raised class ElectricChar2(DessiaObject): From 8881108c6f16a54636c3bddbaedf21e5cc7cc088 Mon Sep 17 00:00:00 2001 From: Steven Masfaraud Date: Thu, 24 Nov 2022 15:21:40 +0100 Subject: [PATCH 08/11] fix: add checks to ci --- tests/ci_tests.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/ci_tests.py b/tests/ci_tests.py index 636391a04..27035a963 100644 --- a/tests/ci_tests.py +++ b/tests/ci_tests.py @@ -15,6 +15,7 @@ 'jsonschema.py', 'sampling.py', 'markdowns.py', + 'checks.py', # Workflows 'workflow/blocks.py', 'workflow/workflow_with_models.py', From 2d15abff9f3c25c86ad1dc6b55084e8a23197c9a Mon Sep 17 00:00:00 2001 From: Steven Masfaraud Date: Thu, 24 Nov 2022 16:58:23 +0100 Subject: [PATCH 09/11] fix: pylint c extension --- .pylintrc | 2 +- dessia_common/checks.py | 10 ++++++++-- dessia_common/core.py | 7 +++++-- dessia_common/workflow/core.py | 2 +- tests/checks.py | 8 ++++---- 5 files changed, 19 insertions(+), 10 deletions(-) diff --git a/.pylintrc b/.pylintrc index 21f7c926d..577b1bea9 100644 --- a/.pylintrc +++ b/.pylintrc @@ -42,7 +42,7 @@ suggestion-mode=yes # Allow loading of arbitrary C extensions. Extensions are imported into the # active Python interpreter and may run arbitrary code. -unsafe-load-any-extension=no +unsafe-load-any-extension=yes [MESSAGES CONTROL] diff --git a/dessia_common/checks.py b/dessia_common/checks.py index bfc9c0c3d..37a132210 100644 --- a/dessia_common/checks.py +++ b/dessia_common/checks.py @@ -49,10 +49,16 @@ def __repr__(self): def __add__(self, other_checklist): return self.__class__(self.checks + other_checklist.checks) - def raise_if_above_level(self, level='error'): + def checks_above_level(self, level='error'): + checks = [] for check in self.checks: if LEVEL_TO_INT[check.level] >= LEVEL_TO_INT[level]: - raise ValueError(f'Check: {check} is above level "{level}"') + checks.append(check) + return checks + + def raise_if_above_level(self, level='error'): + for check in self.checks_above_level(level=level): + raise ValueError(f'Check: {check} is above level "{level}"') def is_int(value, level='error'): diff --git a/dessia_common/core.py b/dessia_common/core.py index a61387f2e..8774ca90a 100644 --- a/dessia_common/core.py +++ b/dessia_common/core.py @@ -395,7 +395,7 @@ def load_from_file(cls, filepath: str): return cls.dict_to_object(dict_) - def checks(self, level='error'): + def check_list(self, level='error'): check_list = CheckList([]) check_list += self._check_platform(level=level) @@ -414,6 +414,9 @@ def checks(self, level='error'): return check_list + def is_valid(self, level='error'): + return not self.check_list().checks_above_level(level=level) + def copy(self, deep=True, memo=None): if deep: return deepcopy(self, memo=memo) @@ -538,7 +541,7 @@ def to_markdown(self) -> str: class_=self.__class__.__name__, table=md_writer.object_table(self)) - def _performance_analysis(self): + def performance_analysis(self): """ Prints time of rendering some commons operations (serialization, hash, displays) """ diff --git a/dessia_common/workflow/core.py b/dessia_common/workflow/core.py index 5b46d2c40..c3e6d7c22 100644 --- a/dessia_common/workflow/core.py +++ b/dessia_common/workflow/core.py @@ -1343,7 +1343,7 @@ def plot(self, **kwargs): file.write(rendered_template.encode('utf-8')) webbrowser.open('file://' + temp_file) - def is_valid(self): + def is_valid(self, level='error'): """ Tell if the workflow is valid: * check type compatibility of pipes inputs/outputs diff --git a/tests/checks.py b/tests/checks.py index a77ee55c7..11d670afc 100644 --- a/tests/checks.py +++ b/tests/checks.py @@ -17,8 +17,8 @@ def __init__(self, capacity: float, number_cells: int, # return False # return True - def checks(self, level='info'): - check_list = DessiaObject.checks(self, level=level) + def check_list(self, level='info'): + check_list = DessiaObject.check_list(self, level=level) check_list += checks.is_float(self.capacity, level=level) check_list += checks.is_int(self.number_cells, level=level) @@ -28,12 +28,12 @@ def checks(self, level='info'): battery = Battery(3., 2, 'Good name') -check_list = battery.checks() +check_list = battery.check_list() print(check_list) check_list.raise_if_above_level('error') battery = Battery(None, 22.2, 1) -check_list2 = battery.checks() +check_list2 = battery.check_list() print(check_list2) raised = False From 92a2206254d46bb58145afd4d6d560a5870cdfae Mon Sep 17 00:00:00 2001 From: Steven Masfaraud Date: Wed, 30 Nov 2022 15:02:47 +0100 Subject: [PATCH 10/11] fix: script --- tests/models_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/models_test.py b/tests/models_test.py index 9d1508413..197d0b337 100644 --- a/tests/models_test.py +++ b/tests/models_test.py @@ -33,7 +33,7 @@ system1.save_export_to_file('xlsx', 'generic_xlsx') os.path.isfile('generic_xlsx.xlsx') -checks = system1.checks() +check_list = system1.check_list() system1.save_to_file('system1') system1_lff = dc.DessiaObject.load_from_file('system1.json') From b5682cfebf6989f56b945acb727ac7b5d7d9833f Mon Sep 17 00:00:00 2001 From: Steven Masfaraud Date: Wed, 30 Nov 2022 16:57:36 +0100 Subject: [PATCH 11/11] fix: performance analysis enhancement --- dessia_common/core.py | 10 ++++++---- tests/workflow/power_simulation.py | 2 +- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/dessia_common/core.py b/dessia_common/core.py index 255df8e1e..eba96b0e9 100644 --- a/dessia_common/core.py +++ b/dessia_common/core.py @@ -546,26 +546,28 @@ def performance_analysis(self): """ Prints time of rendering some commons operations (serialization, hash, displays) """ + print(f'### Performance analysis of object {self} ###') data_hash_time = time.time() self._data_hash() data_hash_time = time.time() - data_hash_time - print(f'Data hash time: {round(data_hash_time, 3)} seconds') + print(f'\t- data hash time: {round(data_hash_time, 3)} seconds') todict_time = time.time() dict_ = self.to_dict() todict_time = time.time() - todict_time - print(f'to_dict time: {round(todict_time, 3)} seconds') + print(f'\t- to_dict time: {round(todict_time, 3)} seconds') dto_time = time.time() self.dict_to_object(dict_) dto_time = time.time() - dto_time - print(f'dict_to_object time: {round(dto_time, 3)} seconds') + print(f'\t- dict_to_object time: {round(dto_time, 3)} seconds') for display_setting in self.display_settings(): display_time = time.time() self._display_from_selector(display_setting.selector) display_time = time.time() - display_time - print(f'Generation of display {display_setting.selector} in: {round(display_time, 6)} seconds') + print(f'\t- generation of display {display_setting.selector} in: {round(display_time, 6)} seconds') + print('\n') def _check_platform(self, level='error'): """ diff --git a/tests/workflow/power_simulation.py b/tests/workflow/power_simulation.py index 5f92cc6f1..b324d4d8b 100644 --- a/tests/workflow/power_simulation.py +++ b/tests/workflow/power_simulation.py @@ -51,7 +51,7 @@ manual_run.to_dict(use_pointers=False) manual_run.jsonschema() -manual_run._performance_analysis() +manual_run.performance_analysis() # Testing that there is no pointer when use_pointers=False d = workflow_run.to_dict(use_pointers=False)