diff --git a/CHANGELOG.md b/CHANGELOG.md index a69b544b..e6173e02 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,9 @@ Log of changes in the versions +## v1.1.1 +- bugfix: Setting a default value for toolbox validators in convention yaml file was not working. Fixed it. + ## v1.1.0 - simplified and clean up much code, especially convention sub package - added identifier utils diff --git a/h5rdmtoolbox/convention/core.py b/h5rdmtoolbox/convention/core.py index aecc96d1..89520082 100644 --- a/h5rdmtoolbox/convention/core.py +++ b/h5rdmtoolbox/convention/core.py @@ -4,7 +4,6 @@ import h5py import inspect import pathlib -import pydantic import re import shutil import sys @@ -13,9 +12,9 @@ from pydoc import locate from typing import Union, List, Dict, Tuple +from h5rdmtoolbox import errors from h5rdmtoolbox.repository import RepositoryInterface from h5rdmtoolbox.wrapper import ds_decoder -from h5rdmtoolbox import errors from . import cfg from . import consts from . import errors @@ -399,7 +398,8 @@ def register(self): add_convention(self) def validate(self, file_or_filename: Union[str, pathlib.Path, "File"]) -> List[Dict]: - """Checks a file for compliance with the convention + """Checks a file for compliance with the convention. It will NOT raise an error but + return a list of invalid attributes. Parameters ---------- @@ -469,6 +469,7 @@ def _validate_convention(name, node): with File(file_or_filename, 'r') as f: logger.debug(f'Checking file {file_or_filename} for compliance with convention {self.name}') + _validate_convention('/', f) f.visititems(_validate_convention) return failed diff --git a/h5rdmtoolbox/convention/generate.py b/h5rdmtoolbox/convention/generate.py index c156822a..ee5503b4 100644 --- a/h5rdmtoolbox/convention/generate.py +++ b/h5rdmtoolbox/convention/generate.py @@ -127,26 +127,41 @@ def write_convention_module_from_yaml(yaml_filename: pathlib.Path, name=None): for k, v in class_definitions.items(): f.write(f'\nclass {k[1:]}(BaseModel):') description = v.get('description', k[1:]) - f.write(f'\n{INDENT}"""{description}"""\n{INDENT}') + f.write(f'\n{INDENT}"""{description}"""\n') validator = {} + _validator_default = {} for kk, vv in v.items(): - _vv = vv.strip('$') - if _vv in toolbox_validators.validators: + # _vv = vv.strip('$') + splitted_validator = vv.strip('$').split('=', 1) + validator_name = splitted_validator[0].strip() + if len(splitted_validator) == 2: + default_value = splitted_validator[1].strip() + else: + default_value = None + _validator_default[kk] = [validator_name, default_value] + + if validator_name in toolbox_validators.validators: try: - if toolbox_validators.validators[_vv].__name__ != 'Annotated': # in py3.8 this will raise an error - validator[kk] = f'toolbox_validators.{toolbox_validators.validators[_vv].__name__}' - used_toolbox_validators[_vv] = f'toolbox_validators.{toolbox_validators.validators[_vv].__name__}' + if toolbox_validators.validators[validator_name].__name__ != 'Annotated': # in py3.8 this will raise an error + validator[kk] = f'toolbox_validators.{toolbox_validators.validators[validator_name].__name__}' + used_toolbox_validators[validator_name] = f'toolbox_validators.{toolbox_validators.validators[validator_name].__name__}' else: - validator[kk] = f'toolbox_validators.validators["{_vv}"]' - used_toolbox_validators[_vv] = f'toolbox_validators.validators["{_vv}"]' + validator[kk] = f'toolbox_validators.validators["{validator_name}"]' + used_toolbox_validators[validator_name] = f'toolbox_validators.validators["{validator_name}"]' except AttributeError: - validator[kk] = f'toolbox_validators.validators["{_vv}"]' - used_toolbox_validators[_vv] = f'toolbox_validators.validators["{_vv}"]' + validator[kk] = f'toolbox_validators.validators["{validator_name}"]' + used_toolbox_validators[validator_name] = f'toolbox_validators.validators["{validator_name}"]' else: validator[kk] = vv - f.write(f'\n{INDENT}'.join([f'{ak}: {av}' for ak, av in validator.items()])) + for ak, av in validator.items(): + validator_value, validator_default = _validator_default[ak] + if validator_default is not None: + f.write(f'{INDENT}{ak}: {used_toolbox_validators.get(validator_name, validator_value)} = {validator_default}\n') + else: + f.write(f'{INDENT}{ak}: {av}\n') + # f.write(f'\n{INDENT}'.join([f'{ak}: {av}' for ak, av in validator.items()])) f.write('\n\n') # write standard attribute classes: @@ -185,7 +200,7 @@ def write_convention_module_from_yaml(yaml_filename: pathlib.Path, name=None): # with open(py_filename, 'a') as f: f.write('\nvalidator_dict = {\n' + INDENT) - f.write(f'\n{INDENT}'.join(f"'{k}': {k[1:]}," for k, v in class_definitions.items())) + f.write(f'\n{INDENT}'.join(f"\n'{k}': {k[1:]}," for k, v in class_definitions.items())) f.write(f'\n{INDENT}'.join(f"'{k}': {v}," for k, v in used_toolbox_validators.items())) # f.write(f"\n{INDENT}'$int': IntValidator, # see h5rdmtoolbox.convention.toolbox_validators") # f.write(f"\n{INDENT}'$str': StringValidator, # see h5rdmtoolbox.convention.toolbox_validators") diff --git a/tests/conventions/simple_cv.yaml b/tests/conventions/simple_cv.yaml index 21fec3c6..6125aca4 100644 --- a/tests/conventions/simple_cv.yaml +++ b/tests/conventions/simple_cv.yaml @@ -1,6 +1,17 @@ __contact__: https://orcid.org/0000-0001-8729-0482 __institution__: https://orcid.org/members/001G000001e5aUTIAY __name__: simple_cv + +$personOrOrganization: + name: str + orcid: $orcid=None + +creator: + validator: $personOrOrganization + description: Name and affiliation of the data creator. + target_method: __init__ + default_value: $NONE + units: default_value: $EMPTY description: The physical unit of the dataset. If dimensionless, the unit is ''. diff --git a/tests/conventions/test_conventions.py b/tests/conventions/test_conventions.py index 188f0f3a..6ae6c005 100644 --- a/tests/conventions/test_conventions.py +++ b/tests/conventions/test_conventions.py @@ -640,19 +640,38 @@ def test_engmeta_example(self): pass def test_read_invalid_attribute(self): - cv = h5tbx.convention.Convention.from_yaml(__this_dir__ / 'simple_cv.yaml') - h5tbx.use(None) - with h5tbx.File() as h5: - h5.create_dataset('ds', data=[1, 2], attrs=dict(units='invalid')) - with h5tbx.use(cv): - with h5tbx.File(h5.hdf_filename) as h5: - with h5tbx.set_config(ignore_get_std_attr_err=True): - with self.assertWarns(h5tbx.warnings.StandardAttributeValidationWarning): - units = h5.ds.units - self.assertEqual(units, 'invalid') - with h5tbx.use(None): - with h5tbx.File(h5.hdf_filename) as h5: + cv = h5tbx.convention.Convention.from_yaml(__this_dir__ / 'simple_cv.yaml', overwrite=True) + # h5tbx.use(None) + # with h5tbx.File() as h5: + # h5.create_dataset('ds', data=[1, 2], attrs=dict(units='invalid')) + # with h5tbx.use(cv): + # with h5tbx.File(h5.hdf_filename) as h5: + # with h5tbx.set_config(ignore_get_std_attr_err=True): + # with self.assertWarns(h5tbx.warnings.StandardAttributeValidationWarning): + # units = h5.ds.units + # self.assertEqual(units, 'invalid') + # with h5tbx.use(None): + # with h5tbx.File(h5.hdf_filename) as h5: + # with self.assertRaises(AttributeError): + # _ = h5.ds.units + # units = h5.ds.attrs['units'] + # self.assertEqual(units, 'invalid') + + with h5tbx.use('simple_cv'): + with h5tbx.File(creator={'name': 'Joe'}) as h5: + self.assertEqual(h5.creator.name, 'Joe') + self.assertEqual(h5.creator.orcid, None) + with h5tbx.File(creator={'name': 'Joe', + 'invalid': '123'}) as h5: + self.assertEqual(h5.creator.name, 'Joe') + self.assertEqual(h5.creator.orcid, None) with self.assertRaises(AttributeError): - _ = h5.ds.units - units = h5.ds.attrs['units'] - self.assertEqual(units, 'invalid') + self.assertEqual(h5.creator.invalid, '123') + with self.assertRaises(h5tbx.errors.StandardAttributeError): + with h5tbx.File(creator={'name': 'Joe', + 'orcid': '123'}) as h5: + pass + with h5tbx.File(creator={'name': 'Joe', + 'orcid': h5tbx.__author_orcid__}) as h5: + self.assertEqual(h5.creator.name, 'Joe') + self.assertEqual(str(h5.creator.orcid), h5tbx.__author_orcid__)