Merge pull request #134 from pycroscopy/gerd-nested-ductionary

nested dictionary and file chooser updated

notebooks/00_basic_usage/plot_dataset.ipynb

@@ -11416,7 +11416,7 @@
  ],
  "metadata": {
   "kernelspec": {
-   "display_name": "Python 3",
+   "display_name": "Python 3 (ipykernel)",
    "language": "python",
    "name": "python3"
   },
@@ -11430,7 +11430,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.8.8"
+   "version": "3.8.11"
   },
   "toc": {
    "base_numbering": 1,
@@ -11447,5 +11447,5 @@
   }
  },
  "nbformat": 4,
- "nbformat_minor": 1
+ "nbformat_minor": 4
 }

notebooks/02_visualization/plot_2d.ipynb

@@ -199,7 +199,7 @@
  ],
  "metadata": {
   "kernelspec": {
-   "display_name": "Python 3",
+   "display_name": "Python 3 (ipykernel)",
    "language": "python",
    "name": "python3"
   },
@@ -213,9 +213,9 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.7.4"
+   "version": "3.8.11"
   }
  },
  "nbformat": 4,
- "nbformat_minor": 2
+ "nbformat_minor": 4
 }

setup.py

@@ -20,6 +20,7 @@
                 'distributed>=2.0.0'
                 'psutil',
                 'six',
+                'ipyfilechooser',
                 'joblib>=0.11.0',
                 'ipywidgets>=5.2.2',
                 'ipython>=5.1.0,<6;python_version<"3.3"',  # IPython 6.0+ does not support Python 2.6, 2.7, 3.0, 3.1, or 3.2

sidpy/__version__.py

@@ -1,2 +1,2 @@
 version = '0.0.6'
-time = '2020-10-18 16:08:00'
+time = '2021-09-19 16:08:00'

sidpy/hdf/hdf_utils.py

@@ -19,7 +19,7 @@
 from sidpy.__version__ import version as sidpy_version
 from sidpy.base.string_utils import validate_single_string_arg, \
     validate_list_of_strings, clean_string_att, get_time_stamp
-from sidpy.base.dict_utils import flatten_dict
+# from sidpy.base.dict_utils import flatten_dict
 
 if sys.version_info.major == 3:
     unicode = str
@@ -402,26 +402,49 @@ def write_simple_attrs(h5_obj, attrs, force_to_str=True, verbose=False):
             if verbose:
                 print('taking the name: {} of Enum: {}'.format(val.name, val))
             val = val.name
+
+        if isinstance(val, list):
+            dictionaries = False
+            for item in val:
+                if isinstance(item, dict):
+                    dictionaries = True
+                    break
+            if dictionaries:
+                new_val = {}
+                for key, item in enumerate(val):
+                    new_val[str(key)] = item
+                val = new_val
+
         if isinstance(val, dict):
-            raise ValueError('provided dictionary was nested, not flat. '
-                             'Flatten dictionary using sidpy.base.dict_utils.'
-                             'flatten_dict before calling sidpy.hdf.hdf_utils.'
-                             'write_simple_attrs')
+            if isinstance(h5_obj, h5py.Dataset):
+                raise ValueError('provided dictionary was nested, not flat. '
+                                 'Flatten dictionary using sidpy.base.dict_utils.'
+                                 'flatten_dict before calling sidpy.hdf.hdf_utils.'
+                                 'write_simple_attrs')
+            else:
+                new_object = h5_obj.create_group(str(key))
+                write_simple_attrs(new_object, val, force_to_str=True, verbose=False)
+
         if verbose:
             print('Writing attribute: {} with value: {}'.format(key, val))
-        clean_val = clean_string_att(val)
-        if verbose:
-            print('Attribute cleaned into: {}'.format(clean_val))
-        try:
-            h5_obj.attrs[key] = clean_val
-        except Exception as excp:
-            if force_to_str:
-                warn('Casting attribute value: {} of type: {} to str'
-                     ''.format(val, type(val)))
-                h5_obj.attrs[key] = str(val)
-            else:
-                raise excp('Could not write attribute value: {} of type: {}'
-                           ''.format(val, type(val)))
+
+        if not (isinstance(val, dict)):  # not sure how this can happen
+            if verbose:
+                print(key,val)
+            clean_val = clean_string_att(val)
+
+            if verbose:
+                print('Attribute cleaned into: {}'.format(clean_val))
+            try:
+                h5_obj.attrs[key] = clean_val
+            except Exception as excp:
+                if force_to_str:
+                    warn('Casting attribute value: {} of type: {} to str'
+                        ''.format(val, type(val)))
+                    h5_obj.attrs[key] = str(val)
+                else:
+                    raise excp('Could not write attribute value: {} of type: {}'
+                            ''.format(val, type(val)))
     if verbose:
         print('Wrote all (simple) attributes to {}: {}\n'
               ''.format(type(h5_obj), h5_obj.name.split('/')[-1]))
@@ -750,8 +773,8 @@ def write_dict_to_h5_group(h5_group, metadata, group_name):
 
     Notes
     -----
-    Nested dictionaries will be flattened until sidpy implements functions
-    to write and read nested dictionaries to and from HDF5 files
+    Writes now (sidpy version 0.0.6) nested dictionaries to HDF5 files.
+    Use h5_group_to_dict to read from HDF5 file.
     """
     if not isinstance(metadata, dict):
         raise TypeError('metadata is not a dict but of type: {}'
@@ -765,6 +788,32 @@ def write_dict_to_h5_group(h5_group, metadata, group_name):
     validate_single_string_arg(group_name, 'group_name')
     group_name = group_name.replace(' ', '_')
     h5_md_group = h5_group.create_group(group_name)
-    flat_dict = flatten_dict(metadata)
-    write_simple_attrs(h5_md_group, flat_dict)
+    # flat_dict = flatten_dict(metadata)
+    write_simple_attrs(h5_md_group, metadata)
     return h5_md_group
+
+
+def h5_group_to_dict(group, group_dict={}):
+    """ 
+    Reads a hdf5 group into a nested dictionary
+    
+    Parameters
+    ----------
+    group: hdf5.Group
+        starting group to read from
+    group_dict: dict
+        group dictionary; mostly needed for recursive reading of nested groups but can be used for initialization
+    Returns
+    -------
+    group_dict: dict
+    """
+
+    if not isinstance(group, h5py.Group):
+        raise TypeError('we need a hypy group to read from')
+    if not isinstance(group_dict, dict):
+        raise TypeError('group_dict needs to be a pytho dictionary')
+
+    group_dict[group.name.split('/')[-1]] = dict(group.attrs)
+    for key in group.keys():
+         h5_group_to_dict(group[key], group_dict[group.name.split('/')[-1]])
+    return group_dict

sidpy/io/interface_utils.py

@@ -10,12 +10,168 @@
 from __future__ import division, print_function, absolute_import, unicode_literals
 import os
 import sys
+import ipyfilechooser
 
 if sys.version_info.major == 3:
     unicode = str
     if sys.version_info.minor < 6:
         ModuleNotFoundError = ValueError
 
+class open_file_dialog(ipyfilechooser.FileChooser):
+    def __init__(self, directory='.'):
+        super().__init__(directory) 
+        self._use_dir_icons = True
+
+
+    def _apply_selection(self):
+        super()._apply_selection()
+        selected = os.path.join(
+            self._selected_path,
+            self._selected_filename
+        )
+
+        if os.path.isfile(selected):
+            self._label.value = self._LBL_TEMPLATE.format(
+                self._selected_filename,
+                'blue'
+            )
+        else:
+            self._label.value = self._LBL_TEMPLATE.format(
+                self._selected_filename,
+                'green'
+            )
+
+
+    def _set_form_values(self, path: str, filename: str) -> None:
+        """Set the form values."""
+        # Disable triggers to prevent selecting an entry in the Select
+        # box from automatically triggering a new event.
+        self._pathlist.unobserve(
+            self._on_pathlist_select,
+            names='value'
+        )
+        self._dircontent.unobserve(
+            self._on_dircontent_select,
+            names='value'
+        )
+        self._filename.unobserve(
+            self._on_filename_change,
+            names='value'
+        )
+
+        # In folder only mode zero out the filename
+        if self._show_only_dirs:
+            filename = ''
+
+        # Set form values
+        self._pathlist.options = ipyfilechooser.utils.get_subpaths(path)
+        self._pathlist.value = path
+        self._filename.value = filename
+
+        # file/folder real names
+        dircontent_real_names = ipyfilechooser.utils.get_dir_contents(
+            path,
+            show_hidden=self._show_hidden,
+            prepend_icons=False,
+            show_only_dirs=self._show_only_dirs,
+            filter_pattern=self._filter_pattern
+        )
+
+        # file/folder display names
+        dircontent_display_names = ipyfilechooser.utils.get_dir_contents(
+            path,
+            show_hidden=self._show_hidden,
+            prepend_icons=self._use_dir_icons,
+            show_only_dirs=self._show_only_dirs,
+            filter_pattern=self._filter_pattern
+        )
+        dircontent_display_names = self.set_display_names(dircontent_real_names, dircontent_display_names)
+
+        # Dict to map real names to display names
+        self._map_name_to_disp = {
+            real_name: disp_name
+            for real_name, disp_name in zip(
+                dircontent_real_names,
+                dircontent_display_names
+            )
+        }
+
+        # Dict to map display names to real names
+        self._map_disp_to_name = {
+            disp_name: real_name
+            for real_name, disp_name in
+            self._map_name_to_disp.items()
+        }
+
+        # Set _dircontent form value to display names
+        self._dircontent.options = dircontent_display_names
+
+        # If the value in the filename Text box equals a value in the
+        # Select box and the entry is a file then select the entry.
+        if ((filename in dircontent_real_names) and
+                os.path.isfile(os.path.join(path, filename))):
+            self._dircontent.value = self._map_name_to_disp[filename]
+        else:
+            self._dircontent.value = None
+
+        # Reenable triggers again
+        self._pathlist.observe(
+            self._on_pathlist_select,
+            names='value'
+        )
+        self._dircontent.observe(
+            self._on_dircontent_select,
+            names='value'
+        )
+        self._filename.observe(
+            self._on_filename_change,
+            names='value'
+        )
+
+        # Update the state of the select button
+        if self._gb.layout.display is None:
+            # Disable the select button if path and filename
+            # - equal an existing folder in the current view
+            # - equal the already selected values
+            # - don't match the provided filter pattern(s)
+            check1 = filename in dircontent_real_names
+            check2 = os.path.isdir(os.path.join(path, filename))
+            check3 = False
+            check4 = False
+
+            # Only check selected if selected is set
+            if ((self._selected_path is not None) and
+                    (self._selected_filename is not None)):
+                selected = os.path.join(
+                    self._selected_path,
+                    self._selected_filename
+                )
+                check3 = os.path.join(path, filename) == selected
+
+            # Ensure only allowed extensions are used
+            if self._filter_pattern:
+                check4 = not ipyfilechooser.utils.match_item(filename, self._filter_pattern)
+
+            if (check1 and check2) or check3 or check4:
+                self._select.disabled = True
+            else:
+                self._select.disabled = False
+
+
+    def set_display_names(self, dircontent_real_names, dircontent_display_names):
+
+        for i in range(len(dircontent_display_names)):
+            name = dircontent_display_names[i]
+            full_name = os.path.join(self._pathlist.value, dircontent_real_names[i])
+            if os.path.isfile(full_name):
+                size = os.path.getsize(full_name) * 2 ** -20
+                basename, extension = os.path.splitext(name)
+                if extension in ['.hf5']:
+                    dircontent_display_names[i] = f" {dircontent_display_names[i]:50}  -- {size:.1f} MB"
+                else:
+                    dircontent_display_names[i] = dircontent_display_names[i]
+
+        return dircontent_display_names
 
 def check_ssh():
     """
@@ -63,7 +219,7 @@ def get_QT_app():
     return _instance
 
 
-def openfile_dialog(file_types="All files (*)", multiple_files=False,
+def openfile_dialog_QT(file_types="All files (*)", multiple_files=False,
                     file_path='.', caption="Select a file..."):
     """
     Opens a File dialog which is used in open_file() function