Data transfer status bars (#245)

* using _obj_put and _obj_get in data transfer thread

* adding status bars; bugfixes

* ruff and pylint

* Bugfix: folders and collections are now created correctly.

* variable naming for threads and their slots

* Update ibridgesgui/sync.py

Co-authored-by: qubixes <44498096+qubixes@users.noreply.github.com>

* Update ibridgesgui/popup_widgets.py

Co-authored-by: qubixes <44498096+qubixes@users.noreply.github.com>

* update docs

* some cosmetics

---------

Co-authored-by: Staiger, Christine <christine.staiger@wur.nl>
Co-authored-by: qubixes <44498096+qubixes@users.noreply.github.com>

docs/userdoc.qmd

@@ -30,9 +30,11 @@ You can **navigate to another part of the iRODS collection tree** by editing thi
 
 - The `Create Collection` button will create a collection in the path shown in the navigation field above.
 
-- The buttons `Upload File` and `Upload Folder` open a file dialog to select data to upload. The destination will be the path shown in the navigation field.
+- The `Upload` button opens a new popup window to select data to upload. The destination will be the path shown in the navigation field.
 
-- The `Download` button will download data to your `Downloads` folder. To download data you will need to select an item in the table below.
+- The `Download` button opens a popup window to select the destination on your computer. You will also be able to export the metadata and store it in a separate file along your download. To download data you will need to select an item in the table below.
+
+- With the `Delete` button you can delete the selected collection or data object in the table. **Note, The data will be deleted completely from the iRODS instance.**
 
 ### Listing
 

ibridgesgui/gui_utils.py

@@ -88,8 +88,9 @@ def combine_operations(operations: list[Operations]) -> Operations:
     ops = operations[0]
     ops.create_dir = set().union(*[o.create_dir for o in operations])
     ops.create_collection = set().union(*[o.create_collection for o in operations])
-    _ = [ops.download.extend(o.download) for o in operations]
-    _ = [ops.upload.extend(o.upload) for o in operations]
+    for op in operations[1:]:
+        ops.download.extend(op.download)
+        ops.upload.extend(op.upload)
 
     return ops
 

ibridgesgui/popup_widgets.py

@@ -269,6 +269,7 @@ def save_env_as(self):
             except TypeError:
                 self.error_label.setText("File type needs to be .json")
 
+
 class UploadData(QDialog, Ui_uploadData):
     """Popup window to upload data to browser."""
 
@@ -294,22 +295,22 @@ def __init__(self, logger, session, irods_path):
         self.folder_button.clicked.connect(self.select_folder)
         self.hide_button.clicked.connect(self.close_window)
 
-
     def close_window(self):
         """Close window while data transfer stays in progress."""
         if self.active_upload:
             reply = QMessageBox.critical(
-                        self, "Message",
-                        "Do you want to close the window while the transfer continues?",
-                        QMessageBox.StandardButton.Yes,
-                        QMessageBox.StandardButton.No,
-                    )
+                self,
+                "Message",
+                "Do you want to close the window while the transfer continues?",
+                QMessageBox.StandardButton.Yes,
+                QMessageBox.StandardButton.No,
+            )
             if reply == QMessageBox.StandardButton.Yes:
                 self.active_upload = False
         self.close()
 
     # pylint: disable=C0103
-    def closeEvent(self, evnt): # noqa
+    def closeEvent(self, evnt):  # noqa
         """Override close when download is in process."""
         if self.active_upload:
             evnt.ignore()
@@ -340,19 +341,37 @@ def _get_upload_params(self):
         self._start_upload(local_paths)
 
     def _start_upload(self, lpaths):
-        self._enable_buttons(False)
         self.setCursor(QtGui.QCursor(QtCore.Qt.CursorShape.WaitCursor))
-        self.active_upload = True
         self.error_label.setText(f"Uploading to {str(self.irods_path)} ....")
         env_path = Path("~").expanduser().joinpath(".irods", get_last_ienv_path())
 
         try:
-            ops = combine_operations([upload(self.session, p, self.irods_path,
-                                         overwrite = self.overwrite.isChecked(),
-                                         dry_run = True) for p in lpaths])
+            ops = combine_operations(
+                [
+                    upload(
+                        self.session,
+                        p,
+                        self.irods_path,
+                        overwrite=self.overwrite.isChecked(),
+                        dry_run=True,
+                    )
+                    for p in lpaths
+                ]
+            )
 
-            self.upload_thread = TransferDataThread(env_path, self.logger, ops,
-                                                    overwrite = self.overwrite.isChecked())
+            if len(ops.upload) == 0:
+                self.error_label.setText("Data already present and up to date.")
+                self.setCursor(QtGui.QCursor(QtCore.Qt.CursorShape.ArrowCursor))
+            else:
+                self._enable_buttons(False)
+                self.active_upload = True
+                self.upload_thread = TransferDataThread(
+                    env_path, self.logger, ops, overwrite=self.overwrite.isChecked()
+                )
+                self.upload_thread.result.connect(self._upload_fetch_result)
+                self.upload_thread.finished.connect(self._finish_upload)
+                self.upload_thread.current_progress.connect(self._upload_status)
+                self.upload_thread.start()
 
         except FileExistsError:
             self.error_label.setText("Data already exists. Check 'overwrite' to overwrite.")
@@ -361,33 +380,28 @@ def _start_upload(self, lpaths):
             return
         except Exception as err:
             self.error_label.setText(
-                    f"Could not instantiate a new session from {env_path}: {repr(err)}."
+                f"Could not instantiate a new session from {env_path}: {repr(err)}."
             )
             self.setCursor(QtGui.QCursor(QtCore.Qt.CursorShape.ArrowCursor))
             self._enable_buttons(True)
             return
 
-        self.upload_thread.succeeded.connect(self._upload_end)
-        self.upload_thread.finished.connect(self._finish_upload)
-        self.upload_thread.current_progress.connect(self._upload_status)
-        self.upload_thread.start()
-
     def _finish_upload(self):
-        self._enable_buttons(True)
         self.setCursor(QtGui.QCursor(QtCore.Qt.CursorShape.ArrowCursor))
         del self.upload_thread
 
     def _upload_status(self, state):
-        self.error_label.setText(state)
+        up_size, transferred_size, obj_count, num_objs, obj_failed = state
+        self.progress_bar.setValue(int(transferred_size*100/up_size))
+        text = f"{obj_count} of {num_objs} files; failed: {obj_failed}."
+        self.error_label.setText(text)
 
-    def _upload_end(self, thread_output: dict):
+    def _upload_fetch_result(self, thread_output: dict):
         self.active_upload = False
         if thread_output["error"] == "":
             self.error_label.setText("Upload finished.")
         else:
             self.error_label.setText("Errors occurred during upload. Consult the logs.")
-        self._enable_buttons(True)
-        self.setCursor(QtGui.QCursor(QtCore.Qt.CursorShape.ArrowCursor))
 
     def _enable_buttons(self, enable):
         self.upload_button.setEnabled(enable)
@@ -404,6 +418,7 @@ def _fs_select(self, path_select):
 
         return path
 
+
 class DownloadData(QDialog, Ui_downloadData):
     """Popup window to dowload data from browser."""
 
@@ -423,10 +438,11 @@ def __init__(self, logger, session, irods_path):
 
         self.source_browser.append(self.irods_path_tree())
         self.timestamp = datetime.now().strftime("%m%d%Y-%H%M")
-        self.metadata.setText(
-            f"Store metadata as\nibridges_metadata_{self.irods_path.name}_{self.timestamp}.json")
-
         self.meta_path = None
+        self.meta_download = (
+            f"bridges_metadata_{self.irods_path.name.split('.')[0]}_{self.timestamp}.json"
+        )
+        self.metadata.setText(f"Store metadata as\n{self.meta_download}")
         self.folder_button.clicked.connect(self.select_folder)
         self.download_button.clicked.connect(self._get_download_params)
         self.hide_button.clicked.connect(self.close_window)
@@ -435,38 +451,39 @@ def close_window(self):
         """Close window while data transfer stays in progress."""
         if self.active_download:
             reply = QMessageBox.critical(
-                        self, "Message",
-                        "Do you want to close the window while the transfer continues?",
-                        QMessageBox.StandardButton.Yes,
-                        QMessageBox.StandardButton.No,
-                    )
+                self,
+                "Message",
+                "Do you want to close the window while the transfer continues?",
+                QMessageBox.StandardButton.Yes,
+                QMessageBox.StandardButton.No,
+            )
             if reply == QMessageBox.StandardButton.Yes:
                 self.active_download = False
         self.close()
 
     # pylint: disable=C0103
-    def closeEvent(self, evnt): # noqa
+    def closeEvent(self, evnt):  # noqa
         """Override close when download is in process."""
         if self.active_download:
             evnt.ignore()
 
-
     def irods_path_tree(self):
         """Expand the irods_path if it is a collection."""
         if self.irods_path.collection_exists():
             return "\n".join(
-                    [coll.name for coll in self.irods_path.collection.subcollections]\
-                    + [obj.name for obj in self.irods_path.collection.data_objects ])
+                [coll.name for coll in self.irods_path.collection.subcollections]
+                + [obj.name for obj in self.irods_path.collection.data_objects]
+            )
 
         return str(self.irods_path)
 
-
     def select_folder(self):
         """Select the download destination."""
+        self.error_label.clear()
         select_dir = Path(
-                QFileDialog.getExistingDirectory(
-                    self, "Select Directory", directory=str(Path("~").expanduser())
-                )
+            QFileDialog.getExistingDirectory(
+                self, "Select Directory", directory=str(Path("~").expanduser())
+            )
         )
         if str(select_dir) == "" or str(select_dir) == ".":
             return
@@ -481,12 +498,12 @@ def _get_download_params(self):
 
         if not local_path.is_dir():
             self.error_label.setText(
-                    f"Dowload folder {local_path} dows not exist or is not a folder.")
+                f"Dowload folder {local_path} does not exist or is not a folder."
+            )
             return
 
         if self.metadata.isChecked():
-            self.meta_path = local_path.joinpath(
-                    f"ibridges_metadata_{self.irods_path.name}_{self.timestamp}.json")
+            self.meta_path = local_path.joinpath(self.meta_download)
 
         self._start_download(local_path)
 
@@ -497,48 +514,58 @@ def _enable_buttons(self, enable):
         self.metadata.setEnabled(enable)
 
     def _start_download(self, local_path):
-        self.active_download = True
         self.setCursor(QtGui.QCursor(QtCore.Qt.CursorShape.WaitCursor))
-        self._enable_buttons(False)
         self.error_label.setText(f"Downloading to {local_path} ....")
         env_path = Path("~").expanduser().joinpath(".irods", get_last_ienv_path())
         try:
-            ops = download(self.session, self.irods_path, local_path,
-                           overwrite = self.overwrite.isChecked(),
-                           metadata = self.meta_path, dry_run=True)
-            self.download_thread = TransferDataThread(env_path, self.logger, ops,
-                                                      overwrite=self.overwrite.isChecked())
+            ops = download(
+                self.session,
+                self.irods_path,
+                local_path,
+                overwrite=self.overwrite.isChecked(),
+                metadata=self.meta_path,
+                dry_run=True,
+            )
+
+            if len(ops.download) == 0 and len(ops.meta_download) == 0:
+                self.error_label.setText("Data already present and up to date.")
+                self.setCursor(QtGui.QCursor(QtCore.Qt.CursorShape.ArrowCursor))
+            else:
+                self._enable_buttons(False)
+                self.active_download = True
+                self.download_thread = TransferDataThread(
+                    env_path, self.logger, ops, overwrite=self.overwrite.isChecked()
+                )
+                self.download_thread.result.connect(self._download_fetch_result)
+                self.download_thread.finished.connect(self._finish_download)
+                self.download_thread.current_progress.connect(self._download_status)
+                self.download_thread.start()
         except FileExistsError:
             self.error_label.setText("Data already exists. Check 'overwrite' to overwrite.")
             self.setCursor(QtGui.QCursor(QtCore.Qt.CursorShape.ArrowCursor))
             self._enable_buttons(True)
             return
         except Exception as err:
             self.error_label.setText(
-                    f"Could not instantiate a new session from {env_path}: {repr(err)}."
+                f"Could not instantiate thread from {env_path}: {repr(err)}."
             )
             self.setCursor(QtGui.QCursor(QtCore.Qt.CursorShape.ArrowCursor))
             self._enable_buttons(True)
             return
 
-        self.download_thread.succeeded.connect(self._download_end)
-        self.download_thread.finished.connect(self._finish_download)
-        self.download_thread.current_progress.connect(self._download_status)
-        self.download_thread.start()
-
     def _finish_download(self):
-        self._enable_buttons(True)
         self.setCursor(QtGui.QCursor(QtCore.Qt.CursorShape.ArrowCursor))
         del self.download_thread
 
     def _download_status(self, state):
-        self.error_label.setText(state)
+        down_size, transferred_size, obj_count, num_objs, obj_failed = state
+        self.progress_bar.setValue(int(transferred_size*100/down_size))
+        text = f"{obj_count} of {num_objs} files; failed: {obj_failed}."
+        self.error_label.setText(text)
 
-    def _download_end(self, thread_output: dict):
+    def _download_fetch_result(self, thread_output: dict):
         self.active_download = False
         if thread_output["error"] == "":
             self.error_label.setText("Download finished.")
         else:
             self.error_label.setText("Errors occurred during download. Consult the logs.")
-        self._enable_buttons(True)
-        self.setCursor(QtGui.QCursor(QtCore.Qt.CursorShape.ArrowCursor))

ibridgesgui/search.py

@@ -229,7 +229,7 @@ def _start_download(self, irods_paths, folder, overwrite):
             )
             self.setCursor(QtGui.QCursor(QtCore.Qt.CursorShape.ArrowCursor))
             return
-        self.download_thread.succeeded.connect(self._download_end)
+        self.download_thread.result.connect(self._download_fetch_result)
         self.download_thread.finished.connect(self._finish_download)
         self.download_thread.current_progress.connect(self._download_status)
         self.download_thread.start()
@@ -242,9 +242,11 @@ def _finish_download(self):
         del self.download_thread
 
     def _download_status(self, state):
-        self.error_label.setText(state)
+        _, _, obj_count, num_objs, obj_failed = state
+        text = f"{obj_count} of {num_objs} files; failed: {obj_failed}."
+        self.error_label.setText(text)
 
-    def _download_end(self, thread_output: dict):
+    def _download_fetch_result(self, thread_output: dict):
         if thread_output["error"] == "":
             self.error_label.setText("Download finished.")
         else:
@@ -269,7 +271,7 @@ def _start_search(self, key_vals, path, checksum):
                 "Could not instantiate a new session from{env_path}.Check configuration"
             )
             return
-        self.search_thread.succeeded.connect(self._fetch_results)
+        self.search_thread.result.connect(self._fetch_results)
         self.search_thread.finished.connect(self._finish_search)
         self.search_thread.start()