Error handling during conversion; fixed env bug to allow vips to be a…

…ccessible from subprocess (andreped#22)

* Implemented run_wrapper() method to handle verbosity and fix env bug

* use wrapper in convert methods; handle errors gracefully

* Skip images that fail

* Linted code

* Bump v0.1.2

setup.py

@@ -6,7 +6,7 @@
 
 setup(
     name="vsi2tif",
-    version="0.1.1",
+    version="0.1.2",
     author="André Pedersen, Sebastian Krossa, Erik Smistad",
     author_email="andrped94@gmail.com",
     license="MIT",

vsi2tif/src/convert.py

@@ -1,7 +1,9 @@
+import logging
 import os
-import subprocess as sp
 from tempfile import TemporaryDirectory
 
+from .utils import run_wrapper
+
 
 def cellsens2raw(
     input_path: str,
@@ -22,10 +24,10 @@ def cellsens2raw(
         f"{bfconvert} -tilex {tz} -tiley {tz} -nogroup -no-upgrade -overwrite -bigtiff -series {plane} "
         f"-compression {compression} {input_path} {output_path}"
     )
-    if verbose == 0:
-        sp.check_call(cmd, shell=True, env={"BF_MAX_MEM": f"{max_mem}g"}, stdout=sp.DEVNULL, stderr=sp.STDOUT)
-    else:
-        sp.check_call(cmd, shell=True, env={"BF_MAX_MEM": f"{max_mem}g"})
+    try:
+        run_wrapper(cmd=cmd, verbose=verbose, max_mem=max_mem)
+    except RuntimeError as e:
+        logging.error(e)
 
 
 def raw2tif(input_path: str, output_path: str, compression: str = "jpeg", quality: int = 85, verbose: int = 1) -> None:
@@ -37,10 +39,10 @@ def raw2tif(input_path: str, output_path: str, compression: str = "jpeg", qualit
     cmd = (
         f"vips tiffsave {input_path} {output_path} --bigtiff --tile --pyramid --compression={compression} --Q={quality}"
     )
-    if verbose == 0:
-        sp.check_call(cmd, shell=True, stdout=sp.DEVNULL, stderr=sp.STDOUT)
-    else:
-        sp.check_call(cmd, shell=True)
+    try:
+        run_wrapper(cmd=cmd, verbose=verbose)
+    except RuntimeError as e:
+        logging.error(e)
 
 
 def cellsens2tif(
@@ -60,7 +62,16 @@ def cellsens2tif(
         bigtiff_path = os.path.join(temp_dir, "temporary.btf")
 
         # first convert from Olympus format to raw TIFF
-        cellsens2raw(input_path, bigtiff_path, bfconvert, "LZW", tz, plane, max_mem, verbose)
+        try:
+            cellsens2raw(input_path, bigtiff_path, bfconvert, "LZW", tz, plane, max_mem, verbose)
+        except Exception as e:
+            logging.error(f"Failed to convert Olympus file to BigTIFF. Skipping image: {input_path}")
+            raise e
 
         # construct tiled, pyramidal TIFF
-        raw2tif(bigtiff_path, output_path, compression, quality, verbose)
+        try:
+            raw2tif(bigtiff_path, output_path, compression, quality, verbose)
+        except Exception as e:
+            logging.error(f"Failed to convert BigTIFF to tiled, pyramidal TIFF. Skipping image: {input_path}")
+            logging.error(e)
+            raise e

vsi2tif/src/process.py

@@ -50,4 +50,9 @@ def cellsens2tif_batch(
         curr_input_path = os.path.join(root, file)
         curr_output_path = (output_path + "/" + curr_input_path.split(input_path)[-1]).replace(".vsi", ".tif")
 
-        cellsens2tif(curr_input_path, curr_output_path, bfconvert, compression, tz, plane, quality, max_mem, verbose)
+        try:
+            cellsens2tif(
+                curr_input_path, curr_output_path, bfconvert, compression, tz, plane, quality, max_mem, verbose
+            )
+        except Exception:
+            continue

vsi2tif/src/utils.py

@@ -0,0 +1,37 @@
+import logging
+import os
+import subprocess as sp
+
+
+def run_wrapper(cmd: str, verbose: int = 0, max_mem: int = 4):
+    # merge current environment with the new BF_MAX_MEM variable
+    env = os.environ.copy()
+    env["BF_MAX_MEM"] = f"{max_mem}g"
+
+    if verbose == 0:
+        # capture the output silently when verbose is disabled
+        result = sp.run(cmd, shell=True, env=env, capture_output=True, text=True)
+
+        # check if the process failed
+        if result.returncode != 0:
+            raise RuntimeError(f"Command failed with error: {result.stderr}")
+
+    else:
+        # stream output in real-time if verbose is enabled
+        process = sp.Popen(cmd, shell=True, env=env, stdout=sp.PIPE, stderr=sp.PIPE, text=True)
+
+        # stream stdout in real-time
+        for stdout_line in iter(process.stdout.readline, ""):
+            logging.info(stdout_line.strip())  # Log each line of output
+        process.stdout.close()
+
+        # wait for process to finish
+        process.wait()
+
+        # check if the process failed
+        if process.returncode != 0:
+            stderr_output = process.stderr.read().strip()
+            process.stderr.close()
+            raise RuntimeError(f"Command failed with error: {stderr_output}")
+
+        process.stderr.close()