Merge pull request #267 from dchaley/issue266/useGsFastcopy

Use gs_fastcopy

deepcell_imaging/benchmark_utils.py

@@ -41,15 +41,19 @@ def get_gce_region():
 def get_gce_is_preemptible():
     # Call the metadata server.
     try:
-        metadata_server = "http://metadata/computeMetadata/v1/instance/scheduling/preemptible"
+        metadata_server = (
+            "http://metadata/computeMetadata/v1/instance/scheduling/preemptible"
+        )
         metadata_flavor = {"Metadata-Flavor": "Google"}
 
         # This comes back like this: projects/1234567890/machineTypes/n2-standard-8
         preemptible_str = requests.get(metadata_server, headers=metadata_flavor).text
         return preemptible_str.lower() == "true"
     except Exception as e:
         exception_string = traceback.format_exc()
-        logging.warning("Error getting preemptible, assuming false. Error: " + exception_string)
+        logging.warning(
+            "Error getting preemptible, assuming false. Error: " + exception_string
+        )
         return False
 
 
@@ -58,7 +62,9 @@ def get_gpu_info():
     import tensorflow as tf
 
     gpu_devices = tf.config.experimental.list_physical_devices("GPU")
-    gpu_details = [tf.config.experimental.get_device_details(gpu) for gpu in gpu_devices]
+    gpu_details = [
+        tf.config.experimental.get_device_details(gpu) for gpu in gpu_devices
+    ]
 
     gpus_by_name = {
         k: list(v) for k, v in groupby(gpu_details, key=lambda x: x["device_name"])

deepcell_imaging/mesmer_app.py

@@ -99,7 +99,7 @@ def preprocess_image(model_input_shape, image, image_mpp):
 def predict(model, image, batch_size):
     logger = logging.getLogger(__name__)
     model_image_shape = model.input_shape[1:]
-    pad_mode = 'constant'
+    pad_mode = "constant"
 
     # TODO: we need to validate the input. But what validations?
 
@@ -108,12 +108,8 @@ def predict(model, image, batch_size):
 
     # Run images through model
     t = timeit.default_timer()
-    output_tiles = batch_predict(
-        model=model, tiles=tiles, batch_size=batch_size
-    )
-    logger.debug(
-        "Model prediction finished in %s s", timeit.default_timer() - t
-    )
+    output_tiles = batch_predict(model=model, tiles=tiles, batch_size=batch_size)
+    logger.debug("Model prediction finished in %s s", timeit.default_timer() - t)
 
     # Untile images
     output_images = _untile_output(output_tiles, tiles_info, model_image_shape)
@@ -122,7 +118,13 @@ def predict(model, image, batch_size):
     return format_output_mesmer(output_images)
 
 
-def postprocess(output_images, input_shape, compartment="whole-cell", whole_cell_kwargs={}, nuclear_kwargs={}):
+def postprocess(
+    output_images,
+    input_shape,
+    compartment="whole-cell",
+    whole_cell_kwargs={},
+    nuclear_kwargs={},
+):
     logger = logging.getLogger(__name__)
 
     # TODO: We need to validate the input (the output_images parameter)
@@ -179,7 +181,7 @@ def postprocess(output_images, input_shape, compartment="whole-cell", whole_cell
         "Post-processed results with %s in %s s",
         mesmer_postprocess.__name__,
         timeit.default_timer() - t,
-        )
+    )
 
     # Resize label_image back to original resolution if necessary
     return _resize_output(label_image, input_shape)
@@ -346,7 +348,7 @@ def format_output_mesmer(output_list):
 
 
 def mesmer_postprocess(
-        model_output, compartment="whole-cell", whole_cell_kwargs=None, nuclear_kwargs=None
+    model_output, compartment="whole-cell", whole_cell_kwargs=None, nuclear_kwargs=None
 ):
     """Postprocess Mesmer output to generate predictions for distinct cellular compartments
 
@@ -414,7 +416,7 @@ def batch_predict(model, tiles, batch_size):
 
     # loop through each batch
     for i in range(0, tiles.shape[0], batch_size):
-        batch_inputs = tiles[i: i + batch_size, ...]
+        batch_inputs = tiles[i : i + batch_size, ...]
 
         batch_outputs = model.predict(batch_inputs, batch_size=batch_size)
 
@@ -430,7 +432,6 @@ def batch_predict(model, tiles, batch_size):
 
         # save each batch to corresponding index in output list
         for j, batch_out in enumerate(batch_outputs):
-            output_tiles[j][i: i + batch_size, ...] = batch_out
+            output_tiles[j][i : i + batch_size, ...] = batch_out
 
     return output_tiles
-

deepcell_imaging/mesmer_no_postprocess.py

@@ -1,9 +1,10 @@
-
 from deepcell.applications import Application, Mesmer
 from deepcell.applications.mesmer import mesmer_preprocess, format_output_mesmer
 
+
 def noop(model_output, *args, **kwargs):
-    return model_output['whole-cell'][-1]
+    return model_output["whole-cell"][-1]
+
 
 class MesmerNoPostprocess(Mesmer):
     def __init__(self, model):
@@ -20,4 +21,5 @@ def __init__(self, model):
             postprocessing_fn=noop,
             format_model_output_fn=format_output_mesmer,
             dataset_metadata=Mesmer.dataset_metadata,
-            model_metadata=Mesmer.model_metadata)
+            model_metadata=Mesmer.model_metadata,
+        )

deepcell_imaging/numpy_utils.py

@@ -9,9 +9,9 @@ def npz_headers(npz):
     """Takes a path to an .npz file, which is a Zip archive of .npy files.
     Generates a sequence of (name, shape, np.dtype).
     """
-    with zipfile.ZipFile(smart_open.open(npz, mode='rb')) as archive:
+    with zipfile.ZipFile(smart_open.open(npz, mode="rb")) as archive:
         for name in archive.namelist():
-            if not name.endswith('.npy'):
+            if not name.endswith(".npy"):
                 continue
 
             npy = archive.open(name)

requirements.txt

@@ -1,6 +1,7 @@
 deepcell==0.12.9
 google-cloud-bigquery
 google-cloud-notebooks
+gs-fastcopy
 imagecodecs
 numpy
 protobuf==3.20.3

run-sample.py

@@ -15,9 +15,13 @@
         input_channels = loader["input_channels"]
 
 model = tf.keras.models.load_model(mesmer_app.model_path)
-preprocessed_image = mesmer_app.preprocess_image(model.input_shape, input_channels[np.newaxis, ...], image_mpp=None)
+preprocessed_image = mesmer_app.preprocess_image(
+    model.input_shape, input_channels[np.newaxis, ...], image_mpp=None
+)
 inferred_images = mesmer_app.predict(model, preprocessed_image, batch_size=4)
-predictions = mesmer_app.postprocess(inferred_images, input_channels[np.newaxis, ...].shape, compartment='whole-cell')
+predictions = mesmer_app.postprocess(
+    inferred_images, input_channels[np.newaxis, ...].shape, compartment="whole-cell"
+)
 
 print(input_channels.shape)
 print(predictions.shape)