From fa13d42ba49757195b989653f1059e1f2acb0af9 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Tue, 12 Oct 2021 23:32:59 +0000 Subject: [PATCH 01/12] Added save and load methods --- starfish/core/types/_spot_finding_results.py | 79 ++++++++++++++++++++ 1 file changed, 79 insertions(+) diff --git a/starfish/core/types/_spot_finding_results.py b/starfish/core/types/_spot_finding_results.py index b3dc71d65..bc2352bb0 100644 --- a/starfish/core/types/_spot_finding_results.py +++ b/starfish/core/types/_spot_finding_results.py @@ -6,6 +6,7 @@ from starfish.core.types import Axes, Coordinates, SpotAttributes from starfish.core.util.logging import Log +import json AXES_ORDER = (Axes.ROUND, Axes.CH) @@ -109,6 +110,84 @@ def values(self): """ return self._results.values() + def save(self, output_dir_name: str) -> None: + """Save spot finding results to series of files. + + Parameters + ---------- + output_dir_name: str + Location to save all files. + + """ + json_data = {} + + coords = {} + for key in self.physical_coord_ranges.keys(): + path = "{}coords_{}.nc".format(output_dir_name, key) + coords[key] = path + self.physical_coord_ranges[key].to_netcdf(path) + json_data["physical_coord_ranges"] = coords + + path = "{}log.arr" + json_data["log"] = path.format(output_dir_name) + with open(path.format(output_dir_name), "w") as f: + f.write(self.log.encode()) + + spot_attrs = {} + for key in self._results.keys(): + path = "{}spots_{}_{}.nc".format(output_dir_name, key[0], key[1]) + spot_attrs["{}_{}".format(key[0], key[1])] = path + self._results[key].spot_attrs.save(path) + json_data["spot_attrs"] = spot_attrs + + save = json.dumps(json_data) + with open("{}SpotFindingResults.json".format(output_dir_name), "w") as f: + f.write(save) + + @classmethod + def load(cls, json_file: str): + """Load serialized spot finding results. + + Parameters: + ----------- + json_file: str + json file to read + + Returns: + -------- + SpotFindingResults: + Object containing loaded results + + """ + fl = open(json_file) + data = json.load(fl) + + with open(data["log"]) as f: + log = Log.decode(f.read()) + + rename_axes = { + 'x': Coordinates.X.value, + 'y': Coordinates.Y.value, + 'z': Coordinates.Z.value + } + coords = {} + for coord, path in data["physical_coord_ranges"].items(): + coords[rename_axes[coord]] = xr.load_dataarray(path) + + spot_attributes_list = [] + for key, path in data["spot_attrs"].items(): + zero = int(key.split("_")[0]) + one = int(key.split("_")[1]) + index = {AXES_ORDER[0]: zero, AXES_ORDER[1]: one} + spots = SpotAttributes.load(path) + spot_attributes_list.append((PerImageSliceSpotResults(spots, extras=None),index)) + + return SpotFindingResults( + imagestack_coords = coords, + log = log, + spot_attributes_list = spot_attributes_list + ) + @property def round_labels(self): """ From 5efe4dd656e07436711bf01d4918be6db111096b Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Wed, 13 Oct 2021 19:18:52 +0000 Subject: [PATCH 02/12] Fixed dict typing error from validation --- starfish/core/types/_spot_finding_results.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/starfish/core/types/_spot_finding_results.py b/starfish/core/types/_spot_finding_results.py index bc2352bb0..060656f2c 100644 --- a/starfish/core/types/_spot_finding_results.py +++ b/starfish/core/types/_spot_finding_results.py @@ -1,5 +1,5 @@ from dataclasses import dataclass -from typing import Any, Hashable, Mapping, MutableMapping, Optional, Sequence, Tuple +from typing import Any, Hashable, Mapping, MutableMapping, Optional, Sequence, Tuple, Dict import xarray as xr @@ -119,7 +119,7 @@ def save(self, output_dir_name: str) -> None: Location to save all files. """ - json_data = {} + json_data: Dict[str, Any] = {} coords = {} for key in self.physical_coord_ranges.keys(): @@ -129,7 +129,8 @@ def save(self, output_dir_name: str) -> None: json_data["physical_coord_ranges"] = coords path = "{}log.arr" - json_data["log"] = path.format(output_dir_name) + json_data["log"] = {} + json_data["log"]["path"] = path.format(output_dir_name) with open(path.format(output_dir_name), "w") as f: f.write(self.log.encode()) @@ -162,7 +163,7 @@ def load(cls, json_file: str): fl = open(json_file) data = json.load(fl) - with open(data["log"]) as f: + with open(data["log"]["path"]) as f: log = Log.decode(f.read()) rename_axes = { From a1db934f9968af86e65ad3710b371271c2a10488 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Wed, 20 Oct 2021 01:59:19 +0000 Subject: [PATCH 03/12] Fixed error when reloading log, added unit test --- starfish/core/types/_spot_finding_results.py | 3 +- starfish/core/types/test/test_saving_spots.py | 68 +++++++++++++++++++ 2 files changed, 70 insertions(+), 1 deletion(-) create mode 100644 starfish/core/types/test/test_saving_spots.py diff --git a/starfish/core/types/_spot_finding_results.py b/starfish/core/types/_spot_finding_results.py index 060656f2c..5b61f96c6 100644 --- a/starfish/core/types/_spot_finding_results.py +++ b/starfish/core/types/_spot_finding_results.py @@ -164,7 +164,8 @@ def load(cls, json_file: str): data = json.load(fl) with open(data["log"]["path"]) as f: - log = Log.decode(f.read()) + txt = json.load(f) + log = Log.decode(str(txt['log'])) rename_axes = { 'x': Coordinates.X.value, diff --git a/starfish/core/types/test/test_saving_spots.py b/starfish/core/types/test/test_saving_spots.py new file mode 100644 index 000000000..f7d1ffefd --- /dev/null +++ b/starfish/core/types/test/test_saving_spots.py @@ -0,0 +1,68 @@ +import os +import tempfile +import pytest +import pandas as pd +import numpy as np +import xarray as xr + +from starfish.types import Axes, Coordinates, CoordinateValue, Features +from starfish.core.types import SpotFindingResults, PerImageSliceSpotResults, SpotAttributes +from starfish.core.util.logging import Log + +def dummy_spots() -> SpotFindingResults: + rounds = 4 + channels = 3 + spot_count = 100 + img_dim = {'x':2048, 'y':2048, 'z':29} + + coords = {} + renameAxes = { + 'x': Coordinates.X.value, + 'y': Coordinates.Y.value, + 'z': Coordinates.Z.value + } + for dim in img_dim.keys(): + coords[renameAxes[dim]] = xr.DataArray(np.arange(0,1,img_dim[dim])) + + log = Log() + + spot_attributes_list = [] + for r in range(rounds): + for c in range(channels): + index = {Axes.ROUND: r, Axes.CH: c} + spots = SpotAttributes(pd.DataFrame( + np.random.randint(0,100,size=(spot_count,4)), + columns=[Axes.X.value, + Axes.Y.value, + Axes.ZPLANE.value, + Features.SPOT_RADIUS] + )) + spot_attributes_list.append( + (PerImageSliceSpotResults(spots, extras=None),index) + ) + + return SpotFindingResults( + imagestack_coords = coords, + log = log, + spot_attributes_list = spot_attributes_list + ) + +def test_saving_spots() -> None: + data = dummy_spots() + + #test serialization + tempdir = tempfile.mkdtemp() + print(tempdir) + data.save(tempdir+"/") + + #load back into memory + data2 = SpotFindingResults.load(os.path.join(tempdir, 'SpotFindingResults.json')) + + #ensure all items are equal + assert data.keys() == data2.keys() + assert data._log.encode() == data2._log.encode() + for ax in data.physical_coord_ranges.keys(): + np.testing.assert_equal(data.physical_coord_ranges[ax].to_numpy(), data2.physical_coord_ranges[ax].to_numpy()) + for k in data._results.keys(): + np.testing.assert_array_equal(data._results[k].spot_attrs.data, + data2._results[k].spot_attrs.data) From 32bc3b1553667f2d060f2961bcd17735948689f8 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Wed, 20 Oct 2021 23:44:12 +0000 Subject: [PATCH 04/12] Fixed flake8 errors --- starfish/core/types/_spot_finding_results.py | 15 +++--- starfish/core/types/test/test_saving_spots.py | 51 ++++++++++--------- 2 files changed, 34 insertions(+), 32 deletions(-) diff --git a/starfish/core/types/_spot_finding_results.py b/starfish/core/types/_spot_finding_results.py index 5b61f96c6..e00d76432 100644 --- a/starfish/core/types/_spot_finding_results.py +++ b/starfish/core/types/_spot_finding_results.py @@ -1,12 +1,13 @@ from dataclasses import dataclass -from typing import Any, Hashable, Mapping, MutableMapping, Optional, Sequence, Tuple, Dict +from typing import Any, Dict, Hashable, Mapping, MutableMapping, Optional, Sequence, Tuple import xarray as xr +import json + from starfish.core.types import Axes, Coordinates, SpotAttributes from starfish.core.util.logging import Log -import json AXES_ORDER = (Axes.ROUND, Axes.CH) @@ -148,7 +149,7 @@ def save(self, output_dir_name: str) -> None: @classmethod def load(cls, json_file: str): """Load serialized spot finding results. - + Parameters: ----------- json_file: str @@ -182,12 +183,12 @@ def load(cls, json_file: str): one = int(key.split("_")[1]) index = {AXES_ORDER[0]: zero, AXES_ORDER[1]: one} spots = SpotAttributes.load(path) - spot_attributes_list.append((PerImageSliceSpotResults(spots, extras=None),index)) + spot_attributes_list.append((PerImageSliceSpotResults(spots, extras=None), index)) return SpotFindingResults( - imagestack_coords = coords, - log = log, - spot_attributes_list = spot_attributes_list + imagestack_coords=coords, + log=log, + spot_attributes_list=spot_attributes_list ) @property diff --git a/starfish/core/types/test/test_saving_spots.py b/starfish/core/types/test/test_saving_spots.py index f7d1ffefd..9e8c5e157 100644 --- a/starfish/core/types/test/test_saving_spots.py +++ b/starfish/core/types/test/test_saving_spots.py @@ -1,20 +1,20 @@ import os import tempfile -import pytest -import pandas as pd + import numpy as np +import pandas as pd import xarray as xr -from starfish.types import Axes, Coordinates, CoordinateValue, Features -from starfish.core.types import SpotFindingResults, PerImageSliceSpotResults, SpotAttributes +from starfish.types import Axes, Coordinates, Features +from starfish.core.types import PerImageSliceSpotResults, SpotAttributes, SpotFindingResults from starfish.core.util.logging import Log def dummy_spots() -> SpotFindingResults: rounds = 4 channels = 3 spot_count = 100 - img_dim = {'x':2048, 'y':2048, 'z':29} - + img_dim = {'x': 2048, 'y': 2048, 'z': 29} + coords = {} renameAxes = { 'x': Coordinates.X.value, @@ -22,47 +22,48 @@ def dummy_spots() -> SpotFindingResults: 'z': Coordinates.Z.value } for dim in img_dim.keys(): - coords[renameAxes[dim]] = xr.DataArray(np.arange(0,1,img_dim[dim])) - + coords[renameAxes[dim]] = xr.DataArray(np.arange(0, 1, img_dim[dim])) + log = Log() - + spot_attributes_list = [] for r in range(rounds): for c in range(channels): index = {Axes.ROUND: r, Axes.CH: c} spots = SpotAttributes(pd.DataFrame( - np.random.randint(0,100,size=(spot_count,4)), + np.random.randint(0, 100, size=(spot_count, 4)), columns=[Axes.X.value, Axes.Y.value, Axes.ZPLANE.value, Features.SPOT_RADIUS] )) spot_attributes_list.append( - (PerImageSliceSpotResults(spots, extras=None),index) + (PerImageSliceSpotResults(spots, extras=None), index) ) - + return SpotFindingResults( - imagestack_coords = coords, - log = log, - spot_attributes_list = spot_attributes_list + imagestack_coords=coords, + log=log, + spot_attributes_list=spot_attributes_list ) def test_saving_spots() -> None: data = dummy_spots() - - #test serialization + + # test serialization tempdir = tempfile.mkdtemp() print(tempdir) - data.save(tempdir+"/") - - #load back into memory + data.save(tempdir + "/") + + # load back into memory data2 = SpotFindingResults.load(os.path.join(tempdir, 'SpotFindingResults.json')) - - #ensure all items are equal + + # ensure all items are equal assert data.keys() == data2.keys() - assert data._log.encode() == data2._log.encode() + assert data._log.encode() == data2._log.encode() for ax in data.physical_coord_ranges.keys(): - np.testing.assert_equal(data.physical_coord_ranges[ax].to_numpy(), data2.physical_coord_ranges[ax].to_numpy()) + np.testing.assert_equal(data.physical_coord_ranges[ax].to_numpy(), + data2.physical_coord_ranges[ax].to_numpy()) for k in data._results.keys(): np.testing.assert_array_equal(data._results[k].spot_attrs.data, - data2._results[k].spot_attrs.data) + data2._results[k].spot_attrs.data) From 8ac2682a00b68fa37f5f23bf83a5ee442814e439 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Thu, 21 Oct 2021 00:01:39 +0000 Subject: [PATCH 05/12] Fixed imports --- starfish/core/types/_spot_finding_results.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/starfish/core/types/_spot_finding_results.py b/starfish/core/types/_spot_finding_results.py index e00d76432..bc2912323 100644 --- a/starfish/core/types/_spot_finding_results.py +++ b/starfish/core/types/_spot_finding_results.py @@ -1,11 +1,12 @@ from dataclasses import dataclass from typing import Any, Dict, Hashable, Mapping, MutableMapping, Optional, Sequence, Tuple -import xarray as xr - import json -from starfish.core.types import Axes, Coordinates, SpotAttributes +import xarray as xr + +from starfish.core.types import PerImageSliceSpotResults, SpotAttributes, SpotFindingResults +from starfish.types import Axes, Coordinates, SpotAttributes from starfish.core.util.logging import Log From 12c2ecc3e71201233802fecb4f8985b835760cc9 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Thu, 21 Oct 2021 00:05:51 +0000 Subject: [PATCH 06/12] Fixed import grouping --- starfish/core/types/_spot_finding_results.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/starfish/core/types/_spot_finding_results.py b/starfish/core/types/_spot_finding_results.py index bc2912323..18e3fced6 100644 --- a/starfish/core/types/_spot_finding_results.py +++ b/starfish/core/types/_spot_finding_results.py @@ -1,14 +1,12 @@ from dataclasses import dataclass -from typing import Any, Dict, Hashable, Mapping, MutableMapping, Optional, Sequence, Tuple - import json +from typing import Any, Dict, Hashable, Mapping, MutableMapping, Optional, Sequence, Tuple import xarray as xr from starfish.core.types import PerImageSliceSpotResults, SpotAttributes, SpotFindingResults -from starfish.types import Axes, Coordinates, SpotAttributes from starfish.core.util.logging import Log - +from starfish.types import Axes, Coordinates, SpotAttributes AXES_ORDER = (Axes.ROUND, Axes.CH) From d7d2d34f46db4b9af18407f00b5404ad0b20d113 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Thu, 21 Oct 2021 00:09:19 +0000 Subject: [PATCH 07/12] Fixed import order --- starfish/core/types/_spot_finding_results.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/starfish/core/types/_spot_finding_results.py b/starfish/core/types/_spot_finding_results.py index 18e3fced6..b5ac6cb2a 100644 --- a/starfish/core/types/_spot_finding_results.py +++ b/starfish/core/types/_spot_finding_results.py @@ -1,12 +1,12 @@ -from dataclasses import dataclass import json +from dataclasses import dataclass from typing import Any, Dict, Hashable, Mapping, MutableMapping, Optional, Sequence, Tuple import xarray as xr +from starfish.types import Axes, Coordinates, SpotAttributes from starfish.core.types import PerImageSliceSpotResults, SpotAttributes, SpotFindingResults from starfish.core.util.logging import Log -from starfish.types import Axes, Coordinates, SpotAttributes AXES_ORDER = (Axes.ROUND, Axes.CH) From 052676dbff88d886e5fd253b23b934bfc9a653d0 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Thu, 21 Oct 2021 00:12:39 +0000 Subject: [PATCH 08/12] Resolving lint import error --- starfish/core/types/_spot_finding_results.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/starfish/core/types/_spot_finding_results.py b/starfish/core/types/_spot_finding_results.py index b5ac6cb2a..12bacb676 100644 --- a/starfish/core/types/_spot_finding_results.py +++ b/starfish/core/types/_spot_finding_results.py @@ -4,8 +4,8 @@ import xarray as xr -from starfish.types import Axes, Coordinates, SpotAttributes from starfish.core.types import PerImageSliceSpotResults, SpotAttributes, SpotFindingResults +from starfish.types import Axes, Coordinates, SpotAttributes from starfish.core.util.logging import Log AXES_ORDER = (Axes.ROUND, Axes.CH) From 0cc4e84f509a222a210ad85e94747ac36b3175e6 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Thu, 21 Oct 2021 00:18:30 +0000 Subject: [PATCH 09/12] Rearranged imports for flake --- starfish/core/types/_spot_finding_results.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/starfish/core/types/_spot_finding_results.py b/starfish/core/types/_spot_finding_results.py index 12bacb676..bccf036d0 100644 --- a/starfish/core/types/_spot_finding_results.py +++ b/starfish/core/types/_spot_finding_results.py @@ -5,8 +5,8 @@ import xarray as xr from starfish.core.types import PerImageSliceSpotResults, SpotAttributes, SpotFindingResults -from starfish.types import Axes, Coordinates, SpotAttributes from starfish.core.util.logging import Log +from starfish.types import Axes, Coordinates, SpotAttributes AXES_ORDER = (Axes.ROUND, Axes.CH) From b67e4688f047f4479b70b340b22df75c3bbeea4d Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Fri, 22 Oct 2021 02:52:32 +0000 Subject: [PATCH 10/12] Fixed imports, changed json paths to be relative to that dir --- starfish/core/types/_spot_finding_results.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/starfish/core/types/_spot_finding_results.py b/starfish/core/types/_spot_finding_results.py index bccf036d0..e7dfc6a91 100644 --- a/starfish/core/types/_spot_finding_results.py +++ b/starfish/core/types/_spot_finding_results.py @@ -1,12 +1,12 @@ import json +import os from dataclasses import dataclass from typing import Any, Dict, Hashable, Mapping, MutableMapping, Optional, Sequence, Tuple import xarray as xr -from starfish.core.types import PerImageSliceSpotResults, SpotAttributes, SpotFindingResults +from starfish.core.types import Axes, Coordinates, SpotAttributes from starfish.core.util.logging import Log -from starfish.types import Axes, Coordinates, SpotAttributes AXES_ORDER = (Axes.ROUND, Axes.CH) @@ -121,28 +121,31 @@ def save(self, output_dir_name: str) -> None: """ json_data: Dict[str, Any] = {} + os.chdir(os.path.dirname(output_dir_name)) + base_name = os.path.basename(output_dir_name) + coords = {} for key in self.physical_coord_ranges.keys(): - path = "{}coords_{}.nc".format(output_dir_name, key) + path = "{}coords_{}.nc".format(base_name, key) coords[key] = path self.physical_coord_ranges[key].to_netcdf(path) json_data["physical_coord_ranges"] = coords path = "{}log.arr" json_data["log"] = {} - json_data["log"]["path"] = path.format(output_dir_name) - with open(path.format(output_dir_name), "w") as f: + json_data["log"]["path"] = path.format(base_name) + with open(path.format(base_name), "w") as f: f.write(self.log.encode()) spot_attrs = {} for key in self._results.keys(): - path = "{}spots_{}_{}.nc".format(output_dir_name, key[0], key[1]) + path = "{}spots_{}_{}.nc".format(base_name, key[0], key[1]) spot_attrs["{}_{}".format(key[0], key[1])] = path self._results[key].spot_attrs.save(path) json_data["spot_attrs"] = spot_attrs save = json.dumps(json_data) - with open("{}SpotFindingResults.json".format(output_dir_name), "w") as f: + with open("{}SpotFindingResults.json".format(base_name), "w") as f: f.write(save) @classmethod @@ -162,6 +165,7 @@ def load(cls, json_file: str): """ fl = open(json_file) data = json.load(fl) + os.chdir(os.path.dirname(json_file)) with open(data["log"]["path"]) as f: txt = json.load(f) From 409e2f50c92f386b24b8382020bc458f95d498c5 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Fri, 22 Oct 2021 23:11:27 +0000 Subject: [PATCH 11/12] Fixed loading of log json --- starfish/core/types/_spot_finding_results.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/starfish/core/types/_spot_finding_results.py b/starfish/core/types/_spot_finding_results.py index e7dfc6a91..eb4171619 100644 --- a/starfish/core/types/_spot_finding_results.py +++ b/starfish/core/types/_spot_finding_results.py @@ -168,8 +168,9 @@ def load(cls, json_file: str): os.chdir(os.path.dirname(json_file)) with open(data["log"]["path"]) as f: - txt = json.load(f) - log = Log.decode(str(txt['log'])) + txt = json.load(f)['log'] + txt = json.dumps(txt) + log = Log.decode(txt) rename_axes = { 'x': Coordinates.X.value, From 0357662dbf26249a44ae5765d185ad6ddd74d1dc Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Tue, 26 Oct 2021 19:32:45 +0000 Subject: [PATCH 12/12] Fixed issue with changing pwd while saving and loading --- starfish/core/types/_spot_finding_results.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/starfish/core/types/_spot_finding_results.py b/starfish/core/types/_spot_finding_results.py index eb4171619..a9a910262 100644 --- a/starfish/core/types/_spot_finding_results.py +++ b/starfish/core/types/_spot_finding_results.py @@ -121,6 +121,7 @@ def save(self, output_dir_name: str) -> None: """ json_data: Dict[str, Any] = {} + pwd = os.getcwd() os.chdir(os.path.dirname(output_dir_name)) base_name = os.path.basename(output_dir_name) @@ -148,6 +149,8 @@ def save(self, output_dir_name: str) -> None: with open("{}SpotFindingResults.json".format(base_name), "w") as f: f.write(save) + os.chdir(pwd) + @classmethod def load(cls, json_file: str): """Load serialized spot finding results. @@ -165,6 +168,8 @@ def load(cls, json_file: str): """ fl = open(json_file) data = json.load(fl) + pwd = os.getcwd() + os.chdir(os.path.dirname(json_file)) with open(data["log"]["path"]) as f: @@ -189,6 +194,8 @@ def load(cls, json_file: str): spots = SpotAttributes.load(path) spot_attributes_list.append((PerImageSliceSpotResults(spots, extras=None), index)) + os.chdir(pwd) + return SpotFindingResults( imagestack_coords=coords, log=log,