From 35ee06f3db98a55a448378b35d0ed124db1fc50d Mon Sep 17 00:00:00 2001 From: Juliane Reinhardt Date: Wed, 19 Aug 2020 10:21:45 -0700 Subject: [PATCH 01/29] fixing error messages --- xicam/spectral/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/xicam/spectral/__init__.py b/xicam/spectral/__init__.py index 775e420..0833e75 100644 --- a/xicam/spectral/__init__.py +++ b/xicam/spectral/__init__.py @@ -4,6 +4,7 @@ from xicam.plugins import GUIPlugin, GUILayout from xicam.plugins.guiplugin import PanelState from xicam.gui.widgets.imageviewmixins import XArrayView, DepthPlot, BetterTicks, BetterLayout, BetterPlots +from xicam.gui.widgets.dynimageview import DynImageView import logging from xicam.gui.widgets.library import LibraryWidget from databroker.core import BlueskyRun From b3eb0ba26aec1d390ea5a7134308c6f962d21919 Mon Sep 17 00:00:00 2001 From: Juliane Reinhardt Date: Wed, 19 Aug 2020 10:52:51 -0700 Subject: [PATCH 02/29] testing --- xicam/spectral/__init__.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/xicam/spectral/__init__.py b/xicam/spectral/__init__.py index 19e294a..7dfb5f9 100644 --- a/xicam/spectral/__init__.py +++ b/xicam/spectral/__init__.py @@ -7,6 +7,7 @@ from xicam.gui.widgets.dynimageview import DynImageView import logging from xicam.gui.widgets.library import LibraryWidget +from xicam.gui.widgets.linearworkfloweditor import WorkflowEditor from databroker.core import BlueskyRun from xarray import DataArray @@ -41,15 +42,15 @@ def __init__(self): self.catalog_viewer = CatalogViewerBlend() self.results_viewer = DynImageView() # self.workflow_widget = WorkflowEditor() - # self.library_viewer = LibraryWidget() + self.library_viewer = LibraryWidget() - self._workflow = StxmWorkflow() # Create a workflow - self._workflow_editor = WorkflowEditor(workflow=self._workflow) - self._workflow_editor.sigRunWorkflow.connect(self.run_workflow) + # self._workflow = StxmWorkflow() # Create a workflow + # self._workflow_editor = WorkflowEditor(workflow=self._workflow) + # self._workflow_editor.sigRunWorkflow.connect(self.run_workflow) - catalog_viewer_layout = GUILayout(self.catalog_viewer, - right=self._workflow_editor, - bottom=self.results_viewer) + # catalog_viewer_layout = GUILayout(self.catalog_viewer, + # right=self._workflow_editor, + # bottom=self.results_viewer) self.stages = { "Acquire": GUILayout(QWidget()), From ee6a74a5df81ff6950b4d7c49f572bf40c9cef74 Mon Sep 17 00:00:00 2001 From: Juliane Reinhardt Date: Wed, 19 Aug 2020 11:02:51 -0700 Subject: [PATCH 03/29] back to working state --- xicam/spectral/__init__.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/xicam/spectral/__init__.py b/xicam/spectral/__init__.py index 7dfb5f9..30772d0 100644 --- a/xicam/spectral/__init__.py +++ b/xicam/spectral/__init__.py @@ -8,6 +8,7 @@ import logging from xicam.gui.widgets.library import LibraryWidget from xicam.gui.widgets.linearworkfloweditor import WorkflowEditor +from .workflows.workflows import StxmWorkflow from databroker.core import BlueskyRun from xarray import DataArray @@ -41,21 +42,21 @@ class SpectralPlugin(GUIPlugin): def __init__(self): self.catalog_viewer = CatalogViewerBlend() self.results_viewer = DynImageView() - # self.workflow_widget = WorkflowEditor() + self.workflow_widget = WorkflowEditor self.library_viewer = LibraryWidget() - # self._workflow = StxmWorkflow() # Create a workflow - # self._workflow_editor = WorkflowEditor(workflow=self._workflow) - # self._workflow_editor.sigRunWorkflow.connect(self.run_workflow) + self._workflow = StxmWorkflow() # Create a workflow + self._workflow_editor = WorkflowEditor(workflow=self._workflow) + self._workflow_editor.sigRunWorkflow.connect(self.run_workflow) - # catalog_viewer_layout = GUILayout(self.catalog_viewer, - # right=self._workflow_editor, - # bottom=self.results_viewer) + catalog_viewer_layout = GUILayout(self.catalog_viewer, + right=self._workflow_editor, + bottom=self.results_viewer) self.stages = { "Acquire": GUILayout(QWidget()), "Library": GUILayout(left=PanelState.Disabled, lefttop=PanelState.Disabled, center=self.library_viewer, right=self.catalog_viewer), - "Map": GUILayout(self.catalog_viewer), + "Map": GUILayout(self.catalog_viewer, right=self._workflow_editor, bottom=self.results_viewer), "Decomposition": GUILayout(QWidget()), "Clustering": GUILayout(QWidget()), } From 5fa3dd8907a3240d553e106eb62ed0767efd6c8c Mon Sep 17 00:00:00 2001 From: Juliane Reinhardt Date: Wed, 19 Aug 2020 12:39:21 -0700 Subject: [PATCH 04/29] add operations --- xicam/spectral/workflows/workflows.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/xicam/spectral/workflows/workflows.py b/xicam/spectral/workflows/workflows.py index deba5bb..e47066c 100644 --- a/xicam/spectral/workflows/workflows.py +++ b/xicam/spectral/workflows/workflows.py @@ -1,6 +1,7 @@ from shop.correction.register import RegisterOperation -from shop.correction.filter import MedianFilterOperation, WienerFilterOperation +from shop.correction.filter import MedianFilterOperation, WienerFilterOperation, Denoise, Despike +from shop.correction.opitcal_density import CalcOpticalDensity # from ..operations.register import RegisterOperation # from shop.correction.filter import median_filter, wiener_filter @@ -16,10 +17,15 @@ def __init__(self): register = RegisterOperation() wiener = MedianFilterOperation() median = WienerFilterOperation() + denoise = Denoise() + #TODO: how can calcOD receive IO map in workflow editor + calcOD = CalcOpticalDensity() # Add operation to the workflow self.add_operations(register) self.add_operations(wiener) self.add_operations(median) + self.add_operation(denoise) + self.add_operation(calcOD) #connect one operation's output with another operation's input # self.add_link() From ee2a3a5fda4b0ba2957bcd9c5b3918c319fa6115 Mon Sep 17 00:00:00 2001 From: Juliane Reinhardt Date: Thu, 27 Aug 2020 12:42:54 -0700 Subject: [PATCH 05/29] change to pystxmtools package for import of operations --- .gitignore | 1 + xicam/spectral/ingestors/__init__.py | 2 +- xicam/spectral/workflows/workflows.py | 14 +++++++++----- 3 files changed, 11 insertions(+), 6 deletions(-) diff --git a/.gitignore b/.gitignore index 65dd56b..04cad7f 100644 --- a/.gitignore +++ b/.gitignore @@ -9,3 +9,4 @@ build/ dist/ *.egg-info/ .vscode +venv diff --git a/xicam/spectral/ingestors/__init__.py b/xicam/spectral/ingestors/__init__.py index 7c352db..2af0144 100644 --- a/xicam/spectral/ingestors/__init__.py +++ b/xicam/spectral/ingestors/__init__.py @@ -101,7 +101,7 @@ def ingest_cxi(path): 'entry_1/data_1/translation': sample_y} )] - # Compose bluseksy run + # Compose bluesky run run_bundle = event_model.compose_run() # type: event_model.ComposeRunBundle #Create start document diff --git a/xicam/spectral/workflows/workflows.py b/xicam/spectral/workflows/workflows.py index e47066c..06a0b74 100644 --- a/xicam/spectral/workflows/workflows.py +++ b/xicam/spectral/workflows/workflows.py @@ -1,7 +1,8 @@ -from shop.correction.register import RegisterOperation -from shop.correction.filter import MedianFilterOperation, WienerFilterOperation, Denoise, Despike -from shop.correction.opitcal_density import CalcOpticalDensity +from pystxmtools.corrections.register import RegisterOperation +from pystxmtools.corrections.filter import MedianFilterOperation, WienerFilterOperation, Denoise, Despike +from pystxmtools.corrections.optical_density import CalcOpticalDensity +from pystxmtools.corrections.fitting import LeastSquaresFit # from ..operations.register import RegisterOperation # from shop.correction.filter import median_filter, wiener_filter @@ -19,13 +20,16 @@ def __init__(self): median = WienerFilterOperation() denoise = Denoise() #TODO: how can calcOD receive IO map in workflow editor - calcOD = CalcOpticalDensity() + calc_OD = CalcOpticalDensity() + lstsq_fit = LeastSquaresFit() + # Add operation to the workflow self.add_operations(register) self.add_operations(wiener) self.add_operations(median) self.add_operation(denoise) - self.add_operation(calcOD) + self.add_operation(calc_OD) + self.add_operation(lstsq_fit) #connect one operation's output with another operation's input # self.add_link() From 4d04629e2e3ac71a2885ba8201a752a195e11cf3 Mon Sep 17 00:00:00 2001 From: Juliane Reinhardt Date: Mon, 31 Aug 2020 16:50:26 -0700 Subject: [PATCH 06/29] add entry points for pystxm operations --- setup.py | 11 +++++++++++ xicam/spectral/workflows/workflows.py | 5 +++-- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index f4b98c9..fcced50 100644 --- a/setup.py +++ b/setup.py @@ -39,6 +39,17 @@ entry_points={ "xicam.plugins.GUIPlugin": ["spectral = xicam.spectral:SpectralPlugin"], "databroker.ingestors": ["application/x-hdf5 = xicam.spectral.ingestors:ingest_nxSTXM"], + "xicam.plugins.OperationPlugin": ["wiener_filter = pystxmtools.corrections.filter:WienerFilterOperation", + "median_filter = pystxmtools.corrections.filter:MedianFilterOperation", + "nl_means_filter = pystxmtools.corrections.filter:nlMeansFilter", + "despike = pystxmtools.corrections.filter:Despike", + "denoise = pystxmtools.corrections.filter:Denoise", + "calc_optical_density = pystxmtools.corrections.optical_density:CalcOpticalDensity", + "get_I0_mask = pystxmtools.corrections.optical_density:CalcI0Mask", + "register_stack = pystxmtools.corrections.register:RegisterOperation", + "lin_fit_spectra = pystxmtools.corrections.fitting:LinFitSpectra", + "lstsq_fit = pystxmtools.corrections.fitting:LeastSquaresFit", + "nn_lstsq = pystxmtools.corrections.fitting:NNLeastSquaresFit"] # "databroker.handlers": [ # "JPEG = xicam.catalog_viewer.image_handlers:JPEGHandler", # "TIFF = xicam.catalog_viewer.image_handlers:TIFFHandler", diff --git a/xicam/spectral/workflows/workflows.py b/xicam/spectral/workflows/workflows.py index 06a0b74..49bee90 100644 --- a/xicam/spectral/workflows/workflows.py +++ b/xicam/spectral/workflows/workflows.py @@ -20,7 +20,7 @@ def __init__(self): median = WienerFilterOperation() denoise = Denoise() #TODO: how can calcOD receive IO map in workflow editor - calc_OD = CalcOpticalDensity() + # calc_OD = CalcOpticalDensity() lstsq_fit = LeastSquaresFit() # Add operation to the workflow @@ -28,8 +28,9 @@ def __init__(self): self.add_operations(wiener) self.add_operations(median) self.add_operation(denoise) - self.add_operation(calc_OD) + # self.add_operation(calc_OD) self.add_operation(lstsq_fit) + # self.auto_connect_all() #connect one operation's output with another operation's input # self.add_link() From b296a7cab4bab7e8b9bb8a96e1937f3096220789 Mon Sep 17 00:00:00 2001 From: Juliane Reinhardt Date: Mon, 31 Aug 2020 16:56:56 -0700 Subject: [PATCH 07/29] remove ingestor parts --> see different PR --- xicam/spectral/ingestors/__init__.py | 73 ---------------------------- 1 file changed, 73 deletions(-) diff --git a/xicam/spectral/ingestors/__init__.py b/xicam/spectral/ingestors/__init__.py index 2af0144..38cd06d 100644 --- a/xicam/spectral/ingestors/__init__.py +++ b/xicam/spectral/ingestors/__init__.py @@ -67,79 +67,6 @@ def ingest_nxSTXM(paths): yield 'stop', run_bundle.compose_stop() -def ingest_cxi(path): - - assert len(path) == 1 - path = path[0] - - h5 = h5py.File(path, 'r') - - data = h5['entry_1']['data_1']['data'][()] # data array - results = h5['entry_1']['image_1']['data'][()] # data array - translation = h5['entry_1']['data_1']['translation'][()] # data coords - sample_x = translation[:,0] #these are the scan positions not the x/y dimensions of the image or diffraction pattern - sample_y = translation[:,1] - # sample_z = translation[:,2] - - energy = h5['entry_1']['instrument_1']['source_1']['energy'][()] #energy in Joule - wavelength = speed_of_light*h/energy - # energy_eV = energy/e - energy_eV = np.empty(sample_x) - energy_eV.fill(energy/e) - - ### Create data array - #TODO energy, coords len not matching number of frames - xarray = DataArray(data, dims=['E (eV)', 'y (μm)', 'x (μm)'], coords=[energy_eV, sample_y, sample_x]) - dask_data = da.from_array(xarray) - - ### describe projections and create databroker run catalog (=run_bundle) - projections = [('NXcxi_ptycho', - {'entry_1/data_1/data': ('primary', 'raw'), - 'entry_1/instrument_1/source_1/energy': energy, - #TODO how to define sample coords from cxi - 'entry_1/data_1/translation': sample_x, - 'entry_1/data_1/translation': sample_y} - )] - - # Compose bluesky run - run_bundle = event_model.compose_run() # type: event_model.ComposeRunBundle - - #Create start document - start_doc = run_bundle.start_doc - start_doc["sample_name"] = Path(paths[0]).resolve().stem - - #TODO add projections once defined - #start_doc["projections"] = projections - yield 'start', start_doc - - # Compose descriptor - source = 'nx_cxi' - frame_data_keys = {'raw': {'source': source, - 'dtype': 'number', - 'dims': xarray.dims, - # 'coords': [energy, sample_y, sample_x], - 'shape': data.shape}} - frame_stream_name = 'primary' - frame_stream_bundle = run_bundle.compose_descriptor(data_keys=frame_data_keys, - name=frame_stream_name, - # configuration=_metadata(path) - ) - yield 'descriptor', frame_stream_bundle.descriptor_doc - - yield 'event', frame_stream_bundle.compose_event(data={'raw': dask_data}, - timestamps={'raw': time.time()}) - - #create stop document - yield 'stop', run_bundle.compose_stop() - - # return run_bundle - - -def ingest_hdr(path): - assert len(path) == 1 - path = path[0] - - h5 = h5py.File(path, 'r') # From f9698d9959ff9d1d65806c62ca4a50044576b253 Mon Sep 17 00:00:00 2001 From: Juliane Reinhardt Date: Fri, 4 Sep 2020 12:14:00 -0700 Subject: [PATCH 08/29] test example --- xicam/tests/test_examples.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/xicam/tests/test_examples.py b/xicam/tests/test_examples.py index 9691fca..d4d3019 100644 --- a/xicam/tests/test_examples.py +++ b/xicam/tests/test_examples.py @@ -7,7 +7,7 @@ from skimage import data from scipy.ndimage.interpolation import shift -from shop.correction.register import register_frames_stack +from pystxmtools.corrections.register import register_frames_stack @fixture def test_stack(): @@ -17,14 +17,15 @@ def test_stack(): stack = [] for n in range(n_frames): stack.append(shift(image, shifts[n])) - return np.asarray(stack) + return np.asarray(stack), shifts def test_register_frame_stack(test_stack): - "Test something in register_frame_stack " - aligned_frames = register_frames_stack(test_stack) + aligned_frames = register_frames_stack(test_stack[0], mode='translation') + shifts_calc = register_frames_stack(test_stack[1]) print(aligned_frames.shape) print(test_stack.shape) - assert aligned_frames.shape[0] == test_stack.shape[0] + assert aligned_frames.shape[0] == test_stack[0].shape[0] + assert shifts_calc == test_stack[1] From ae411b439f79f2d5625b797b8b3c2e5035f3a475 Mon Sep 17 00:00:00 2001 From: Juliane Reinhardt Date: Fri, 11 Sep 2020 12:03:08 -0700 Subject: [PATCH 09/29] use apply signature and new imports of operation --- setup.py | 2 +- xicam/spectral/__init__.py | 6 +++--- xicam/spectral/workflows/workflows.py | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/setup.py b/setup.py index fcced50..95383f1 100644 --- a/setup.py +++ b/setup.py @@ -40,7 +40,7 @@ "xicam.plugins.GUIPlugin": ["spectral = xicam.spectral:SpectralPlugin"], "databroker.ingestors": ["application/x-hdf5 = xicam.spectral.ingestors:ingest_nxSTXM"], "xicam.plugins.OperationPlugin": ["wiener_filter = pystxmtools.corrections.filter:WienerFilterOperation", - "median_filter = pystxmtools.corrections.filter:MedianFilterOperation", + "median_filter = pystxmtools.corrections.filter:median_filter_operation", "nl_means_filter = pystxmtools.corrections.filter:nlMeansFilter", "despike = pystxmtools.corrections.filter:Despike", "denoise = pystxmtools.corrections.filter:Denoise", diff --git a/xicam/spectral/__init__.py b/xicam/spectral/__init__.py index 30772d0..e6c85d2 100644 --- a/xicam/spectral/__init__.py +++ b/xicam/spectral/__init__.py @@ -6,7 +6,7 @@ from xicam.gui.widgets.imageviewmixins import XArrayView, DepthPlot, BetterTicks, BetterLayout, BetterPlots from xicam.gui.widgets.dynimageview import DynImageView import logging -from xicam.gui.widgets.library import LibraryWidget +#from xicam.gui.widgets.library import LibraryWidget from xicam.gui.widgets.linearworkfloweditor import WorkflowEditor from .workflows.workflows import StxmWorkflow from databroker.core import BlueskyRun @@ -43,7 +43,7 @@ def __init__(self): self.catalog_viewer = CatalogViewerBlend() self.results_viewer = DynImageView() self.workflow_widget = WorkflowEditor - self.library_viewer = LibraryWidget() + # self.library_viewer = LibraryWidget() self._workflow = StxmWorkflow() # Create a workflow self._workflow_editor = WorkflowEditor(workflow=self._workflow) @@ -55,7 +55,7 @@ def __init__(self): self.stages = { "Acquire": GUILayout(QWidget()), - "Library": GUILayout(left=PanelState.Disabled, lefttop=PanelState.Disabled, center=self.library_viewer, right=self.catalog_viewer), + # "Library": GUILayout(left=PanelState.Disabled, lefttop=PanelState.Disabled, center=self.library_viewer, right=self.catalog_viewer), "Map": GUILayout(self.catalog_viewer, right=self._workflow_editor, bottom=self.results_viewer), "Decomposition": GUILayout(QWidget()), "Clustering": GUILayout(QWidget()), diff --git a/xicam/spectral/workflows/workflows.py b/xicam/spectral/workflows/workflows.py index 49bee90..a21f0ce 100644 --- a/xicam/spectral/workflows/workflows.py +++ b/xicam/spectral/workflows/workflows.py @@ -1,6 +1,6 @@ from pystxmtools.corrections.register import RegisterOperation -from pystxmtools.corrections.filter import MedianFilterOperation, WienerFilterOperation, Denoise, Despike +from pystxmtools.corrections.filter import median_filter_operation, WienerFilterOperation, Denoise, Despike from pystxmtools.corrections.optical_density import CalcOpticalDensity from pystxmtools.corrections.fitting import LeastSquaresFit @@ -16,7 +16,7 @@ def __init__(self): # Create instances of operations register = RegisterOperation() - wiener = MedianFilterOperation() + wiener = median_filter_operation() median = WienerFilterOperation() denoise = Denoise() #TODO: how can calcOD receive IO map in workflow editor From b11de6279d512032128dc2904fa537dc0a01fe4b Mon Sep 17 00:00:00 2001 From: Juliane Reinhardt Date: Sun, 13 Sep 2020 16:40:40 -0700 Subject: [PATCH 10/29] adapt new decorator written pystxm operations --- setup.py | 18 +++++++++--------- xicam/spectral/workflows/workflows.py | 16 ++++++++-------- 2 files changed, 17 insertions(+), 17 deletions(-) diff --git a/setup.py b/setup.py index 95383f1..6243fbc 100644 --- a/setup.py +++ b/setup.py @@ -39,17 +39,17 @@ entry_points={ "xicam.plugins.GUIPlugin": ["spectral = xicam.spectral:SpectralPlugin"], "databroker.ingestors": ["application/x-hdf5 = xicam.spectral.ingestors:ingest_nxSTXM"], - "xicam.plugins.OperationPlugin": ["wiener_filter = pystxmtools.corrections.filter:WienerFilterOperation", + "xicam.plugins.OperationPlugin": ["wiener_filter = pystxmtools.corrections.filter:wiener_filter_operation", "median_filter = pystxmtools.corrections.filter:median_filter_operation", - "nl_means_filter = pystxmtools.corrections.filter:nlMeansFilter", - "despike = pystxmtools.corrections.filter:Despike", - "denoise = pystxmtools.corrections.filter:Denoise", - "calc_optical_density = pystxmtools.corrections.optical_density:CalcOpticalDensity", - "get_I0_mask = pystxmtools.corrections.optical_density:CalcI0Mask", + "nl_means_filter = pystxmtools.corrections.filter:nl_means_filter_operation", + "despike = pystxmtools.corrections.filter:despike_operation", + "denoise = pystxmtools.corrections.filter:denoise_operation", + "calc_optical_density = pystxmtools.corrections.optical_density:calc_opt_density_operation", + "get_I0_mask = pystxmtools.corrections.optical_density:calc_I0_mask_operation", "register_stack = pystxmtools.corrections.register:RegisterOperation", - "lin_fit_spectra = pystxmtools.corrections.fitting:LinFitSpectra", - "lstsq_fit = pystxmtools.corrections.fitting:LeastSquaresFit", - "nn_lstsq = pystxmtools.corrections.fitting:NNLeastSquaresFit"] + "lin_fit_spectra = pystxmtools.corrections.fitting:lin_fit_operation", + "lstsq_fit = pystxmtools.corrections.fitting:lsq_fit_operation", + "nn_lstsq = pystxmtools.corrections.fitting:nn_lsq_fit_operation"] # "databroker.handlers": [ # "JPEG = xicam.catalog_viewer.image_handlers:JPEGHandler", # "TIFF = xicam.catalog_viewer.image_handlers:TIFFHandler", diff --git a/xicam/spectral/workflows/workflows.py b/xicam/spectral/workflows/workflows.py index a21f0ce..4a6363c 100644 --- a/xicam/spectral/workflows/workflows.py +++ b/xicam/spectral/workflows/workflows.py @@ -1,8 +1,8 @@ from pystxmtools.corrections.register import RegisterOperation -from pystxmtools.corrections.filter import median_filter_operation, WienerFilterOperation, Denoise, Despike -from pystxmtools.corrections.optical_density import CalcOpticalDensity -from pystxmtools.corrections.fitting import LeastSquaresFit +from pystxmtools.corrections.filter import median_filter_operation, wiener_filter_operation, denoise_operation, despike_operation +from pystxmtools.corrections.optical_density import calc_opt_density_operation +from pystxmtools.corrections.fitting import lsq_fit_operation # from ..operations.register import RegisterOperation # from shop.correction.filter import median_filter, wiener_filter @@ -17,18 +17,18 @@ def __init__(self): # Create instances of operations register = RegisterOperation() wiener = median_filter_operation() - median = WienerFilterOperation() - denoise = Denoise() + median = wiener_filter_operation() + denoise = denoise_operation() #TODO: how can calcOD receive IO map in workflow editor - # calc_OD = CalcOpticalDensity() - lstsq_fit = LeastSquaresFit() + calc_OD = calc_opt_density_operation() + lstsq_fit = lsq_fit_operation() # Add operation to the workflow self.add_operations(register) self.add_operations(wiener) self.add_operations(median) self.add_operation(denoise) - # self.add_operation(calc_OD) + self.add_operation(calc_OD) self.add_operation(lstsq_fit) # self.auto_connect_all() #connect one operation's output with another operation's input From 6b88e3cc30bb6a084ef860190ea089d6980636aa Mon Sep 17 00:00:00 2001 From: Juliane Reinhardt Date: Sun, 13 Sep 2020 21:18:55 -0700 Subject: [PATCH 11/29] modify imports for register_operation --- setup.py | 2 +- xicam/spectral/workflows/workflows.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/setup.py b/setup.py index 6243fbc..05a9dc3 100644 --- a/setup.py +++ b/setup.py @@ -46,7 +46,7 @@ "denoise = pystxmtools.corrections.filter:denoise_operation", "calc_optical_density = pystxmtools.corrections.optical_density:calc_opt_density_operation", "get_I0_mask = pystxmtools.corrections.optical_density:calc_I0_mask_operation", - "register_stack = pystxmtools.corrections.register:RegisterOperation", + "register_stack = pystxmtools.corrections.register:register_operation", "lin_fit_spectra = pystxmtools.corrections.fitting:lin_fit_operation", "lstsq_fit = pystxmtools.corrections.fitting:lsq_fit_operation", "nn_lstsq = pystxmtools.corrections.fitting:nn_lsq_fit_operation"] diff --git a/xicam/spectral/workflows/workflows.py b/xicam/spectral/workflows/workflows.py index 4a6363c..d66bf7c 100644 --- a/xicam/spectral/workflows/workflows.py +++ b/xicam/spectral/workflows/workflows.py @@ -1,5 +1,5 @@ -from pystxmtools.corrections.register import RegisterOperation +from pystxmtools.corrections.register import register_operation from pystxmtools.corrections.filter import median_filter_operation, wiener_filter_operation, denoise_operation, despike_operation from pystxmtools.corrections.optical_density import calc_opt_density_operation from pystxmtools.corrections.fitting import lsq_fit_operation @@ -15,7 +15,7 @@ def __init__(self): super(StxmWorkflow, self).__init__(name="STXM Workflow") # Create instances of operations - register = RegisterOperation() + register = register_operation() wiener = median_filter_operation() median = wiener_filter_operation() denoise = denoise_operation() From a2bb9c7252f90c46d73f6bc9b5ec769c8bbea120 Mon Sep 17 00:00:00 2001 From: Juliane Reinhardt Date: Wed, 16 Sep 2020 00:18:20 -0700 Subject: [PATCH 12/29] messing with the test --- xicam/tests/test_examples.py | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/xicam/tests/test_examples.py b/xicam/tests/test_examples.py index d4d3019..37e3e4e 100644 --- a/xicam/tests/test_examples.py +++ b/xicam/tests/test_examples.py @@ -13,19 +13,22 @@ def test_stack(): image = data.camera() n_frames = 10 - shifts = [(round(random.uniform(1, 50), 2), round(random.uniform(1, 50), 2)) for i in range(n_frames)] - stack = [] + # shifts = (round(random.uniform(1, 50), 2), round(random.uniform(1, 50), 2)) + im_shift_stack = [] for n in range(n_frames): - stack.append(shift(image, shifts[n])) - return np.asarray(stack), shifts + shifts = (round(random.uniform(1, 50), 2), round(random.uniform(1, 50), 2)) + el = [shift(image, shifts), 0] + im_shift_stack.append(el) + return np.asarray(im_shift_stack) + def test_register_frame_stack(test_stack): - aligned_frames = register_frames_stack(test_stack[0], mode='translation') - shifts_calc = register_frames_stack(test_stack[1]) - print(aligned_frames.shape) - print(test_stack.shape) + "Test something in register_frame_stack " + aligned_frames, calc_shifts = register_frames_stack(frames=test_stack[:,0]) - assert aligned_frames.shape[0] == test_stack[0].shape[0] - assert shifts_calc == test_stack[1] + assert aligned_frames.shape[0] == test_stack[:,0].shape[0] + assert calc_shifts == test_stack[:,1] +# if __name__ == "__main__": +# test_stack() From 73f37dbbf9de31538ae0a3f97815dcbff5f79736 Mon Sep 17 00:00:00 2001 From: Juliane Reinhardt Date: Wed, 16 Sep 2020 00:18:40 -0700 Subject: [PATCH 13/29] messing with the test --- xicam/tests/test_examples.py | 30 ++++++++++++++++++++++-------- 1 file changed, 22 insertions(+), 8 deletions(-) diff --git a/xicam/tests/test_examples.py b/xicam/tests/test_examples.py index 37e3e4e..733f0f5 100644 --- a/xicam/tests/test_examples.py +++ b/xicam/tests/test_examples.py @@ -9,7 +9,7 @@ from pystxmtools.corrections.register import register_frames_stack -@fixture +# @fixture def test_stack(): image = data.camera() n_frames = 10 @@ -17,18 +17,32 @@ def test_stack(): im_shift_stack = [] for n in range(n_frames): shifts = (round(random.uniform(1, 50), 2), round(random.uniform(1, 50), 2)) - el = [shift(image, shifts), 0] + el = [shift(image, shifts), shifts] im_shift_stack.append(el) return np.asarray(im_shift_stack) -def test_register_frame_stack(test_stack): +def test_register_frame_stack(): "Test something in register_frame_stack " - aligned_frames, calc_shifts = register_frames_stack(frames=test_stack[:,0]) + image = data.camera() + n_frames = 10 + # shifts = (round(random.uniform(1, 50), 2), round(random.uniform(1, 50), 2)) + im_stack = [] + shift_stack = [] + for n in range(n_frames-1): + shifts = [round(random.uniform(1, 50), 2), round(random.uniform(1, 50), 2)] + shift_stack.append(shifts) + + shifted_frames = shift(image, shifts) + im_stack.append(shifted_frames) + # aligned_frames, calc_shifts = register_frames_stack(frames=test_stack[:,0]) + aligned_frames, calc_shifts = register_frames_stack(np.asarray(im_stack)) + + return aligned_frames, calc_shifts, shift_stack, shifts, np.asarray(im_stack) - assert aligned_frames.shape[0] == test_stack[:,0].shape[0] - assert calc_shifts == test_stack[:,1] + # assert calc_shifts == shift_stack + # assert calc_shifts == test_stack[:,1] -# if __name__ == "__main__": -# test_stack() +if __name__ == "__main__": + test_stack() From d651fb997d852e87bfbfc75a9e137b924cd91a5f Mon Sep 17 00:00:00 2001 From: Juliane Date: Wed, 21 Oct 2020 19:50:09 -0700 Subject: [PATCH 14/29] add vanessa's operations to entrypoints --- setup.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index a149c5f..9f67649 100644 --- a/setup.py +++ b/setup.py @@ -54,9 +54,11 @@ "calc_optical_density = pystxmtools.corrections.optical_density:calc_opt_density_operation", "get_I0_mask = pystxmtools.corrections.optical_density:calc_I0_mask_operation", "register_stack = pystxmtools.corrections.register:register_operation", + "mosaic_registration = pystxmtools.corrections.mosaic_registration:mosaic_register_operation", "lin_fit_spectra = pystxmtools.corrections.fitting:lin_fit_operation", "lstsq_fit = pystxmtools.corrections.fitting:lsq_fit_operation", - "nn_lstsq = pystxmtools.corrections.fitting:nn_lsq_fit_operation"] + "nn_lstsq = pystxmtools.corrections.fitting:nn_lsq_fit_operation", + "cropping = pystxmtools.corrections.cropping:cropping_operation"] # "databroker.handlers": [ # "JPEG = xicam.catalog_viewer.image_handlers:JPEGHandler", # "TIFF = xicam.catalog_viewer.image_handlers:TIFFHandler", From 45f423f5f2e70b01d0d78e938c2194c0098d4e79 Mon Sep 17 00:00:00 2001 From: Juliane Reinhardt Date: Mon, 26 Oct 2020 16:17:41 -0700 Subject: [PATCH 15/29] execute iterative false --- xicam/spectral/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xicam/spectral/__init__.py b/xicam/spectral/__init__.py index bd774f6..3db1126 100644 --- a/xicam/spectral/__init__.py +++ b/xicam/spectral/__init__.py @@ -91,7 +91,7 @@ def __init__(self): callback_slot=self.append_treatment, finished_slot=self.show_treatment, kwargs_callable=self.treatment_kwargs, - execute_iterative=True) + execute_iterative=False) self.stages = { "Acquire": GUILayout(QWidget()), From d2752cd381cf1fcd6461dda999301137573b8012 Mon Sep 17 00:00:00 2001 From: Juliane Reinhardt Date: Wed, 28 Oct 2020 00:43:10 -0700 Subject: [PATCH 16/29] get array data from image intent if projector return list --- xicam/spectral/stages/__init__.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/xicam/spectral/stages/__init__.py b/xicam/spectral/stages/__init__.py index c5143ea..9a7dfd7 100644 --- a/xicam/spectral/stages/__init__.py +++ b/xicam/spectral/stages/__init__.py @@ -1,5 +1,6 @@ import numpy as np import xarray as xr + from databroker.in_memory import BlueskyInMemoryCatalog from qtpy.QtWidgets import QLabel, QComboBox, QHBoxLayout, QWidget, QSpacerItem, QSizePolicy @@ -15,9 +16,10 @@ from xicam.gui.widgets.linearworkfloweditor import WorkflowEditor from databroker.core import BlueskyRun from xicam.core.execution import Workflow +from xicam.core.intents import Intent from xicam.plugins import GUIPlugin from ..widgets.image_viewers import CatalogViewerBlend -from ..projectors import project_nxSTXM, project_all +from ..projectors import project_all class SpectralBase(GUIPlugin): @@ -44,7 +46,12 @@ def treatment_kwargs(self, workflow): # FIXME: Putting this here for now... self.current_data = None - return {'data': project_all(self.current_catalog)} + projected_data = project_all(self.current_catalog) + #FIXME how to handle different projector outputs such as list of intents or actual data array + if isinstance(projected_data, list): + return {'data': projected_data[0].image} + else: + return {'data': projected_data} def append_treatment(self, result_set): if self.current_data is None: @@ -59,12 +66,11 @@ def appendCatalog(self, run_catalog: BlueskyRun, **kwargs): # catalog.metadata.update(self.schema()) ensemble = Ensemble() ensemble.append_catalog(run_catalog) - self.ensemble_model.add_ensemble(ensemble, project_nxSTXM) + self.ensemble_model.add_ensemble(ensemble, project_all) try: - # Apply nxSTXM projection - xdata = project_nxSTXM(run_catalog) - + # Apply projection + xdata = project_all(run_catalog) self.catalog_viewer.setData(xdata, view_dims=('y (μm)', 'x (μm)')) except Exception as e: From 64a2367ebd9c13d900ab8a721f0fb4d742cd0c05 Mon Sep 17 00:00:00 2001 From: Juliane Reinhardt Date: Thu, 29 Oct 2020 23:35:55 -0700 Subject: [PATCH 17/29] Ron's suggestions --- xicam/spectral/projectors/__init__.py | 6 +++--- xicam/spectral/stages/__init__.py | 3 ++- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/xicam/spectral/projectors/__init__.py b/xicam/spectral/projectors/__init__.py index 8bfbb3f..381e784 100644 --- a/xicam/spectral/projectors/__init__.py +++ b/xicam/spectral/projectors/__init__.py @@ -16,7 +16,7 @@ def project_nxSTXM(run_catalog: BlueskyRun): xdata = np.squeeze(xdata) xdata = xdata.assign_coords({xdata.dims[0]: energy, xdata.dims[2]: sample_x, xdata.dims[1]: sample_y}) - + #TODO return intents return xdata @@ -40,8 +40,8 @@ def project_nxCXI_ptycho(run_catalog: BlueskyRun): rec_data_trans = rec_data_trans.assign_coords( {rec_data_trans.dims[0]: energy, rec_data_trans.dims[1]: coords_y, rec_data_trans.dims[2]: coords_x}) - - return [ImageIntent(item_name='Transmission Reconstruction', image=rec_data_trans), + #TODO add imagemixins to display complex + return [ImageIntent(item_name='hyperspectral_data', image=rec_data_trans), # ImageIntent(image=rec_data_phase, item_name='phase reconstruction') ] diff --git a/xicam/spectral/stages/__init__.py b/xicam/spectral/stages/__init__.py index 9a7dfd7..3ec4df7 100644 --- a/xicam/spectral/stages/__init__.py +++ b/xicam/spectral/stages/__init__.py @@ -49,7 +49,8 @@ def treatment_kwargs(self, workflow): projected_data = project_all(self.current_catalog) #FIXME how to handle different projector outputs such as list of intents or actual data array if isinstance(projected_data, list): - return {'data': projected_data[0].image} + data_intent = next(filter(lambda intent: intent.name=='hyperspectral_data', projected_data)) + return {'data': data_intent.image} else: return {'data': projected_data} From 1af93be1d2dd08be1b3c34cdb35f642fa68aea81 Mon Sep 17 00:00:00 2001 From: Juliane Reinhardt Date: Fri, 30 Oct 2020 00:01:47 -0700 Subject: [PATCH 18/29] get array data from Image Intent for workflow operation --- xicam/spectral/projectors/__init__.py | 3 +-- xicam/spectral/stages/__init__.py | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/xicam/spectral/projectors/__init__.py b/xicam/spectral/projectors/__init__.py index 381e784..a7fd26a 100644 --- a/xicam/spectral/projectors/__init__.py +++ b/xicam/spectral/projectors/__init__.py @@ -16,8 +16,7 @@ def project_nxSTXM(run_catalog: BlueskyRun): xdata = np.squeeze(xdata) xdata = xdata.assign_coords({xdata.dims[0]: energy, xdata.dims[2]: sample_x, xdata.dims[1]: sample_y}) - #TODO return intents - return xdata + return [ImageIntent(item_name='hyperspectral_data', image=xdata)] def project_nxCXI_ptycho(run_catalog: BlueskyRun): diff --git a/xicam/spectral/stages/__init__.py b/xicam/spectral/stages/__init__.py index 3ec4df7..627453f 100644 --- a/xicam/spectral/stages/__init__.py +++ b/xicam/spectral/stages/__init__.py @@ -47,9 +47,8 @@ def treatment_kwargs(self, workflow): # FIXME: Putting this here for now... self.current_data = None projected_data = project_all(self.current_catalog) - #FIXME how to handle different projector outputs such as list of intents or actual data array if isinstance(projected_data, list): - data_intent = next(filter(lambda intent: intent.name=='hyperspectral_data', projected_data)) + data_intent = next(filter(lambda intent: intent.item_name=='hyperspectral_data', iter(projected_data))) return {'data': data_intent.image} else: return {'data': projected_data} From aa9065ee478adb4b48ecdafaac3a02a0af969053 Mon Sep 17 00:00:00 2001 From: Juliane Reinhardt Date: Wed, 9 Dec 2020 21:32:33 -0800 Subject: [PATCH 19/29] add decomposition pca to entry points --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index 9f67649..3e7f5cb 100644 --- a/setup.py +++ b/setup.py @@ -51,6 +51,7 @@ "nl_means_filter = pystxmtools.corrections.filter:nl_means_filter_operation", "despike = pystxmtools.corrections.filter:despike_operation", "denoise = pystxmtools.corrections.filter:denoise_operation", + "decomposition = xicam.spectral.operations.decomposition:pca", "calc_optical_density = pystxmtools.corrections.optical_density:calc_opt_density_operation", "get_I0_mask = pystxmtools.corrections.optical_density:calc_I0_mask_operation", "register_stack = pystxmtools.corrections.register:register_operation", From bfcbbdb34ec14ed32495ccc57ccec536e8c8a517 Mon Sep 17 00:00:00 2001 From: Juliane Reinhardt Date: Thu, 10 Dec 2020 00:11:12 -0800 Subject: [PATCH 20/29] add switch for energy axis being last or first axis --- xicam/spectral/operations/decomposition.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/xicam/spectral/operations/decomposition.py b/xicam/spectral/operations/decomposition.py index b9f4a78..cb21ca7 100644 --- a/xicam/spectral/operations/decomposition.py +++ b/xicam/spectral/operations/decomposition.py @@ -35,11 +35,15 @@ class svd_solver(enum.Enum): @intent(PlotIntent, "Component 0", match_key='components', output_map={"y": "component0", 'x': 'energy'}, labels={'left': 'PCA 1', 'bottom': 'Energy (eV)'}) @intent(PlotIntent, "Component 1", match_key='components', output_map={"y": "component1", 'x': 'energy'}, labels={'left': 'PCA 2', 'bottom': 'Energy (eV)'}) @intent(PlotIntent, "Component 2", match_key='components', output_map={"y": "component2", 'x': 'energy'}, labels={'left': 'PCA 3', 'bottom': 'Energy (eV)'}) -def pca(data:np.ndarray, n_components:Union[int, float, str]=3, copy:bool=True, whiten:bool=False, svd_solver:svd_solver='auto', tol:float=0.0, iterated_power:Union[str, int]='auto', random_state:int=None): +def pca(data:np.ndarray, energy_axis_last:bool = True, n_components:Union[int, float, str]=3, copy:bool=True, whiten:bool=False, svd_solver:svd_solver='auto', tol:float=0.0, iterated_power:Union[str, int]='auto', random_state:int=None): pca = PCA(n_components, copy=copy, whiten=whiten, svd_solver=svd_solver, tol=tol, iterated_power=iterated_power, random_state=random_state) - pca.fit(np.asarray(data).reshape(-1, data.shape[2])) - images = pca.transform(np.asarray(data).reshape(-1, data.shape[2])).reshape(data.shape[0],data.shape[1], 3) + if energy_axis_last: + pca.fit(np.asarray(data).reshape(-1, data.shape[2])) + images = pca.transform(np.asarray(data).reshape(-1, data.shape[2])).reshape(data.shape[0],data.shape[1], 3) + else: + pca.fit(np.asarray(data).reshape(-1, data.shape[0])) + images = pca.transform(np.asarray(data).reshape(-1, data.shape[0])).reshape(data.shape[1], data.shape[2], 3) return images, images[:,:,0], images[:,:,1], images[:,:,2], pca.explained_variance_ratio_, pca.components_[0], pca.components_[1], pca.components_[2], data.coords['E (eV)'] From 69e521d5fb9c28ec1d6fed69dd5061fed2887ec6 Mon Sep 17 00:00:00 2001 From: Juliane Reinhardt Date: Thu, 10 Dec 2020 00:14:51 -0800 Subject: [PATCH 21/29] formatting --- xicam/spectral/operations/decomposition.py | 28 +++++++++++++++------- 1 file changed, 20 insertions(+), 8 deletions(-) diff --git a/xicam/spectral/operations/decomposition.py b/xicam/spectral/operations/decomposition.py index cb21ca7..3d89ca2 100644 --- a/xicam/spectral/operations/decomposition.py +++ b/xicam/spectral/operations/decomposition.py @@ -26,17 +26,28 @@ class svd_solver(enum.Enum): *[f'component{i}' for i in range(3)], 'energy') @visible('data', False) -# @input_names('data', 'Number of components', 'Copy', 'Whiten', 'SVD Solver', 'Tolerance', 'Iterated Power', 'Random State') +# @input_names('data', 'energy axis is last', 'Number of components', 'Copy', 'Whiten', 'SVD Solver', 'Tolerance', 'Iterated Power', 'Random State') @intent(PairPlotIntent, "Pair Plot", output_map={"transform_data": "transform_data"}) @intent(ImageIntent, "PCA Transform0", output_map={"image": "image0"}) @intent(ImageIntent, "PCA Transform1", output_map={"image": "image1"}) @intent(ImageIntent, "PCA Transform2", output_map={"image": "image2"}) -@intent(BarIntent, "Variance Ratio", x=[1,2,3], width=1, output_map={"height": "variance"}, labels={'left': 'Variance Ratio'}) -@intent(PlotIntent, "Component 0", match_key='components', output_map={"y": "component0", 'x': 'energy'}, labels={'left': 'PCA 1', 'bottom': 'Energy (eV)'}) -@intent(PlotIntent, "Component 1", match_key='components', output_map={"y": "component1", 'x': 'energy'}, labels={'left': 'PCA 2', 'bottom': 'Energy (eV)'}) -@intent(PlotIntent, "Component 2", match_key='components', output_map={"y": "component2", 'x': 'energy'}, labels={'left': 'PCA 3', 'bottom': 'Energy (eV)'}) -def pca(data:np.ndarray, energy_axis_last:bool = True, n_components:Union[int, float, str]=3, copy:bool=True, whiten:bool=False, svd_solver:svd_solver='auto', tol:float=0.0, iterated_power:Union[str, int]='auto', random_state:int=None): - pca = PCA(n_components, copy=copy, whiten=whiten, svd_solver=svd_solver, tol=tol, iterated_power=iterated_power, random_state=random_state) +@intent(BarIntent, "Variance Ratio", x=[1,2,3], width=1, output_map={"height": "variance"}, + labels={'left': 'Variance Ratio'}) +@intent(PlotIntent, "Component 0", match_key='components', output_map={"y": "component0", 'x': 'energy'}, + labels={'left': 'PCA 1', 'bottom': 'Energy (eV)'}) +@intent(PlotIntent, "Component 1", match_key='components', output_map={"y": "component1", 'x': 'energy'}, + labels={'left': 'PCA 2', 'bottom': 'Energy (eV)'}) +@intent(PlotIntent, "Component 2", match_key='components', output_map={"y": "component2", 'x': 'energy'}, + labels={'left': 'PCA 3', 'bottom': 'Energy (eV)'}) +def pca(data:np.ndarray, + energy_axis_last:bool = True, + n_components:Union[int, float, str]=3, + copy:bool=True, whiten:bool=False, + svd_solver:svd_solver='auto', tol:float=0.0, + iterated_power:Union[str, int]='auto', + random_state:int=None): + pca = PCA(n_components, copy=copy, whiten=whiten, svd_solver=svd_solver, tol=tol, iterated_power=iterated_power, + random_state=random_state) if energy_axis_last: pca.fit(np.asarray(data).reshape(-1, data.shape[2])) @@ -45,7 +56,8 @@ def pca(data:np.ndarray, energy_axis_last:bool = True, n_components:Union[int, f pca.fit(np.asarray(data).reshape(-1, data.shape[0])) images = pca.transform(np.asarray(data).reshape(-1, data.shape[0])).reshape(data.shape[1], data.shape[2], 3) - return images, images[:,:,0], images[:,:,1], images[:,:,2], pca.explained_variance_ratio_, pca.components_[0], pca.components_[1], pca.components_[2], data.coords['E (eV)'] + return images, images[:,:,0], images[:,:,1], images[:,:,2], pca.explained_variance_ratio_, pca.components_[0], \ + pca.components_[1], pca.components_[2], data.coords['E (eV)'] class init(enum.Enum): From d6684762cdb67b4f653dcb5fa13be03e4c752fdd Mon Sep 17 00:00:00 2001 From: Juliane Reinhardt Date: Mon, 25 Jan 2021 10:43:38 -0800 Subject: [PATCH 22/29] some fixes in projector and adding entry points for more operations --- setup.py | 13 ++++++++++--- xicam/spectral/ingestors/NXcxi_ptycho.py | 3 ++- xicam/spectral/operations/decomposition.py | 2 +- xicam/spectral/projectors/__init__.py | 12 ++++++++++-- 4 files changed, 23 insertions(+), 7 deletions(-) diff --git a/setup.py b/setup.py index 3e7f5cb..a4f661f 100644 --- a/setup.py +++ b/setup.py @@ -52,14 +52,21 @@ "despike = pystxmtools.corrections.filter:despike_operation", "denoise = pystxmtools.corrections.filter:denoise_operation", "decomposition = xicam.spectral.operations.decomposition:pca", - "calc_optical_density = pystxmtools.corrections.optical_density:calc_opt_density_operation", + "calc_optical_density = " + "pystxmtools.corrections.optical_density:calc_opt_density_operation", "get_I0_mask = pystxmtools.corrections.optical_density:calc_I0_mask_operation", "register_stack = pystxmtools.corrections.register:register_operation", - "mosaic_registration = pystxmtools.corrections.mosaic_registration:mosaic_register_operation", + "mosaic_registration = " + "pystxmtools.corrections.mosaic_registration:mosaic_register_operation", "lin_fit_spectra = pystxmtools.corrections.fitting:lin_fit_operation", "lstsq_fit = pystxmtools.corrections.fitting:lsq_fit_operation", "nn_lstsq = pystxmtools.corrections.fitting:nn_lsq_fit_operation", - "cropping = pystxmtools.corrections.cropping:cropping_operation"] + "cropping = pystxmtools.corrections.cropping:cropping_operation", + "pic_mapping = pystxmtools.corrections.pic_functions:pic_mapping_operation", + "plot_amplitude_azimuth = " + "pystxmtools.corrections.pic_functions:plot_amplitude_azimuth_operation", + "plot_rgb_map = pystxmtools.corrections.pic_functions:plot_rgb_map_operation", + ] # "databroker.handlers": [ # "JPEG = xicam.catalog_viewer.image_handlers:JPEGHandler", # "TIFF = xicam.catalog_viewer.image_handlers:TIFFHandler", diff --git a/xicam/spectral/ingestors/NXcxi_ptycho.py b/xicam/spectral/ingestors/NXcxi_ptycho.py index e56bf6a..2b35814 100644 --- a/xicam/spectral/ingestors/NXcxi_ptycho.py +++ b/xicam/spectral/ingestors/NXcxi_ptycho.py @@ -66,7 +66,8 @@ def ingest_cxi(paths): sorted_entry_list = sorted(h5_entry_dict.items(), key=lambda item: item[1]) # TODO check if energies are "continuous" or have interruptions # the removal of last for entries is only temporary due to the example dataset - del sorted_entry_list[-4:] + # FIXME what to do if some energies are 'out of range'? + # del sorted_entry_list[-4:] frames_stack = [] rec_stack = [] diff --git a/xicam/spectral/operations/decomposition.py b/xicam/spectral/operations/decomposition.py index 3d89ca2..9164400 100644 --- a/xicam/spectral/operations/decomposition.py +++ b/xicam/spectral/operations/decomposition.py @@ -51,7 +51,7 @@ def pca(data:np.ndarray, if energy_axis_last: pca.fit(np.asarray(data).reshape(-1, data.shape[2])) - images = pca.transform(np.asarray(data).reshape(-1, data.shape[2])).reshape(data.shape[0],data.shape[1], 3) + images = pca.transform(np.asarray(data).reshape(-1, data.shape[2])).reshape(data.shape[0], data.shape[1], 3) else: pca.fit(np.asarray(data).reshape(-1, data.shape[0])) images = pca.transform(np.asarray(data).reshape(-1, data.shape[0])).reshape(data.shape[1], data.shape[2], 3) diff --git a/xicam/spectral/projectors/__init__.py b/xicam/spectral/projectors/__init__.py index c9a1a5c..9144b9d 100644 --- a/xicam/spectral/projectors/__init__.py +++ b/xicam/spectral/projectors/__init__.py @@ -39,8 +39,16 @@ def project_nxCXI_ptycho(run_catalog: BlueskyRun): rec_data_phase = getattr(run_catalog, phase_rec_stream).to_dask()[phase_rec_field] rec_data_phase = np.squeeze(rec_data_phase) - rec_data_trans = rec_data_trans.assign_coords( - {rec_data_trans.dims[0]: energy, rec_data_trans.dims[1]: coords_y, rec_data_trans.dims[2]: coords_x}) + if rec_data_trans.dims[0] == 'E (eV)': + rec_data_trans = rec_data_trans.assign_coords({rec_data_trans.dims[0]: energy, + rec_data_trans.dims[1]: coords_y, + rec_data_trans.dims[2]: coords_x}) + elif rec_data_trans.dims[0] == 'y (nm)': + rec_data_trans = rec_data_trans.assign_coords({rec_data_trans.dims[0]: coords_y, + rec_data_trans.dims[1]: coords_x}) + + + #TODO add imagemixins to display complex return [ImageIntent(item_name='hyperspectral_data', image=rec_data_trans), # ImageIntent(image=rec_data_phase, item_name='phase reconstruction') From 4ed29ef285b6565f1896e84454eb5422d35a53ed Mon Sep 17 00:00:00 2001 From: Juliane Reinhardt Date: Tue, 2 Feb 2021 12:18:14 -1000 Subject: [PATCH 23/29] add multiple intent display names for splitview --- xicam/spectral/projectors/__init__.py | 7 ++----- xicam/spectral/stages/__init__.py | 3 ++- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/xicam/spectral/projectors/__init__.py b/xicam/spectral/projectors/__init__.py index 280c575..fba9cdc 100644 --- a/xicam/spectral/projectors/__init__.py +++ b/xicam/spectral/projectors/__init__.py @@ -14,12 +14,10 @@ def project_nxSTXM(run_catalog: BlueskyRun): energy = projection['irmap/DATA/energy']['value'] xdata = getattr(run_catalog, stream).to_dask()[field] # type: xr.DataArray - xdata = np.squeeze(xdata) - xdata = xdata.assign_coords({xdata.dims[0]: energy, xdata.dims[2]: sample_x, xdata.dims[1]: sample_y}) - return xdata.transpose('y (μm)', 'x (μm)', ...) + return [ImageIntent(item_name='IR maps', image=xdata),] def project_nxCXI_ptycho(run_catalog: BlueskyRun): @@ -49,9 +47,8 @@ def project_nxCXI_ptycho(run_catalog: BlueskyRun): rec_data_trans.dims[1]: coords_x}) - #TODO add imagemixins to display complex - return [ImageIntent(item_name='hyperspectral_data', image=rec_data_trans), + return [ImageIntent(item_name='ptychography data', image=rec_data_trans), # ImageIntent(image=rec_data_phase, item_name='phase reconstruction') ] diff --git a/xicam/spectral/stages/__init__.py b/xicam/spectral/stages/__init__.py index 390757f..ed9cdc9 100644 --- a/xicam/spectral/stages/__init__.py +++ b/xicam/spectral/stages/__init__.py @@ -44,12 +44,13 @@ def __init__(self): super(SpectralBase, self).__init__() def treatment_kwargs(self, workflow): + intent_name = ['hyperspectral_data', 'ptychography data', 'IR maps'] # FIXME: Putting this here for now... self.current_data = None projected_data = project_all(self.current_catalog) if isinstance(projected_data, list): - data_intent = next(filter(lambda intent: intent.item_name=='hyperspectral_data', iter(projected_data))) + data_intent = next(filter(lambda intent: intent.item_name in intent_name, iter(projected_data))) return {'data': data_intent.image} else: return {'data': projected_data} From be18c84f312d81abbd3f9d04a57a57a42bc59195 Mon Sep 17 00:00:00 2001 From: Juliane Reinhardt Date: Tue, 2 Feb 2021 12:19:49 -1000 Subject: [PATCH 24/29] change decomposition to energy axis = first --- xicam/spectral/ingestors/NXcxi_ptycho.py | 2 +- xicam/spectral/operations/decomposition.py | 10 +++++++--- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/xicam/spectral/ingestors/NXcxi_ptycho.py b/xicam/spectral/ingestors/NXcxi_ptycho.py index 2b35814..0d322ce 100644 --- a/xicam/spectral/ingestors/NXcxi_ptycho.py +++ b/xicam/spectral/ingestors/NXcxi_ptycho.py @@ -67,7 +67,7 @@ def ingest_cxi(paths): # TODO check if energies are "continuous" or have interruptions # the removal of last for entries is only temporary due to the example dataset # FIXME what to do if some energies are 'out of range'? - # del sorted_entry_list[-4:] + del sorted_entry_list[-4:] frames_stack = [] rec_stack = [] diff --git a/xicam/spectral/operations/decomposition.py b/xicam/spectral/operations/decomposition.py index d598e49..d763ef7 100644 --- a/xicam/spectral/operations/decomposition.py +++ b/xicam/spectral/operations/decomposition.py @@ -37,9 +37,13 @@ class svd_solver(enum.Enum): def pca(data:np.ndarray, n_components:Union[int, float, str]=3, copy:bool=True, whiten:bool=False, svd_solver:svd_solver='auto', tol:float=0.0, iterated_power:Union[str, int]='auto', random_state:int=None): pca = PCA(n_components, copy=copy, whiten=whiten, svd_solver=svd_solver, tol=tol, iterated_power=iterated_power, random_state=random_state) - pca.fit(np.asarray(data).reshape(-1, data.shape[2])) - images = pca.transform(np.asarray(data).reshape(-1, data.shape[2])).reshape(data.shape[0],data.shape[1], n_components) - + # try: + # print('Energy axis last') + # pca.fit(np.asarray(data).reshape(-1, data.shape[2])) + # images = pca.transform(np.asarray(data).reshape(-1, data.shape[2])).reshape(data.shape[0], data.shape[1], 3) + # except: + pca.fit(np.asarray(data).reshape(-1, data.shape[0])) + images = pca.transform(np.asarray(data).reshape(-1, data.shape[0])).reshape(data.shape[1], data.shape[2], 3) return images, pca.explained_variance_ratio_, pca.components_, data.coords['E (eV)'], np.arange(images.shape[-1]) From ffeb104e306c007d45d4b48f981a8a0604467390 Mon Sep 17 00:00:00 2001 From: Juliane Reinhardt Date: Thu, 4 Feb 2021 21:08:51 -1000 Subject: [PATCH 25/29] minors --- xicam/spectral/projectors/__init__.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/xicam/spectral/projectors/__init__.py b/xicam/spectral/projectors/__init__.py index fba9cdc..1cc609c 100644 --- a/xicam/spectral/projectors/__init__.py +++ b/xicam/spectral/projectors/__init__.py @@ -45,8 +45,6 @@ def project_nxCXI_ptycho(run_catalog: BlueskyRun): elif rec_data_trans.dims[0] == 'y (nm)': rec_data_trans = rec_data_trans.assign_coords({rec_data_trans.dims[0]: coords_y, rec_data_trans.dims[1]: coords_x}) - - #TODO add imagemixins to display complex return [ImageIntent(item_name='ptychography data', image=rec_data_trans), # ImageIntent(image=rec_data_phase, item_name='phase reconstruction') From d979e2362f62a2d97e2f94a2d4e89b94b39fbb39 Mon Sep 17 00:00:00 2001 From: Ronald Pandolfi Date: Fri, 5 Feb 2021 13:45:03 -0800 Subject: [PATCH 26/29] Trim comments --- xicam/spectral/operations/decomposition.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/xicam/spectral/operations/decomposition.py b/xicam/spectral/operations/decomposition.py index d763ef7..cb84caa 100644 --- a/xicam/spectral/operations/decomposition.py +++ b/xicam/spectral/operations/decomposition.py @@ -37,11 +37,6 @@ class svd_solver(enum.Enum): def pca(data:np.ndarray, n_components:Union[int, float, str]=3, copy:bool=True, whiten:bool=False, svd_solver:svd_solver='auto', tol:float=0.0, iterated_power:Union[str, int]='auto', random_state:int=None): pca = PCA(n_components, copy=copy, whiten=whiten, svd_solver=svd_solver, tol=tol, iterated_power=iterated_power, random_state=random_state) - # try: - # print('Energy axis last') - # pca.fit(np.asarray(data).reshape(-1, data.shape[2])) - # images = pca.transform(np.asarray(data).reshape(-1, data.shape[2])).reshape(data.shape[0], data.shape[1], 3) - # except: pca.fit(np.asarray(data).reshape(-1, data.shape[0])) images = pca.transform(np.asarray(data).reshape(-1, data.shape[0])).reshape(data.shape[1], data.shape[2], 3) return images, pca.explained_variance_ratio_, pca.components_, data.coords['E (eV)'], np.arange(images.shape[-1]) From 6d521e6bb10dc8763e099b0ba9b3fdd3ea3abb98 Mon Sep 17 00:00:00 2001 From: Juliane Reinhardt Date: Fri, 5 Feb 2021 22:07:36 -1000 Subject: [PATCH 27/29] remove entry points and more to pystxm-tools --- setup.py | 21 --------------------- 1 file changed, 21 deletions(-) diff --git a/setup.py b/setup.py index a4f661f..0664ce2 100644 --- a/setup.py +++ b/setup.py @@ -46,27 +46,6 @@ "databroker.ingestors": ["application/x-hdf5 = xicam.spectral.ingestors.nxSTXM:ingest_nxSTXM", 'application/x-fits = xicam.spectral.ingestors.arpes_fits:ingest_NXarpes', 'application/x-cxi = xicam.spectral.ingestors:ingest_cxi'], - "xicam.plugins.OperationPlugin": ["wiener_filter = pystxmtools.corrections.filter:wiener_filter_operation", - "median_filter = pystxmtools.corrections.filter:median_filter_operation", - "nl_means_filter = pystxmtools.corrections.filter:nl_means_filter_operation", - "despike = pystxmtools.corrections.filter:despike_operation", - "denoise = pystxmtools.corrections.filter:denoise_operation", - "decomposition = xicam.spectral.operations.decomposition:pca", - "calc_optical_density = " - "pystxmtools.corrections.optical_density:calc_opt_density_operation", - "get_I0_mask = pystxmtools.corrections.optical_density:calc_I0_mask_operation", - "register_stack = pystxmtools.corrections.register:register_operation", - "mosaic_registration = " - "pystxmtools.corrections.mosaic_registration:mosaic_register_operation", - "lin_fit_spectra = pystxmtools.corrections.fitting:lin_fit_operation", - "lstsq_fit = pystxmtools.corrections.fitting:lsq_fit_operation", - "nn_lstsq = pystxmtools.corrections.fitting:nn_lsq_fit_operation", - "cropping = pystxmtools.corrections.cropping:cropping_operation", - "pic_mapping = pystxmtools.corrections.pic_functions:pic_mapping_operation", - "plot_amplitude_azimuth = " - "pystxmtools.corrections.pic_functions:plot_amplitude_azimuth_operation", - "plot_rgb_map = pystxmtools.corrections.pic_functions:plot_rgb_map_operation", - ] # "databroker.handlers": [ # "JPEG = xicam.catalog_viewer.image_handlers:JPEGHandler", # "TIFF = xicam.catalog_viewer.image_handlers:TIFFHandler", From 24d99630396f23f66c9e6a3dfb6e879328ee5e9c Mon Sep 17 00:00:00 2001 From: Juliane Reinhardt Date: Fri, 5 Feb 2021 22:13:27 -1000 Subject: [PATCH 28/29] remove extra logic, keep only energy stack data in projector --- xicam/spectral/projectors/__init__.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/xicam/spectral/projectors/__init__.py b/xicam/spectral/projectors/__init__.py index 1cc609c..52a7fcf 100644 --- a/xicam/spectral/projectors/__init__.py +++ b/xicam/spectral/projectors/__init__.py @@ -38,13 +38,11 @@ def project_nxCXI_ptycho(run_catalog: BlueskyRun): rec_data_phase = getattr(run_catalog, phase_rec_stream).to_dask()[phase_rec_field] rec_data_phase = np.squeeze(rec_data_phase) - if rec_data_trans.dims[0] == 'E (eV)': - rec_data_trans = rec_data_trans.assign_coords({rec_data_trans.dims[0]: energy, + + rec_data_trans = rec_data_trans.assign_coords({rec_data_trans.dims[0]: energy, rec_data_trans.dims[1]: coords_y, rec_data_trans.dims[2]: coords_x}) - elif rec_data_trans.dims[0] == 'y (nm)': - rec_data_trans = rec_data_trans.assign_coords({rec_data_trans.dims[0]: coords_y, - rec_data_trans.dims[1]: coords_x}) + #TODO add imagemixins to display complex return [ImageIntent(item_name='ptychography data', image=rec_data_trans), # ImageIntent(image=rec_data_phase, item_name='phase reconstruction') From bdb432dd299d7bf80df4a02e66a64548afab7727 Mon Sep 17 00:00:00 2001 From: Juliane Reinhardt Date: Tue, 2 Mar 2021 11:39:54 -1000 Subject: [PATCH 29/29] list to project all --- xicam/spectral/stages/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xicam/spectral/stages/__init__.py b/xicam/spectral/stages/__init__.py index ed9cdc9..6c464fa 100644 --- a/xicam/spectral/stages/__init__.py +++ b/xicam/spectral/stages/__init__.py @@ -68,7 +68,7 @@ def appendCatalog(self, run_catalog: BlueskyRun, **kwargs): # catalog.metadata.update(self.schema()) ensemble = Ensemble() ensemble.append_catalog(run_catalog) - self.ensemble_model.add_ensemble(ensemble, project_all) + self.ensemble_model.add_ensemble(ensemble, [project_all]) try: # Apply nxSTXM projection