From 5e92669f7a5fdeb7e98c2fee5b11e35631b078e4 Mon Sep 17 00:00:00 2001 From: davidmcdonagh Date: Thu, 18 Jul 2024 11:00:08 +0100 Subject: [PATCH 1/8] Additional methods for FormatISISSXD to extract data more efficiently. --- newsfragments/XXX.misc | 1 + src/dxtbx/format/FormatISISSXD.py | 56 ++++++++++++++++++++++++++++++- 2 files changed, 56 insertions(+), 1 deletion(-) create mode 100644 newsfragments/XXX.misc diff --git a/newsfragments/XXX.misc b/newsfragments/XXX.misc new file mode 100644 index 000000000..5574bbb6f --- /dev/null +++ b/newsfragments/XXX.misc @@ -0,0 +1 @@ +Additional methods for FormatISISSXD to extract data more efficiently. diff --git a/src/dxtbx/format/FormatISISSXD.py b/src/dxtbx/format/FormatISISSXD.py index 00299d515..264ac44be 100644 --- a/src/dxtbx/format/FormatISISSXD.py +++ b/src/dxtbx/format/FormatISISSXD.py @@ -3,10 +3,11 @@ from typing import List, Tuple import h5py +import numpy as np import cctbx.array_family.flex as flex -from dxtbx import IncorrectFormatError +from dxtbx import IncorrectFormatError, flumpy from dxtbx.format.FormatHDF5 import FormatHDF5 from dxtbx.model import Detector, Goniometer, Scan from dxtbx.model.beam import BeamFactory, PolychromaticBeam, Probe @@ -44,6 +45,9 @@ def get_name(image_file: str) -> str: return get_name(image_file) == "SXD" + def get_instrument_name(self) -> str: + return "SXD" + def get_experiment_title(self) -> str: return self._nxs_file["raw_data_1"]["title"][0].decode() @@ -313,3 +317,53 @@ def get_raw_data(self, index: int, use_loaded_data=True) -> Tuple[flex.int]: raw_data.append(panel_data) return tuple(raw_data) + + def get_flattened_data(self, scale_data: bool = True) -> Tuple[flex.int]: + """ + Image data summed along the time-of-flight direction + """ + + panel_size = self._get_image_size() + total_pixels = panel_size[0] * panel_size[1] + # Panel positions are offset by 4 in raw_data array + # See p24 of https://www.isis.stfc.ac.uk/Pages/sxd-user-guide6683.pdf + idx_offset = 4 + max_val = None + num_tof_bins = len(self._get_time_of_flight()) + raw_data = [] + for panel_idx in range(self._get_num_panels()): + start_idx = (panel_idx * total_pixels) + (panel_idx * idx_offset) + end_idx = start_idx + total_pixels + panel_data = flex.int( + self._nxs_file["raw_data_1/detector_1/counts"][0, start_idx:end_idx, :] + ) + panel_max_val = max(panel_data) + if max_val is None or max_val < panel_max_val: + max_val = panel_max_val + panel_data.reshape(flex.grid(panel_size[0], panel_size[1], num_tof_bins)) + panel_data = np.sum(flumpy.to_numpy(panel_data).T, axis=2) + raw_data.append(panel_data) + + if scale_data: + return tuple([(i / max_val).tolist() for i in raw_data]) + + return tuple([i.tolist() for i in raw_data]) + + def get_flattened_pixel_data( + self, panel_idx: int, x: int, y: int + ) -> Tuple[Tuple, Tuple]: + time_channels = self._get_time_of_flight() + panel_size = self._get_image_size() + height = panel_size[1] + total_pixels = panel_size[0] * panel_size[1] + # Panel positions are offset by 4 in raw_data array + # See p24 of https://www.isis.stfc.ac.uk/Pages/sxd-user-guide6683.pdf + idx_offset = 4 + idx = (panel_idx * total_pixels) + (panel_idx * idx_offset) + y * height + x + return ( + time_channels, + tuple(self._nxs_file["raw_data_1/detector_1/counts"][0, idx, :].tolist()), + ) + + def get_proton_charge(self): + return float(self._nxs_file["raw_data_1/proton_charge"][0]) From 7e1eba84bfb18efcf8e4b774000a3edb124ccc8c Mon Sep 17 00:00:00 2001 From: DiamondLightSource-build-server Date: Thu, 18 Jul 2024 10:02:57 +0000 Subject: [PATCH 2/8] Rename newsfragments/XXX.misc to newsfragments/743.misc --- newsfragments/{XXX.misc => 743.misc} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename newsfragments/{XXX.misc => 743.misc} (100%) diff --git a/newsfragments/XXX.misc b/newsfragments/743.misc similarity index 100% rename from newsfragments/XXX.misc rename to newsfragments/743.misc From 7964b545e2ded14ddd2aa5cb8248ab3ed813e009 Mon Sep 17 00:00:00 2001 From: davidmcdonagh Date: Thu, 18 Jul 2024 22:55:46 +0100 Subject: [PATCH 3/8] Fix panels being flipped and panel 0 being upsidedown in some experiments. --- src/dxtbx/format/FormatISISSXD.py | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/src/dxtbx/format/FormatISISSXD.py b/src/dxtbx/format/FormatISISSXD.py index 264ac44be..05cdb4486 100644 --- a/src/dxtbx/format/FormatISISSXD.py +++ b/src/dxtbx/format/FormatISISSXD.py @@ -7,7 +7,7 @@ import cctbx.array_family.flex as flex -from dxtbx import IncorrectFormatError, flumpy +from dxtbx import IncorrectFormatError from dxtbx.format.FormatHDF5 import FormatHDF5 from dxtbx.model import Detector, Goniometer, Scan from dxtbx.model.beam import BeamFactory, PolychromaticBeam, Probe @@ -334,18 +334,22 @@ def get_flattened_data(self, scale_data: bool = True) -> Tuple[flex.int]: for panel_idx in range(self._get_num_panels()): start_idx = (panel_idx * total_pixels) + (panel_idx * idx_offset) end_idx = start_idx + total_pixels - panel_data = flex.int( - self._nxs_file["raw_data_1/detector_1/counts"][0, start_idx:end_idx, :] - ) - panel_max_val = max(panel_data) + panel_data = self._nxs_file["raw_data_1/detector_1/counts"][ + 0, start_idx:end_idx, : + ] + panel_max_val = np.max(panel_data) if max_val is None or max_val < panel_max_val: max_val = panel_max_val - panel_data.reshape(flex.grid(panel_size[0], panel_size[1], num_tof_bins)) - panel_data = np.sum(flumpy.to_numpy(panel_data).T, axis=2) + panel_data = np.reshape( + panel_data, (panel_size[0], panel_size[1], num_tof_bins) + ) + panel_data = np.flipud(np.sum(panel_data, axis=2)) + if panel_idx == 0 and self._panel_0_flipped(): + panel_data = np.flipud(panel_data) raw_data.append(panel_data) - if scale_data: - return tuple([(i / max_val).tolist() for i in raw_data]) + if scale_data: + return tuple([(i / max_val).tolist() for i in raw_data]) return tuple([i.tolist() for i in raw_data]) From 175a177af65f16989dff39c057682a24278fc8c7 Mon Sep 17 00:00:00 2001 From: davidmcdonagh Date: Tue, 23 Jul 2024 21:25:46 +0100 Subject: [PATCH 4/8] kick ci --- src/dxtbx/format/FormatISISSXD.py | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/src/dxtbx/format/FormatISISSXD.py b/src/dxtbx/format/FormatISISSXD.py index 264ac44be..05cdb4486 100644 --- a/src/dxtbx/format/FormatISISSXD.py +++ b/src/dxtbx/format/FormatISISSXD.py @@ -7,7 +7,7 @@ import cctbx.array_family.flex as flex -from dxtbx import IncorrectFormatError, flumpy +from dxtbx import IncorrectFormatError from dxtbx.format.FormatHDF5 import FormatHDF5 from dxtbx.model import Detector, Goniometer, Scan from dxtbx.model.beam import BeamFactory, PolychromaticBeam, Probe @@ -334,18 +334,22 @@ def get_flattened_data(self, scale_data: bool = True) -> Tuple[flex.int]: for panel_idx in range(self._get_num_panels()): start_idx = (panel_idx * total_pixels) + (panel_idx * idx_offset) end_idx = start_idx + total_pixels - panel_data = flex.int( - self._nxs_file["raw_data_1/detector_1/counts"][0, start_idx:end_idx, :] - ) - panel_max_val = max(panel_data) + panel_data = self._nxs_file["raw_data_1/detector_1/counts"][ + 0, start_idx:end_idx, : + ] + panel_max_val = np.max(panel_data) if max_val is None or max_val < panel_max_val: max_val = panel_max_val - panel_data.reshape(flex.grid(panel_size[0], panel_size[1], num_tof_bins)) - panel_data = np.sum(flumpy.to_numpy(panel_data).T, axis=2) + panel_data = np.reshape( + panel_data, (panel_size[0], panel_size[1], num_tof_bins) + ) + panel_data = np.flipud(np.sum(panel_data, axis=2)) + if panel_idx == 0 and self._panel_0_flipped(): + panel_data = np.flipud(panel_data) raw_data.append(panel_data) - if scale_data: - return tuple([(i / max_val).tolist() for i in raw_data]) + if scale_data: + return tuple([(i / max_val).tolist() for i in raw_data]) return tuple([i.tolist() for i in raw_data]) From 9aa0aba021a6d6dc6f159bfa2bc26a1fb50c2db3 Mon Sep 17 00:00:00 2001 From: davidmcdonagh Date: Wed, 24 Jul 2024 11:00:59 +0100 Subject: [PATCH 5/8] kick ci From 46560784ef2664265318095f051800c179911cd6 Mon Sep 17 00:00:00 2001 From: davidmcdonagh Date: Thu, 25 Jul 2024 16:13:52 +0100 Subject: [PATCH 6/8] Typo in FormatISISSXD get_goniometer --- src/dxtbx/format/FormatISISSXD.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/dxtbx/format/FormatISISSXD.py b/src/dxtbx/format/FormatISISSXD.py index 05cdb4486..6d92681a8 100644 --- a/src/dxtbx/format/FormatISISSXD.py +++ b/src/dxtbx/format/FormatISISSXD.py @@ -66,7 +66,7 @@ def get_goniometer(self, idx: int = None) -> Goniometer: try: experiment_title = self.get_experiment_title() if "w=" in experiment_title: - angle = float(experiment_title().split("w=")[1].split()[0]) + angle = float(experiment_title.split("w=")[1].split()[0]) elif "wccr" in experiment_title: angle = float(experiment_title.split("wccr=")[1].split()[0]) elif "wtl" in experiment_title: From 773e4a2d7cdefc59a0f781290052fd7793b7491e Mon Sep 17 00:00:00 2001 From: davidmcdonagh Date: Tue, 30 Jul 2024 14:05:23 +0100 Subject: [PATCH 7/8] Fixed flattened images not being scaled correctly. --- src/dxtbx/format/FormatISISSXD.py | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/src/dxtbx/format/FormatISISSXD.py b/src/dxtbx/format/FormatISISSXD.py index 6d92681a8..9b3c5086a 100644 --- a/src/dxtbx/format/FormatISISSXD.py +++ b/src/dxtbx/format/FormatISISSXD.py @@ -318,7 +318,9 @@ def get_raw_data(self, index: int, use_loaded_data=True) -> Tuple[flex.int]: return tuple(raw_data) - def get_flattened_data(self, scale_data: bool = True) -> Tuple[flex.int]: + def get_flattened_data( + self, image_range: None | Tuple = None, scale_data: bool = True + ) -> Tuple[flex.int]: """ Image data summed along the time-of-flight direction """ @@ -337,15 +339,27 @@ def get_flattened_data(self, scale_data: bool = True) -> Tuple[flex.int]: panel_data = self._nxs_file["raw_data_1/detector_1/counts"][ 0, start_idx:end_idx, : ] - panel_max_val = np.max(panel_data) - if max_val is None or max_val < panel_max_val: - max_val = panel_max_val panel_data = np.reshape( panel_data, (panel_size[0], panel_size[1], num_tof_bins) ) - panel_data = np.flipud(np.sum(panel_data, axis=2)) + if image_range is not None: + assert ( + len(image_range) == 2 + ), "expected image_range to be only two values" + assert ( + image_range[0] >= 0 and image_range[0] < image_range[1] + ), "image_range[0] out of range" + assert image_range[1] <= num_tof_bins, "image_range[1] out of range" + panel_data = np.flipud( + np.sum(panel_data[:, :, image_range[0] : image_range[1]], axis=2) + ) + else: + panel_data = np.flipud(np.sum(panel_data, axis=2)) if panel_idx == 0 and self._panel_0_flipped(): panel_data = np.flipud(panel_data) + panel_max_val = np.max(panel_data) + if max_val is None or max_val < panel_max_val: + max_val = panel_max_val raw_data.append(panel_data) if scale_data: From d0a744f144f1f7ee7cd44e5a86adca4cebea2911 Mon Sep 17 00:00:00 2001 From: davidmcdonagh Date: Wed, 31 Jul 2024 22:34:06 +0100 Subject: [PATCH 8/8] Refactored FormatISISSXD goniometer. --- src/dxtbx/format/FormatISISSXD.py | 47 +++++++++++++++++++++---------- 1 file changed, 32 insertions(+), 15 deletions(-) diff --git a/src/dxtbx/format/FormatISISSXD.py b/src/dxtbx/format/FormatISISSXD.py index 6d92681a8..549fe3064 100644 --- a/src/dxtbx/format/FormatISISSXD.py +++ b/src/dxtbx/format/FormatISISSXD.py @@ -59,23 +59,26 @@ def get_experiment_description(self) -> str: run_number = self.get_experiment_run_number() return f"{title} ({run_number})" - def get_goniometer(self, idx: int = None) -> Goniometer: - rotation_axis = (0.0, 1.0, 0.0) - fixed_rotation = (1.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 1.0) - goniometer = GoniometerFactory.make_goniometer(rotation_axis, fixed_rotation) + def get_goniometer_phi_angle(self) -> float: try: experiment_title = self.get_experiment_title() if "w=" in experiment_title: - angle = float(experiment_title.split("w=")[1].split()[0]) + return float(experiment_title.split("w=")[1].split()[0]) elif "wccr" in experiment_title: - angle = float(experiment_title.split("wccr=")[1].split()[0]) + return float(experiment_title.split("wccr=")[1].split()[0]) elif "wtl" in experiment_title: - angle = float(experiment_title.split("wtl=")[1].split()[0]) + return float(experiment_title.split("wtl=")[1].split()[0]) else: - return goniometer - goniometer.rotate_around_origin(rotation_axis, angle) + return 0 except (ValueError, IndexError): - pass + return 0 + + def get_goniometer(self, idx: int = None) -> Goniometer: + rotation_axis = (0.0, 1.0, 0.0) + fixed_rotation = (1.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 1.0) + goniometer = GoniometerFactory.make_goniometer(rotation_axis, fixed_rotation) + angle = self.get_goniometer_phi_angle() + goniometer.rotate_around_origin(rotation_axis, angle) return goniometer def get_detector(self, index: int = None) -> Detector: @@ -318,7 +321,9 @@ def get_raw_data(self, index: int, use_loaded_data=True) -> Tuple[flex.int]: return tuple(raw_data) - def get_flattened_data(self, scale_data: bool = True) -> Tuple[flex.int]: + def get_flattened_data( + self, image_range: None | Tuple = None, scale_data: bool = True + ) -> Tuple[flex.int]: """ Image data summed along the time-of-flight direction """ @@ -337,15 +342,27 @@ def get_flattened_data(self, scale_data: bool = True) -> Tuple[flex.int]: panel_data = self._nxs_file["raw_data_1/detector_1/counts"][ 0, start_idx:end_idx, : ] - panel_max_val = np.max(panel_data) - if max_val is None or max_val < panel_max_val: - max_val = panel_max_val panel_data = np.reshape( panel_data, (panel_size[0], panel_size[1], num_tof_bins) ) - panel_data = np.flipud(np.sum(panel_data, axis=2)) + if image_range is not None: + assert ( + len(image_range) == 2 + ), "expected image_range to be only two values" + assert ( + image_range[0] >= 0 and image_range[0] < image_range[1] + ), "image_range[0] out of range" + assert image_range[1] <= num_tof_bins, "image_range[1] out of range" + panel_data = np.flipud( + np.sum(panel_data[:, :, image_range[0] : image_range[1]], axis=2) + ) + else: + panel_data = np.flipud(np.sum(panel_data, axis=2)) if panel_idx == 0 and self._panel_0_flipped(): panel_data = np.flipud(panel_data) + panel_max_val = np.max(panel_data) + if max_val is None or max_val < panel_max_val: + max_val = panel_max_val raw_data.append(panel_data) if scale_data: