From 713ce930286d8af970079252e4716793586f2b6a Mon Sep 17 00:00:00 2001 From: Nicholas Devenish Date: Thu, 23 May 2024 09:16:46 +0100 Subject: [PATCH 1/3] Fix old I03 data not processing (#737) Some data taken with the Eiger pre-2020 was written: - With a 64-bit VDS pointing to 32-bit data arrays, - Without the metafile being linked in the main file, - Without the eiger self-reported data type being in the metafile. Solving these allows DIALS to know that the VDS is wrong, and that it is safe to convert down to 32-bit. Soves dials/dials#2669. --- newsfragments/737.bugfix | 1 + src/dxtbx/format/FormatNXmxDLS.py | 56 ++++++++++++++++++++++++------- 2 files changed, 45 insertions(+), 12 deletions(-) create mode 100644 newsfragments/737.bugfix diff --git a/newsfragments/737.bugfix b/newsfragments/737.bugfix new file mode 100644 index 000000000..96481eadc --- /dev/null +++ b/newsfragments/737.bugfix @@ -0,0 +1 @@ +Fix case where old I03 Eiger nexus data (pre-2020) would fail to process. diff --git a/src/dxtbx/format/FormatNXmxDLS.py b/src/dxtbx/format/FormatNXmxDLS.py index 842907efc..9070463ed 100644 --- a/src/dxtbx/format/FormatNXmxDLS.py +++ b/src/dxtbx/format/FormatNXmxDLS.py @@ -1,10 +1,10 @@ from __future__ import annotations +import contextlib import datetime -import os +import logging from functools import cached_property from pathlib import Path -from typing import Union import h5py import numpy as np @@ -12,37 +12,66 @@ from dxtbx.format.FormatNXmx import FormatNXmx +logger = logging.getLogger(__name__) -def get_bit_depth_from_meta(meta_file_name): + +def get_bit_depth_from_meta(meta_file_name: Path) -> int: with h5py.File(meta_file_name) as f: - return int(f["/_dectris/bit_depth_image"][()]) + with contextlib.suppress(KeyError): + return int(f["/_dectris/bit_depth_image"][()]) + + # This might be a very old meta file, that only had 'datatype'? + if "datatype" in f: + unique_vals = np.unique(f["datatype"]) + if not len(unique_vals) == 1: + raise RuntimeError( + f"Error: Could not determine bit depth from metafile {meta_file_name} (multiple bit depths)" + ) + return np.dtype(unique_vals[0]).itemsize * 8 + + raise RuntimeError("Cannot determine metafile bit depth") -def find_meta_filename(master_like: Union[str, Path]) -> Union[str, Path]: +def find_meta_filename(master_like: Path) -> Path: """ Find the path to the '..._meta.h5' file in the same directory as the master file. Args: master_like: File path of the master HDF5 file. + Returns: File path of the HDF5 metadata '..._meta.h5' file. + + Raises: + RuntimeError: If no meta-file was found. """ - def _local_visit(name): + def _local_visit(name) -> Path | None: obj = f[name] if not hasattr(obj, "keys"): return None for k in obj.keys(): kclass = obj.get(k, getlink=True, getclass=True) if kclass is h5py._hl.group.ExternalLink: - kfile = obj.get(k, getlink=True).filename - if kfile.split(".")[0].endswith("meta"): + kfile = master_like.parent / obj.get(k, getlink=True).filename + if kfile.stem.endswith("meta"): return kfile + return None - master_dir = os.path.split(master_like)[0] + master_like = Path(master_like) with h5py.File(master_like) as f: meta_filename = f.visit(_local_visit) - return os.path.join(master_dir, meta_filename) + if meta_filename is None: + # Try a fallback of looking for a file named the same but "meta.h5" + look_for = master_like.with_name( + master_like.stem.removesuffix("_master") + "_meta.h5" + ) + if look_for.is_file(): + meta_filename = look_for + else: + raise RuntimeError(f"Could not find h5 meta file for {master_like}") + + return meta_filename class FormatNXmxDLS(FormatNXmx): @@ -76,11 +105,14 @@ def _start(self): # See https://jira.diamond.ac.uk/browse/MXGDA-3674 try: self._bit_depth_readout = get_bit_depth_from_meta(self._meta) - except Exception: + except RuntimeError: + logger.warning( + "Could not determine bit depth of legacy image data: Falling back to default of 16" + ) self._bit_depth_readout = 16 @cached_property - def _meta(self): + def _meta(self) -> Path: return find_meta_filename(self._image_file) def _get_nxmx(self, fh: h5py.File): From f49d03252a685c3b06e5e625a51c16c93113648c Mon Sep 17 00:00:00 2001 From: Jenkins Date: Thu, 23 May 2024 09:29:41 +0100 Subject: [PATCH 2/3] =?UTF-8?q?Bump=20version:=203.18.0=20=E2=86=92=203.19?= =?UTF-8?q?.1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.cfg | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 1b418c045..b6b91840a 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 3.18.0 +current_version = 3.19.1 commit = True tag = False parse = (?P\d+)\.(?P\d+)\.(?P[a-z]+)?(?P\d+)? diff --git a/setup.py b/setup.py index e77727f9d..5a051100a 100644 --- a/setup.py +++ b/setup.py @@ -8,7 +8,7 @@ # Static version number which is updated by bump2version # Do not change this manually - use 'bump2version ' -__version_tag__ = "3.18.0" +__version_tag__ = "3.19.1" setup_kwargs = { "name": "dxtbx", From 53e401d0cef8c96fe5a305655339e999042304fe Mon Sep 17 00:00:00 2001 From: Jenkins Date: Thu, 23 May 2024 09:29:42 +0100 Subject: [PATCH 3/3] Towncrier for 3.19.1 towncrier --name "DIALS" --version 3.19.1 --- CHANGELOG.rst | 9 +++++++++ newsfragments/737.bugfix | 1 - 2 files changed, 9 insertions(+), 1 deletion(-) delete mode 100644 newsfragments/737.bugfix diff --git a/CHANGELOG.rst b/CHANGELOG.rst index a3b734a5b..b62a37748 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,3 +1,12 @@ +DIALS 3.19.1 (2024-05-23) +========================= + +Bugfixes +-------- + +- Fix case where old I03 Eiger nexus data (pre-2020) would fail to process. (`#737 `_) + + dxtbx 3.19.0 (2024-04-17) ========================= diff --git a/newsfragments/737.bugfix b/newsfragments/737.bugfix deleted file mode 100644 index 96481eadc..000000000 --- a/newsfragments/737.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix case where old I03 Eiger nexus data (pre-2020) would fail to process.