From 29f50075632395ffe448a6ad55b446b7d0f5dc8d Mon Sep 17 00:00:00 2001 From: Nicholas Devenish Date: Wed, 22 May 2024 11:51:23 +0100 Subject: [PATCH 1/8] Rework metafile search to be slightly more exhaustive --- src/dxtbx/format/FormatNXmxDLS.py | 33 +++++++++++++++++++++---------- 1 file changed, 23 insertions(+), 10 deletions(-) diff --git a/src/dxtbx/format/FormatNXmxDLS.py b/src/dxtbx/format/FormatNXmxDLS.py index 842907efc..2f7cf5eff 100644 --- a/src/dxtbx/format/FormatNXmxDLS.py +++ b/src/dxtbx/format/FormatNXmxDLS.py @@ -1,10 +1,8 @@ from __future__ import annotations import datetime -import os from functools import cached_property from pathlib import Path -from typing import Union import h5py import numpy as np @@ -18,31 +16,46 @@ def get_bit_depth_from_meta(meta_file_name): return int(f["/_dectris/bit_depth_image"][()]) -def find_meta_filename(master_like: Union[str, Path]) -> Union[str, Path]: +def find_meta_filename(master_like: Path) -> Path: """ Find the path to the '..._meta.h5' file in the same directory as the master file. Args: master_like: File path of the master HDF5 file. + Returns: File path of the HDF5 metadata '..._meta.h5' file. + + Raises: + RuntimeError: If no meta-file was found. """ - def _local_visit(name): + def _local_visit(name) -> Path | None: obj = f[name] if not hasattr(obj, "keys"): return None for k in obj.keys(): kclass = obj.get(k, getlink=True, getclass=True) if kclass is h5py._hl.group.ExternalLink: - kfile = obj.get(k, getlink=True).filename - if kfile.split(".")[0].endswith("meta"): + kfile = master_like.parent / obj.get(k, getlink=True).filename + if kfile.stem.endswith("meta"): return kfile + return None - master_dir = os.path.split(master_like)[0] + master_like = Path(master_like) with h5py.File(master_like) as f: meta_filename = f.visit(_local_visit) - return os.path.join(master_dir, meta_filename) + if meta_filename is None: + # Try a fallback of looking for a file named the same but "meta.h5" + look_for = master_like.with_name( + master_like.stem.removesuffix("_master") + "_meta.h5" + ) + if look_for.is_file(): + meta_filename = look_for + else: + raise RuntimeError(f"Could not find h5 meta file for {master_like}") + + return meta_filename class FormatNXmxDLS(FormatNXmx): @@ -76,11 +89,11 @@ def _start(self): # See https://jira.diamond.ac.uk/browse/MXGDA-3674 try: self._bit_depth_readout = get_bit_depth_from_meta(self._meta) - except Exception: + except RuntimeError: self._bit_depth_readout = 16 @cached_property - def _meta(self): + def _meta(self) -> Path: return find_meta_filename(self._image_file) def _get_nxmx(self, fh: h5py.File): From 605cc99d973843efa2d5367d3fdbef55cd10fde5 Mon Sep 17 00:00:00 2001 From: Nicholas Devenish Date: Wed, 22 May 2024 12:06:13 +0100 Subject: [PATCH 2/8] More exhaustive checking of meta data type --- src/dxtbx/format/FormatNXmxDLS.py | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/src/dxtbx/format/FormatNXmxDLS.py b/src/dxtbx/format/FormatNXmxDLS.py index 2f7cf5eff..0502d15fd 100644 --- a/src/dxtbx/format/FormatNXmxDLS.py +++ b/src/dxtbx/format/FormatNXmxDLS.py @@ -1,5 +1,6 @@ from __future__ import annotations +import contextlib import datetime from functools import cached_property from pathlib import Path @@ -11,9 +12,21 @@ from dxtbx.format.FormatNXmx import FormatNXmx -def get_bit_depth_from_meta(meta_file_name): +def get_bit_depth_from_meta(meta_file_name: Path) -> int: with h5py.File(meta_file_name) as f: - return int(f["/_dectris/bit_depth_image"][()]) + with contextlib.suppress(KeyError): + return int(f["/_dectris/bit_depth_image"][()]) + + # This might be a very old meta file, that only had 'datatype'? + if "datatype" in f: + unique_vals = np.unique(f["datatype"]) + if not len(unique_vals) == 1: + raise RuntimeError( + f"Error: Could not determine bit depth from metafile {meta_file_name} (multiple bit depths)" + ) + return np.dtype(unique_vals[0]).itemsize * 8 + + raise RuntimeError("Cannot determine metafile bit depth") def find_meta_filename(master_like: Path) -> Path: From 362b5f4c2fd11ffbbf5126b5190fc4ea5b1152c6 Mon Sep 17 00:00:00 2001 From: Nicholas Devenish Date: Wed, 22 May 2024 12:06:46 +0100 Subject: [PATCH 3/8] Add news --- newsfragments/XXX.bugfix | 1 + 1 file changed, 1 insertion(+) create mode 100644 newsfragments/XXX.bugfix diff --git a/newsfragments/XXX.bugfix b/newsfragments/XXX.bugfix new file mode 100644 index 000000000..7946486b9 --- /dev/null +++ b/newsfragments/XXX.bugfix @@ -0,0 +1 @@ +Fix case where old I03 nexus data (pre-2020) would fail to process. From b33da08b475e23508ce85e1bfdc55989fd783725 Mon Sep 17 00:00:00 2001 From: DiamondLightSource-build-server Date: Wed, 22 May 2024 11:09:45 +0000 Subject: [PATCH 4/8] Rename newsfragments/XXX.bugfix to newsfragments/737.bugfix --- newsfragments/{XXX.bugfix => 737.bugfix} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename newsfragments/{XXX.bugfix => 737.bugfix} (100%) diff --git a/newsfragments/XXX.bugfix b/newsfragments/737.bugfix similarity index 100% rename from newsfragments/XXX.bugfix rename to newsfragments/737.bugfix From 8a5785450ca6bdd2d9ba8b9916294d93fa57ca9b Mon Sep 17 00:00:00 2001 From: Nicholas Devenish Date: Wed, 22 May 2024 12:29:12 +0100 Subject: [PATCH 5/8] Remove redundant conversion --- src/dxtbx/format/FormatNXmxDLS.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/dxtbx/format/FormatNXmxDLS.py b/src/dxtbx/format/FormatNXmxDLS.py index 0502d15fd..0a8706beb 100644 --- a/src/dxtbx/format/FormatNXmxDLS.py +++ b/src/dxtbx/format/FormatNXmxDLS.py @@ -54,7 +54,6 @@ def _local_visit(name) -> Path | None: return kfile return None - master_like = Path(master_like) with h5py.File(master_like) as f: meta_filename = f.visit(_local_visit) From ab534d517f92ab653cf1e9aee98ef9f5079c2404 Mon Sep 17 00:00:00 2001 From: Nicholas Devenish Date: Wed, 22 May 2024 13:11:29 +0100 Subject: [PATCH 6/8] Revert "Remove redundant conversion" This reverts commit 8a5785450ca6bdd2d9ba8b9916294d93fa57ca9b. --- src/dxtbx/format/FormatNXmxDLS.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/dxtbx/format/FormatNXmxDLS.py b/src/dxtbx/format/FormatNXmxDLS.py index 0a8706beb..0502d15fd 100644 --- a/src/dxtbx/format/FormatNXmxDLS.py +++ b/src/dxtbx/format/FormatNXmxDLS.py @@ -54,6 +54,7 @@ def _local_visit(name) -> Path | None: return kfile return None + master_like = Path(master_like) with h5py.File(master_like) as f: meta_filename = f.visit(_local_visit) From 0c8b36ad02f8e5352b3bf4a578de85b2d1ae8790 Mon Sep 17 00:00:00 2001 From: Nicholas Devenish Date: Thu, 23 May 2024 08:47:19 +0100 Subject: [PATCH 7/8] Tweak news --- newsfragments/737.bugfix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/newsfragments/737.bugfix b/newsfragments/737.bugfix index 7946486b9..96481eadc 100644 --- a/newsfragments/737.bugfix +++ b/newsfragments/737.bugfix @@ -1 +1 @@ -Fix case where old I03 nexus data (pre-2020) would fail to process. +Fix case where old I03 Eiger nexus data (pre-2020) would fail to process. From d0934b3bca352f1cc8b9861ca3edf083346a5c33 Mon Sep 17 00:00:00 2001 From: Nicholas Devenish Date: Thu, 23 May 2024 08:51:19 +0100 Subject: [PATCH 8/8] Add log message when bit depth cannot be found --- src/dxtbx/format/FormatNXmxDLS.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/dxtbx/format/FormatNXmxDLS.py b/src/dxtbx/format/FormatNXmxDLS.py index 0502d15fd..9070463ed 100644 --- a/src/dxtbx/format/FormatNXmxDLS.py +++ b/src/dxtbx/format/FormatNXmxDLS.py @@ -2,6 +2,7 @@ import contextlib import datetime +import logging from functools import cached_property from pathlib import Path @@ -11,6 +12,8 @@ from dxtbx.format.FormatNXmx import FormatNXmx +logger = logging.getLogger(__name__) + def get_bit_depth_from_meta(meta_file_name: Path) -> int: with h5py.File(meta_file_name) as f: @@ -103,6 +106,9 @@ def _start(self): try: self._bit_depth_readout = get_bit_depth_from_meta(self._meta) except RuntimeError: + logger.warning( + "Could not determine bit depth of legacy image data: Falling back to default of 16" + ) self._bit_depth_readout = 16 @cached_property