Skip to content

Commit

Permalink
Fix old I03 data not processing (#737)
Browse files Browse the repository at this point in the history
Some data taken with the Eiger pre-2020 was written:
- With a 64-bit VDS pointing to 32-bit data arrays,
- Without the metafile being linked in the main file,
- Without the eiger self-reported data type being in the metafile.

Solving these allows DIALS to know that the VDS is wrong, and that it is
safe to convert down to 32-bit.

Soves dials/dials#2669.
  • Loading branch information
ndevenish authored May 23, 2024
1 parent 88c7db3 commit c4fc39f
Show file tree
Hide file tree
Showing 2 changed files with 45 additions and 12 deletions.
1 change: 1 addition & 0 deletions newsfragments/737.bugfix
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Fix case where old I03 Eiger nexus data (pre-2020) would fail to process.
56 changes: 44 additions & 12 deletions src/dxtbx/format/FormatNXmxDLS.py
Original file line number Diff line number Diff line change
@@ -1,48 +1,77 @@
from __future__ import annotations

import contextlib
import datetime
import os
import logging
from functools import cached_property
from pathlib import Path
from typing import Union

import h5py
import numpy as np
import nxmx

from dxtbx.format.FormatNXmx import FormatNXmx

logger = logging.getLogger(__name__)

def get_bit_depth_from_meta(meta_file_name):

def get_bit_depth_from_meta(meta_file_name: Path) -> int:
with h5py.File(meta_file_name) as f:
return int(f["/_dectris/bit_depth_image"][()])
with contextlib.suppress(KeyError):
return int(f["/_dectris/bit_depth_image"][()])

# This might be a very old meta file, that only had 'datatype'?
if "datatype" in f:
unique_vals = np.unique(f["datatype"])
if not len(unique_vals) == 1:
raise RuntimeError(
f"Error: Could not determine bit depth from metafile {meta_file_name} (multiple bit depths)"
)
return np.dtype(unique_vals[0]).itemsize * 8

raise RuntimeError("Cannot determine metafile bit depth")


def find_meta_filename(master_like: Union[str, Path]) -> Union[str, Path]:
def find_meta_filename(master_like: Path) -> Path:
"""
Find the path to the '..._meta.h5' file in the same directory as the master file.
Args:
master_like: File path of the master HDF5 file.
Returns:
File path of the HDF5 metadata '..._meta.h5' file.
Raises:
RuntimeError: If no meta-file was found.
"""

def _local_visit(name):
def _local_visit(name) -> Path | None:
obj = f[name]
if not hasattr(obj, "keys"):
return None
for k in obj.keys():
kclass = obj.get(k, getlink=True, getclass=True)
if kclass is h5py._hl.group.ExternalLink:
kfile = obj.get(k, getlink=True).filename
if kfile.split(".")[0].endswith("meta"):
kfile = master_like.parent / obj.get(k, getlink=True).filename
if kfile.stem.endswith("meta"):
return kfile
return None

master_dir = os.path.split(master_like)[0]
master_like = Path(master_like)
with h5py.File(master_like) as f:
meta_filename = f.visit(_local_visit)

return os.path.join(master_dir, meta_filename)
if meta_filename is None:
# Try a fallback of looking for a file named the same but "meta.h5"
look_for = master_like.with_name(
master_like.stem.removesuffix("_master") + "_meta.h5"
)
if look_for.is_file():
meta_filename = look_for
else:
raise RuntimeError(f"Could not find h5 meta file for {master_like}")

return meta_filename


class FormatNXmxDLS(FormatNXmx):
Expand Down Expand Up @@ -76,11 +105,14 @@ def _start(self):
# See https://jira.diamond.ac.uk/browse/MXGDA-3674
try:
self._bit_depth_readout = get_bit_depth_from_meta(self._meta)
except Exception:
except RuntimeError:
logger.warning(
"Could not determine bit depth of legacy image data: Falling back to default of 16"
)
self._bit_depth_readout = 16

@cached_property
def _meta(self):
def _meta(self) -> Path:
return find_meta_filename(self._image_file)

def _get_nxmx(self, fh: h5py.File):
Expand Down

0 comments on commit c4fc39f

Please sign in to comment.