diff --git a/.azure-pipelines/azure-pipelines.yml b/.azure-pipelines/azure-pipelines.yml index ccd5c6918..c5ec71f44 100644 --- a/.azure-pipelines/azure-pipelines.yml +++ b/.azure-pipelines/azure-pipelines.yml @@ -14,11 +14,11 @@ stages: steps: - checkout: none - # Use Python >=3.8 for syntax validation + # Use Python >=3.9 for syntax validation - task: UsePythonVersion@0 displayName: Set up python inputs: - versionSpec: 3.8 + versionSpec: 3.9 # Run syntax validation on a shallow clone - bash: | @@ -78,7 +78,7 @@ stages: vmImage: ubuntu-20.04 timeoutInMinutes: 60 variables: - PYTHON_VERSION: 3.8 + PYTHON_VERSION: 3.9 steps: - template: unix-build.yml @@ -88,10 +88,10 @@ stages: vmImage: ubuntu-20.04 strategy: matrix: - python38: - PYTHON_VERSION: 3.8 python39: PYTHON_VERSION: 3.9 + python311: + PYTHON_VERSION: 3.11 timeoutInMinutes: 60 steps: - template: unix-build.yml @@ -101,10 +101,10 @@ stages: vmImage: macOS-latest strategy: matrix: - python38: - PYTHON_VERSION: 3.8 python39: PYTHON_VERSION: 3.9 + python311: + PYTHON_VERSION: 3.11 timeoutInMinutes: 60 steps: - template: unix-build.yml @@ -114,8 +114,10 @@ stages: vmImage: windows-2019 strategy: matrix: - python38: - PYTHON_VERSION: 3.8 + python39: + PYTHON_VERSION: 3.9 + python311: + PYTHON_VERSION: 3.11 timeoutInMinutes: 20 steps: - template: windows-build.yml diff --git a/.azure-pipelines/bootstrap.py b/.azure-pipelines/bootstrap.py index d475d7709..becd7a72b 100644 --- a/.azure-pipelines/bootstrap.py +++ b/.azure-pipelines/bootstrap.py @@ -847,7 +847,7 @@ def run(): "--python", help="Install this minor version of Python (default: %(default)s)", default="3.9", - choices=("3.8", "3.9", "3.10"), + choices=("3.9", "3.10", "3.11"), ) parser.add_argument( "--branch", diff --git a/.azure-pipelines/ci-conda-env.txt b/.azure-pipelines/ci-conda-env.txt index 78a2235a6..a9ffd554e 100644 --- a/.azure-pipelines/ci-conda-env.txt +++ b/.azure-pipelines/ci-conda-env.txt @@ -2,7 +2,7 @@ conda-forge::boost conda-forge::boost-cpp conda-forge::bzip2 conda-forge::c-compiler<1.5 -conda-forge::cctbx-base==2022.12 +conda-forge::cctbx-base==2023.5 conda-forge::conda conda-forge::cxx-compiler<1.5 conda-forge::python-dateutil @@ -18,10 +18,10 @@ conda-forge::matplotlib-base conda-forge::mrcfile conda-forge::numpy conda-forge::nxmx +conda-forge::orderedset conda-forge::pillow>=5.4.1 conda-forge::pint conda-forge::pip -conda-forge::procrunner conda-forge::psutil conda-forge::pyrtf conda-forge::pybind11 diff --git a/.bumpversion.cfg b/.bumpversion.cfg index d3de94e98..64494b80e 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 3.15.dev +current_version = 3.16.dev commit = True tag = False parse = (?P\d+)\.(?P\d+)\.(?P[a-z]+)?(?P\d+)? diff --git a/AUTHORS b/AUTHORS index 5dc2d823f..4ba6be00e 100644 --- a/AUTHORS +++ b/AUTHORS @@ -14,6 +14,7 @@ Graeme Winter Ian Rees Iris Young James Beilsten-Edmands +James Hester James Holton James Parkhurst Johan Hattne diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 734063096..101191acd 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,3 +1,44 @@ +DIALS 3.15.1 (2023-06-29) +========================= + +Bugfixes +-------- + +- ``dxtbx.dlsnxs2cbf``: Fix import overwritten by local variable. (`#641 `_) + + +dxtbx 3.15.0 (2023-06-13) +========================= + +Features +-------- + +- Support for Bruker Photon detectors has been extended to include Photon-III. (`#637 `_) + + +Bugfixes +-------- + +- Rigaku Saturn SMV images with multi-axis crystal goniometers are now handledi, instead of being silently ignored. With thanks to James Hester for this contribution. (`#617 `_) +- FormatCBFFull: If rotation angles are decreasing, then invert the rotation axis as well as the angles, to be consistent. (`#623 `_) +- Bugfix for CCTBX bootstrapped environments, without conda. (`#630 `_) + + +Misc +---- + +- `#625 `_, `#636 `_, `#639 `_ + + +DIALS 3.14.2 (2023-05-16) +========================= + +Bugfixes +-------- + +- Compatibility fix for the DECTRIS Eiger FileWriter. Recent FileWriter versions split bit depth metadata into two separate items, ``bit_depth_readout`` from the NXmx standard, and the new ``bit_depth_image`` field. This adds support for the latter, and now passes the metadata through into image conversion. (`#632 `_) + + dxtbx 3.14.0 (2023-04-12) ========================= diff --git a/SConscript b/SConscript index 5b69aa630..fe962f34f 100644 --- a/SConscript +++ b/SConscript @@ -119,7 +119,7 @@ if not env_etc.no_boost_python and hasattr(env_etc, "boost_adaptbx_include"): env = env_no_includes_boost_python_ext.Clone() # Don't surface warnings from system or cctbx_project headers - system_includes = [x for x in env_etc.conda_cpppath if x] + system_includes = [x for x in env_etc.conda_cpppath if x] if libtbx.env.build_options.use_conda else [] system_includes.append(str(Path(env_etc.scitbx_dist).parent)) env.Append(CXXFLAGS=[f"-isystem{x}" for x in system_includes]) env.Append(SHCXXFLAGS=[f"-isystem{x}" for x in system_includes]) diff --git a/libtbx_config b/libtbx_config index 0f9b1ae2e..e511ce2b2 100644 --- a/libtbx_config +++ b/libtbx_config @@ -3,7 +3,6 @@ "python_required": [ "dials-data>=2.0.30", "pint", - "procrunner>=1.0.2", "pytest>=4.5,<5.0", ], } diff --git a/libtbx_refresh.py b/libtbx_refresh.py index f80786519..eef867e0f 100644 --- a/libtbx_refresh.py +++ b/libtbx_refresh.py @@ -1,6 +1,7 @@ from __future__ import annotations import contextlib +import importlib import inspect import io import os @@ -17,11 +18,6 @@ except ModuleNotFoundError: pass -try: - import pkg_resources -except ModuleNotFoundError: - pkg_resources = None - def _install_setup_readonly_fallback(package_name: str): """ @@ -67,9 +63,8 @@ def _install_setup_readonly_fallback(package_name: str): if import_path not in sys.path: sys.path.insert(0, import_path) - # ...and add to the existing pkg_resources working_set - if pkg_resources: - pkg_resources.working_set.add_entry(import_path) + # ...and make sure it is picked up by the import system + importlib.invalidate_caches() # This is already generated by this point, but will get picked up # on the second libtbx.refresh. diff --git a/newsfragments/625.misc b/newsfragments/625.misc deleted file mode 100644 index ae1df20fe..000000000 --- a/newsfragments/625.misc +++ /dev/null @@ -1 +0,0 @@ -Add array_family to install environment diff --git a/newsfragments/640.misc b/newsfragments/640.misc new file mode 100644 index 000000000..8183d5860 --- /dev/null +++ b/newsfragments/640.misc @@ -0,0 +1 @@ +Use Python built-in ``subprocess`` in place of external dependency ``procrunner`` for running CLI tests. diff --git a/newsfragments/642.misc b/newsfragments/642.misc new file mode 100644 index 000000000..b8a2436d1 --- /dev/null +++ b/newsfragments/642.misc @@ -0,0 +1 @@ +Drop Python 3.8 support in line with https://dials.github.io/kb/proposals/dc3. Add Python 3.11 support. diff --git a/newsfragments/643.misc b/newsfragments/643.misc new file mode 100644 index 000000000..0780b3d13 --- /dev/null +++ b/newsfragments/643.misc @@ -0,0 +1 @@ +Move from deprecated pkg_resources to importlib. diff --git a/pytest.ini b/pytest.ini index a3adf716c..a89237729 100644 --- a/pytest.ini +++ b/pytest.ini @@ -5,6 +5,7 @@ filterwarnings = ignore:numpy.dtype size changed:RuntimeWarning ignore:Datablocks are deprecated:UserWarning ignore:Deprecated call to `pkg_resources.declare_namespace:DeprecationWarning + ignore:`product` is deprecated as of NumPy:DeprecationWarning:h5py|numpy junit_family = legacy markers = regression: dxtbx regression test diff --git a/setup.cfg b/setup.cfg index 8c0ba29aa..3a93e3ec8 100644 --- a/setup.cfg +++ b/setup.cfg @@ -8,8 +8,9 @@ classifiers = Operating System :: Microsoft :: Windows Operating System :: POSIX :: Linux Programming Language :: Python :: 3 - Programming Language :: Python :: 3.8 Programming Language :: Python :: 3.9 + Programming Language :: Python :: 3.10 + Programming Language :: Python :: 3.11 [flake8] # Black disagrees with flake8 on a few points. Ignore those. diff --git a/setup.py b/setup.py index bb8b3d8fd..8213d9921 100644 --- a/setup.py +++ b/setup.py @@ -8,7 +8,7 @@ # Static version number which is updated by bump2version # Do not change this manually - use 'bump2version ' -__version_tag__ = "3.15.dev" +__version_tag__ = "3.16.dev" setup_kwargs = { "name": "dxtbx", diff --git a/src/dxtbx/array_family/flex_table_suite.h b/src/dxtbx/array_family/flex_table_suite.h index a8e500b19..b7132717b 100644 --- a/src/dxtbx/array_family/flex_table_suite.h +++ b/src/dxtbx/array_family/flex_table_suite.h @@ -12,7 +12,6 @@ #include #include #include -#include #include #include #include @@ -20,6 +19,8 @@ #include #include +#include "ref_pickle_double_buffered.h" + namespace dxtbx { namespace af { namespace flex_table_suite { using namespace boost::python; diff --git a/src/dxtbx/array_family/ref_pickle_double_buffered.h b/src/dxtbx/array_family/ref_pickle_double_buffered.h new file mode 100644 index 000000000..12477ea56 --- /dev/null +++ b/src/dxtbx/array_family/ref_pickle_double_buffered.h @@ -0,0 +1,8 @@ +#ifndef SCITBX_ARRAY_FAMILY_BOOST_PYTHON_REF_PICKLE_DOUBLE_BUFFERED_H_WRAPPER +#define SCITBX_ARRAY_FAMILY_BOOST_PYTHON_REF_PICKLE_DOUBLE_BUFFERED_H_WRAPPER + +// This header does not have an include guard + +#include + +#endif \ No newline at end of file diff --git a/src/dxtbx/command_line/dlsnxs2cbf.py b/src/dxtbx/command_line/dlsnxs2cbf.py index 7c6cc4dbc..742fb2534 100644 --- a/src/dxtbx/command_line/dlsnxs2cbf.py +++ b/src/dxtbx/command_line/dlsnxs2cbf.py @@ -2,7 +2,7 @@ Convert a NXmx-format NeXus file to a set of CBF-format image files. Note that this tool does not produce full imgCIF-format files, only -Dectris-style mini-CBF files consisting of a plain text simplified +DECTRIS-style mini-CBF files consisting of a plain text simplified header and the binary compressed image data. The simplified header does not contain a full description of the experiment geometry and some metadata, including the detector orientation, are unspecified. As @@ -18,7 +18,7 @@ import dxtbx.util.dlsnxs2cbf -parser = argparse.ArgumentParser(description=__doc__) +parser = argparse.ArgumentParser(description=__doc__, prog="dxtbx.dlsnxs2cbf") parser.add_argument( "nexus_file", metavar="nexus-file", help="Input NeXus file.", type=Path ) diff --git a/src/dxtbx/format/FormatBrukerPhotonII.py b/src/dxtbx/format/FormatBrukerPhoton.py similarity index 87% rename from src/dxtbx/format/FormatBrukerPhotonII.py rename to src/dxtbx/format/FormatBrukerPhoton.py index 073f54bfd..7ccb2b4e7 100644 --- a/src/dxtbx/format/FormatBrukerPhotonII.py +++ b/src/dxtbx/format/FormatBrukerPhoton.py @@ -18,7 +18,7 @@ from dxtbx.format.FormatBruker import FormatBruker -class FormatBrukerPhotonII(FormatBruker): +class FormatBrukerPhoton(FormatBruker): @staticmethod def understand(image_file): @@ -32,6 +32,7 @@ def understand(image_file): dettype = header_dic.get("DETTYPE") if dettype is None: return False + # We support Photon-II and Photon-III detectors if not dettype.startswith("CMOS-PHOTONII"): return False @@ -44,9 +45,9 @@ def _start(self): except OSError: return False - self.header_dict = FormatBrukerPhotonII.parse_header(header_lines) + self.header_dict = FormatBrukerPhoton.parse_header(header_lines) - # The Photon II format can't currently use BrukerImage, see + # The Photon II/III format can't currently use BrukerImage, see # https://github.com/cctbx/cctbx_project/issues/65 # from iotbx.detectors.bruker import BrukerImage # self.detectorbase = BrukerImage(self._image_file) @@ -73,24 +74,21 @@ def _goniometer(self): axes, angles, names, scan_axis ) - @staticmethod - def _estimate_gain(wavelength): - """Estimate the detector gain based on values provided by Bruker. Each ADU - corresponds to 36.6 electrons. The X-ray conversion results in deposited - charge according to the following table for typical home sources: - - In (0.5136 A): 359.6893 e/X-ray - Ag (0.5609 A): 329.3748 e/X-ray - Mo (0.7107 A): 259.9139 e/X-ray - Ga (1.3414 A): 137.6781 e/X-ray - Cu (1.5418 A): 119.8156 e/X-ray - - This fits the linear model (1/G) = -0.0000193358 + 0.1981607255 * wavelength - extremely well. + def _calculate_gain(self, wavelength): + """The CCDPARM header item contains 5 items: + 1. readnoise + 2. e/ADU + 3. e/photon + 4. bias + 5. full scale + The gain in ADU/X-ray is given by (e/photon) / (e/ADU). """ - inv_gain = -0.0000193358 + 0.1981607255 * wavelength - assert inv_gain > 0.1 - return 1.0 / inv_gain + ccdparm = self.header_dict["CCDPARM"].split() + e_ADU = float(ccdparm[1]) + e_photon = float(ccdparm[2]) + if e_ADU == 0: + return 1.0 + return e_photon / e_ADU def _detector(self): # goniometer angles in ANGLES are 2-theta, omega, phi, chi (FIXED) @@ -123,7 +121,7 @@ def _detector(self): # Not a CCD, but is an integrating detector. Photon II has a 90 um Gadox # scintillator. - gain = self._estimate_gain(float(self.header_dict["WAVELEN"].split()[0])) + gain = self._calculate_gain(float(self.header_dict["WAVELEN"].split()[0])) return self._detector_factory.complex( "CCD", origin.elems, @@ -136,9 +134,15 @@ def _detector(self): ) def _beam(self): + """Assume home source, so make unpolarized beam""" wavelength = float(self.header_dict["WAVELEN"].split()[0]) - return self._beam_factory.simple(wavelength) + return self._beam_factory.make_polarized_beam( + sample_to_source=(0.0, 0.0, 1.0), + wavelength=wavelength, + polarization=(0, 1, 0), + polarization_fraction=0.5, + ) def _scan(self): @@ -257,4 +261,4 @@ def get_raw_data(self): if __name__ == "__main__": for arg in sys.argv[1:]: - print(FormatBrukerPhotonII.understand(arg)) + print(FormatBrukerPhoton.understand(arg)) diff --git a/src/dxtbx/format/FormatNXmxEigerFilewriter.py b/src/dxtbx/format/FormatNXmxEigerFilewriter.py index e06240d11..7e8eb2924 100644 --- a/src/dxtbx/format/FormatNXmxEigerFilewriter.py +++ b/src/dxtbx/format/FormatNXmxEigerFilewriter.py @@ -28,6 +28,19 @@ def __init__(self, image_file, **kwargs): """Initialise the image structure from the given file.""" super().__init__(image_file, **kwargs) + def _start(self): + super()._start() + try: + # This is (currently) a DECTRIS-specific non-standard item that + # we will use in preference to bit_depth_readout (see below) + self._bit_depth_image = int( + self._cached_file_handle["/entry/instrument/detector/bit_depth_image"][ + () + ] + ) + except KeyError: + self._bit_depth_image = None + def _get_nxmx(self, fh: h5py.File): nxmx_obj = nxmx.NXmx(fh) nxentry = nxmx_obj.entries[0] @@ -47,14 +60,21 @@ def get_raw_data(self, index): nxmx_obj = self._get_nxmx(self._cached_file_handle) nxdata = nxmx_obj.entries[0].data[0] nxdetector = nxmx_obj.entries[0].instruments[0].detectors[0] - raw_data = get_raw_data(nxdata, nxdetector, index) - if self._bit_depth_readout: + + # Prefer bit_depth_image over bit_depth_readout since the former + # actually corresponds to the bit depth of the images as stored on + # disk. See also: + # https://www.dectris.com/support/downloads/header-docs/nexus/ + bit_depth = self._bit_depth_image or self._bit_depth_readout + raw_data = get_raw_data(nxdata, nxdetector, index, bit_depth) + + if bit_depth: # if 32 bit then it is a signed int, I think if 8, 16 then it is # unsigned with the highest two values assigned as masking values - if self._bit_depth_readout == 32: + if bit_depth == 32: top = 2**31 else: - top = 2**self._bit_depth_readout + top = 2**bit_depth for data in raw_data: d1d = data.as_1d() d1d.set_selected(d1d == top - 1, -1) @@ -63,7 +83,10 @@ def get_raw_data(self, index): def get_raw_data( - nxdata: nxmx.NXdata, nxdetector: nxmx.NXdetector, index: int + nxdata: nxmx.NXdata, + nxdetector: nxmx.NXdetector, + index: int, + bit_depth: int | None = None, ) -> tuple[flex.float | flex.double | flex.int, ...]: """Return the raw data for an NXdetector. @@ -85,6 +108,8 @@ def get_raw_data( all_data = [] sliced_outer = data[index] for module_slices in get_detector_module_slices(nxdetector): - data_as_flex = _dataset_as_flex(sliced_outer, tuple(module_slices)) + data_as_flex = _dataset_as_flex( + sliced_outer, tuple(module_slices), bit_depth=bit_depth + ) all_data.append(data_as_flex) return tuple(all_data) diff --git a/src/dxtbx/format/FormatSMVRigakuSaturn.py b/src/dxtbx/format/FormatSMVRigakuSaturn.py index c44343a2a..ecd80d0f3 100644 --- a/src/dxtbx/format/FormatSMVRigakuSaturn.py +++ b/src/dxtbx/format/FormatSMVRigakuSaturn.py @@ -10,6 +10,7 @@ from iotbx.detectors.saturn import SaturnImage from scitbx import matrix +from scitbx.array_family import flex from dxtbx.format.FormatSMVRigaku import FormatSMVRigaku @@ -85,12 +86,45 @@ def _goniometer(self): """Initialize the structure for the goniometer - this will need to correctly compose the axes given in the image header. In this case this is made rather straightforward as the image header has the - calculated rotation axis stored in it. We could work from the + calculated rotation axis stored in it. We work from the rest of the header and construct a goniometer model.""" - axis = tuple(map(float, self._header_dictionary["ROTATION_VECTOR"].split())) + head_dict = self._header_dictionary + + names = [e.strip() for e in head_dict["CRYSTAL_GONIO_NAMES"].split()] + values = [float(e) for e in head_dict["CRYSTAL_GONIO_VALUES"].split()] + units = [e.strip() for e in head_dict["CRYSTAL_GONIO_UNITS"].split()] + axis_elts = [float(e) for e in head_dict["CRYSTAL_GONIO_VECTORS"].split()] + + rot_axis = tuple(map(float, head_dict["ROTATION_VECTOR"].split())) + scan_axis = head_dict["ROTATION_AXIS_NAME"].strip() + axes = [matrix.col(axis_elts[3 * j : 3 * (j + 1)]) for j in range(len(units))] + + # Take only elements that have corresponding units of 'deg' (which is + # probably all of them). + filt = [e == "deg" for e in units] + values = [e for e, f in zip(values, filt) if f] + names = [e for e, f in zip(names, filt) if f] + axes = [e for e, f in zip(axes, filt) if f] + + # Multi-axis gonio requires axes in order as viewed from crystal to gonio + # base. Assume the SMV header records them in reverse order. + + axes = flex.vec3_double(reversed(axes)) + names = flex.std_string(reversed(names)) + values = flex.double(reversed(values)) + scan_axis = flex.first_index(names, scan_axis) + assert scan_axis is not None + gonio = self._goniometer_factory.make_multi_axis_goniometer( + axes, values, names, scan_axis + ) + + # The calculated rotation axis is also recorded in the header. We + # use this to check that the goniometer is as expected + for e1, e2 in zip(rot_axis, gonio.get_rotation_axis()): + assert abs(e1 - e2) < 1e-6 - return self._goniometer_factory.known_axis(axis) + return gonio def _detector(self): """Return a model for the detector, allowing for two-theta offsets diff --git a/src/dxtbx/format/Registry.py b/src/dxtbx/format/Registry.py index b7034a22e..03dc39e0d 100644 --- a/src/dxtbx/format/Registry.py +++ b/src/dxtbx/format/Registry.py @@ -5,12 +5,11 @@ """ from __future__ import annotations +import importlib.metadata import os import typing from typing import Callable -import pkg_resources - from dxtbx.util import get_url_scheme if typing.TYPE_CHECKING: @@ -35,7 +34,7 @@ def get_format_class_index() -> dict[str, tuple[Callable[[], type[Format]], list """ if not hasattr(get_format_class_index, "cache"): class_index = {} - for e in pkg_resources.iter_entry_points("dxtbx.format"): + for e in importlib.metadata.entry_points()["dxtbx.format"]: if ":" in e.name: format_name, base_classes_str = e.name.split(":", 1) base_classes = tuple(base_classes_str.split(",")) diff --git a/src/dxtbx/model/__init__.py b/src/dxtbx/model/__init__.py index 69048e762..26a2fc4d8 100644 --- a/src/dxtbx/model/__init__.py +++ b/src/dxtbx/model/__init__.py @@ -5,11 +5,12 @@ import os import sys +from orderedset import OrderedSet + import boost_adaptbx.boost.python import cctbx.crystal import cctbx.sgtbx import cctbx.uctbx -from libtbx.containers import OrderedSet from scitbx import matrix from scitbx.array_family import flex diff --git a/src/dxtbx/model/experiment_list.py b/src/dxtbx/model/experiment_list.py index e943818ee..ec836614f 100644 --- a/src/dxtbx/model/experiment_list.py +++ b/src/dxtbx/model/experiment_list.py @@ -3,6 +3,7 @@ import collections import copy import errno +import importlib.metadata import itertools import json import logging @@ -11,8 +12,6 @@ import pickle from typing import Any, Callable, Generator, Iterable -import pkg_resources - import dxtbx.datablock from dxtbx.datablock import ( BeamComparison, @@ -323,7 +322,6 @@ def decode(self): # a sensible experiment. el = ExperimentList() for eobj in self._obj["experiment"]: - # Get the models identifier = eobj.get("identifier", "") beam = self._lookup_model("beam", eobj) @@ -467,7 +465,7 @@ def _lookup_model(self, name, experiment_dict): @staticmethod def _scaling_model_from_dict(obj): """Get the scaling model from a dictionary.""" - for entry_point in pkg_resources.iter_entry_points("dxtbx.scaling_model_ext"): + for entry_point in importlib.metadata.entry_points()["dxtbx.scaling_model_ext"]: if entry_point.name == obj["__id__"]: return entry_point.load().from_dict(obj) diff --git a/src/dxtbx/model/goniometer.py b/src/dxtbx/model/goniometer.py index 3cde825aa..be4016d25 100644 --- a/src/dxtbx/model/goniometer.py +++ b/src/dxtbx/model/goniometer.py @@ -396,6 +396,7 @@ def imgCIF_H(cbf_handle): axis_vectors = {} angles = {} scan_axis = None + scan_axis_reversed = False cbf_handle.find_category(b"axis") for i in range(cbf_handle.count_rows()): cbf_handle.find_column(b"equipment") @@ -431,6 +432,8 @@ def imgCIF_H(cbf_handle): "More than one scan axis is defined: not currently supported." ) scan_axis = axis_name + if increment < 0: + scan_axis_reversed = True cbf_handle.next_row() if not len(axis_vectors) == len(angles): raise ValueError( @@ -454,6 +457,11 @@ def imgCIF_H(cbf_handle): # If no scan_axis, probably a still shot ⇒ scan axis arbitrary, set to 0. scan_axis = ordered_axes.index(scan_axis) if scan_axis else 0 + # invert axis since the axis _values_ will be inverted in scan.py::imgCIF_H + if scan_axis_reversed: + a0, a1, a2 = axis_vectors[scan_axis] + axis_vectors[scan_axis] = (-a0, -a1, -a2) + # construct a multi-axis goniometer gonio = GoniometerFactory.multi_axis( axis_vectors, angles, flex.std_string(ordered_axes), scan_axis diff --git a/src/dxtbx/model/profile.py b/src/dxtbx/model/profile.py index 0fbb0434b..ba3852586 100644 --- a/src/dxtbx/model/profile.py +++ b/src/dxtbx/model/profile.py @@ -1,9 +1,8 @@ from __future__ import annotations +import importlib.metadata import logging -import pkg_resources - class ProfileModelFactory: """ @@ -17,7 +16,7 @@ def from_dict(obj): """ if obj is None: return None - for entry_point in pkg_resources.iter_entry_points("dxtbx.profile_model"): + for entry_point in importlib.metadata.entry_points()["dxtbx.profile_model"]: if entry_point.name == obj["__id__"]: return entry_point.load().from_dict(obj) logging.getLogger("dxtbx.model.profile").warn( diff --git a/src/dxtbx/util/dlsnxs2cbf.py b/src/dxtbx/util/dlsnxs2cbf.py index a78e2f584..011afe42d 100644 --- a/src/dxtbx/util/dlsnxs2cbf.py +++ b/src/dxtbx/util/dlsnxs2cbf.py @@ -12,11 +12,12 @@ from tqdm import tqdm import dxtbx.model +import dxtbx.nexus from dxtbx.ext import compress -def compute_cbf_header(nxmx: nxmx.NXmx, nn: int): - nxentry = nxmx.entries[0] +def compute_cbf_header(nxmx_obj: nxmx.NXmx, nn: int): + nxentry = nxmx_obj.entries[0] nxsample = nxentry.samples[0] nxinstrument = nxentry.instruments[0] nxdetector = nxinstrument.detectors[0] @@ -145,11 +146,11 @@ def make_cbf( with h5py.File(in_name) as f: start_tag = binascii.unhexlify("0c1a04d5") - nxmx = nxmx.NXmx(f) - nxsample = nxmx.entries[0].samples[0] - nxinstrument = nxmx.entries[0].instruments[0] + nxmx_obj = nxmx.NXmx(f) + nxsample = nxmx_obj.entries[0].samples[0] + nxinstrument = nxmx_obj.entries[0].instruments[0] nxdetector = nxinstrument.detectors[0] - nxdata = nxmx.entries[0].data[0] + nxdata = nxmx_obj.entries[0].data[0] dependency_chain = nxmx.get_dependency_chain(nxsample.depends_on) scan_axis = None @@ -183,7 +184,7 @@ def make_cbf( print(f"Writing images to {template}{'#' * num_digits}.cbf:") for j in tqdm(range(num_images), unit=" images"): - header = compute_cbf_header(nxmx, j) + header = compute_cbf_header(nxmx_obj, j) (data,) = dxtbx.nexus.get_raw_data(nxdata, nxdetector, j) if bit_depth_readout: # if 32 bit then it is a signed int, I think if 8, 16 then it is diff --git a/tests/command_line/test_average.py b/tests/command_line/test_average.py index 38d1c901e..bedbc2d48 100644 --- a/tests/command_line/test_average.py +++ b/tests/command_line/test_average.py @@ -1,13 +1,15 @@ from __future__ import annotations -import procrunner +import shutil +import subprocess + import pytest import dxtbx @pytest.mark.parametrize("use_mpi", [True, False]) -def test_average(dials_data, tmpdir, use_mpi): +def test_average(dials_data, tmp_path, use_mpi): # averager uses cbf handling code in the xfel module pytest.importorskip("xfel") @@ -22,16 +24,16 @@ def test_average(dials_data, tmpdir, use_mpi): command = "mpirun" mpargs = "-n 2 dxtbx.image_average --mpi=True".split() else: - command = "dxtbx.image_average" + command = shutil.which("dxtbx.image_average") mpargs = "-n 2".split() - result = procrunner.run( + result = subprocess.run( [command] + mpargs + "-v -a avg.cbf -s stddev.cbf -m max.cbf".split() + [data], - working_directory=tmpdir, + cwd=tmp_path, ) assert not result.returncode and not result.stderr h5 = dxtbx.load(data).get_detector() - cbf = dxtbx.load(tmpdir.join("avg.cbf")).get_detector() + cbf = dxtbx.load(tmp_path / "avg.cbf").get_detector() assert h5.is_similar_to(cbf) assert h5[0].get_gain() == cbf[0].get_gain() diff --git a/tests/command_line/test_dlsnxs2cbf.py b/tests/command_line/test_dlsnxs2cbf.py index 5bf7673fd..954b3947c 100644 --- a/tests/command_line/test_dlsnxs2cbf.py +++ b/tests/command_line/test_dlsnxs2cbf.py @@ -69,10 +69,9 @@ def test_dlsnxs2cbf_deleted_axis(dials_data, tmp_path, remove_axis): make_cbf(tmp_path / master, template=str(tmp_path / "image_%04d.cbf")) -@pytest.mark.xfail(reason="Broken for old data while collecting new data") def test_dlsnxs2cbf_help(capsys): with pytest.raises(SystemExit): run(["-h"]) captured = capsys.readouterr() - assert parser.description in captured.out - assert "Template cbf output name e.g. 'image_%04d.cbf'" in captured.out + assert parser.description.splitlines()[0] in captured.out + assert "usage: dxtbx.dlsnxs2cbf" in captured.out diff --git a/tests/format/test_FormatNXmxDLS16M.py b/tests/format/test_FormatNXmxDLS16M.py index be60d4681..56ecc1434 100644 --- a/tests/format/test_FormatNXmxDLS16M.py +++ b/tests/format/test_FormatNXmxDLS16M.py @@ -98,6 +98,7 @@ def test_rotation_scan_i04_2022_run_5(master_h5): assert masker.get_mask(imageset.get_detector(), 100)[0].count(False) == 1110799 +@pytest.mark.skip(reason="Data erased from CM visit") @pytest.mark.parametrize( "master_h5", [ @@ -116,6 +117,7 @@ def test_masked_i04_32bit(master_h5): assert flex.max(imageset[0][0]) != 0x7FFFFFFF +@pytest.mark.skip(reason="Data erased from CM visit") @pytest.mark.parametrize( "master_h5", [ diff --git a/tests/format/test_FormatNXmxDLS16MI03.py b/tests/format/test_FormatNXmxDLS16MI03.py index 0dc5f4833..6a71dc3b6 100644 --- a/tests/format/test_FormatNXmxDLS16MI03.py +++ b/tests/format/test_FormatNXmxDLS16MI03.py @@ -11,6 +11,7 @@ pytest.importorskip("h5py") +@pytest.mark.skip(reason="Data erased from CM visit") @pytest.mark.parametrize( "master_h5,masked_count", [