Merge branch 'main' into 633-imageset-slicing

.azure-pipelines/azure-pipelines.yml

@@ -14,11 +14,11 @@ stages:
  steps:
  - checkout: none
 
- # Use Python >=3.8 for syntax validation
+ # Use Python >=3.9 for syntax validation
  - task: UsePythonVersion@0
  displayName: Set up python
  inputs:
- versionSpec: 3.8
+ versionSpec: 3.9
 
  # Run syntax validation on a shallow clone
  - bash: |
@@ -78,7 +78,7 @@ stages:
  vmImage: ubuntu-20.04
  timeoutInMinutes: 60
  variables:
- PYTHON_VERSION: 3.8
+ PYTHON_VERSION: 3.9
  steps:
  - template: unix-build.yml
 
@@ -88,10 +88,10 @@ stages:
  vmImage: ubuntu-20.04
  strategy:
  matrix:
- python38:
- PYTHON_VERSION: 3.8
  python39:
  PYTHON_VERSION: 3.9
+ python311:
+ PYTHON_VERSION: 3.11
  timeoutInMinutes: 60
  steps:
  - template: unix-build.yml
@@ -101,10 +101,10 @@ stages:
  vmImage: macOS-latest
  strategy:
  matrix:
- python38:
- PYTHON_VERSION: 3.8
  python39:
  PYTHON_VERSION: 3.9
+ python311:
+ PYTHON_VERSION: 3.11
  timeoutInMinutes: 60
  steps:
  - template: unix-build.yml
@@ -114,8 +114,10 @@ stages:
  vmImage: windows-2019
  strategy:
  matrix:
- python38:
- PYTHON_VERSION: 3.8
+ python39:
+ PYTHON_VERSION: 3.9
+ python311:
+ PYTHON_VERSION: 3.11
  timeoutInMinutes: 20
  steps:
  - template: windows-build.yml

.azure-pipelines/bootstrap.py

@@ -847,7 +847,7 @@ def run():
  "--python",
  help="Install this minor version of Python (default: %(default)s)",
  default="3.9",
- choices=("3.8", "3.9", "3.10"),
+ choices=("3.9", "3.10", "3.11"),
  )
  parser.add_argument(
  "--branch",

.azure-pipelines/ci-conda-env.txt

@@ -2,7 +2,7 @@ conda-forge::boost
 conda-forge::boost-cpp
 conda-forge::bzip2
 conda-forge::c-compiler<1.5
-conda-forge::cctbx-base==2022.12
+conda-forge::cctbx-base==2023.5
 conda-forge::conda
 conda-forge::cxx-compiler<1.5
 conda-forge::python-dateutil
@@ -18,10 +18,10 @@ conda-forge::matplotlib-base
 conda-forge::mrcfile
 conda-forge::numpy
 conda-forge::nxmx
+conda-forge::orderedset
 conda-forge::pillow>=5.4.1
 conda-forge::pint
 conda-forge::pip
-conda-forge::procrunner
 conda-forge::psutil
 conda-forge::pyrtf
 conda-forge::pybind11

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 3.15.dev
+current_version = 3.16.dev
 commit = True
 tag = False
 parse = (?P<major>\d+)\.(?P<minor>\d+)\.(?P<release>[a-z]+)?(?P<patch>\d+)?

AUTHORS

@@ -14,6 +14,7 @@ Graeme Winter
 Ian Rees
 Iris Young
 James Beilsten-Edmands
+James Hester
 James Holton
 James Parkhurst
 Johan Hattne

CHANGELOG.rst

@@ -1,3 +1,44 @@
+DIALS 3.15.1 (2023-06-29)
+=========================
+
+Bugfixes
+--------
+
+- ``dxtbx.dlsnxs2cbf``: Fix import overwritten by local variable. (`#641 <https://github.com/cctbx/dxtbx/issues/641>`_)
+
+
+dxtbx 3.15.0 (2023-06-13)
+=========================
+
+Features
+--------
+
+- Support for Bruker Photon detectors has been extended to include Photon-III. (`#637 <https://github.com/cctbx/dxtbx/issues/637>`_)
+
+
+Bugfixes
+--------
+
+- Rigaku Saturn SMV images with multi-axis crystal goniometers are now handledi, instead of being silently ignored. With thanks to James Hester for this contribution. (`#617 <https://github.com/cctbx/dxtbx/issues/617>`_)
+- FormatCBFFull: If rotation angles are decreasing, then invert the rotation axis as well as the angles, to be consistent. (`#623 <https://github.com/cctbx/dxtbx/issues/623>`_)
+- Bugfix for CCTBX bootstrapped environments, without conda. (`#630 <https://github.com/cctbx/dxtbx/issues/630>`_)
+
+
+Misc
+----
+
+- `#625 <https://github.com/cctbx/dxtbx/issues/625>`_, `#636 <https://github.com/cctbx/dxtbx/issues/636>`_, `#639 <https://github.com/cctbx/dxtbx/issues/639>`_
+
+
+DIALS 3.14.2 (2023-05-16)
+=========================
+
+Bugfixes
+--------
+
+- Compatibility fix for the DECTRIS Eiger FileWriter. Recent FileWriter versions split bit depth metadata into two separate items, ``bit_depth_readout`` from the NXmx standard, and the new ``bit_depth_image`` field. This adds support for the latter, and now passes the metadata through into image conversion. (`#632 <https://github.com/cctbx/dxtbx/issues/632>`_)
+
+
 dxtbx 3.14.0 (2023-04-12)
 =========================
 

SConscript

@@ -119,7 +119,7 @@ if not env_etc.no_boost_python and hasattr(env_etc, "boost_adaptbx_include"):
  env = env_no_includes_boost_python_ext.Clone()
 
  # Don't surface warnings from system or cctbx_project headers
- system_includes = [x for x in env_etc.conda_cpppath if x]
+ system_includes = [x for x in env_etc.conda_cpppath if x] if libtbx.env.build_options.use_conda else []
  system_includes.append(str(Path(env_etc.scitbx_dist).parent))
  env.Append(CXXFLAGS=[f"-isystem{x}" for x in system_includes])
  env.Append(SHCXXFLAGS=[f"-isystem{x}" for x in system_includes])

libtbx_config

@@ -3,7 +3,6 @@
  "python_required": [
  "dials-data>=2.0.30",
  "pint",
- "procrunner>=1.0.2",
  "pytest>=4.5,<5.0",
  ],
 }

libtbx_refresh.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import contextlib
+import importlib
 import inspect
 import io
 import os
@@ -17,11 +18,6 @@
 except ModuleNotFoundError:
  pass
 
-try:
- import pkg_resources
-except ModuleNotFoundError:
- pkg_resources = None
-
 
 def _install_setup_readonly_fallback(package_name: str):
  """
@@ -67,9 +63,8 @@ def _install_setup_readonly_fallback(package_name: str):
  if import_path not in sys.path:
  sys.path.insert(0, import_path)
 
- # ...and add to the existing pkg_resources working_set
- if pkg_resources:
- pkg_resources.working_set.add_entry(import_path)
+ # ...and make sure it is picked up by the import system
+ importlib.invalidate_caches()
 
  # This is already generated by this point, but will get picked up
  # on the second libtbx.refresh.

newsfragments/640.misc

@@ -0,0 +1 @@
+Use Python built-in ``subprocess`` in place of external dependency ``procrunner`` for running CLI tests.

newsfragments/642.misc

@@ -0,0 +1 @@
+Drop Python 3.8 support in line with https://dials.github.io/kb/proposals/dc3. Add Python 3.11 support.

newsfragments/643.misc

@@ -0,0 +1 @@
+Move from deprecated pkg_resources to importlib.

pytest.ini

@@ -5,6 +5,7 @@ filterwarnings =
  ignore:numpy.dtype size changed:RuntimeWarning
  ignore:Datablocks are deprecated:UserWarning
  ignore:Deprecated call to `pkg_resources.declare_namespace:DeprecationWarning
+ ignore:`product` is deprecated as of NumPy:DeprecationWarning:h5py|numpy
 junit_family = legacy
 markers =
  regression: dxtbx regression test

setup.cfg

@@ -8,8 +8,9 @@ classifiers =
  Operating System :: Microsoft :: Windows
  Operating System :: POSIX :: Linux
  Programming Language :: Python :: 3
- Programming Language :: Python :: 3.8
  Programming Language :: Python :: 3.9
+ Programming Language :: Python :: 3.10
+ Programming Language :: Python :: 3.11
 
 [flake8]
 # Black disagrees with flake8 on a few points. Ignore those.

setup.py

@@ -8,7 +8,7 @@
 
 # Static version number which is updated by bump2version
 # Do not change this manually - use 'bump2version <major/minor/patch/release>'
-__version_tag__ = "3.15.dev"
+__version_tag__ = "3.16.dev"
 
 setup_kwargs = {
  "name": "dxtbx",

src/dxtbx/array_family/flex_table_suite.h

@@ -12,14 +12,15 @@
 #include <boost/python/suite/indexing/vector_indexing_suite.hpp>
 #include <boost/mpl/for_each.hpp>
 #include <scitbx/array_family/flex_types.h>
-#include <scitbx/array_family/boost_python/ref_pickle_double_buffered.h>
 #include <scitbx/boost_python/slice.h>
 #include <scitbx/boost_python/utils.h>
 #include <dxtbx/array_family/flex_table.h>
 #include <scitbx/array_family/shared.h>
 #include <scitbx/array_family/versa.h>
 #include <dxtbx/error.h>
 
+#include "ref_pickle_double_buffered.h"
+
 namespace dxtbx { namespace af { namespace flex_table_suite {
 
  using namespace boost::python;

src/dxtbx/array_family/ref_pickle_double_buffered.h

@@ -0,0 +1,8 @@
+#ifndef SCITBX_ARRAY_FAMILY_BOOST_PYTHON_REF_PICKLE_DOUBLE_BUFFERED_H_WRAPPER
+#define SCITBX_ARRAY_FAMILY_BOOST_PYTHON_REF_PICKLE_DOUBLE_BUFFERED_H_WRAPPER
+
+// This header does not have an include guard
+
+#include <scitbx/array_family/boost_python/ref_pickle_double_buffered.h>
+
+#endif

src/dxtbx/command_line/dlsnxs2cbf.py

@@ -2,7 +2,7 @@
 Convert a NXmx-format NeXus file to a set of CBF-format image files.
 
 Note that this tool does not produce full imgCIF-format files, only
-Dectris-style mini-CBF files consisting of a plain text simplified
+DECTRIS-style mini-CBF files consisting of a plain text simplified
 header and the binary compressed image data. The simplified header
 does not contain a full description of the experiment geometry and some
 metadata, including the detector orientation, are unspecified. As
@@ -18,7 +18,7 @@
 
 import dxtbx.util.dlsnxs2cbf
 
-parser = argparse.ArgumentParser(description=__doc__)
+parser = argparse.ArgumentParser(description=__doc__, prog="dxtbx.dlsnxs2cbf")
 parser.add_argument(
  "nexus_file", metavar="nexus-file", help="Input NeXus file.", type=Path
 )

src/dxtbx/format/FormatBrukerPhotonII.py → src/dxtbx/format/FormatBrukerPhoton.py

@@ -18,7 +18,7 @@
 from dxtbx.format.FormatBruker import FormatBruker
 
 
-class FormatBrukerPhotonII(FormatBruker):
+class FormatBrukerPhoton(FormatBruker):
  @staticmethod
  def understand(image_file):
 
@@ -32,6 +32,7 @@ def understand(image_file):
  dettype = header_dic.get("DETTYPE")
  if dettype is None:
  return False
+ # We support Photon-II and Photon-III detectors
  if not dettype.startswith("CMOS-PHOTONII"):
  return False
 
@@ -44,9 +45,9 @@ def _start(self):
  except OSError:
  return False
 
- self.header_dict = FormatBrukerPhotonII.parse_header(header_lines)
+ self.header_dict = FormatBrukerPhoton.parse_header(header_lines)
 
- # The Photon II format can't currently use BrukerImage, see
+ # The Photon II/III format can't currently use BrukerImage, see
  # https://github.com/cctbx/cctbx_project/issues/65
  # from iotbx.detectors.bruker import BrukerImage
  # self.detectorbase = BrukerImage(self._image_file)
@@ -73,24 +74,21 @@ def _goniometer(self):
  axes, angles, names, scan_axis
  )
 
- @staticmethod
- def _estimate_gain(wavelength):
- """Estimate the detector gain based on values provided by Bruker. Each ADU
- corresponds to 36.6 electrons. The X-ray conversion results in deposited
- charge according to the following table for typical home sources:
-
- In (0.5136 A): 359.6893 e/X-ray
- Ag (0.5609 A): 329.3748 e/X-ray
- Mo (0.7107 A): 259.9139 e/X-ray
- Ga (1.3414 A): 137.6781 e/X-ray
- Cu (1.5418 A): 119.8156 e/X-ray
-
- This fits the linear model (1/G) = -0.0000193358 + 0.1981607255 * wavelength
- extremely well.
+ def _calculate_gain(self, wavelength):
+ """The CCDPARM header item contains 5 items:
+ 1. readnoise
+ 2. e/ADU
+ 3. e/photon
+ 4. bias
+ 5. full scale
+ The gain in ADU/X-ray is given by (e/photon) / (e/ADU).
  """
- inv_gain = -0.0000193358 + 0.1981607255 * wavelength
- assert inv_gain > 0.1
- return 1.0 / inv_gain
+ ccdparm = self.header_dict["CCDPARM"].split()
+ e_ADU = float(ccdparm[1])
+ e_photon = float(ccdparm[2])
+ if e_ADU == 0:
+ return 1.0
+ return e_photon / e_ADU
 
  def _detector(self):
  # goniometer angles in ANGLES are 2-theta, omega, phi, chi (FIXED)
@@ -123,7 +121,7 @@ def _detector(self):
 
  # Not a CCD, but is an integrating detector. Photon II has a 90 um Gadox
  # scintillator.
- gain = self._estimate_gain(float(self.header_dict["WAVELEN"].split()[0]))
+ gain = self._calculate_gain(float(self.header_dict["WAVELEN"].split()[0]))
  return self._detector_factory.complex(
  "CCD",
  origin.elems,
@@ -136,9 +134,15 @@ def _detector(self):
  )
 
  def _beam(self):
+ """Assume home source, so make unpolarized beam"""
  wavelength = float(self.header_dict["WAVELEN"].split()[0])
 
- return self._beam_factory.simple(wavelength)
+ return self._beam_factory.make_polarized_beam(
+ sample_to_source=(0.0, 0.0, 1.0),
+ wavelength=wavelength,
+ polarization=(0, 1, 0),
+ polarization_fraction=0.5,
+ )
 
  def _scan(self):
 
@@ -257,4 +261,4 @@ def get_raw_data(self):
 
 if __name__ == "__main__":
  for arg in sys.argv[1:]:
- print(FormatBrukerPhotonII.understand(arg))
+ print(FormatBrukerPhoton.understand(arg))