Merge branch 'main' into remove-datablock

.azure-pipelines/azure-pipelines.yml

@@ -14,11 +14,11 @@ stages:
     steps:
       - checkout: none
 
-      # Use Python >=3.8 for syntax validation
+      # Use Python >=3.9 for syntax validation
       - task: UsePythonVersion@0
         displayName: Set up python
         inputs:
-          versionSpec: 3.8
+          versionSpec: 3.9
 
       # Run syntax validation on a shallow clone
       - bash: |
@@ -78,7 +78,7 @@ stages:
         vmImage: ubuntu-20.04
       timeoutInMinutes: 60
       variables:
-        PYTHON_VERSION: 3.8
+        PYTHON_VERSION: 3.9
       steps:
       - template: unix-build.yml
 
@@ -88,10 +88,10 @@ stages:
         vmImage: ubuntu-20.04
       strategy:
         matrix:
-          python38:
-            PYTHON_VERSION: 3.8
           python39:
             PYTHON_VERSION: 3.9
+          python311:
+            PYTHON_VERSION: 3.11
       timeoutInMinutes: 60
       steps:
       - template: unix-build.yml
@@ -101,10 +101,10 @@ stages:
         vmImage: macOS-latest
       strategy:
         matrix:
-          python38:
-            PYTHON_VERSION: 3.8
           python39:
             PYTHON_VERSION: 3.9
+          python311:
+            PYTHON_VERSION: 3.11
       timeoutInMinutes: 60
       steps:
       - template: unix-build.yml
@@ -114,8 +114,10 @@ stages:
         vmImage: windows-2019
       strategy:
         matrix:
-          python38:
-            PYTHON_VERSION: 3.8
+          python39:
+            PYTHON_VERSION: 3.9
+          python311:
+            PYTHON_VERSION: 3.11
       timeoutInMinutes: 20
       steps:
       - template: windows-build.yml

.azure-pipelines/bootstrap.py

@@ -847,7 +847,7 @@ def run():
         "--python",
         help="Install this minor version of Python (default: %(default)s)",
         default="3.9",
-        choices=("3.8", "3.9", "3.10"),
+        choices=("3.9", "3.10", "3.11"),
     )
     parser.add_argument(
         "--branch",

.azure-pipelines/ci-conda-env.txt

@@ -2,7 +2,7 @@ conda-forge::boost
 conda-forge::boost-cpp
 conda-forge::bzip2
 conda-forge::c-compiler<1.5
-conda-forge::cctbx-base==2022.12
+conda-forge::cctbx-base==2023.5
 conda-forge::conda
 conda-forge::cxx-compiler<1.5
 conda-forge::python-dateutil
@@ -18,10 +18,10 @@ conda-forge::matplotlib-base
 conda-forge::mrcfile
 conda-forge::numpy
 conda-forge::nxmx
+conda-forge::orderedset
 conda-forge::pillow>=5.4.1
 conda-forge::pint
 conda-forge::pip
-conda-forge::procrunner
 conda-forge::psutil
 conda-forge::pyrtf
 conda-forge::pybind11

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 3.15.dev
+current_version = 3.16.dev
 commit = True
 tag = False
 parse = (?P<major>\d+)\.(?P<minor>\d+)\.(?P<release>[a-z]+)?(?P<patch>\d+)?

AUTHORS

@@ -14,6 +14,7 @@ Graeme Winter
 Ian Rees
 Iris Young
 James Beilsten-Edmands
+James Hester
 James Holton
 James Parkhurst
 Johan Hattne

CHANGELOG.rst

@@ -1,3 +1,35 @@
+DIALS 3.15.1 (2023-06-29)
+=========================
+
+Bugfixes
+--------
+
+- ``dxtbx.dlsnxs2cbf``: Fix import overwritten by local variable. (`#641 <https://github.com/cctbx/dxtbx/issues/641>`_)
+
+
+dxtbx 3.15.0 (2023-06-13)
+=========================
+
+Features
+--------
+
+- Support for Bruker Photon detectors has been extended to include Photon-III. (`#637 <https://github.com/cctbx/dxtbx/issues/637>`_)
+
+
+Bugfixes
+--------
+
+- Rigaku Saturn SMV images with multi-axis crystal goniometers are now handledi, instead of being silently ignored. With thanks to James Hester for this contribution. (`#617 <https://github.com/cctbx/dxtbx/issues/617>`_)
+- FormatCBFFull: If rotation angles are decreasing, then invert the rotation axis as well as the angles, to be consistent. (`#623 <https://github.com/cctbx/dxtbx/issues/623>`_)
+- Bugfix for CCTBX bootstrapped environments, without conda. (`#630 <https://github.com/cctbx/dxtbx/issues/630>`_)
+
+
+Misc
+----
+
+- `#625 <https://github.com/cctbx/dxtbx/issues/625>`_, `#636 <https://github.com/cctbx/dxtbx/issues/636>`_, `#639 <https://github.com/cctbx/dxtbx/issues/639>`_
+
+
 DIALS 3.14.2 (2023-05-16)
 =========================
 

SConscript

@@ -119,7 +119,7 @@ if not env_etc.no_boost_python and hasattr(env_etc, "boost_adaptbx_include"):
     env = env_no_includes_boost_python_ext.Clone()
 
     # Don't surface warnings from system or cctbx_project headers
-    system_includes = [x for x in env_etc.conda_cpppath if x]
+    system_includes = [x for x in env_etc.conda_cpppath if x] if libtbx.env.build_options.use_conda else []
     system_includes.append(str(Path(env_etc.scitbx_dist).parent))
     env.Append(CXXFLAGS=[f"-isystem{x}" for x in system_includes])
     env.Append(SHCXXFLAGS=[f"-isystem{x}" for x in system_includes])

libtbx_config

@@ -3,7 +3,6 @@
     "python_required": [
         "dials-data>=2.0.30",
         "pint",
-        "procrunner>=1.0.2",
         "pytest>=4.5,<5.0",
     ],
 }

libtbx_refresh.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import contextlib
+import importlib
 import inspect
 import io
 import os
@@ -17,11 +18,6 @@
 except ModuleNotFoundError:
     pass
 
-try:
-    import pkg_resources
-except ModuleNotFoundError:
-    pkg_resources = None
-
 
 def _install_setup_readonly_fallback(package_name: str):
     """
@@ -67,9 +63,8 @@ def _install_setup_readonly_fallback(package_name: str):
     if import_path not in sys.path:
         sys.path.insert(0, import_path)
 
-    # ...and add to the existing pkg_resources working_set
-    if pkg_resources:
-        pkg_resources.working_set.add_entry(import_path)
+    # ...and make sure it is picked up by the import system
+    importlib.invalidate_caches()
 
     # This is already generated by this point, but will get picked up
     # on the second libtbx.refresh.

newsfragments/299.bugfix

@@ -0,0 +1 @@
+Panel geometry definitions in PHIL are merged by panel id before constructing panels

newsfragments/640.misc

@@ -0,0 +1 @@
+Use Python built-in ``subprocess`` in place of external dependency ``procrunner`` for running CLI tests.

newsfragments/642.misc

@@ -0,0 +1 @@
+Drop Python 3.8 support in line with https://dials.github.io/kb/proposals/dc3. Add Python 3.11 support.

newsfragments/643.misc

@@ -0,0 +1 @@
+Move from deprecated pkg_resources to importlib.

pytest.ini

@@ -5,6 +5,7 @@ filterwarnings =
     ignore:numpy.dtype size changed:RuntimeWarning
     ignore:Datablocks are deprecated:UserWarning
     ignore:Deprecated call to `pkg_resources.declare_namespace:DeprecationWarning
+    ignore:`product` is deprecated as of NumPy:DeprecationWarning:h5py|numpy
 junit_family = legacy
 markers =
     regression: dxtbx regression test

setup.cfg

@@ -8,8 +8,9 @@ classifiers =
     Operating System :: Microsoft :: Windows
     Operating System :: POSIX :: Linux
     Programming Language :: Python :: 3
-    Programming Language :: Python :: 3.8
     Programming Language :: Python :: 3.9
+    Programming Language :: Python :: 3.10
+    Programming Language :: Python :: 3.11
 
 [flake8]
 # Black disagrees with flake8 on a few points. Ignore those.

setup.py

@@ -8,7 +8,7 @@
 
 # Static version number which is updated by bump2version
 # Do not change this manually - use 'bump2version <major/minor/patch/release>'
-__version_tag__ = "3.15.dev"
+__version_tag__ = "3.16.dev"
 
 setup_kwargs = {
     "name": "dxtbx",

src/dxtbx/command_line/dlsnxs2cbf.py

@@ -2,7 +2,7 @@
 Convert a NXmx-format NeXus file to a set of CBF-format image files.
 
 Note that this tool does not produce full imgCIF-format files, only
-Dectris-style mini-CBF files consisting of a plain text simplified
+DECTRIS-style mini-CBF files consisting of a plain text simplified
 header and the binary compressed image data.  The simplified header
 does not contain a full description of the experiment geometry and some
 metadata, including the detector orientation, are unspecified.  As
@@ -18,7 +18,7 @@
 
 import dxtbx.util.dlsnxs2cbf
 
-parser = argparse.ArgumentParser(description=__doc__)
+parser = argparse.ArgumentParser(description=__doc__, prog="dxtbx.dlsnxs2cbf")
 parser.add_argument(
     "nexus_file", metavar="nexus-file", help="Input NeXus file.", type=Path
 )

src/dxtbx/format/FormatBrukerPhotonII.py → src/dxtbx/format/FormatBrukerPhoton.py

@@ -18,7 +18,7 @@
 from dxtbx.format.FormatBruker import FormatBruker
 
 
-class FormatBrukerPhotonII(FormatBruker):
+class FormatBrukerPhoton(FormatBruker):
     @staticmethod
     def understand(image_file):
 
@@ -32,6 +32,7 @@ def understand(image_file):
         dettype = header_dic.get("DETTYPE")
         if dettype is None:
             return False
+        # We support Photon-II and Photon-III detectors
         if not dettype.startswith("CMOS-PHOTONII"):
             return False
 
@@ -44,9 +45,9 @@ def _start(self):
         except OSError:
             return False
 
-        self.header_dict = FormatBrukerPhotonII.parse_header(header_lines)
+        self.header_dict = FormatBrukerPhoton.parse_header(header_lines)
 
-        # The Photon II format can't currently use BrukerImage, see
+        # The Photon II/III format can't currently use BrukerImage, see
         # https://github.com/cctbx/cctbx_project/issues/65
         # from iotbx.detectors.bruker import BrukerImage
         # self.detectorbase = BrukerImage(self._image_file)
@@ -73,24 +74,21 @@ def _goniometer(self):
             axes, angles, names, scan_axis
         )
 
-    @staticmethod
-    def _estimate_gain(wavelength):
-        """Estimate the detector gain based on values provided by Bruker. Each ADU
-        corresponds to 36.6 electrons. The X-ray conversion results in deposited
-        charge according to the following table for typical home sources:
-
-        In (0.5136 A): 359.6893 e/X-ray
-        Ag (0.5609 A): 329.3748 e/X-ray
-        Mo (0.7107 A): 259.9139 e/X-ray
-        Ga (1.3414 A): 137.6781 e/X-ray
-        Cu (1.5418 A): 119.8156 e/X-ray
-
-        This fits the linear model (1/G) = -0.0000193358 + 0.1981607255 * wavelength
-        extremely well.
+    def _calculate_gain(self, wavelength):
+        """The CCDPARM header item contains 5 items:
+            1. readnoise
+            2. e/ADU
+            3. e/photon
+            4. bias
+            5. full scale
+        The gain in ADU/X-ray is given by (e/photon) / (e/ADU).
         """
-        inv_gain = -0.0000193358 + 0.1981607255 * wavelength
-        assert inv_gain > 0.1
-        return 1.0 / inv_gain
+        ccdparm = self.header_dict["CCDPARM"].split()
+        e_ADU = float(ccdparm[1])
+        e_photon = float(ccdparm[2])
+        if e_ADU == 0:
+            return 1.0
+        return e_photon / e_ADU
 
     def _detector(self):
         # goniometer angles in ANGLES are 2-theta, omega, phi, chi (FIXED)
@@ -123,7 +121,7 @@ def _detector(self):
 
         # Not a CCD, but is an integrating detector. Photon II has a 90 um Gadox
         # scintillator.
-        gain = self._estimate_gain(float(self.header_dict["WAVELEN"].split()[0]))
+        gain = self._calculate_gain(float(self.header_dict["WAVELEN"].split()[0]))
         return self._detector_factory.complex(
             "CCD",
             origin.elems,
@@ -136,9 +134,15 @@ def _detector(self):
         )
 
     def _beam(self):
+        """Assume home source, so make unpolarized beam"""
         wavelength = float(self.header_dict["WAVELEN"].split()[0])
 
-        return self._beam_factory.simple(wavelength)
+        return self._beam_factory.make_polarized_beam(
+            sample_to_source=(0.0, 0.0, 1.0),
+            wavelength=wavelength,
+            polarization=(0, 1, 0),
+            polarization_fraction=0.5,
+        )
 
     def _scan(self):
 
@@ -257,4 +261,4 @@ def get_raw_data(self):
 
 if __name__ == "__main__":
     for arg in sys.argv[1:]:
-        print(FormatBrukerPhotonII.understand(arg))
+        print(FormatBrukerPhoton.understand(arg))