Merge remote-tracking branch 'origin/main' into remove-datablock

.azure-pipelines/unix-build.yml

@@ -38,7 +38,7 @@ steps:
     set -ux
     mkdir build
     cd build
-    cmake ../modules/dxtbx
+    cmake ../modules/dxtbx -DCMAKE_UNITY_BUILD=true
     cmake --build . --target install
     pip install ../modules/dxtbx
   displayName: Build dxtbx

.azure-pipelines/windows-build.yml

@@ -51,7 +51,7 @@ steps:
 
     mkdir build
     cd build
-    cmake ../modules/dxtbx
+    cmake ../modules/dxtbx -DCMAKE_UNITY_BUILD=true
     if %errorlevel% neq 0 exit /b %errorlevel%
 
     cmake --build . --config Release

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 3.14.dev
+current_version = 3.15.dev
 commit = True
 tag = False
 parse = (?P<major>\d+)\.(?P<minor>\d+)\.(?P<release>[a-z]+)?(?P<patch>\d+)?

CHANGELOG.rst

@@ -1,3 +1,34 @@
+DIALS 3.14.2 (2023-05-16)
+=========================
+
+Bugfixes
+--------
+
+- Compatibility fix for the DECTRIS Eiger FileWriter. Recent FileWriter versions split bit depth metadata into two separate items, ``bit_depth_readout`` from the NXmx standard, and the new ``bit_depth_image`` field. This adds support for the latter, and now passes the metadata through into image conversion. (`#632 <https://github.com/cctbx/dxtbx/issues/632>`_)
+
+
+dxtbx 3.14.0 (2023-04-12)
+=========================
+
+Features
+--------
+
+- ``flumpy``: Add support for conversion of ``flex.miller_index`` arrays to/from numpy. (`#618 <https://github.com/cctbx/dxtbx/issues/618>`_)
+
+
+Bugfixes
+--------
+
+- Flumpy: Prefer returning ``flex.int`` instead of ``flex.long`` when they are the same size. This solves ambiguous behaviour when reading images on Windows platforms. (`#607 <https://github.com/cctbx/dxtbx/issues/607>`_)
+- ``dxtbx.plot_detector_models``: Fix display of multiple single-panel detector models. (`#610 <https://github.com/cctbx/dxtbx/issues/610>`_)
+
+
+Misc
+----
+
+- `#604 <https://github.com/cctbx/dxtbx/issues/604>`_, `#608 <https://github.com/cctbx/dxtbx/issues/608>`_, `#609 <https://github.com/cctbx/dxtbx/issues/609>`_, `#611 <https://github.com/cctbx/dxtbx/issues/611>`_, `#614 <https://github.com/cctbx/dxtbx/issues/614>`_
+
+
 dxtbx 3.13.0 (2023-01-26)
 =========================
 

SConscript

@@ -2,6 +2,7 @@ import sys
 import os
 import libtbx.load_env
 from libtbx.env_config import get_boost_library_with_python_version
+from pathlib import Path
 
 try:
     # Check if we have a "modernized" pycbf
@@ -31,6 +32,7 @@ env_etc.dxtbx_common_includes = [
 
 Import("env_no_includes_boost_python_ext")
 env = env_no_includes_boost_python_ext.Clone()
+
 if libtbx.env.build_options.use_conda:
     boost_python = get_boost_library_with_python_version(
         "boost_python", env_etc.conda_libpath
@@ -51,7 +53,6 @@ env_etc.dxtbx_hdf5_libs = ["hdf5"]
 env_etc.dxtbx_hdf5_lib_paths = []
 
 if sys.platform == "win32" and env_etc.compiler == "win32_cl":
-
     if libtbx.env.build_options.use_conda:
         env_etc.dxtbx_hdf5_libs = ["hdf5"]
         env_etc.cppdefines = {"H5_BUILT_AS_DYNAMIC_LIB": 1}
@@ -116,6 +117,13 @@ else:
 if not env_etc.no_boost_python and hasattr(env_etc, "boost_adaptbx_include"):
     Import("env_no_includes_boost_python_ext")
     env = env_no_includes_boost_python_ext.Clone()
+
+    # Don't surface warnings from system or cctbx_project headers
+    system_includes = [x for x in env_etc.conda_cpppath if x]
+    system_includes.append(str(Path(env_etc.scitbx_dist).parent))
+    env.Append(CXXFLAGS=[f"-isystem{x}" for x in system_includes])
+    env.Append(SHCXXFLAGS=[f"-isystem{x}" for x in system_includes])
+
     env_etc.enable_more_warnings(env=env)
     env_etc.include_registry.append(
         env=env,

newsfragments/623.bugfix

@@ -0,0 +1,2 @@
+FormatCBFFull: if rotation angles are decreasing not increasing we invert the angles, now invert rotation axis as well.
+

newsfragments/625.misc

@@ -0,0 +1 @@
+Add array_family to install environment

newsfragments/636.misc

@@ -0,0 +1 @@
+Use a wrapper for ref_pickle_double_buffered.h that correctly guards against multiple inclusion.

setup.py

@@ -8,7 +8,7 @@
 
 # Static version number which is updated by bump2version
 # Do not change this manually - use 'bump2version <major/minor/patch/release>'
-__version_tag__ = "3.14.dev"
+__version_tag__ = "3.15.dev"
 
 setup_kwargs = {
     "name": "dxtbx",
@@ -26,7 +26,7 @@
     "package_dir": {"": "src"},
     "package_data": {
         "": ["*"],
-        "dxtbx": ["boost_python/*", "example/*", "py.typed"],
+        "dxtbx": ["array_family/*", "boost_python/*", "example/*", "py.typed"],
         "dxtbx.format": ["boost_python/*"],
         "dxtbx.masking": ["boost_python/*"],
         "dxtbx.model": ["boost_python/*"],

src/dxtbx/array_family/flex_table_suite.h

@@ -12,14 +12,15 @@
 #include <boost/python/suite/indexing/vector_indexing_suite.hpp>
 #include <boost/mpl/for_each.hpp>
 #include <scitbx/array_family/flex_types.h>
-#include <scitbx/array_family/boost_python/ref_pickle_double_buffered.h>
 #include <scitbx/boost_python/slice.h>
 #include <scitbx/boost_python/utils.h>
 #include <dxtbx/array_family/flex_table.h>
 #include <scitbx/array_family/shared.h>
 #include <scitbx/array_family/versa.h>
 #include <dxtbx/error.h>
 
+#include "ref_pickle_double_buffered.h"
+
 namespace dxtbx { namespace af { namespace flex_table_suite {
 
   using namespace boost::python;

src/dxtbx/array_family/ref_pickle_double_buffered.h

@@ -0,0 +1,8 @@
+#ifndef SCITBX_ARRAY_FAMILY_BOOST_PYTHON_REF_PICKLE_DOUBLE_BUFFERED_H_WRAPPER
+#define SCITBX_ARRAY_FAMILY_BOOST_PYTHON_REF_PICKLE_DOUBLE_BUFFERED_H_WRAPPER
+
+// This header does not have an include guard
+
+#include <scitbx/array_family/boost_python/ref_pickle_double_buffered.h>
+
+#endif

src/dxtbx/boost_python/flumpy.cc

@@ -24,6 +24,7 @@
 #include <scitbx/vec2.h>
 #include <scitbx/vec3.h>
 #include <scitbx/mat3.h>
+#include <cctbx/miller.h>
 
 using boost::optional;
 using std::cout;
@@ -43,7 +44,8 @@ using grid = af::versa<T, af::flex_grid<>>;
     grid<long>, grid<int8_t>, grid<int16_t>, grid<int32_t>, grid<int64_t>,            \
     grid<float>, grid<double>, grid<scitbx::vec3<double>>, grid<scitbx::vec3<int>>,   \
     grid<scitbx::vec2<double>>, grid<scitbx::mat3<double>>,                           \
-    grid<af::tiny<std::size_t, 2>>, grid<std::complex<double>>
+    grid<af::tiny<std::size_t, 2>>, grid<std::complex<double>>,                       \
+    grid<cctbx::miller::index<int>>
 // Unwrapped, and possibly doesn't make sense:
 // void wrap_flex_std_string(); - differently sized per element
 // void wrap_flex_sym_mat3_double(); - nonlinear memory layout
@@ -99,6 +101,31 @@ py::buffer_info get_buffer_specific(grid<scitbx::vec3<T>> flex) {
                          strides);
 }
 
+template <typename T>
+py::buffer_info get_buffer_specific(grid<cctbx::miller::index<T>> flex) {
+  std::vector<size_t> dim_sizes;
+  for (auto size : flex.accessor().all()) {
+    dim_sizes.push_back(size);
+  }
+  dim_sizes.push_back(3);
+
+  std::vector<size_t> strides;
+  for (int i = 0; i < dim_sizes.size(); ++i) {
+    auto stride = sizeof(T);
+    for (int j = i + 1; j < dim_sizes.size(); ++j) {
+      stride *= dim_sizes[j];
+    }
+    strides.push_back(stride);
+  }
+
+  return py::buffer_info(&flex.front(),
+                         sizeof(T),
+                         py::format_descriptor<T>::format(),
+                         dim_sizes.size(),
+                         dim_sizes,
+                         strides);
+}
+
 template <typename T>
 py::buffer_info get_buffer_specific(grid<scitbx::vec2<T>> flex) {
   std::vector<size_t> dim_sizes;
@@ -486,6 +513,11 @@ py::object from_numpy(py::object array) {
 /// More structured arrays need to be explicitly requested
 template <template <class> class VecType>
 py::object vec_from_numpy(py::array np_array) {
+  // Check that this array is contiguous
+  if (!is_array_c_contiguous(np_array)) {
+    throw ERR_NON_CONTIGUOUS;
+  }
+
   static_assert(VecType<int>::fixed_size == 2 || VecType<int>::fixed_size == 3,
                 "Only vec2/vec3 supported");
   // Only accept arrays whose last dimension is the size of this object
@@ -495,6 +527,11 @@ py::object vec_from_numpy(py::array np_array) {
   }
 
   auto dtype = np_array.attr("dtype").attr("char").cast<char>();
+  // If we are on windows, where int and long are the same size - prefer 'i'.
+  // This is because flex arrays are only bound to 'int' on this platform.
+  if (dtype == 'l' && (sizeof(long) == sizeof(int))) {
+    dtype = 'i';
+  }
 
   std::string accepted_types = VecType<int>::fixed_size == 2 ? "dQ" : "di";
   if (accepted_types.find(dtype) == std::string::npos) {
@@ -526,11 +563,6 @@ py::object vec_from_numpy(py::array np_array) {
 /// Decide which sized vector we want to convert to, and hand off to the
 /// specialization
 py::object vecs_from_numpy(py::array np_array) {
-  // Check that this array is contiguous
-  if (!is_array_c_contiguous(np_array)) {
-    throw ERR_NON_CONTIGUOUS;
-  }
-
   if (np_array.shape(np_array.ndim() - 1) == 3) {
     return vec_from_numpy<scitbx::vec3>(np_array);
   } else if (np_array.shape(np_array.ndim() - 1) == 2) {
@@ -540,6 +572,19 @@ py::object vecs_from_numpy(py::array np_array) {
     "Invalid input array: last numpy dimension must be 2 or 3 to convert to vector");
 }
 
+/// Create a versa<miller_index> from numpy
+py::object miller_index_from_numpy(py::array np_array) {
+  // miller_index is ONLY bound as 'int', but on windows we accept the same-sized long
+  auto dtype = np_array.attr("dtype").attr("char").cast<char>();
+  std::string accepted_types = sizeof(long) == sizeof(int) ? "il" : "i";
+  if (accepted_types.find(dtype) == std::string::npos) {
+    throw std::invalid_argument(
+      std::string("miller_index only supports int32 or intc types - cannot convert '")
+      + std::to_string(dtype) + "'");
+  }
+  return vec_from_numpy<cctbx::miller::index>(np_array);
+}
+
 py::object mat3_from_numpy(py::array np_array) {
   // Check that this array is contiguous
   if (!is_array_c_contiguous(np_array)) {
@@ -586,8 +631,12 @@ PYBIND11_MODULE(dxtbx_flumpy, m) {
         &vecs_from_numpy,
         "Convert a numpy object to a flex.vec2 or .vec3, depending on input array");
   m.def("mat3_from_numpy", &mat3_from_numpy, "Convert a numpy object to a flex.mat3");
+  m.def("miller_index_from_numpy",
+        &miller_index_from_numpy,
+        "Convert a numpy object to a flex.miller_index");
 
   // Make sure that we have imported flex - cannot do boost::python conversions
   // otherwise
   pybind11::module::import("scitbx.array_family.flex");
+  pybind11::module::import("cctbx.array_family.flex");
 }

src/dxtbx/command_line/plot_detector_models.py

@@ -4,6 +4,7 @@
 
 import os
 import sys
+from collections.abc import Sequence
 
 import matplotlib.pyplot as plt
 import numpy as np
@@ -66,6 +67,7 @@ def do_3d_projection(self, renderer=None):
 def plot_group(
     g, color, ax, orthographic=False, show_origin_vectors=True, panel_numbers=True
 ):
+    panel_corners = []
     # recursively plot a detector group
     p = g.parent()
     if show_origin_vectors:
@@ -90,7 +92,11 @@ def plot_group(
     if g.is_group():
         for c in g:
             # plot all the children
-            plot_group(c, color, ax, orthographic, show_origin_vectors, panel_numbers)
+            panel_corners.append(
+                plot_group(
+                    c, color, ax, orthographic, show_origin_vectors, panel_numbers
+                )
+            )
     else:
         # plot the panel boundaries
         size = g.get_image_size()
@@ -102,6 +108,7 @@ def plot_group(
         v2 = p3 - p0
         vcen = ((v2 / 2) + (v1 / 2)) + p0
         z = list(zip(p0, p1, p2, p3, p0))
+        panel_corners.append((p0.elems, p1.elems, p2.elems, p3.elems))
 
         if orthographic:
             ax.plot(z[0], z[1], color=color)
@@ -115,6 +122,7 @@ def plot_group(
             if panel_numbers:
                 # Annotate with panel numbers
                 ax.text(vcen[0], vcen[1], vcen[2], "%d" % g.index())
+    return panel_corners
 
 
 def plot_image_plane_projection(detector, color, ax, panel_numbers=True):
@@ -140,6 +148,14 @@ def plot_image_plane_projection(detector, color, ax, panel_numbers=True):
             ax.text(vcen[0], vcen[1], "%d" % panel.index())
 
 
+def flatten(xs):
+    for x in xs:
+        if isinstance(x, Sequence) and not isinstance(x, (str, bytes)):
+            yield from flatten(x)
+        else:
+            yield x
+
+
 def run(args=None):
     dxtbx.util.encode_output_as_utf8()
     args = args or sys.argv[1:]
@@ -159,6 +175,7 @@ def run(args=None):
     colormap = plt.cm.gist_ncar
     colors = [colormap(i) for i in np.linspace(0, 0.9, len(files))]
     min_z = max_z = None
+    ax = None
     for file_name, color in zip(files, colors):
 
         # read the data and get the detector models
@@ -173,15 +190,15 @@ def run(args=None):
             detectors = detectors[0:1]
         for detector in detectors:
             # plot the hierarchy
-            if params.orthographic:
+            if ax is None and params.orthographic:
                 ax = fig.gca()
-            else:
+            elif ax is None:
                 ax = fig.add_subplot(projection="3d")
 
             if params.orthographic and params.project_onto == "image_plane":
                 plot_image_plane_projection(detector, color, ax, params.panel_numbers)
             else:
-                plot_group(
+                panel_corners = plot_group(
                     detector.hierarchy(),
                     color,
                     ax,
@@ -191,7 +208,9 @@ def run(args=None):
                 )
 
                 if not params.orthographic:
-                    all_z = [p.get_origin()[2] for p in detector]
+                    # flatten the nested list of panel coordinates and then select
+                    # all the z coordinates (i.e. every 3rd value)
+                    all_z = list(flatten(panel_corners))[2::3]
                     if min_z is None:
                         min_z = min(all_z)
                         max_z = max(all_z)