Merge branch 'main' into add_properties_to_scan_test_changes

toastisme · Jul 25, 2023 · a07022f · a07022f
2 parents 2320a42 + 8117959
commit a07022f
Show file tree

Hide file tree

Showing 167 changed files with 4,587 additions and 4,127 deletions.
diff --git a/.azure-pipelines/azure-pipelines.yml b/.azure-pipelines/azure-pipelines.yml
@@ -14,11 +14,11 @@ stages:
     steps:
       - checkout: none
 
-      # Use Python >=3.8 for syntax validation
+      # Use Python >=3.9 for syntax validation
       - task: UsePythonVersion@0
         displayName: Set up python
         inputs:
-          versionSpec: 3.8
+          versionSpec: 3.9
 
       # Run syntax validation on a shallow clone
       - bash: |
@@ -75,8 +75,8 @@ stages:
       vmImage: ubuntu-20.04
     strategy:
       matrix:
-        python38:
-          PYTHON_VERSION: 3.8
+        python39:
+          PYTHON_VERSION: 3.9
     timeoutInMinutes: 150
     steps:
     # Dependencies required for building gltbx
@@ -94,8 +94,8 @@ stages:
       vmImage: macOS-latest
     strategy:
       matrix:
-        python38:
-          PYTHON_VERSION: 3.8
+        python39:
+          PYTHON_VERSION: 3.9
     timeoutInMinutes: 250
     steps:
     - template: unix-build.yml
@@ -107,10 +107,8 @@ stages:
       vmImage: windows-latest
     strategy:
       matrix:
-        python38:
-          PYTHON_VERSION: 3.8
-      # python39:
-      #   PYTHON_VERSION: 3.9
+        python39:
+          PYTHON_VERSION: 3.9
     timeoutInMinutes: 150
     steps:
     - template: windows-build.yml

diff --git a/.conda-envs/linux.txt b/.conda-envs/linux.txt
@@ -29,7 +29,6 @@ conda-forge::pandas
 conda-forge::pillow>=5.4.1
 conda-forge::pint
 conda-forge::pip
-conda-forge::procrunner>=2.2.0
 conda-forge::psutil
 conda-forge::pybind11
 conda-forge::pyrtf

diff --git a/.conda-envs/macos.txt b/.conda-envs/macos.txt
@@ -30,7 +30,6 @@ conda-forge::pandas
 conda-forge::pillow>=5.4.1
 conda-forge::pint
 conda-forge::pip
-conda-forge::procrunner>=2.2.0
 conda-forge::psutil
 conda-forge::pthread-stubs
 conda-forge::pybind11

diff --git a/.conda-envs/windows.txt b/.conda-envs/windows.txt
@@ -29,7 +29,6 @@ conda-forge::pandas
 conda-forge::pillow>=5.4.1
 conda-forge::pint
 conda-forge::pip
-conda-forge::procrunner>=2.2.0
 conda-forge::psutil
 conda-forge::pybind11
 conda-forge::pyrtf

diff --git a/.github/workflows/pytype-pull.yml b/.github/workflows/pytype-pull.yml
@@ -29,7 +29,7 @@ jobs:
 
     - name: Install external packages for type analysis
       run: |
-        pip install procrunner pytest
+        pip install pytest
 
     - name: Copy analysis script from merged branch
       run: |

diff --git a/CHANGELOG.rst b/CHANGELOG.rst
@@ -1,3 +1,58 @@
+DIALS 3.15.1 (2023-06-29)
+=========================
+
+Bugfixes
+--------
+
+- ``dials.export_bitmaps``: Fix the ``resolution_rings.fontsize=`` feature to work on Mac, and more reliably across platforms. (`#2441 <https://github.com/dials/dials/issues/2441>`_)
+
+
+DIALS 3.15.0 (2023-06-13)
+=========================
+
+Features
+--------
+
+- ``dials.refine``: Independent groups of experiments are now refined separately by default. This may not then produce identical output, but results are believed to be of equivalent quality. The previous behaviour can be selected by using ``separate_independent_sets=False``. (`#2336 <https://github.com/dials/dials/issues/2336>`_)
+- Python 3.11 is now supported as a bootstrap target. (`#2338 <https://github.com/dials/dials/issues/2338>`_)
+- Support ``IOBS/SIGIOBS`` and ``FOBS/SIGFOBS`` columns when reading reference intensities from MTZ files. (`#2386 <https://github.com/dials/dials/issues/2386>`_)
+- ``dials.export_bitmaps``: Optionally overlay ice rings. (`#2393 <https://github.com/dials/dials/issues/2393>`_)
+- ``export_bitmaps`` API code has been separated out of CLI script into the new ``dials.util.export_bitmaps`` module. (`#2417 <https://github.com/dials/dials/issues/2417>`_)
+- ``dials.reindex``: Allow reindexing against a reference file (mtz, pdb, cif) from which intensities can be extracted/generated (`#2420 <https://github.com/dials/dials/issues/2420>`_)
+- ``dials.refine``: new option ``auto_reduction.ignore=True`` allows avoiding the parameter auto-reduction procedure for special cases. (`#2431 <https://github.com/dials/dials/issues/2431>`_)
+
+
+Bugfixes
+--------
+
+- ``dials.export_bitmaps``: Correctly display the requested number of resolution rings. (`#2397 <https://github.com/dials/dials/issues/2397>`_)
+- More directly support cctbx bootstrap environments without fixed-version dependencies. (`#2409 <https://github.com/dials/dials/issues/2409>`_)
+- ``dials.scale``: Fix crash in html plotting for scaling and filtering when R-pim/R-merge values contain 'None' values (`#2410 <https://github.com/dials/dials/issues/2410>`_)
+- ``dials.scale``, ``dials.cosym``: Use bulk solvent model when calculating intensities from a reference file. (`#2414 <https://github.com/dials/dials/issues/2414>`_)
+- ``dials.integrate``: Fix multiprocessing on Windows with Python 3.11. (`#2426 <https://github.com/dials/dials/issues/2426>`_)
+- ``dials.ssx_integrate``: Fix error in partiality calculation for ellipsoid integration algorithm. (`#2429 <https://github.com/dials/dials/issues/2429>`_)
+- ``dials.import``: Don't print filename template for every image of an ``ImageSequence``. (`#2432 <https://github.com/dials/dials/issues/2432>`_)
+
+
+Deprecations and Removals
+-------------------------
+
+- Moved ``dials::af::flex_table`` and associated tooling to dxtbx. Functions that are specific to ``reflection_table`` have been moved to ``reflection_table_suite.h``. (`#2113 <https://github.com/dials/dials/issues/2113>`_)
+- ``dials.find_spots``: Remove ``force_2d=`` option, which did not work as expected. (`#2422 <https://github.com/dials/dials/issues/2422>`_)
+
+
+Misc
+----
+
+- `#2402 <https://github.com/dials/dials/issues/2402>`_, `#2403 <https://github.com/dials/dials/issues/2403>`_, `#2419 <https://github.com/dials/dials/issues/2419>`_, `#2424 <https://github.com/dials/dials/issues/2424>`_, `#2425 <https://github.com/dials/dials/issues/2425>`_, `#2427 <https://github.com/dials/dials/issues/2427>`_, `#2428 <https://github.com/dials/dials/issues/2428>`_
+
+
+DIALS 3.14.2 (2023-05-16)
+=========================
+
+No significant changes.
+
+
 DIALS 3.14.1 (2023-04-20)
 =========================
 

diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -36,15 +36,6 @@ if(NOT TARGET Boost::python)
     set_target_properties(Boost::python PROPERTIES INTERFACE_LINK_LIBRARIES Python::Module)
 endif()
 
-# Python 3.8 aliases snprintf and vsnprintf on windows. This causes failures on more
-# recent compilers. Check for this, and set the variable if the symbol is present.
-if(${Python_VERSION} VERSION_LESS "3.9" AND WIN32)
-    check_cxx_symbol_exists(snprintf cstdio HAVE_SNPRINTF)
-    if (HAVE_SNPRINTF)
-        target_compile_definitions(Python::Module INTERFACE HAVE_SNPRINTF)
-    endif()
-endif()
-
 # Put the libraries into lib/ so that we can run this in-place in a TBX install
 set(CMAKE_LIBRARY_OUTPUT_DIRECTORY  "${CMAKE_BINARY_DIR}/lib")
 

diff --git a/SConscript b/SConscript
@@ -13,7 +13,7 @@ if not env_etc.no_boost_python and hasattr(env_etc, "boost_adaptbx_include"):
     env = env_no_includes_boost_python_ext.Clone()
     env_etc.enable_more_warnings(env=env)
 
-    system_includes = [x for x in env_etc.conda_cpppath if x]
+    system_includes = [x for x in env_etc.conda_cpppath if x] if libtbx.env.build_options.use_conda else []
     system_includes.append(str(Path(env_etc.scitbx_dist).parent))
     env.Append(CXXFLAGS=[f"-isystem{x}" for x in system_includes])
     env.Append(SHCXXFLAGS=[f"-isystem{x}" for x in system_includes])
@@ -31,6 +31,11 @@ if not env_etc.no_boost_python and hasattr(env_etc, "boost_adaptbx_include"):
         env_etc.dials_include,
     ]
 
+    # Handle cctbx bootstrap builds that pull a fixed msgpack version into modules/
+    msgpack = Path(libtbx.env.dist_path("dials")).parent / "msgpack-3.1.1" / "include"
+    if msgpack.is_dir():
+        include_paths.append(str(msgpack))
+
     if libtbx.env.build_options.use_conda:
         boost_python = get_boost_library_with_python_version(
             "boost_python", env_etc.conda_libpath

diff --git a/conftest.py b/conftest.py
@@ -9,6 +9,7 @@
 import multiprocessing
 import os
 import sys
+import warnings
 from pathlib import Path
 
 import pytest
@@ -20,6 +21,33 @@
 collect_ignore = []
 
 
+def _build_filterwarnings_string() -> str:
+    """
+    Given the set of active warnings, build a PYTHONWARNINGS string.
+
+    This lets us set the PYTHONWARNINGS environment variable so that
+    warning filters are passed down to subprocesses.
+    """
+    filter_parts = []
+    for action, regex, category, modregex, line in warnings.filters:
+        if action != "ignore":
+            continue
+        if category.__module__ != "builtins":
+            continue
+        this_action = [
+            action,
+            regex.pattern if regex else "",
+            category.__name__,
+        ]
+        if modregex is not None:
+            this_action.append(modregex.pattern)
+            if line:
+                this_action.append(line)
+        filter_parts.append(":".join(str(x) for x in this_action))
+
+    return ",".join(filter_parts)
+
+
 def pytest_configure(config):
     if not config.pluginmanager.hasplugin("dials_data"):
 
@@ -31,6 +59,8 @@ def dials_data():
     config.addinivalue_line(
         "markers", "xfel: Mark test to run xfail if xfel module is missing"
     )
+    # Ensure that subprocesses get the warnings filters
+    os.environ["PYTHONWARNINGS"] = _build_filterwarnings_string()
 
 
 def pytest_collection_modifyitems(config, items):

diff --git a/doc/examples/test_boilerplate.py b/doc/examples/test_boilerplate.py
@@ -5,10 +5,10 @@
 from boilerplate import run
 
 
-def test_boilerplate(dials_data, tmpdir):
+def test_boilerplate(dials_data, tmp_path, monkeypatch):
+    monkeypatch.chdir(tmp_path)
     x4wide = dials_data("x4wide_processed", pathlib=True)
-
-    with tmpdir.as_cwd(), mock.patch("boilerplate.dials.util.log"):
+    with mock.patch("boilerplate.dials.util.log"):
         run(
             args=[
                 str(x4wide / "AUTOMATIC_DEFAULT_scaled.expt"),
@@ -18,4 +18,4 @@ def test_boilerplate(dials_data, tmpdir):
             ]
         )
 
-    assert tmpdir.join("stronger.refl").check()
+    assert (tmp_path / "stronger.refl").is_file()
diff --git a/doc/sphinx/documentation/data_files.rst b/doc/sphinx/documentation/data_files.rst
@@ -211,13 +211,19 @@ An example of a short file is shown below.
     ]
   }
 
+Opening an experiment list file from Python can be done like this:
+
+.. code-block:: python
+
+  from dxtbx.model.experiment_list import ExperimentList
+  experiments = ExperimentList.from_file("indexed.expt")
+
 .. _reflection_pickle:
 
 Reflection files
 ----------------
 
-The reflection files are saved in python's "pickle" format. This is a binary
-format that is convenient for serializing python classes. The reflection files
+The reflection files are saved in MessagePack format. The reflection files
 will contain a table with some or all of the following columns.
 
 
@@ -282,3 +288,10 @@ will contain a table with some or all of the following columns.
 +-------------------------------+------------------------------------------------------+
 | shoebox                       | shoebox data/mask/background struct                  |
 +-------------------------------+------------------------------------------------------+
+
+Opening a reflection file in Python looks like this:
+
+.. code-block:: python
+
+  from dials.array_family import flex
+  reflections = flex.reflection_table.from_file("indexed.refl")
diff --git a/installer/bootstrap.py b/installer/bootstrap.py
@@ -44,7 +44,7 @@
 allowed_ssh_connections = {}
 concurrent_git_connection_limit = threading.Semaphore(5)
 
-_prebuilt_cctbx_base = "2023.1"  # February 2023 release
+_prebuilt_cctbx_base = "2023.5"  # June 2023 release
 
 
 def make_executable(filepath):
@@ -1272,7 +1272,7 @@ def run():
         "--python",
         help="Install this minor version of Python (default: %(default)s)",
         default="3.10",
-        choices=("3.8", "3.9", "3.10", "3.11"),
+        choices=("3.9", "3.10", "3.11"),
     )
     parser.add_argument(
         "--branch",

diff --git a/newsfragments/2338.feature b/newsfragments/2338.feature
diff --git a/newsfragments/2393.feature b/newsfragments/2393.feature
diff --git a/newsfragments/2397.bugfix b/newsfragments/2397.bugfix
diff --git a/newsfragments/2402.misc b/newsfragments/2402.misc
diff --git a/newsfragments/2403.misc b/newsfragments/2403.misc
diff --git a/newsfragments/2433.misc b/newsfragments/2433.misc
@@ -0,0 +1,2 @@
+Use Python built-in ``subprocess`` in place of external dependency ``procrunner`` for running CLI tests.
+Prefer ``tmp_path`` pytest fixture over ``tmpdir`` (the former uses more modern ``pathlib.Path`` objects).
diff --git a/newsfragments/2437.bugfix b/newsfragments/2437.bugfix
@@ -0,0 +1 @@
+Fix import template tests on Windows.
diff --git a/newsfragments/2438.misc b/newsfragments/2438.misc
@@ -0,0 +1 @@
+Mark xfel dependency on image grouping test.
diff --git a/newsfragments/2439.bugfix b/newsfragments/2439.bugfix
@@ -0,0 +1 @@
+``dials.reindex``: Fix crash for reindexing against a reference file when more than one input file
diff --git a/newsfragments/2444.doc b/newsfragments/2444.doc
@@ -0,0 +1 @@
+Update data files documentation to refer to MessagePack rather than pickle format.
diff --git a/newsfragments/2445.misc b/newsfragments/2445.misc
@@ -0,0 +1 @@
+Drop Python 3.8 support in line with https://dials.github.io/kb/proposals/dc3.
diff --git a/newsfragments/2455.feature b/newsfragments/2455.feature
@@ -0,0 +1 @@
+``dials.scale``: by default, absorption correction is switched off if all scattering angles are such that cos(2ϑ) ~ 1
diff --git a/newsfragments/2456.feature b/newsfragments/2456.feature
@@ -0,0 +1 @@
+``dials.index``: extend the stills indexer option ``indexing.refinement_protocol.mode=None`` to also operate on the scans indexer.
diff --git a/newsfragments/2457.bugfix b/newsfragments/2457.bugfix
@@ -0,0 +1 @@
+``dials.index``: fail with a useful error message when there are too few spots for indexing.
diff --git a/newsfragments/2458.bugfix b/newsfragments/2458.bugfix
@@ -0,0 +1 @@
+``dials.image_viewer``: a non-functional "Show mask" checkbox has been removed from the Mask tool.
diff --git a/newsfragments/2461.feature b/newsfragments/2461.feature
@@ -0,0 +1 @@
+``dials.export_bitmaps``: Set ``resolution_rings.d_spacings`` to manually specify the size of resolution rings.
diff --git a/newsfragments/2462.misc b/newsfragments/2462.misc
@@ -0,0 +1 @@
+Temporarily skip test_model_background on macos.
diff --git a/pytest.ini b/pytest.ini
@@ -8,4 +8,7 @@ filterwarnings =
     ignore:warn_ungrouped_named_tokens_in_collection:UserWarning
     ignore:numpy.dtype size changed:RuntimeWarning
     ignore:numpy.ufunc size changed:RuntimeWarning
+    ignore:`product` is deprecated as of NumPy:DeprecationWarning:h5py|numpy
+    ignore:pkg_resources is deprecated as an API:DeprecationWarning
+
 junit_family = legacy
diff --git a/setup.py b/setup.py
@@ -44,8 +44,9 @@
         "Operating System :: Microsoft :: Windows",
         "Operating System :: POSIX",
         "Programming Language :: Python :: 3",
-        "Programming Language :: Python :: 3.8",
         "Programming Language :: Python :: 3.9",
+        "Programming Language :: Python :: 3.10",
+        "Programming Language :: Python :: 3.11",
         "Programming Language :: Python :: Implementation :: CPython",
     ],
     "entry_points": {

diff --git a/src/dials/algorithms/indexing/indexer.py b/src/dials/algorithms/indexing/indexer.py
@@ -171,7 +171,7 @@
       .help = "refine_shells: refine in increasing resolution cutoffs after indexing."
               "repredict_only: do not refine after indexing, just update spot"
               "predictions."
-              "None: turns off all forms of refinement (currently only applies to stills_indexer)"
+              "None: do not refine and do not update spot predictions."
     n_macro_cycles = 5
       .type = int(value_min=1)
       .help = "Maximum number of macro cycles of refinement, reindexing all"
@@ -642,6 +642,11 @@ def index(self):
                         spherical_relp=self.all_params.refinement.parameterisation.spherical_relp_model,
                     )
                     ref_predictor(refined_reflections)
+                elif self.params.refinement_protocol.mode is None:
+                    refined_experiments, refined_reflections = (
+                        experiments,
+                        reflections_for_refinement,
+                    )
                 else:
                     try:
                         refined_experiments, refined_reflections = self.refine(