Merge remote-tracking branch 'origin/main' into spot_finding_f_strings

.azure-pipelines/azure-pipelines.yml

@@ -37,10 +37,10 @@ stages:
 
  - bash: |
  set -eux
- pip install flake8
+ pip install ruff
  cd repository
- python .azure-pipelines/flake8-validation.py
- displayName: Flake8 validation
+ python .azure-pipelines/lint-validation.py
+ displayName: Ruff validation
 
  # Set up constants for further build steps
  - bash: |

.azure-pipelines/flake8-validation.py → .azure-pipelines/lint-validation.py

@@ -3,18 +3,14 @@
 import os
 import subprocess
 
-# Flake8 validation
-known_bad = {
- "src/dials/algorithms/rs_mapper/__init__.py": {"F401", "F403"},
- "src/dials/algorithms/shoebox/__init__.py": {"F401", "F403"},
- "src/dials/nexus/__init__.py": {"F401", "F403"},
- "src/dials/test/command_line/test_generate_distortion_maps.py": {"F841"},
-}
+# Ruff validation
+known_bad = {}
 failures = 0
 try:
- flake8 = subprocess.run(
+ process = subprocess.run(
  [
- "flake8",
+ "ruff",
+ "check",
  "--exit-zero",
  ],
  capture_output=True,
@@ -24,14 +20,14 @@
  )
 except (subprocess.CalledProcessError, subprocess.TimeoutExpired) as e:
  print(
- "##vso[task.logissue type=error;]flake8 validation failed with",
+ "##vso[task.logissue type=error;]Ruff validation failed with",
  str(e.__class__.__name__),
  )
  print(e.stdout)
  print(e.stderr)
- print("##vso[task.complete result=Failed;]flake8 validation failed")
+ print("##vso[task.complete result=Failed;]Ruff validation failed")
  exit()
-for line in flake8.stdout.split("\n"):
+for line in process.stdout.split("\n"):
  if ":" not in line:
  continue
  filename, lineno, column, error = line.split(":", maxsplit=3)
@@ -47,5 +43,5 @@
  )
 
 if failures:
- print(f"##vso[task.logissue type=warning]Found {failures} flake8 violation(s)")
- print(f"##vso[task.complete result=Failed;]Found {failures} flake8 violation(s)")
+ print(f"##vso[task.logissue type=warning]Found {failures} Ruff violation(s)")
+ print(f"##vso[task.complete result=Failed;]Found {failures} Ruff violation(s)")

.azure-pipelines/unix-build-cmake.yml

@@ -28,7 +28,7 @@ steps:
  dials-data
  pytest-cov
  pytest-timeout" >> modules/dials/${{ parameters.conda_environment }}
- python modules/dials/installer/bootstrap.py base --clean --python $(PYTHON_VERSION) --prebuilt-cctbx
+ python modules/dials/installer/bootstrap.py base --clean --python $(PYTHON_VERSION) --cmake
  displayName: Create python $(PYTHON_VERSION) environment
  workingDirectory: $(Pipeline.Workspace)
 

.azure-pipelines/windows-build.yml

@@ -29,7 +29,7 @@ steps:
 
  mv ci-conda-env.txt modules/dials/.conda-envs/windows.txt
 
- python3 modules/dials/installer/bootstrap.py base --clean --python $(PYTHON_VERSION) --prebuilt-cctbx
+ python3 modules/dials/installer/bootstrap.py base --clean --python $(PYTHON_VERSION) --cmake
 
  displayName: Create python $(PYTHON_VERSION) environment
  workingDirectory: $(Pipeline.Workspace)

.clang-format

@@ -26,4 +26,5 @@ IndentCaseLabels: false
 NamespaceIndentation: Inner
 # Don't sort includes
 SortIncludes: false
+AllowShortLoopsOnASingleLine: false
 ---

.conda-envs/linux.txt

@@ -17,6 +17,7 @@ conda-forge::iota
 conda-forge::jinja2
 conda-forge::libboost-devel
 conda-forge::libboost-python-devel
+conda-forge::libglu
 conda-forge::matplotlib-base>=3.0.2
 conda-forge::mesa-libgl-devel-cos7-x86_64
 conda-forge::mrcfile
@@ -32,6 +33,7 @@ conda-forge::pint
 conda-forge::pip
 conda-forge::psutil
 conda-forge::pybind11
+conda-forge::pyopengl
 conda-forge::pyrtf
 conda-forge::pytest
 conda-forge::pytest-forked

.conda-envs/macos.txt

@@ -33,6 +33,7 @@ conda-forge::pip
 conda-forge::psutil
 conda-forge::pthread-stubs
 conda-forge::pybind11
+conda-forge::pyopengl
 conda-forge::pyrtf
 conda-forge::pytest
 conda-forge::pytest-forked
@@ -52,6 +53,6 @@ conda-forge::sqlite
 conda-forge::tabulate
 conda-forge::tqdm
 conda-forge::urllib3
-conda-forge::wxpython>=4.2.0
+conda-forge::wxpython>=4.2.0=*_5
 conda-forge::xz
 conda-forge::zlib

.conda-envs/windows.txt

@@ -31,6 +31,7 @@ conda-forge::pint
 conda-forge::pip
 conda-forge::psutil
 conda-forge::pybind11
+conda-forge::pyopengl
 conda-forge::pyrtf
 conda-forge::pytest
 conda-forge::pytest-forked

.github/workflows/newsfragments.yml

@@ -10,10 +10,10 @@ jobs:
  rename-news:
  name: Newsfragment
  runs-on: ubuntu-latest
- if: github.event.pull_request.draft == false
+ if: (github.event.pull_request.draft == false) && (! startswith(github.event.pull_request.title, '[pre-commit.ci]'))
  steps:
  - name: Check out the repository
- uses: actions/checkout@v2
+ uses: actions/checkout@v4
  with:
  ref: ${{ github.event.pull_request.head.sha }}
  token: ${{ secrets.USER_TOKEN }}

.pre-commit-config.yaml

@@ -1,7 +1,11 @@
+ci:
+ skip: [no-images]
+ autoupdate_schedule: quarterly
+
 repos:
 # Syntax validation and some basic sanity checks
 - repo: https://github.com/pre-commit/pre-commit-hooks
- rev: v4.1.0
+ rev: v4.6.0
  hooks:
  - id: check-merge-conflict
  - id: check-ast
@@ -14,46 +18,21 @@ repos:
  - id: no-commit-to-branch
  name: "Don't commit to 'main' directly"
 
-# Automatically sort imports
-- repo: https://github.com/PyCQA/isort
- rev: 5.12.0
- hooks:
- - id: isort
- args: [
- '-a', 'from __future__ import annotations', # 3.7-3.11
- '--rm', 'from __future__ import absolute_import', # -3.0
- '--rm', 'from __future__ import division', # -3.0
- '--rm', 'from __future__ import generator_stop', # -3.7
- '--rm', 'from __future__ import generators', # -2.3
- '--rm', 'from __future__ import nested_scopes', # -2.2
- '--rm', 'from __future__ import print_function', # -3.0
- '--rm', 'from __future__ import unicode_literals', # -3.0
- '--rm', 'from __future__ import with_statement', # -2.6
- ]
- exclude: ^installer/
-
-# Automatic source code formatting
-- repo: https://github.com/psf/black
- rev: 22.3.0
+- repo: https://github.com/astral-sh/ruff-pre-commit
+ rev: v0.6.3
  hooks:
- - id: black
- args: [--safe, --quiet]
- files: \.pyi?$|SConscript$|^libtbx_config$
- types: [file]
+ - id: ruff
+ args: [--fix, --exit-non-zero-on-fix, --show-fixes]
+ - id: ruff-format
+ files: \.pyi?$|SConscript$|^libtbx_config$
+ types: [file]
 
 - repo: https://github.com/pre-commit/mirrors-clang-format
- rev: v13.0.1
+ rev: v18.1.8
  hooks:
  - id: clang-format
  files: \.c(c|pp|xx)?$|\.h(pp)?$
 
-# Linting
-- repo: https://github.com/PyCQA/flake8
- rev: 4.0.1
- hooks:
- - id: flake8
- additional_dependencies: ['flake8-comprehensions==3.8.0']
-
 # Give a specific warning for added image files
 - repo: local
  hooks:

CHANGELOG.rst

@@ -1,3 +1,90 @@
+DIALS 3.21.1 (2024-08-23)
+=========================
+
+Bugfixes
+--------
+
+- CMake build can now be used for release branches. (`#2727 <https://github.com/dials/dials/issues/2727>`_)
+
+
+DIALS 3.21.0 (2024-08-20)
+=========================
+
+Features
+--------
+
+- Add classes to support time-of-flight and Laue indexing and refinement. (`#2662 <https://github.com/dials/dials/issues/2662>`_)
+- ``dials.symmetry``: Allow free selection of ``significance_level`` in the range [0,1]. (`#2696 <https://github.com/dials/dials/issues/2696>`_)
+- ``dials.export``: Add support for exporting still data in mmcif format that GEMMI can read. (`#2709 <https://github.com/dials/dials/issues/2709>`_)
+
+
+Bugfixes
+--------
+
+- ``dials.index``: Fix a ``pink_indexer`` error that caused failures for images with electron diffraction geometry. (`#2680 <https://github.com/dials/dials/issues/2680>`_)
+- ``dials.correlation_matrix``: Correctly select datasets for output json after filtering, when used by multiplex. (`#2681 <https://github.com/dials/dials/issues/2681>`_)
+- ``dials.index``: Avoid mm to px conversion when the ``refinement_protocol`` is set to do no refinement, as the required data are not available. (`#2687 <https://github.com/dials/dials/issues/2687>`_)
+- Avoid deprecated ``matplotlib.cm.get_cmap`` calls (`#2688 <https://github.com/dials/dials/issues/2688>`_)
+- Change Docker base image to rockylinux:8 as centos:7 is EOL (`#2689 <https://github.com/dials/dials/issues/2689>`_)
+- ``dials.image_viewer``: Increase the maximum resolution for the ring tool radius. This was too small for new detectors. (`#2697 <https://github.com/dials/dials/issues/2697>`_)
+- ``dials.index``: Fix potential crash in max_cell estimation when all spots are at the resolution of ice rings. (`#2699 <https://github.com/dials/dials/issues/2699>`_)
+- ``dials.merge``: Use GEMMI to output merged MTZs, for consistency with ``dials.export`` (`#2700 <https://github.com/dials/dials/issues/2700>`_)
+- ``dials.reciprocal_lattice_viewer``: Fix middle mouse drag to translate function. (`#2707 <https://github.com/dials/dials/issues/2707>`_)
+- Performance improvement for selections from large reflection tables. For a table containing 165k experiment identifiers the speedup is 1000x (12 minutes per call). (`#2718 <https://github.com/dials/dials/issues/2718>`_)
+
+
+Improved Documentation
+----------------------
+
+- Docstrings and type hints are added to the ``reindex_experiments`` and ``reindex_reflections`` functions to make it easier to use these outside the ``dials.reindex`` program. (`#2683 <https://github.com/dials/dials/issues/2683>`_)
+- Improvements to the small molecule tutorial. (`#2692 <https://github.com/dials/dials/issues/2692>`_)
+
+
+Deprecations and Removals
+-------------------------
+
+- `dials.import`: remove useless as_grid_scan option as misleading. To import as stills set scan.oscillation=0,0 (`#2454 <https://github.com/dials/dials/issues/2454>`_)
+
+
+Misc
+----
+
+- `#2675 <https://github.com/dials/dials/issues/2675>`_, `#2702 <https://github.com/dials/dials/issues/2702>`_, `#2714 <https://github.com/dials/dials/issues/2714>`_, `#2715 <https://github.com/dials/dials/issues/2715>`_, `#2723 <https://github.com/dials/dials/issues/2723>`_
+
+
+DIALS 3.20.0 (2024-06-19)
+=========================
+
+Features
+--------
+
+- New tool: ``dials.correlation_matrix``: A new command-line tool for correlation and cosine similarity clustering of multi-crystal datasets, independent of ``xia2.multiplex``. It provides HTML output, including clustering heatmaps, dendrograms and corresponding ``dials.cosym`` graphs. (`#2632 <https://github.com/dials/dials/issues/2632>`_)
+- DIALS is now compatible with Python 3.12. (`#2651 <https://github.com/dials/dials/issues/2651>`_)
+- ``dials.scale``: Add filtering options to default basic error model to allow error modelling of stills data. (`#2654 <https://github.com/dials/dials/issues/2654>`_)
+- ``dials.cosym``: Add alternative weighting during cosym CC calculation with a new ``cc_weights=`` option. (`#2666 <https://github.com/dials/dials/issues/2666>`_)
+- ``dials.correlation_matrix``: Add tables with cluster information to html output. (`#2671 <https://github.com/dials/dials/issues/2671>`_)
+- New tool: ``dials.split_still_data`` for splitting dials-processed still data based on image number (e.g. dose series). (`#2672 <https://github.com/dials/dials/issues/2672>`_)
+
+
+Bugfixes
+--------
+
+- ``dials.find_rotation_axis``: Correctly set the orientation of the rotation axis for a multi-axis goniometer. (`#2658 <https://github.com/dials/dials/issues/2658>`_)
+- Fix ``dials.show`` beam checks for time of flight experiments. (`#2660 <https://github.com/dials/dials/issues/2660>`_)
+- When masking, raise an error if ``d_min > d_max`` (where no spots would be found). (`#2664 <https://github.com/dials/dials/issues/2664>`_)
+- ``dials.cosym``: Make function return structure correctly, in the recently added ``cc_weights=`` option. (`#2668 <https://github.com/dials/dials/issues/2668>`_)
+- ``dials.find_rotation_axis``: Add reflection selection criteria, to avoid runs that use a very large amount of memory. (`#2670 <https://github.com/dials/dials/issues/2670>`_)
+- ``dials.cosym``: For ``cc_weights=sigma``, ensure correct filtering based on min_pairs parameters. (`#2673 <https://github.com/dials/dials/issues/2673>`_)
+- ``dials.cosym``: Fix to give more accurate cc calculation when running with a ``space_group=`` set. (`#2674 <https://github.com/dials/dials/issues/2674>`_)
+- Fix ``flex.reflection_table.concat``, to not modify in-place. (`#2679 <https://github.com/dials/dials/issues/2679>`_)
+
+
+Misc
+----
+
+- `#2613 <https://github.com/dials/dials/issues/2613>`_, `#2616 <https://github.com/dials/dials/issues/2616>`_, `#2641 <https://github.com/dials/dials/issues/2641>`_, `#2667 <https://github.com/dials/dials/issues/2667>`_
+
+
 DIALS 3.19.1 (2024-05-23)
 =========================
 

CMakeLists.txt

@@ -1,4 +1,4 @@
-cmake_minimum_required(VERSION 3.20 FATAL_ERROR)
+cmake_minimum_required(VERSION 3.20...3.30 FATAL_ERROR)
 
 project(dials)
 
@@ -91,6 +91,7 @@ install(
  dials_pychef_ext
  dials_refinement_helpers_ext
  dials_scaling_ext
+ dials_tof_scaling_ext
  dials_util_ext
  dials_util_streambuf_test_ext
  dials_viewer_ext

CONTRIBUTING.md

@@ -101,7 +101,7 @@ message that the code in question is special and care should be taken.
  install` - if in the libtbx ecosystem - or manually install the hooks with
  `pre-commit install`. These use the [pre-commit] tool and ensure that
  various sanity checks are run before commit, including import order,
- formatting, syntax compatibility, basic flake8 checks, lack of conflict
+ formatting, syntax compatibility, basic Ruff checks, lack of conflict
  markers and file size limits. Basically, most of the essential rules will be
  checked automatically by this.
 - **We format python code with [black]**. This means that while writing code
@@ -110,11 +110,11 @@ message that the code in question is special and care should be taken.
  black (the pre-commit hook will help do this for you), but if for some reason
  you miss this, the whole codebase is auto-cleaned once a week. Most IDEs
  and editors have support for running formatters like black automatically.
-- **Avoid introducing new flake8 warnings** - if you feel that it's appropriate
+- **Avoid introducing new Ruff warnings** - if you feel that it's appropriate
  to ignore a warning, mark it up explicitly with a [noqa] comment. The most
  important subset of checks are run as part of the pre-commit checks, but
  please try to resolve any other valid warnings shown with a normal run of
- flake8. The configuration in the repository turns off any warnings that
+ Ruff. The configuration in the repository turns off any warnings that
  disagree with our standard practice.
 - **We format C++ code with [clang-format]**. We use a configuration for style
  broadly compatible with what our existing prevailing style was. We don't
@@ -127,7 +127,7 @@ message that the code in question is special and care should be taken.
 [black]: https://github.com/psf/black
 [isort]: https://github.com/PyCQA/isort
 [clang-format]: https://clang.llvm.org/docs/ClangFormat.html
-[noqa]: http://flake8.pycqa.org/en/3.7.7/user/violations.html#in-line-ignoring-errors
+[noqa]: http://Ruff.pycqa.org/en/3.7.7/user/violations.html#in-line-ignoring-errors
 [PEP8]: https://www.python.org/dev/peps/pep-0008
 [Google-style]: https://sphinxcontrib-napoleon.readthedocs.io/en/latest/example_google.html
 [Zen of Python]: https://www.python.org/dev/peps/pep-0020/#the-zen-of-python

Dockerfile

@@ -1,16 +1,25 @@
-# Build dials
-FROM centos:7 as builder
+FROM rockylinux:9 as builder
 
-RUN yum install -y git
+RUN dnf install -y 'dnf-command(config-manager)' && \
+ dnf config-manager --enable crb && \
+ dnf install -y git python3 mesa-libGL-devel ninja-build
 WORKDIR /dials
 COPY installer/bootstrap.py .
-RUN python bootstrap.py
+ENV PIP_ROOT_USER_ACTION=ignore
+ENV CMAKE_GENERATOR=Ninja
+RUN python3 bootstrap.py --cmake
+RUN /dials/conda_base/bin/cmake --install build
+RUN /dials/conda_base/bin/python3 -mpip install modules/dxtbx modules/dials modules/xia2
 
 # Copy to final image
-FROM centos:7
-COPY ./docker-entrypoint.sh .
-COPY --from=builder /dials /dials
-RUN chmod 0755 /docker-entrypoint.sh
+FROM rockylinux:9
+RUN dnf install -y glibc-locale-source
+RUN localedef -i en_US -f UTF-8 en_US.UTF-8
+RUN echo "LANG=\"en_US.UTF-8\"" > /etc/locale.conf
+ENV LANG en_US.UTF-8
 
-ENTRYPOINT ["/docker-entrypoint.sh"]
-CMD ["dials.version"]
+RUN mkdir /dials
+COPY --from=builder /dials/conda_base /dials/conda_base
+COPY --from=builder /dials/dials /dials
+ENV PATH="/dials/conda_base/bin:$PATH"
+CMD ["dials.version"]

build.py

@@ -5,6 +5,7 @@
 out of a setup.py, but mainly because at the moment it's how poetry
 offloads the unresolved build phases.
 """
+
 from __future__ import annotations
 
 import ast

doc/sphinx/button.py

@@ -9,6 +9,7 @@
  '<a href="{{ link }}" class="button"><span>{{ text }}</span></a>'
 )
 
+
 # placeholder node for document graph
 class button_node(nodes.General, nodes.Element):
  pass