From 674bddda8a79fdfd535c86e9140645cc8c4991ac Mon Sep 17 00:00:00 2001 From: Nicholas Devenish Date: Tue, 20 Aug 2024 11:35:18 +0100 Subject: [PATCH 01/10] re-sort the existing dependency file --- .azure-pipelines/ci-conda-env.txt | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.azure-pipelines/ci-conda-env.txt b/.azure-pipelines/ci-conda-env.txt index 23081b8ff..d0c400140 100644 --- a/.azure-pipelines/ci-conda-env.txt +++ b/.azure-pipelines/ci-conda-env.txt @@ -5,14 +5,13 @@ conda-forge::c-compiler conda-forge::cctbx-base conda-forge::conda conda-forge::cxx-compiler -conda-forge::python-dateutil conda-forge::dials-data conda-forge::docutils conda-forge::eigen conda-forge::future conda-forge::h5py>=3.1 -conda-forge::hdf5plugin conda-forge::hdf5 +conda-forge::hdf5plugin conda-forge::matplotlib-base conda-forge::mrcfile conda-forge::natsort @@ -23,15 +22,16 @@ conda-forge::pillow>=5.4.1 conda-forge::pint conda-forge::pip conda-forge::psutil -conda-forge::pyrtf conda-forge::pybind11 +conda-forge::pyrtf +conda-forge::pytest conda-forge::pytest-azurepipelines conda-forge::pytest-cov conda-forge::pytest-forked conda-forge::pytest-mock conda-forge::pytest-timeout conda-forge::pytest-xdist -conda-forge::pytest +conda-forge::python-dateutil conda-forge::requests conda-forge::scipy conda-forge::scons From 3e04ab7c12440e93f04ae5e86bcdfca204f09574 Mon Sep 17 00:00:00 2001 From: Nicholas Devenish Date: Tue, 20 Aug 2024 12:00:05 +0100 Subject: [PATCH 02/10] Reduce CI dependency list --- .azure-pipelines/ci-conda-env.txt | 20 +------------------- 1 file changed, 1 insertion(+), 19 deletions(-) diff --git a/.azure-pipelines/ci-conda-env.txt b/.azure-pipelines/ci-conda-env.txt index d0c400140..af2640e90 100644 --- a/.azure-pipelines/ci-conda-env.txt +++ b/.azure-pipelines/ci-conda-env.txt @@ -1,29 +1,17 @@ -conda-forge::boost -conda-forge::boost-cpp -conda-forge::bzip2 -conda-forge::c-compiler conda-forge::cctbx-base -conda-forge::conda conda-forge::cxx-compiler -conda-forge::dials-data -conda-forge::docutils -conda-forge::eigen -conda-forge::future conda-forge::h5py>=3.1 conda-forge::hdf5 conda-forge::hdf5plugin +conda-forge::libboost-devel conda-forge::matplotlib-base conda-forge::mrcfile conda-forge::natsort conda-forge::numpy<2 conda-forge::nxmx conda-forge::orderedset -conda-forge::pillow>=5.4.1 conda-forge::pint -conda-forge::pip -conda-forge::psutil conda-forge::pybind11 -conda-forge::pyrtf conda-forge::pytest conda-forge::pytest-azurepipelines conda-forge::pytest-cov @@ -32,11 +20,5 @@ conda-forge::pytest-mock conda-forge::pytest-timeout conda-forge::pytest-xdist conda-forge::python-dateutil -conda-forge::requests conda-forge::scipy -conda-forge::scons -conda-forge::setuptools -conda-forge::six -conda-forge::tabulate conda-forge::tqdm -conda-forge::xz From 0fe288d1b416acef5f3966ac091a99627c2ef6b5 Mon Sep 17 00:00:00 2001 From: Nicholas Devenish Date: Tue, 20 Aug 2024 12:06:22 +0100 Subject: [PATCH 03/10] Add the dxtbx dependencies file --- .pre-commit-config.yaml | 1 + dependencies.yaml | 91 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 92 insertions(+) create mode 100644 dependencies.yaml diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 2024e7323..5c52d77ef 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -11,6 +11,7 @@ repos: args: ['--maxkb=200'] - id: check-yaml args: ['--allow-multiple-documents'] + exclude: dependencies.yaml - id: no-commit-to-branch name: "Don't commit to 'main' directly" diff --git a/dependencies.yaml b/dependencies.yaml new file mode 100644 index 000000000..638ac8936 --- /dev/null +++ b/dependencies.yaml @@ -0,0 +1,91 @@ +# # Conda dependencies for dxtbx on all platforms. +# +# This file is interpreted by bootstrap. it uses a HIGHLY limited subset of +# yaml and conda-build selector preprocessing. As such, the file itself is not +# YAML without at minimum preprocessing, and may need full conda-build +# templating to be parseable as yaml directly. +# +# ## Splitting Dependencies +# +# Dependencies are now split here into the categories of build, host, run, as +# per https://docs.conda.io/projects/conda-build/en/latest/resources/define-metadata.html#requirements-section +# +# Briefly, +# - `build` is for build and development tools +# - `host` is complicated, but can broadly be considered "build-time" +# dependencies that are expected to be present on the targeted platform. +# Please see above conda-build documentation for details, but if you are +# unsure where a dependency goes, it's probably safest to put it here. +# - `run` are dependencies not used at build-time, but are required for +# running. Most pure-python packages go here. +# - `test` are dependencies only used to run tests. +# +# ## Filtering dependencies by platform and target +# +# Before use (by bootstrap OR conda), this file is passed through a +# preprocessor. This handles all lines of the form `#[expr]`, and each +# line is filtered out on a line-by-line basis if the expression +# evaluates to false. +# +# Only the last `# [expression]` is stripped and evaluated, which allows +# passing further selection expressions through for onward processing. +# +# Expression values supported: +# - Platform e.g. `osx`, `win` or `linux` +# - `bootstrap`: Whether we are targeting bootstrap.py. This allows us +# to have advanced conda-build syntax (e.g. jinja templates) while +# ignoring these when generating the bootstrap dependency list. +# - `prebuilt_cctbx`: Whether we are building cctbx. This is true for +# bootstrap with CMake and for conda-build builds. +# - Compound expressions with `and` e.g. `osx and bootstrap`. This will +# work with `not`-expressions, but nothing more complex. +# + +build: + - cmake # [prebuilt_cctbx] + - ninja # [prebuilt_cctbx] + - python # [build_platform != target_platform] # [not bootstrap] + - cross-python_{{ target_platform }} # [build_platform != target_platform] # [not bootstrap] + - numpy # [build_platform != target_platform] # [not bootstrap] + - pybind11 # [build_platform != target_platform] # [not bootstrap] + - {{ compiler('cxx') }} # [not bootstrap] + - {{ stdlib('c') }} # [not bootstrap] + - cxx-compiler # [bootstrap] + +host: + - cctbx-base # [prebuilt_cctbx and bootstrap] + - cctbx-base >=2023.10 # [not bootstrap] + - hdf5 + - libboost-devel + - libboost-python-devel + - numpy >=1.19,<1.21|>=1.21.5|<2 #[bootstrap] + - numpy # [not bootstrap] + - pip + - pybind11 + - python + +run: + - {{ pin_compatible('cctbx-base') }} # [not bootstrap] + - {{ pin_compatible('hdf5') }} # [not bootstrap] + - h5py >=3.1.0 + - hdf5plugin + - matplotlib-base >=3.0.2 + - mrcfile + - natsort + - {{ pin_compatible('numpy') }} # [not bootstrap] + - nxmx + - orderedset + - pint + - pycbf # [prebuilt_cctbx] + - python + - python-dateutil >=2.7.0 + - scipy + - tqdm + +test: + - dials-data + - pip + - pytest + - pytest-mock + - pytest-nunit # [win] + - pytest-xdist From 32e52a0d2a70221c7313df20044dc2dcb91738a0 Mon Sep 17 00:00:00 2001 From: Nicholas Devenish Date: Tue, 20 Aug 2024 14:19:05 +0100 Subject: [PATCH 04/10] Switch to using new dependency file in CI --- .azure-pipelines/ci-conda-env.txt | 24 -- .../parse_dependency_selectors.py | 365 ++++++++++++++++++ .azure-pipelines/unix-build.yml | 6 +- .azure-pipelines/windows-build.yml | 9 +- 4 files changed, 375 insertions(+), 29 deletions(-) delete mode 100644 .azure-pipelines/ci-conda-env.txt create mode 100644 .azure-pipelines/parse_dependency_selectors.py diff --git a/.azure-pipelines/ci-conda-env.txt b/.azure-pipelines/ci-conda-env.txt deleted file mode 100644 index af2640e90..000000000 --- a/.azure-pipelines/ci-conda-env.txt +++ /dev/null @@ -1,24 +0,0 @@ -conda-forge::cctbx-base -conda-forge::cxx-compiler -conda-forge::h5py>=3.1 -conda-forge::hdf5 -conda-forge::hdf5plugin -conda-forge::libboost-devel -conda-forge::matplotlib-base -conda-forge::mrcfile -conda-forge::natsort -conda-forge::numpy<2 -conda-forge::nxmx -conda-forge::orderedset -conda-forge::pint -conda-forge::pybind11 -conda-forge::pytest -conda-forge::pytest-azurepipelines -conda-forge::pytest-cov -conda-forge::pytest-forked -conda-forge::pytest-mock -conda-forge::pytest-timeout -conda-forge::pytest-xdist -conda-forge::python-dateutil -conda-forge::scipy -conda-forge::tqdm diff --git a/.azure-pipelines/parse_dependency_selectors.py b/.azure-pipelines/parse_dependency_selectors.py new file mode 100644 index 000000000..f18ddb46d --- /dev/null +++ b/.azure-pipelines/parse_dependency_selectors.py @@ -0,0 +1,365 @@ +#!/usr/bin/env python +# ruff: noqa: I002 + +import logging +import os +import re +import sys +from collections import namedtuple + +VALID_SECTIONS = ["build", "host", "run", "test"] # type: list[SectionName] +Dependency = namedtuple("Dependency", ["name", "version", "raw_line"]) + +try: + from typing import Any, Literal, TypeAlias # noqa: F401 + + SectionName = Literal["build", "host", "run", "test"] # type: TypeAlias + Dependencies = dict[SectionName, list[Dependency]] # type: TypeAlias + +except ImportError: + pass + + +re_selector = re.compile(r"# *\[([^#]+)]$") +re_pin = re.compile(r"""{{ *pin_compatible *\( *['"]([^'"]+)['"]""") + + +def _native_platform(): + # type: () -> Literal["osx", "win", "linux"] + """Gets the native platform name for selection purposes""" + if sys.platform == "darwin": + return "osx" + elif os.name == "nt": + return "win" + elif sys.platform.startswith("linux"): + return "linux" + + +def _split_dependency_line(line): + """Split a single line into (name, version, raw_line) parts""" + # type: (str) -> Dependency + + # Lines that are templated get ignored here + if "{" in line: + return Dependency(None, None, line) + pending = line + # Strip off the comment/selector + if "#" in line: + pending = pending[: pending.index("#")].strip() + # If we have a version spec and no space, this is an error + if " " not in pending and (set(pending) & set("><=!")): + raise RuntimeError( + "Error: Versioned requirement '%s' has no space" % (pending,) + ) + vers = None + if " " in pending: + pending, vers = pending.split(" ", 1) + return Dependency(pending, vers, line) + + +def _merge_dependency_lists(source, merge_into): + # type: (list[Dependency], list[Dependency]) -> None + """ + Merge two lists of dependencies into one unified list. + + This will replace unversioned dependencies with versioned + dependencies, merge dependencies with identical versions, and + leave in place depenencies with versions specified. + + Lines from the source list that don't have a dependency name + will be added as long as they don't have a duplicate line in the + target list. + """ + indices = {x[0]: i for i, x in enumerate(merge_into)} + for pkg, ver, line in source: + if pkg is None: + # Lines that don't define a package always get added, as long + # as we don't have an identical line already. + if not any(x.raw_line == line for x in merge_into): + merge_into.append(Dependency(pkg, ver, line)) + elif pkg in indices: + # This already exists in the target. Should we replace it? + other_ver = merge_into[indices[pkg]][1] + if not other_ver and ver: + logging.debug( + "Merging '{}' over {}".format(line, merge_into[indices[pkg]]) + ) + merge_into[indices[pkg]] = Dependency(pkg, ver, line) + elif other_ver and ver and ver != other_ver: + raise RuntimeError( + "Cannot merge conflicting requirements for %s: '%s' and '%s' - only know how to merge if these are the same, or one is unbound." + % (pkg, ver, other_ver) + ) + else: + merge_into.append(Dependency(pkg, ver, line)) + indices[pkg] = len(merge_into) - 1 + + +# def _merge_dependency_dictionaries(sources): +# # type: (list[dict[str, Dependency]]) -> dict[str, Dependency] +# """Merge multiple parsed dependency dictionaries into one.""" +# Evidently WIP? + + +class DependencySelectorParser(object): + """ + Parse simple conda-build selectors syntax, with optional variables. + + Supported: + - Variables linux, osx, win, in addition to anything passed into __init__ + - Variable inversion e.g. "not osx" + - Basic "And" combinations e.g. "bootstrap and not osx" + """ + + def __init__(self, **kwargs): + self._vars = dict(kwargs) + if kwargs.get("platform", None) is None: + kwargs["platform"] = _native_platform() + self._vars.update( + { + "osx": kwargs["platform"] == "osx", + "linux": kwargs["platform"] == "linux", + "win": kwargs["platform"] == "win", + } + ) + + def _parse_expression(self, fragment, pos=0): + # type: (str, int) -> bool + """Recursively parse an expression or fragment of an expression.""" + if fragment in self._vars: + return self._vars[fragment] + if " and " in fragment: + left, right = fragment.split(" and ", 1) + return self._parse_expression(left, pos) and self._parse_expression( + right, pos + fragment.index(" and ") + ) + if fragment.startswith("not "): + return not self._parse_expression(fragment[4:].strip(), pos + 4) + raise ValueError("Could not parse selector fragment '" + fragment + "'") + + def preprocess(self, data): + # type: (str) -> str + """Apply preprocessing selectors to raw file data""" + output_lines = [] + for line in data.splitlines(): + match = re_selector.search(line) + + if match: + if self._parse_expression(match.group(1)): + output_lines.append(line) + elif re_pin.search(line): + # Ignore pin_compatible dependencies + continue + else: + output_lines.append(line) + return "\n".join(output_lines) + + def parse_file(self, filename): + # type: (str) -> Dependencies + """ + Parse a dependency file into a structured dictionary. + + The dictionary has structure: + { + "section": [ + ("dependency_name", "dependency_version", "raw_line"), + ... + ] + } + """ + with open(filename, "rt") as f: + data = self.preprocess(f.read()) + output = {} # type: Dependencies + current_section = None # type: SectionName | None + for n, line in enumerate(data.splitlines()): + if "#" in line: + line = line[: line.index("#")] + line = line.strip() + if line.endswith(":"): + new_section = line[:-1].strip() + assert new_section in VALID_SECTIONS + current_section = new_section + output[current_section] = [] + elif line.startswith("-"): + if not current_section: + raise RuntimeError( + "Error parsing " + + filename + + ":" + + str(n + 1) + + "; No current section on line '" + + line + + "'" + ) + assert current_section in VALID_SECTIONS + req = _split_dependency_line(line[1:].strip()) + the_list = output.setdefault(current_section, []) + the_list.append(req) + else: + if line: + raise RuntimeError( + "Error parsing " + + filename + + ":" + + str(n + 1) + + "; Uncategorised line '" + + line + + "'" + ) + return output + + def parse_files(self, filenames): + # type: (list[str | os.PathLike]) -> Dependencies + """Parse and merge multiple dependency files.""" + reqs = {} # type: Dependencies + for source in filenames: + source_reqs = self.parse_file(str(source)) + # Now, merge this into the previous results + for section, items in source_reqs.items(): + _merge_dependency_lists(items, reqs.setdefault(section, [])) + return reqs + + +def preprocess_for_bootstrap(paths, prebuilt_cctbx, platform=None, sections=None): + # type: (list[str | os.PathLike], bool, str | None, list[SectionName]|None) -> list[str] + """ + Do dependency file preprocessing, intended for bootstrap.py. + + Args: + paths: List of dependency list files to merge + prebuilt_cctbx: Whether this is processing for a prebuilt CCTBX + distribution, or not. + platform: + The platform to process the dependencies for. Default: Current. + sections: + Which sections to process (build, host, run, test). Default: All. + + Returns: + A list of dependency strings, suitable for passing to conda/mamba install. + """ + parser = DependencySelectorParser( + prebuilt_cctbx=prebuilt_cctbx, + bootstrap=True, + platform=platform or _native_platform(), + ) + reqs = parser.parse_files(paths) + merged_req = [] + for section, items in reqs.items(): + if section in sections or not sections: + _merge_dependency_lists(items, merged_req) + + output_lines = [] + for pkg, ver, _ in sorted(merged_req, key=lambda x: x[0]): + if pkg == "python": + # Bootstrap handles this dependency implicitly + continue + output_lines.append("conda-forge::" + pkg + (ver or "")) + return output_lines + + +def test_parser(): + parser = DependencySelectorParser(bootstrap=True, prebuilt_cctbx=False) + assert parser._parse_expression("osx") + assert parser._parse_expression("bootstrap") + assert parser._parse_expression("osx and bootstrap") + assert not parser._parse_expression("linux and bootstrap") + assert not parser._parse_expression("prebuilt_cctbx and osx and not bootstrap") + + +if __name__ == "__main__": + from argparse import ArgumentParser + + parser = ArgumentParser() + # parser.add_argument( + # "kind", + # choices=["bootstrap", "conda-build"], + # help="Choose the target for handling dependency lists", + # metavar="KIND", + # ) + parser.add_argument( + "--conda-build", + help="Generate structured conda-build-style output", + action="store_true", + ) + parser.add_argument( + "-p", + "--platform", + choices=["osx", "linux", "win"], + help="Choose the target for handling bootstrap dependency lists. Default: {}".format( + _native_platform() + ), + ) + parser.add_argument( + "--prebuilt-cctbx", + help="Mark as using prebuilt cctbx. Implied by conda-build.", + action="store_true", + ) + parser.add_argument( + "--build", + help="Include build section in output", + dest="sections", + action="append_const", + const="build", + ) + parser.add_argument( + "--host", + help="Include build section in output", + dest="sections", + action="append_const", + const="host", + ) + parser.add_argument( + "--test", + help="Include build section in output", + dest="sections", + action="append_const", + const="test", + ) + parser.add_argument( + "--run", + help="Include build section in output", + dest="sections", + action="append_const", + const="run", + ) + parser.add_argument( + "-v", "--verbose", help="Show debugging output", action="store_true" + ) + parser.add_argument("sources", nargs="+", help="Dependency files to merge") + args = parser.parse_args() + if not args.sections: + args.sections = VALID_SECTIONS + + logging.basicConfig( + level=logging.DEBUG if args.verbose else logging.INFO, format="%(message)s" + ) + if not args.conda_build: + print( + "\n".join( + preprocess_for_bootstrap( + args.sources, + prebuilt_cctbx=args.prebuilt_cctbx, + platform=args.platform, + sections=args.sections, + ) + ) + ) + else: + if args.platform: + sys.exit("Error: Can only specify platform with plain-list mode.") + deps = DependencySelectorParser(bootstrap=False, prebuilt_cctbx=True) + reqs = deps.parse_files(args.sources) + output = [] + for section in VALID_SECTIONS: + if section not in reqs or not reqs[section] or section not in args.sections: + continue + output.append(section + ":") + output.extend( + " - " + x.raw_line + for x in sorted( + reqs[section], + key=lambda x: (0 if x.raw_line.startswith("{{") else 1, x.raw_line), + ) + ) + + print("\n".join(output)) diff --git a/.azure-pipelines/unix-build.yml b/.azure-pipelines/unix-build.yml index 45a65570f..77275a320 100644 --- a/.azure-pipelines/unix-build.yml +++ b/.azure-pipelines/unix-build.yml @@ -17,7 +17,11 @@ steps: # Extract the dials-data version so we can correctly cache regression data. - bash: | set -eux - echo pycbf >> modules/dxtbx/.azure-pipelines/ci-conda-env.txt + python3 .azure-pipelines/parse_dependency_selectors.py dependencies.yaml --prebuilt-cctbx > .azure-pipelines/ci-conda-env.txt + printf "conda-forge::pytest-azurepipelines" >> modules/dxtbx/.azure-pipelines/ci-conda-env.txt + printf "conda-forge::pytest-cov" >> modules/dxtbx/.azure-pipelines/ci-conda-env.txt + printf "conda-forge::pytest-forked" >> modules/dxtbx/.azure-pipelines/ci-conda-env.txt + printf "conda-forge::pytest-timeout" >> modules/dxtbx/.azure-pipelines/ci-conda-env.txt python3 modules/dxtbx/.azure-pipelines/bootstrap.py base --clean --python $(PYTHON_VERSION) set +ux diff --git a/.azure-pipelines/windows-build.yml b/.azure-pipelines/windows-build.yml index 1b031eabf..1e93bca75 100644 --- a/.azure-pipelines/windows-build.yml +++ b/.azure-pipelines/windows-build.yml @@ -24,10 +24,11 @@ steps: set -eux # Remove compilers from conda-env, as prebuilt cctbx appears to use # the system configured compilers and so the conda-forge settings conflict - grep -v compiler modules/dxtbx/.azure-pipelines/ci-conda-env.txt > ci-conda-env.txt - echo pycbf >> ci-conda-env.txt - echo cmake >> ci-conda-env.txt - mv ci-conda-env.txt modules/dxtbx/.azure-pipelines/ci-conda-env.txt + python3 .azure-pipelines/parse_dependency_selectors.py dependencies.yaml --prebuilt-cctbx | grep -v compiler > modules/dxtbx/.azure-pipelines/ci-conda-env.txt + printf "conda-forge::pytest-azurepipelines" >> modules/dxtbx/.azure-pipelines/ci-conda-env.txt + printf "conda-forge::pytest-cov" >> modules/dxtbx/.azure-pipelines/ci-conda-env.txt + printf "conda-forge::pytest-forked" >> modules/dxtbx/.azure-pipelines/ci-conda-env.txt + printf "conda-forge::pytest-timeout" >> modules/dxtbx/.azure-pipelines/ci-conda-env.txt python3 modules/dxtbx/.azure-pipelines/bootstrap.py base --clean --python $(PYTHON_VERSION) From 798295d17936ffb5c9bd86ff49fe5d3fc0c89898 Mon Sep 17 00:00:00 2001 From: Nicholas Devenish Date: Tue, 20 Aug 2024 14:24:43 +0100 Subject: [PATCH 05/10] Move extra CI dependencies to their own source --- .azure-pipelines/ci-dependencies.yaml | 6 ++++++ ..._dependency_selectors.py => parse_dependencies.py} | 6 ------ .azure-pipelines/unix-build.yml | 10 +++++----- .azure-pipelines/windows-build.yml | 11 ++++++----- 4 files changed, 17 insertions(+), 16 deletions(-) create mode 100644 .azure-pipelines/ci-dependencies.yaml rename .azure-pipelines/{parse_dependency_selectors.py => parse_dependencies.py} (98%) diff --git a/.azure-pipelines/ci-dependencies.yaml b/.azure-pipelines/ci-dependencies.yaml new file mode 100644 index 000000000..1a44b1793 --- /dev/null +++ b/.azure-pipelines/ci-dependencies.yaml @@ -0,0 +1,6 @@ +# Extra dependendencies used in addition for this CI testing +test: + - pytest-azurepipelines + - pytest-cov + - pytest-forked + - pytest-timeout \ No newline at end of file diff --git a/.azure-pipelines/parse_dependency_selectors.py b/.azure-pipelines/parse_dependencies.py similarity index 98% rename from .azure-pipelines/parse_dependency_selectors.py rename to .azure-pipelines/parse_dependencies.py index f18ddb46d..2a8ece864 100644 --- a/.azure-pipelines/parse_dependency_selectors.py +++ b/.azure-pipelines/parse_dependencies.py @@ -270,12 +270,6 @@ def test_parser(): from argparse import ArgumentParser parser = ArgumentParser() - # parser.add_argument( - # "kind", - # choices=["bootstrap", "conda-build"], - # help="Choose the target for handling dependency lists", - # metavar="KIND", - # ) parser.add_argument( "--conda-build", help="Generate structured conda-build-style output", diff --git a/.azure-pipelines/unix-build.yml b/.azure-pipelines/unix-build.yml index 77275a320..ee88954e4 100644 --- a/.azure-pipelines/unix-build.yml +++ b/.azure-pipelines/unix-build.yml @@ -17,11 +17,11 @@ steps: # Extract the dials-data version so we can correctly cache regression data. - bash: | set -eux - python3 .azure-pipelines/parse_dependency_selectors.py dependencies.yaml --prebuilt-cctbx > .azure-pipelines/ci-conda-env.txt - printf "conda-forge::pytest-azurepipelines" >> modules/dxtbx/.azure-pipelines/ci-conda-env.txt - printf "conda-forge::pytest-cov" >> modules/dxtbx/.azure-pipelines/ci-conda-env.txt - printf "conda-forge::pytest-forked" >> modules/dxtbx/.azure-pipelines/ci-conda-env.txt - printf "conda-forge::pytest-timeout" >> modules/dxtbx/.azure-pipelines/ci-conda-env.txt + python3 .azure-pipelines/parse_dependencies.py \ + dependencies.yaml \ + .azure-pipelines/ci-dependencies.yaml \ + --prebuilt-cctbx \ + > .azure-pipelines/ci-conda-env.txt python3 modules/dxtbx/.azure-pipelines/bootstrap.py base --clean --python $(PYTHON_VERSION) set +ux diff --git a/.azure-pipelines/windows-build.yml b/.azure-pipelines/windows-build.yml index 1e93bca75..1c22fc53e 100644 --- a/.azure-pipelines/windows-build.yml +++ b/.azure-pipelines/windows-build.yml @@ -24,11 +24,12 @@ steps: set -eux # Remove compilers from conda-env, as prebuilt cctbx appears to use # the system configured compilers and so the conda-forge settings conflict - python3 .azure-pipelines/parse_dependency_selectors.py dependencies.yaml --prebuilt-cctbx | grep -v compiler > modules/dxtbx/.azure-pipelines/ci-conda-env.txt - printf "conda-forge::pytest-azurepipelines" >> modules/dxtbx/.azure-pipelines/ci-conda-env.txt - printf "conda-forge::pytest-cov" >> modules/dxtbx/.azure-pipelines/ci-conda-env.txt - printf "conda-forge::pytest-forked" >> modules/dxtbx/.azure-pipelines/ci-conda-env.txt - printf "conda-forge::pytest-timeout" >> modules/dxtbx/.azure-pipelines/ci-conda-env.txt + python3 .azure-pipelines/parse_dependencies.py \ + modules/dxtbx/dependencies.yaml \ + modules/dxtbx/.azure-pipelines/ci-dependencies.yaml \ + --prebuilt-cctbx \ + | grep -v compiler \ + > modules/dxtbx/.azure-pipelines/ci-conda-env.txt python3 modules/dxtbx/.azure-pipelines/bootstrap.py base --clean --python $(PYTHON_VERSION) From 90fa03dee5e6e9acca978083bfe8b09e3acf67be Mon Sep 17 00:00:00 2001 From: Nicholas Devenish Date: Tue, 20 Aug 2024 14:39:45 +0100 Subject: [PATCH 06/10] Fix directory references in unix CI --- .azure-pipelines/unix-build.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.azure-pipelines/unix-build.yml b/.azure-pipelines/unix-build.yml index ee88954e4..93060f759 100644 --- a/.azure-pipelines/unix-build.yml +++ b/.azure-pipelines/unix-build.yml @@ -17,11 +17,11 @@ steps: # Extract the dials-data version so we can correctly cache regression data. - bash: | set -eux - python3 .azure-pipelines/parse_dependencies.py \ - dependencies.yaml \ - .azure-pipelines/ci-dependencies.yaml \ + python3 modules/dxtbx/.azure-pipelines/parse_dependencies.py \ + modules/dxtbx/dependencies.yaml \ + modules/dxtbx/.azure-pipelines/ci-dependencies.yaml \ --prebuilt-cctbx \ - > .azure-pipelines/ci-conda-env.txt + > modules/dxtbx/.azure-pipelines/ci-conda-env.txt python3 modules/dxtbx/.azure-pipelines/bootstrap.py base --clean --python $(PYTHON_VERSION) set +ux From b899752187281c1ee675b23d34275362a0a53bc6 Mon Sep 17 00:00:00 2001 From: Nicholas Devenish Date: Tue, 20 Aug 2024 14:45:16 +0100 Subject: [PATCH 07/10] Add Conda to CI dependencies I think this is what was letting the easy-activate path work --- .azure-pipelines/ci-dependencies.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.azure-pipelines/ci-dependencies.yaml b/.azure-pipelines/ci-dependencies.yaml index 1a44b1793..787bc0ddb 100644 --- a/.azure-pipelines/ci-dependencies.yaml +++ b/.azure-pipelines/ci-dependencies.yaml @@ -3,4 +3,5 @@ test: - pytest-azurepipelines - pytest-cov - pytest-forked - - pytest-timeout \ No newline at end of file + - pytest-timeout + - conda # This adds easy activation scripts \ No newline at end of file From 0c3f91f501d8e2df9e4e2768d5fb624a293e58b3 Mon Sep 17 00:00:00 2001 From: Nicholas Devenish Date: Tue, 20 Aug 2024 14:51:03 +0100 Subject: [PATCH 08/10] Fix path for windows build --- .azure-pipelines/windows-build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.azure-pipelines/windows-build.yml b/.azure-pipelines/windows-build.yml index 1c22fc53e..b47ef31d1 100644 --- a/.azure-pipelines/windows-build.yml +++ b/.azure-pipelines/windows-build.yml @@ -24,7 +24,7 @@ steps: set -eux # Remove compilers from conda-env, as prebuilt cctbx appears to use # the system configured compilers and so the conda-forge settings conflict - python3 .azure-pipelines/parse_dependencies.py \ + python3 modules/dxtbx/.azure-pipelines/parse_dependencies.py \ modules/dxtbx/dependencies.yaml \ modules/dxtbx/.azure-pipelines/ci-dependencies.yaml \ --prebuilt-cctbx \ From 8ae64b671044516bcd807dbdf39f96fcf8a90599 Mon Sep 17 00:00:00 2001 From: Nicholas Devenish Date: Tue, 20 Aug 2024 15:03:08 +0100 Subject: [PATCH 09/10] Tidy up numpy dependencies --- dependencies.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dependencies.yaml b/dependencies.yaml index 638ac8936..ba2e95410 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -54,12 +54,12 @@ build: host: - cctbx-base # [prebuilt_cctbx and bootstrap] - - cctbx-base >=2023.10 # [not bootstrap] + - cctbx-base >=2024 # [not bootstrap] - hdf5 - libboost-devel - libboost-python-devel - - numpy >=1.19,<1.21|>=1.21.5|<2 #[bootstrap] - - numpy # [not bootstrap] + - numpy >=1.21.5,<2 #[bootstrap] + - numpy # [not bootstrap] - pip - pybind11 - python From 64d11d9fbc30a368e41feaee26f11bc021c4cd3e Mon Sep 17 00:00:00 2001 From: Nicholas Devenish Date: Wed, 21 Aug 2024 13:15:04 +0100 Subject: [PATCH 10/10] Add news --- newsfragments/753.misc | 1 + 1 file changed, 1 insertion(+) create mode 100644 newsfragments/753.misc diff --git a/newsfragments/753.misc b/newsfragments/753.misc new file mode 100644 index 000000000..24e0f6ece --- /dev/null +++ b/newsfragments/753.misc @@ -0,0 +1 @@ +Define dxtbx-only dependencies explicitly, and categorised by purpose, for CI (for now). This will allow us to be smarter about what dependencies are installed where.