diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4d6dbddc5..1b3882ec0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -4,109 +4,132 @@ on: push: branches: - master + - backports-v0.7.x - backports-v0.6.x tags: - v*.*.* pull_request: branches: - master + - backports-v0.7.x - backports-v0.6.x # Run daily at 0:01 UTC schedule: - cron: '1 0 * * *' -jobs: - linter: - runs-on: ${{ matrix.os }} - strategy: - matrix: - os: [ubuntu-latest] - python-version: ["3.10"] - name: linter - flake8/black - python ${{ matrix.python-version }} +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true +jobs: + pre-commit: + name: pre-commit + runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 with: - python-version: ${{ matrix.python-version }} - - name: Lint with flake8 and black - run: | - python -m pip install -q flake8 black - flake8 coffea tests setup.py - black --check --diff coffea tests setup.py + python-version: "3.13" + - uses: pre-commit/action@v3.0.1 + with: + extra_args: --hook-stage manual --all-files test: runs-on: ${{ matrix.os }} - needs: linter + needs: pre-commit strategy: matrix: os: [ubuntu-latest, macOS-latest, windows-latest] - java-version: [8] + java-version: [17] java-distribution: ["corretto"] - python-version: ["3.7", "3.10"] + python-version: ["3.9", "3.11"] + name: test coffea (${{ matrix.os }}) - python ${{ matrix.python-version }}, JDK${{ matrix.java-version }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 + with: + fetch-depth: 0 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - - name: Set up JDK ${{ matrix.java-version }} - if: matrix.os != 'windows-latest' - uses: actions/setup-java@v3 + - name: Set up JDK ${{ matrix.java-distribution }}/${{ matrix.java-version }} + uses: actions/setup-java@v4 with: java-version: ${{ matrix.java-version }} distribution: ${{ matrix.java-distribution }} - - name: Set python 3.6 test settings + - name: Set python test settings run: | - echo "INSTALL_EXTRAS=[dev,parsl,dask,spark]" >> $GITHUB_ENV - if: matrix.python-version == 3.6 + echo "INSTALL_EXTRAS='[dev,parsl,dask,servicex]'" >> $GITHUB_ENV - - name: Set python newer than 3.6 test settings - run: | - echo "INSTALL_EXTRAS=[dev,parsl,dask,spark,servicex]" >> $GITHUB_ENV - if: matrix.python-version != 3.6 + - name: Install uv + run: python -m pip install --upgrade uv - - name: Install dependencies (Linux/MacOS) - if: matrix.os != 'windows-latest' + - name: Install dependencies (Linux) + if: matrix.os == 'ubuntu-latest' + run: | + uv pip install --system --upgrade pip setuptools wheel + # mltool installs + # c.f. https://github.com/astral-sh/uv/issues/3437 + python -m pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cpu + uv pip install --system xgboost + uv pip install --system 'tritonclient[grpc,http]!=2.41.0' + # install checked out coffea + uv pip install --system -q '.[dev,parsl,dask]' --upgrade + uv pip list --system + java -version + - name: Install dependencies (MacOS) + if: matrix.os == 'macOS-latest' run: | - python -m pip install --upgrade pip setuptools wheel - python -m pip install -q -e .${{env.INSTALL_EXTRAS}} - python -m pip list + uv pip install --system --upgrade pip setuptools wheel + # mltool installs + # c.f. https://github.com/astral-sh/uv/issues/3437 + python -m pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cpu + uv pip install --system xgboost + # install checked out coffea + uv pip install --system -q '.[dev,dask]' --upgrade + uv pip list --system java -version - name: Install dependencies (Windows) if: matrix.os == 'windows-latest' run: | - python -m pip install --upgrade pip setuptools wheel - python -m pip install -q -e .[dev] - python -m pip list + uv pip install --system --upgrade pip setuptools wheel + # mltool installs + # c.f. https://github.com/astral-sh/uv/issues/3437 + python -m pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cpu + uv pip install --system xgboost + # install checked out coffea + uv pip install --system -q '.[dev,dask]' 'numpy<2' --upgrade + uv pip list --system + java -version + + - name: Start triton server with example model + if: matrix.os == 'ubuntu-latest' + run: | + docker run -d --rm -p 8000:8000 -p 8001:8001 -p 8002:8002 -v ${{ github.workspace }}/tests/samples/triton_models_test:/models nvcr.io/nvidia/tritonserver:23.04-pyt-python-py3 tritonserver --model-repository=/models + - name: Test with pytest - env: - ARROW_PRE_0_15_IPC_FORMAT: 1 run: | - pytest --cov-report=xml --cov=coffea tests + python -m pytest --cov-report=xml --cov=coffea --deselect=test_taskvine - name: Upload codecov - if: matrix.os == 'ubuntu-latest' && matrix.python-version == 3.10 - run: | - python -m pip install codecov - codecov + if: matrix.os == 'ubuntu-latest' && matrix.python-version == 3.11 + uses: codecov/codecov-action@v5 - name: Install graphviz - if: matrix.os == 'ubuntu-latest' && matrix.python-version == 3.10 - uses: kamiazya/setup-graphviz@v1 + if: matrix.os == 'ubuntu-latest' && matrix.python-version == 3.11 + uses: ts-graphviz/setup-graphviz@v2 - name: Install pandoc - if: matrix.os == 'ubuntu-latest' && matrix.python-version == 3.10 + if: matrix.os == 'ubuntu-latest' && matrix.python-version == 3.11 uses: r-lib/actions/setup-pandoc@v2 - name: Build documentation - if: matrix.os == 'ubuntu-latest' && matrix.python-version == 3.10 + if: matrix.os == 'ubuntu-latest' && matrix.python-version == 3.11 run: | cd docs && make html touch build/html/.nojekyll - name: Deploy documentation - if: github.event_name == 'push' && matrix.os == 'ubuntu-latest' && matrix.python-version == 3.10 - uses: crazy-max/ghaction-github-pages@v3.1.0 + if: github.event_name == 'push' && matrix.os == 'ubuntu-latest' && matrix.python-version == 3.11 + uses: crazy-max/ghaction-github-pages@v4 with: target_branch: gh-pages build_dir: docs/build/html @@ -115,10 +138,10 @@ jobs: testwq: runs-on: ubuntu-latest - needs: linter + needs: pre-commit strategy: matrix: - python-version: ["3.10"] + python-version: ["3.11"] name: test coffea-workqueue steps: @@ -131,20 +154,21 @@ jobs: - name: Test work_queue shell: bash -l {0} run: | - conda create --yes --name coffea-env -c conda-forge python=${{ matrix.python-version }} ndcctools dill conda-pack conda + conda create --yes --name coffea-env -c conda-forge python=${{ matrix.python-version }} ndcctools dill conda-pack conda 'setuptools<71' conda activate coffea-env python -m pip install --ignore-installed . cd tests conda-pack --output coffea-env.tar.gz python wq.py coffea-env.tar.gz + # testskyhookjob: # runs-on: ubuntu-latest -# needs: linter +# needs: pre-commit # name: test coffea-skyhook-job # # steps: -# - uses: actions/checkout@3.2.0 +# - uses: actions/checkout@3 # - name: Test Coffea Skyhook Bindings # shell: bash -l {0} # run: | @@ -163,30 +187,32 @@ jobs: if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') runs-on: ubuntu-latest needs: [test, testwq] + permissions: + id-token: write + attestations: write + contents: read strategy: matrix: - python-version: ["3.10"] + python-version: ["3.11"] name: deploy release steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - name: Build package for PyPI run: | - python -m pip install --upgrade pip setuptools wheel - python setup.py sdist bdist_wheel --universal - - name: Create Release - uses: actions/create-release@v1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_OAUTH }} + pipx run hatch build -t sdist -t wheel + - name: Verify the distribution + run: pipx run twine check --strict dist/* + - name: Generate artifact attestation for sdist and wheel + uses: actions/attest-build-provenance@ef244123eb79f2f7a7e75d99086184180e6d0018 # v1.4.4 with: - tag_name: ${{ github.ref }} - release_name: Release ${{ github.ref }} + subject-path: "dist/coffea-*" - name: Publish package to PyPI - uses: pypa/gh-action-pypi-publish@v1.8.4 + uses: pypa/gh-action-pypi-publish@v1.12.2 with: user: __token__ password: ${{ secrets.PYPI_TOKEN }} diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml new file mode 100644 index 000000000..8fe68c5e8 --- /dev/null +++ b/.github/workflows/pr.yml @@ -0,0 +1,22 @@ +name: "Lint PR" + +on: + pull_request: + types: + - opened + - edited + - synchronize + workflow_dispatch: + +concurrency: + group: semantic-pr-title-${{ github.head_ref }} + cancel-in-progress: true + +jobs: + main: + name: Validate PR title + runs-on: ubuntu-latest + steps: + - uses: amannn/action-semantic-pull-request@v5 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/MANIFEST.in b/MANIFEST.in deleted file mode 100644 index 4fb6c3a3f..000000000 --- a/MANIFEST.in +++ /dev/null @@ -1,3 +0,0 @@ -include README.rst -include LICENSE -include coffea/processor/templates/*.tmpl diff --git a/coffea/__init__.py b/coffea/__init__.py index 949400987..ca02eb30f 100644 --- a/coffea/__init__.py +++ b/coffea/__init__.py @@ -27,11 +27,13 @@ # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -from . import version +from coffea._version import __version__ +import re +import sys -__version__ = version.__version__ +version = __version__ +version_info = tuple(re.split(r"[-\.]", __version__)) -import sys if sys.version_info.major < 3: import warnings diff --git a/coffea/version.py b/coffea/version.py deleted file mode 100644 index 0f23d459f..000000000 --- a/coffea/version.py +++ /dev/null @@ -1,37 +0,0 @@ -#!/usr/bin/env python - -# Copyright (c) 2018, Fermilab -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are met: -# -# * Redistributions of source code must retain the above copyright notice, this -# list of conditions and the following disclaimer. -# -# * Redistributions in binary form must reproduce the above copyright notice, -# this list of conditions and the following disclaimer in the documentation -# and/or other materials provided with the distribution. -# -# * Neither the name of the copyright holder nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -import re - -__version__ = "0.7.23" -version = __version__ -version_info = tuple(re.split(r"[-\.]", __version__)) - -del re diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 000000000..e8843f843 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,126 @@ +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[project] +name = "coffea" +version = "0.7.23" +description = "Tools for doing Collider HEP style analysis with columnar operations" +readme = "README.rst" +license = {file = "LICENSE"} +authors = [ + {name = "Lindsey Gray (Fermilab)", email = "lagray@fnal.gov"} +] +maintainers = [ + {name = "Lindsey Gray (Fermilab)", email = "lagray@fnal.gov"} +] +homepage = "https://github.com/CoffeaTeam/coffea" +repository = "https://github.com/CoffeaTeam/coffea" +documentation = "https://github.com/CoffeaTeam/coffea" +keywords = ["HEP", "Physics", "Data Analysis", "Columnar Operations"] +classifiers = [ + "Development Status :: 4 - Beta", + "Intended Audience :: Developers", + "Intended Audience :: Information Technology", + "Intended Audience :: Science/Research", + "License :: OSI Approved :: BSD License", + "Operating System :: MacOS", + "Operating System :: POSIX", + "Operating System :: Unix", + "Programming Language :: Python", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Topic :: Scientific/Engineering", + "Topic :: Scientific/Engineering :: Information Analysis", + "Topic :: Scientific/Engineering :: Mathematics", + "Topic :: Scientific/Engineering :: Physics", + "Topic :: Software Development", + "Topic :: Utilities", +] +dependencies = [ + "setuptools<71", + "awkward>=1.10.3,<2", + "uproot>=4.1.6,==4.*,!=4.2.4,!=4.3.0,!=4.3.1", + "uproot3-methods>=0.10.0", + "uproot3>=3.14.1", + "correctionlib>=2.0.0", + "pyarrow>=1.0.0", + "fsspec", + "matplotlib>=3", + "numba>=0.56.0", + "numpy>=1.18.0,<2", + "scipy>=1.1.0", + "tqdm>=4.27.0", + "lz4", + "cloudpickle>=1.2.3", + "toml>=0.10.2", + "mplhep>=0.1.18", + "packaging", + "pandas", + "hist>=2", + "typing-extensions; python_version<'3.8'", + "cachetools", + "dataclasses; python_version<'3.7'", +] +python-requires = ">=3.8,<3.12" + +[project.optional-dependencies] +spark = ["ipywidgets", "pyspark>=2.4.1,<3.0.0", "jinja2"] +parsl = ["parsl>=1.1"] +dask = [ + "dask[dataframe]>=2.6.0", + "distributed>=2.6.0", + "bokeh>=1.3.4", + "blosc", +] +servicex = [ + "aiostream", + "tenacity", + "servicex>=2.5.3,<3", + "func-adl_servicex", +] +dev = [ + "flake8", + "black", + "pytest", + "pytest-cov", + "pytest-mpl", + "pytest-asyncio", + "pytest-mock", + "sphinx", + "nbsphinx", + "sphinx-rtd-theme", + "sphinx-automodapi", + "sphinx-copybutton>=0.3.2", + "pyinstrument", + "ipython", +] +rucio = [ + "rucio-clients>=32; python_version>'3.8'", + "rucio-clients<32; python_version<'3.9'", + "pyyaml", +] + +[tool.hatch.build.targets.wheel] +packages = ["coffea"] + +[tool.hatch.build.targets.sdist] +include = [ + "README.rst", + "LICENSE", + "coffea/processor/templates/*.tmpl", +] + +[tool.hatch.build.targets.wheel.hooks.version] +path = "coffea/_version.py" + +[tool.pytest.ini_options] +addopts = "-rfesxX -v" +testpaths = ["tests"] + +[tool.mypy] +files = "src" +python_version = "3.8" +ignore_missing_imports = true diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index ed8ddc264..000000000 --- a/setup.cfg +++ /dev/null @@ -1,9 +0,0 @@ -[tool:pytest] -addopts = -rfesxX -v -testpaths = - tests - -[mypy] -files = src -python_version = 3.6 -ignore_missing_imports = True diff --git a/setup.py b/setup.py deleted file mode 100644 index a3e6a6be7..000000000 --- a/setup.py +++ /dev/null @@ -1,161 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -# Copyright (c) 2018, Fermilab -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are met: -# -# * Redistributions of source code must retain the above copyright notice, this -# list of conditions and the following disclaimer. -# -# * Redistributions in binary form must reproduce the above copyright notice, -# this list of conditions and the following disclaimer in the documentation -# and/or other materials provided with the distribution. -# -# * Neither the name of the copyright holder nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -import os.path - -from setuptools import find_packages -from setuptools import setup - - -def get_version(): - g = {} - exec(open(os.path.join("coffea", "version.py")).read(), g) - return g["__version__"] - - -def get_description(): - description = open("README.rst", "rb").read().decode("utf8", "ignore") - start = description.index(".. inclusion-marker-1-5-do-not-remove") - stop = description.index(".. inclusion-marker-3-do-not-remove") - - # before = "" - # after = """ - # Reference documentation - # ======================= - # """ - - return description[start:stop].strip() # before + + after - - -INSTALL_REQUIRES = [ - "setuptools<71", - "awkward>=1.10.3,<2", - "uproot>=4.1.6,==4.*,!=4.2.4,!=4.3.0,!=4.3.1", - "uproot3-methods>=0.10.0", - "uproot3>=3.14.1", - "correctionlib>=2.0.0", - "pyarrow>=1.0.0", - "fsspec", - "matplotlib>=3", - "numba>=0.56.0", - "numpy>=1.18.0,<2", - "scipy>=1.1.0", - "tqdm>=4.27.0", - "lz4", - "cloudpickle>=1.2.3", - "toml>=0.10.2", - "mplhep>=0.1.18", - "packaging", - "pandas", - "hist>=2", - 'typing-extensions;python_version<"3.8"', - "cachetools", - 'dataclasses;python_version<"3.7"', -] -EXTRAS_REQUIRE = {} -EXTRAS_REQUIRE["spark"] = ["ipywidgets", "pyspark>=2.4.1,<3.0.0", "jinja2"] -EXTRAS_REQUIRE["parsl"] = ["parsl>=1.1"] -EXTRAS_REQUIRE["dask"] = [ - "dask[dataframe]>=2.6.0", - "distributed>=2.6.0", - "bokeh>=1.3.4", - "blosc", -] -EXTRAS_REQUIRE["servicex"] = [ - "aiostream", - "tenacity", - "servicex>=2.5.3,<3", - "func-adl_servicex", -] -EXTRAS_REQUIRE["dev"] = [ - "flake8", - "black", - "pytest", - "pytest-cov", - "pytest-mpl", - "pytest-asyncio", - "pytest-mock", - "sphinx", - "nbsphinx", - "sphinx-rtd-theme", - "sphinx-automodapi", - "sphinx-copybutton>=0.3.2", - "pyinstrument", - "ipython", -] -EXTRAS_REQUIRE["rucio"] = [ - "rucio-clients>=32;python_version>'3.8'", - "rucio-clients<32;python_version<'3.9'", - "pyyaml", -] - -setup( - name="coffea", - version=get_version(), - packages=find_packages(exclude=["tests"]), - scripts=[], - include_package_data=True, - description="Tools for doing Collider HEP style analysis with columnar operations", - long_description=get_description(), - author="Lindsey Gray (Fermilab)", - author_email="lagray@fnal.gov", - maintainer="Lindsey Gray (Fermilab)", - maintainer_email="lagray@fnal.gov", - url="https://github.com/CoffeaTeam/coffea", - download_url="https://github.com/CoffeaTeam/coffea/releases", - license="BSD 3-clause", - test_suite="tests", - install_requires=INSTALL_REQUIRES, - extras_require=EXTRAS_REQUIRE, - classifiers=[ - "Development Status :: 4 - Beta", - "Intended Audience :: Developers", - "Intended Audience :: Information Technology", - "Intended Audience :: Science/Research", - "License :: OSI Approved :: BSD License", - "Operating System :: MacOS", - "Operating System :: POSIX", - "Operating System :: Unix", - "Programming Language :: Python", - "Programming Language :: Python :: 3.8", - "Programming Language :: Python :: 3.9", - "Programming Language :: Python :: 3.10", - "Programming Language :: Python :: 3.11", - "Topic :: Scientific/Engineering", - "Topic :: Scientific/Engineering :: Information Analysis", - "Topic :: Scientific/Engineering :: Mathematics", - "Topic :: Scientific/Engineering :: Physics", - "Topic :: Software Development", - "Topic :: Utilities", - ], - platforms="Any", - python_requires=">=3.8,<3.12", -)