From c9069ac1d285a2a6025bdc64b5403ab7c10a65cf Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 11 Aug 2023 14:47:27 +0000 Subject: [PATCH 1/5] Bump pypa/gh-action-pypi-publish from 1.8.8 to 1.8.10 Bumps [pypa/gh-action-pypi-publish](https://github.com/pypa/gh-action-pypi-publish) from 1.8.8 to 1.8.10. - [Release notes](https://github.com/pypa/gh-action-pypi-publish/releases) - [Commits](https://github.com/pypa/gh-action-pypi-publish/compare/v1.8.8...v1.8.10) --- updated-dependencies: - dependency-name: pypa/gh-action-pypi-publish dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index dc5bbd7ac1..9a387fcbce 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -195,7 +195,7 @@ jobs: python -m pip install pip hatch --upgrade python -m hatch build -t sdist -t wheel - name: Publish package to PyPI - uses: pypa/gh-action-pypi-publish@v1.8.8 + uses: pypa/gh-action-pypi-publish@v1.8.10 with: user: __token__ password: ${{ secrets.PYPI_TOKEN }} From ea0c993a6b11d3c485ab9cfdcfc4690b4bd3974e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jonas=20R=C3=BCbenach?= Date: Thu, 17 Aug 2023 14:09:22 +0200 Subject: [PATCH 2/5] Make LumiMask compatible with dask-awkward --- src/coffea/lumi_tools/lumi_tools.py | 41 +++++++++++++++++++---------- tests/test_lumi_tools.py | 8 ++++++ 2 files changed, 35 insertions(+), 14 deletions(-) diff --git a/src/coffea/lumi_tools/lumi_tools.py b/src/coffea/lumi_tools/lumi_tools.py index 73378e6cce..7418ab0022 100644 --- a/src/coffea/lumi_tools/lumi_tools.py +++ b/src/coffea/lumi_tools/lumi_tools.py @@ -1,6 +1,7 @@ import json import awkward as ak +import dask_awkward as dak from numba import types from numba.typed import Dict @@ -107,9 +108,9 @@ def __call__(self, runs, lumis): Parameters ---------- - runs : numpy.ndarray + runs : numpy.ndarray or awkward.highlevel.Array or dask_awkward.Array Vectorized list of run numbers - lumis : numpy.ndarray + lumis : numpy.ndarray or awkward.highlevel.Array or dask_awkward.Array Vectorized list of lumiSection numbers Returns @@ -118,18 +119,30 @@ def __call__(self, runs, lumis): An array of dtype `bool` where valid (run, lumi) tuples will have their corresponding entry set ``True``. """ - # fill numba typed dict - _masks = Dict.empty(key_type=types.uint32, value_type=types.uint32[:]) - for k, v in self._masks.items(): - _masks[k] = v - - if isinstance(runs, ak.highlevel.Array): - runs = ak.to_numpy(runs) - if isinstance(lumis, ak.highlevel.Array): - lumis = ak.to_numpy(lumis) - mask_out = np.zeros(dtype="bool", shape=runs.shape) - LumiMask._apply_run_lumi_mask_kernel(_masks, runs, lumis, mask_out) - return mask_out + + def apply(runs, lumis): + # fill numba typed dict + _masks = Dict.empty(key_type=types.uint32, value_type=types.uint32[:]) + for k, v in self._masks.items(): + _masks[k] = v + + runs_orig = runs + if isinstance(runs, ak.highlevel.Array): + runs = ak.to_numpy(ak.typetracer.length_zero_if_typetracer(runs)) + if isinstance(lumis, ak.highlevel.Array): + lumis = ak.to_numpy(ak.typetracer.length_zero_if_typetracer(lumis)) + mask_out = np.zeros(dtype="bool", shape=runs.shape) + LumiMask._apply_run_lumi_mask_kernel(_masks, runs, lumis, mask_out) + if isinstance(runs_orig, ak.Array): + mask_out = ak.Array(mask_out) + if ak.backend(runs_orig) == "typetracer": + mask_out = ak.Array(mask_out.layout.to_typetracer(forget_length=True)) + return mask_out + + if isinstance(runs, dak.Array): + return dak.map_partitions(apply, runs, lumis) + else: + return apply(runs, lumis) # This could be run in parallel, but windows does not support it @staticmethod diff --git a/tests/test_lumi_tools.py b/tests/test_lumi_tools.py index ff10b35665..63a2bf62de 100644 --- a/tests/test_lumi_tools.py +++ b/tests/test_lumi_tools.py @@ -2,6 +2,8 @@ from coffea.lumi_tools import LumiData, LumiList, LumiMask from coffea.util import numpy as np +import awkward as ak +import dask_awkward as dak def test_lumidata(): @@ -86,6 +88,12 @@ def test_lumimask(): assert np.all(lumimask(runs, lumis) == lumimask_pickle(runs, lumis)) + runs_dak = dak.from_awkward(ak.Array(runs), 1) + lumis_dak = dak.from_awkward(ak.Array(lumis), 1) + assert np.all( + lumimask(runs_dak, lumis_dak).compute() == lumimask_pickle(runs, lumis) + ) + def test_lumilist(): lumidata = LumiData("tests/samples/lumi_small.csv") From e6bbe76e89217641aa9e727aae808d98add0267d Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 17 Aug 2023 12:19:15 +0000 Subject: [PATCH 3/5] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tests/test_lumi_tools.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_lumi_tools.py b/tests/test_lumi_tools.py index 63a2bf62de..bacf9fa205 100644 --- a/tests/test_lumi_tools.py +++ b/tests/test_lumi_tools.py @@ -1,9 +1,9 @@ +import awkward as ak import cloudpickle +import dask_awkward as dak from coffea.lumi_tools import LumiData, LumiList, LumiMask from coffea.util import numpy as np -import awkward as ak -import dask_awkward as dak def test_lumidata(): From 3cbc9ab694d4328771606417139bd912556c461a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jonas=20R=C3=BCbenach?= Date: Thu, 17 Aug 2023 14:59:40 +0200 Subject: [PATCH 4/5] Make lumi tools tests use Distributed client --- tests/test_lumi_tools.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/tests/test_lumi_tools.py b/tests/test_lumi_tools.py index bacf9fa205..f74113b946 100644 --- a/tests/test_lumi_tools.py +++ b/tests/test_lumi_tools.py @@ -1,6 +1,7 @@ import awkward as ak import cloudpickle import dask_awkward as dak +from dask.distributed import Client from coffea.lumi_tools import LumiData, LumiList, LumiMask from coffea.util import numpy as np @@ -58,6 +59,8 @@ def test_lumidata(): def test_lumimask(): + client = Client() + lumimask = LumiMask( "tests/samples/Cert_294927-306462_13TeV_EOY2017ReReco_Collisions17_JSON.txt" ) @@ -91,9 +94,12 @@ def test_lumimask(): runs_dak = dak.from_awkward(ak.Array(runs), 1) lumis_dak = dak.from_awkward(ak.Array(lumis), 1) assert np.all( - lumimask(runs_dak, lumis_dak).compute() == lumimask_pickle(runs, lumis) + client.compute(lumimask(runs_dak, lumis_dak)).result() + == lumimask_pickle(runs, lumis) ) + client.close() + def test_lumilist(): lumidata = LumiData("tests/samples/lumi_small.csv") From fd788365affcdb4cb3b2af4333fb707716f31659 Mon Sep 17 00:00:00 2001 From: Lindsey Gray Date: Thu, 17 Aug 2023 08:50:20 -0500 Subject: [PATCH 5/5] deps: bump awkward pin to >= 2.3.3 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 31ccee808b..454ed3319d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -37,7 +37,7 @@ classifiers = [ "Topic :: Utilities", ] dependencies = [ - "awkward>=2.3.1", + "awkward>=2.3.3", "uproot>=5.0.10", "dask[array]>=2023.4.0", "dask-awkward>=2023.7.1,!=2023.8.0",