From 6e98b1e16cd0bee968d85f66afe32a41aca0eaf6 Mon Sep 17 00:00:00 2001 From: Lindsey Gray Date: Sat, 24 Feb 2024 08:27:40 -0600 Subject: [PATCH 1/8] fix: adjusted to unannounced breaking change in form mapping --- src/coffea/nanoevents/factory.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/coffea/nanoevents/factory.py b/src/coffea/nanoevents/factory.py index fb7323cad..b2b81c3ac 100644 --- a/src/coffea/nanoevents/factory.py +++ b/src/coffea/nanoevents/factory.py @@ -121,7 +121,16 @@ def __call__(self, form): self, ) - def load_buffers(self, tree, keys, start, stop, interp_options): + def load_buffers( + self, + tree, + keys, + start, + stop, + decompression_executor, + interpretation_executor, + interp_options, + ): from functools import partial from coffea.nanoevents.util import tuple_to_key From cade3eaf796e5f3386f29df3ff9ceaf4b9aab124 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sat, 24 Feb 2024 14:27:56 +0000 Subject: [PATCH 2/8] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/coffea/nanoevents/factory.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/coffea/nanoevents/factory.py b/src/coffea/nanoevents/factory.py index b2b81c3ac..5fada2f44 100644 --- a/src/coffea/nanoevents/factory.py +++ b/src/coffea/nanoevents/factory.py @@ -122,11 +122,11 @@ def __call__(self, form): ) def load_buffers( - self, - tree, - keys, - start, - stop, + self, + tree, + keys, + start, + stop, decompression_executor, interpretation_executor, interp_options, From a8ab1ff32bd85c158adc3c7efb673797ce119aef Mon Sep 17 00:00:00 2001 From: Lindsey Gray Date: Sat, 24 Feb 2024 08:28:48 -0600 Subject: [PATCH 3/8] update pin to uproot 5.3.0 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index f729783f7..f902b96e9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -38,7 +38,7 @@ classifiers = [ ] dependencies = [ "awkward>=2.5.2", - "uproot>=5.2.2", + "uproot>=5.3.0", "dask[array]<2024.2.0,>=2023.4.0", "dask-awkward>=2024.2.0", "dask-histogram>=2024.2.0", From 8ad0f01f2f97328e2bc49379424c1caa9dad3e3a Mon Sep 17 00:00:00 2001 From: Lindsey Gray Date: Sat, 24 Feb 2024 09:30:58 -0600 Subject: [PATCH 4/8] adjust UprootSourceMapping --- src/coffea/nanoevents/mapping/uproot.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/coffea/nanoevents/mapping/uproot.py b/src/coffea/nanoevents/mapping/uproot.py index 19d5070cb..6f705143a 100644 --- a/src/coffea/nanoevents/mapping/uproot.py +++ b/src/coffea/nanoevents/mapping/uproot.py @@ -102,9 +102,11 @@ class UprootSourceMapping(BaseSourceMapping): _fix_awkward_form_of_iter = False def __init__( - self, fileopener, start, stop, cache=None, access_log=None, use_ak_forth=False + self, fileopener, start, stop, cache=None, access_log=None, use_ak_forth=False, decompression_executor=None, interpretation_executor=None ): super().__init__(fileopener, start, stop, cache, access_log, use_ak_forth) + self.decompression_executor = decompression_executor or uproot.source.futures.TrivialExecutor() + self.interpretation_executor = interpretation_executor or uproot.source.futures.TrivialExecutor() @classmethod def _extract_base_form(cls, tree, iteritems_options={}): @@ -173,9 +175,7 @@ def get_column_handle(self, columnsource, name, allow_missing): return columnsource[name] if name in columnsource else None return columnsource[name] - def extract_column( - self, columnhandle, start, stop, allow_missing, use_ak_forth=True - ): + def extract_column(self, columnhandle, start, stop, allow_missing, use_ak_forth=True): # make sure uproot is single-core since our calling context might not be if allow_missing and columnhandle is None: @@ -195,8 +195,8 @@ def extract_column( interp, entry_start=start, entry_stop=stop, - decompression_executor=uproot.source.futures.TrivialExecutor(), - interpretation_executor=uproot.source.futures.TrivialExecutor(), + decompression_executor=self.decompression_executor, + interpretation_executor=self.interpretation_executor, ) if allow_missing: From a037bb680fd2a4e24a3e0631f50ba6ffafa84b0c Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sat, 24 Feb 2024 15:31:10 +0000 Subject: [PATCH 5/8] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/coffea/nanoevents/mapping/uproot.py | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/src/coffea/nanoevents/mapping/uproot.py b/src/coffea/nanoevents/mapping/uproot.py index 6f705143a..078014d08 100644 --- a/src/coffea/nanoevents/mapping/uproot.py +++ b/src/coffea/nanoevents/mapping/uproot.py @@ -102,11 +102,23 @@ class UprootSourceMapping(BaseSourceMapping): _fix_awkward_form_of_iter = False def __init__( - self, fileopener, start, stop, cache=None, access_log=None, use_ak_forth=False, decompression_executor=None, interpretation_executor=None + self, + fileopener, + start, + stop, + cache=None, + access_log=None, + use_ak_forth=False, + decompression_executor=None, + interpretation_executor=None, ): super().__init__(fileopener, start, stop, cache, access_log, use_ak_forth) - self.decompression_executor = decompression_executor or uproot.source.futures.TrivialExecutor() - self.interpretation_executor = interpretation_executor or uproot.source.futures.TrivialExecutor() + self.decompression_executor = ( + decompression_executor or uproot.source.futures.TrivialExecutor() + ) + self.interpretation_executor = ( + interpretation_executor or uproot.source.futures.TrivialExecutor() + ) @classmethod def _extract_base_form(cls, tree, iteritems_options={}): @@ -175,7 +187,9 @@ def get_column_handle(self, columnsource, name, allow_missing): return columnsource[name] if name in columnsource else None return columnsource[name] - def extract_column(self, columnhandle, start, stop, allow_missing, use_ak_forth=True): + def extract_column( + self, columnhandle, start, stop, allow_missing, use_ak_forth=True + ): # make sure uproot is single-core since our calling context might not be if allow_missing and columnhandle is None: From 46253b7c3036a7cb1e3bb840b372130bb6a6e35f Mon Sep 17 00:00:00 2001 From: Lindsey Gray Date: Sat, 24 Feb 2024 09:43:12 -0600 Subject: [PATCH 6/8] provide new options to mapping --- src/coffea/nanoevents/factory.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/coffea/nanoevents/factory.py b/src/coffea/nanoevents/factory.py index 5fada2f44..7e877a6b7 100644 --- a/src/coffea/nanoevents/factory.py +++ b/src/coffea/nanoevents/factory.py @@ -148,6 +148,8 @@ def load_buffers( cache={}, access_log=None, use_ak_forth=True, + decompression_executor=decompression_executor, + interpretation_executor=interpretation_executor, ) mapping.preload_column_source(partition_key[0], partition_key[1], tree) buffer_key = partial(self._key_formatter, tuple_to_key(partition_key)) From 4f28a9ebcf9e4ba9fa86ab0931b1daf6d0129cc4 Mon Sep 17 00:00:00 2001 From: Lindsey Gray Date: Sat, 24 Feb 2024 09:56:23 -0600 Subject: [PATCH 7/8] update factory --- src/coffea/nanoevents/factory.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/coffea/nanoevents/factory.py b/src/coffea/nanoevents/factory.py index 7e877a6b7..cbb71c384 100644 --- a/src/coffea/nanoevents/factory.py +++ b/src/coffea/nanoevents/factory.py @@ -230,6 +230,9 @@ def from_root( use_ak_forth=True, delayed=True, known_base_form=None, + decompression_executor=None, + interpretation_executor=None, + ): """Quickly build NanoEvents from a root file @@ -267,6 +270,10 @@ def from_root( Nanoevents will use dask as a backend to construct a delayed task graph representing your analysis. known_base_form: If the base form of the input file is known ahead of time we can skip opening a single file and parsing metadata. + decompression_executor (None or Executor with a ``submit`` method): + see: https://github.com/scikit-hep/uproot5/blob/main/src/uproot/_dask.py#L109 + interpretation_executor (None or Executor with a ``submit`` method): + see: https://github.com/scikit-hep/uproot5/blob/main/src/uproot/_dask.py#L113 """ if treepath is not uproot._util.unset and not isinstance( @@ -315,6 +322,8 @@ def from_root( filter_branch=_remove_not_interpretable, steps_per_file=steps_per_file, known_base_form=known_base_form, + decompression_executor=decompression_executor, + interpretation_executor=interpretation_executor, **uproot_options, ) From 3f62918f015308e6c3bfd52aad6902e1161e3130 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sat, 24 Feb 2024 15:56:37 +0000 Subject: [PATCH 8/8] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/coffea/nanoevents/factory.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/coffea/nanoevents/factory.py b/src/coffea/nanoevents/factory.py index cbb71c384..67e5b2afb 100644 --- a/src/coffea/nanoevents/factory.py +++ b/src/coffea/nanoevents/factory.py @@ -232,7 +232,6 @@ def from_root( known_base_form=None, decompression_executor=None, interpretation_executor=None, - ): """Quickly build NanoEvents from a root file @@ -273,7 +272,7 @@ def from_root( decompression_executor (None or Executor with a ``submit`` method): see: https://github.com/scikit-hep/uproot5/blob/main/src/uproot/_dask.py#L109 interpretation_executor (None or Executor with a ``submit`` method): - see: https://github.com/scikit-hep/uproot5/blob/main/src/uproot/_dask.py#L113 + see: https://github.com/scikit-hep/uproot5/blob/main/src/uproot/_dask.py#L113 """ if treepath is not uproot._util.unset and not isinstance(