From 365ae22df919892da81abf8c03db731c624586aa Mon Sep 17 00:00:00 2001 From: Michele Claus <31700619+clausmichele@users.noreply.github.com> Date: Mon, 26 Feb 2024 16:23:00 +0100 Subject: [PATCH 1/3] labelled arrays (#155) * Draft for labelled arrays * Logic update * Added label test --------- Co-authored-by: Sean <63349506+SerRichard@users.noreply.github.com> --- .../process_implementations/arrays.py | 15 ++++++++++++--- .../process_implementations/core.py | 8 +++++++- .../process_implementations/cubes/reduce.py | 7 ++++--- tests/test_arrays.py | 18 ++++++++++++++++++ 4 files changed, 41 insertions(+), 7 deletions(-) diff --git a/openeo_processes_dask/process_implementations/arrays.py b/openeo_processes_dask/process_implementations/arrays.py index d86860ea..ebbaeaf8 100644 --- a/openeo_processes_dask/process_implementations/arrays.py +++ b/openeo_processes_dask/process_implementations/arrays.py @@ -7,6 +7,7 @@ import pandas as pd import xarray as xr from numpy.typing import ArrayLike +from openeo_pg_parser_networkx.pg_schema import DateTime from xarray.core.duck_array_ops import isnull, notnull from openeo_processes_dask.process_implementations.cubes.utils import _is_dask_array @@ -43,6 +44,8 @@ def array_element( label: Optional[str] = None, return_nodata: Optional[bool] = False, axis=None, + context=None, + dim_labels=None, ): if index is None and label is None: raise ArrayElementParameterMissing( @@ -55,14 +58,20 @@ def array_element( ) if label is not None: - raise NotImplementedError( - "labelled arrays are currently not implemented. Please use index instead." - ) + if isinstance(label, DateTime): + label = label.to_numpy() + (index,) = np.where(dim_labels == label) + if len(index) == 0: + index = None + else: + index = index[0] try: if index is not None: element = np.take(data, index, axis=axis) return element + else: + raise IndexError except IndexError: if return_nodata: logger.warning( diff --git a/openeo_processes_dask/process_implementations/core.py b/openeo_processes_dask/process_implementations/core.py index e3fa7ce7..bc87ee22 100644 --- a/openeo_processes_dask/process_implementations/core.py +++ b/openeo_processes_dask/process_implementations/core.py @@ -68,7 +68,13 @@ def wrapper( else: resolved_kwargs[k] = arg - special_args = ["axis", "keepdims", "source_transposed_axis", "context"] + special_args = [ + "axis", + "keepdims", + "source_transposed_axis", + "context", + "dim_labels", + ] # Remove 'axis' and keepdims parameter if not expected in function signature. for arg in special_args: if arg not in inspect.signature(f).parameters: diff --git a/openeo_processes_dask/process_implementations/cubes/reduce.py b/openeo_processes_dask/process_implementations/cubes/reduce.py index 5641edc4..7702ea06 100644 --- a/openeo_processes_dask/process_implementations/cubes/reduce.py +++ b/openeo_processes_dask/process_implementations/cubes/reduce.py @@ -21,15 +21,16 @@ def reduce_dimension( f"Provided dimension ({dimension}) not found in data.dims: {data.dims}" ) - positional_parameters = {"data": 0} - named_parameters = {"context": context} + dim_labels = data[dimension].values + positional_parameters = {"data": 0} reduced_data = data.reduce( reducer, dim=dimension, keep_attrs=True, positional_parameters=positional_parameters, - named_parameters=named_parameters, + context=context, + dim_labels=dim_labels, ) # Preset diff --git a/tests/test_arrays.py b/tests/test_arrays.py index 5c9eea64..3d760214 100644 --- a/tests/test_arrays.py +++ b/tests/test_arrays.py @@ -48,6 +48,24 @@ def test_array_element( xr.testing.assert_equal(output_cube, input_cube.isel({"bands": 1}, drop=True)) + # Use a label + _process = partial( + process_registry["array_element"].implementation, + label="B02", + data=ParameterReference(from_parameter="data"), + ) + + output_cube = reduce_dimension(data=input_cube, reducer=_process, dimension="bands") + + general_output_checks( + input_cube=input_cube, + output_cube=output_cube, + verify_attrs=False, + verify_crs=True, + ) + + xr.testing.assert_equal(output_cube, input_cube.loc[{"bands": "B02"}].drop("bands")) + # When the index is out of range, we expect an ArrayElementNotAvailable exception to be thrown _process_not_available = partial( process_registry["array_element"].implementation, From 503d879574548ef14a12152b7afbef791eae2afa Mon Sep 17 00:00:00 2001 From: Sean <63349506+SerRichard@users.noreply.github.com> Date: Mon, 26 Feb 2024 16:23:38 +0100 Subject: [PATCH 2/3] Bug in apply kernel (#231) * bump submodule to include new process specs * pre commit hooks * fix hard coded spatial dimension reference --------- Co-authored-by: sean --- openeo_processes_dask/process_implementations/cubes/apply.py | 2 +- openeo_processes_dask/specs/openeo-processes | 2 +- pyproject.toml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/openeo_processes_dask/process_implementations/cubes/apply.py b/openeo_processes_dask/process_implementations/cubes/apply.py index aa425687..1c59706b 100644 --- a/openeo_processes_dask/process_implementations/cubes/apply.py +++ b/openeo_processes_dask/process_implementations/cubes/apply.py @@ -107,7 +107,7 @@ def apply_kernel( ) def convolve(data, kernel, mode="constant", cval=0, fill_value=0): - dims = ("y", "x") + dims = data.openeo.spatial_dims convolved = lambda data: scipy.ndimage.convolve( data, kernel, mode=mode, cval=cval ) diff --git a/openeo_processes_dask/specs/openeo-processes b/openeo_processes_dask/specs/openeo-processes index 12444d8b..16a0ec6f 160000 --- a/openeo_processes_dask/specs/openeo-processes +++ b/openeo_processes_dask/specs/openeo-processes @@ -1 +1 @@ -Subproject commit 12444d8bbc1a983e6b8df0ba956d057dc2d2224e +Subproject commit 16a0ec6f3a256ffd5044cc6d3c712ce9fd92ff73 diff --git a/pyproject.toml b/pyproject.toml index 002a3a6a..c75d954a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "openeo-processes-dask" -version = "2024.2.2" +version = "2024.2.3" description = "Python implementations of many OpenEO processes, dask-friendly by default." authors = ["Lukas Weidenholzer ", "Sean Hoyal ", "Valentina Hutter "] maintainers = ["EODC Staff "] From 9fcafef12f8c6cc6da302a5f02a870b8fa12c278 Mon Sep 17 00:00:00 2001 From: ValentinaHutter <85164505+ValentinaHutter@users.noreply.github.com> Date: Tue, 27 Feb 2024 13:57:30 +0100 Subject: [PATCH 3/3] Update specs to use datacube instead of rastercube (#232) * update specs * version 2024.2.4 --- openeo_processes_dask/specs/openeo-processes | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/openeo_processes_dask/specs/openeo-processes b/openeo_processes_dask/specs/openeo-processes index 16a0ec6f..ea3d202a 160000 --- a/openeo_processes_dask/specs/openeo-processes +++ b/openeo_processes_dask/specs/openeo-processes @@ -1 +1 @@ -Subproject commit 16a0ec6f3a256ffd5044cc6d3c712ce9fd92ff73 +Subproject commit ea3d202a43e966c8524b524efc1a283e3b15a2df diff --git a/pyproject.toml b/pyproject.toml index c75d954a..ac3c38ff 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "openeo-processes-dask" -version = "2024.2.3" +version = "2024.2.4" description = "Python implementations of many OpenEO processes, dask-friendly by default." authors = ["Lukas Weidenholzer ", "Sean Hoyal ", "Valentina Hutter "] maintainers = ["EODC Staff "]