From f3cd4dd1b3b58746917e273deb5460b6a62dd933 Mon Sep 17 00:00:00 2001 From: koenifra Date: Mon, 12 Aug 2024 09:27:19 +0200 Subject: [PATCH 1/2] Adding 'Detecting Deep Moist Convection' to openeo-processes-dask (#263) * add ddmc and test * update pydantic dependencies * Adding ddmc to openeo-processes-dask * Changing the version in the pyproject and changing the import statement to the right folder * Deleting the unused file example%20file.geoparquet --- .../process_implementations/cubes/__init__.py | 1 + .../process_implementations/cubes/ddmc.py | 86 +++++++++++++++++++ pyproject.toml | 2 +- tests/test_ddmc.py | 75 ++++++++++++++++ 4 files changed, 163 insertions(+), 1 deletion(-) create mode 100644 openeo_processes_dask/process_implementations/cubes/ddmc.py create mode 100644 tests/test_ddmc.py diff --git a/openeo_processes_dask/process_implementations/cubes/__init__.py b/openeo_processes_dask/process_implementations/cubes/__init__.py index 11dce1b5..2021994c 100644 --- a/openeo_processes_dask/process_implementations/cubes/__init__.py +++ b/openeo_processes_dask/process_implementations/cubes/__init__.py @@ -1,6 +1,7 @@ from ._filter import * from .aggregate import * from .apply import * +from .ddmc import * from .general import * from .indices import * from .load import * diff --git a/openeo_processes_dask/process_implementations/cubes/ddmc.py b/openeo_processes_dask/process_implementations/cubes/ddmc.py new file mode 100644 index 00000000..65eb912b --- /dev/null +++ b/openeo_processes_dask/process_implementations/cubes/ddmc.py @@ -0,0 +1,86 @@ +from openeo_processes_dask.process_implementations.arrays import array_element +from openeo_processes_dask.process_implementations.cubes.general import add_dimension +from openeo_processes_dask.process_implementations.cubes.merge import merge_cubes +from openeo_processes_dask.process_implementations.cubes.reduce import reduce_dimension +from openeo_processes_dask.process_implementations.data_model import RasterCube + +__all__ = ["ddmc"] + + +def ddmc( + data: RasterCube, + nir08="nir08", + nir09="nir09", + cirrus="cirrus", + swir16="swir16", + swir22="swir22", + gain=2.5, + target_band=None, +): + dimension = data.openeo.band_dims[0] + if target_band is None: + target_band = dimension + + # Mid-Level Clouds + def MIDCL(data): + # B08 = array_element(data, label=nir08, axis = axis) + + B08 = data.sel(**{dimension: nir08}) + + # B09 = array_element(data, label=nir09, axis = axis) + + B09 = data.sel(**{dimension: nir09}) + + MIDCL = B08 - B09 + + MIDCL_result = MIDCL * gain + + return MIDCL_result + + # Deep moist convection + def DC(data): + # B10 = array_element(data, label=cirrus, axis = axis) + # B12 = array_element(data, label=swir22, axis = axis) + + B10 = data.sel(**{dimension: cirrus}) + B12 = data.sel(**{dimension: swir22}) + + DC = B10 - B12 + + DC_result = DC * gain + + return DC_result + + # low-level cloudiness + def LOWCL(data): + # B10 = array_element(data, label=cirrus, axis = axis) + # B11 = array_element(data, label=swir16, axis = axis) + B10 = data.sel(**{dimension: cirrus}) + B11 = data.sel(**{dimension: swir16}) + + LOWCL = B11 - B10 + + LOWCL_result = LOWCL * gain + + return LOWCL_result + + # midcl = reduce_dimension(data, reducer=MIDCL, dimension=dimension) + midcl = MIDCL(data) + midcl = add_dimension(midcl, name=target_band, label="midcl", type=dimension) + + # dc = reduce_dimension(data, reducer=DC, dimension=dimension) + dc = DC(data) + # dc = add_dimension(dc, target_band, "dc") + dc = add_dimension(dc, target_band, label="dc", type=dimension) + + # lowcl = reduce_dimension(data, reducer=LOWCL, dimension=dimension) + lowcl = LOWCL(data) + lowcl = add_dimension(lowcl, target_band, label="lowcl", type=dimension) + + # ddmc = merge_cubes(merge_cubes(midcl, dc), lowcl) + ddmc1 = merge_cubes(midcl, lowcl) + ddmc1.openeo.add_dim_type(name=target_band, type=dimension) + ddmc = merge_cubes(dc, ddmc1, overlap_resolver=target_band) + + # return a datacube + return ddmc diff --git a/pyproject.toml b/pyproject.toml index f1d43f8e..5888a9e4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -32,7 +32,7 @@ rasterio = { version = "^1.3.4", optional = true } dask-geopandas = { version = ">=0.2.0,<1", optional = true } xgboost = { version = ">=1.5.1", optional = true } rioxarray = { version = ">=0.12.0,<1", optional = true } -openeo-pg-parser-networkx = { version = ">=2023.5.1", optional = true } +openeo-pg-parser-networkx = { version = ">=2024.7", optional = true } odc-geo = { version = ">=0.4.1,<1", optional = true } stac_validator = { version = ">=3.3.1", optional = true } stackstac = { version = ">=0.4.3", optional = true } diff --git a/tests/test_ddmc.py b/tests/test_ddmc.py new file mode 100644 index 00000000..ac65c748 --- /dev/null +++ b/tests/test_ddmc.py @@ -0,0 +1,75 @@ +from functools import partial + +import numpy as np +import pytest +import xarray as xr +from openeo_pg_parser_networkx.pg_schema import ( + BoundingBox, + ParameterReference, + TemporalInterval, +) + +from openeo_processes_dask.process_implementations.cubes.ddmc import ddmc +from openeo_processes_dask.process_implementations.cubes.load import load_stac +from openeo_processes_dask.process_implementations.cubes.reduce import ( + reduce_dimension, + reduce_spatial, +) +from openeo_processes_dask.process_implementations.exceptions import ( + ArrayElementNotAvailable, +) +from tests.general_checks import general_output_checks +from tests.mockdata import create_fake_rastercube + + +@pytest.mark.parametrize("size", [(30, 30, 20, 5)]) +@pytest.mark.parametrize("dtype", [np.float32]) +def test_ddmc_instance_dims( + temporal_interval: TemporalInterval, bounding_box: BoundingBox, random_raster_data +): + input_cube = create_fake_rastercube( + data=random_raster_data, + spatial_extent=bounding_box, + temporal_extent=temporal_interval, + bands=["nir08", "nir09", "cirrus", "swir16", "swir22"], + backend="dask", + ) + + data = ddmc(input_cube) + + assert isinstance(data, xr.DataArray) + assert set(input_cube.dims) == set(data.dims) + + +@pytest.mark.parametrize("size", [(30, 30, 20, 5)]) +@pytest.mark.parametrize("dtype", [np.float32]) +def test_ddmc_target_band( + temporal_interval: TemporalInterval, bounding_box: BoundingBox, random_raster_data +): + input_cube = create_fake_rastercube( + data=random_raster_data, + spatial_extent=bounding_box, + temporal_extent=temporal_interval, + bands=["nir08", "nir09", "cirrus", "swir16", "swir22"], + backend="dask", + ) + + data_band = ddmc(data=input_cube, target_band="ddmc") + assert "ddmc" in data_band.dims + + +@pytest.mark.parametrize("size", [(30, 30, 20, 5)]) +@pytest.mark.parametrize("dtype", [np.float32]) +def test_ddmc_input_cube_exception( + temporal_interval: TemporalInterval, bounding_box: BoundingBox, random_raster_data +): + input_cube_exception = create_fake_rastercube( + data=random_raster_data, + spatial_extent=bounding_box, + temporal_extent=temporal_interval, + bands=["b04", "nir09", "cirrus", "swir16", "swir22"], + backend="dask", + ) + + with pytest.raises(KeyError): + data = ddmc(input_cube_exception) From 4390df5b4e010b9e4964c86bbd84a6c8911cb9d9 Mon Sep 17 00:00:00 2001 From: ValentinaHutter <85164505+ValentinaHutter@users.noreply.github.com> Date: Mon, 12 Aug 2024 10:25:54 +0200 Subject: [PATCH 2/2] Specs 2024.8.0 (#264) * update openeo-processes * bump 2023.9.1 * bump 2023.9.0 * update specs to add ddmc * update imports --- openeo_processes_dask/process_implementations/__init__.py | 1 + openeo_processes_dask/process_implementations/cubes/__init__.py | 1 - openeo_processes_dask/specs/openeo-processes | 2 +- pyproject.toml | 2 +- 4 files changed, 3 insertions(+), 3 deletions(-) diff --git a/openeo_processes_dask/process_implementations/__init__.py b/openeo_processes_dask/process_implementations/__init__.py index 64c4014d..cefe4e3d 100644 --- a/openeo_processes_dask/process_implementations/__init__.py +++ b/openeo_processes_dask/process_implementations/__init__.py @@ -5,6 +5,7 @@ from .arrays import * from .comparison import * from .cubes import * +from .cubes.ddmc import * from .inspect import * from .logic import * from .math import * diff --git a/openeo_processes_dask/process_implementations/cubes/__init__.py b/openeo_processes_dask/process_implementations/cubes/__init__.py index 2021994c..11dce1b5 100644 --- a/openeo_processes_dask/process_implementations/cubes/__init__.py +++ b/openeo_processes_dask/process_implementations/cubes/__init__.py @@ -1,7 +1,6 @@ from ._filter import * from .aggregate import * from .apply import * -from .ddmc import * from .general import * from .indices import * from .load import * diff --git a/openeo_processes_dask/specs/openeo-processes b/openeo_processes_dask/specs/openeo-processes index 8b7bb946..7cf65e3c 160000 --- a/openeo_processes_dask/specs/openeo-processes +++ b/openeo_processes_dask/specs/openeo-processes @@ -1 +1 @@ -Subproject commit 8b7bb94691eef89b3587406a1276ad26350e47ac +Subproject commit 7cf65e3c59af70246d669a1a52592ee52920a1f5 diff --git a/pyproject.toml b/pyproject.toml index 5888a9e4..b4ee8c2c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "openeo-processes-dask" -version = "2024.7.0" +version = "2024.8.0" description = "Python implementations of many OpenEO processes, dask-friendly by default." authors = ["Lukas Weidenholzer ", "Sean Hoyal ", "Valentina Hutter "] maintainers = ["EODC Staff "]