From 2b2527e1f73d0f8eb6ae0558ffffa68befd784d4 Mon Sep 17 00:00:00 2001 From: Kevin Schwarzwald Date: Tue, 13 Feb 2024 17:34:35 -0500 Subject: [PATCH 1/6] add new python versions for tests, fix optional dependencies --- ci/environment-noxesmf.yml | 22 ++++++++ ci/environment-py3.11.yml | 25 +++++++++ ci/environment-py3.12.yml | 25 +++++++++ ...onment-py3.8.yml => environment-py3.9.yml} | 2 +- setup.py | 7 +++ tests/test_core.py | 54 ++++++++++--------- xagg/core.py | 9 ++-- 7 files changed, 116 insertions(+), 28 deletions(-) create mode 100644 ci/environment-noxesmf.yml create mode 100644 ci/environment-py3.11.yml create mode 100644 ci/environment-py3.12.yml rename ci/{environment-py3.8.yml => environment-py3.9.yml} (97%) diff --git a/ci/environment-noxesmf.yml b/ci/environment-noxesmf.yml new file mode 100644 index 0000000..0ad44c2 --- /dev/null +++ b/ci/environment-noxesmf.yml @@ -0,0 +1,22 @@ +name: test_env_xagg_noxe +channels: + - conda-forge +dependencies: + - python=3.12 + ############## Without xesmf / esmpy / esmf explicitly installed, to check optional dependency + - numpy + - scipy + - xarray + - pandas + - netcdf4 + - geopandas >= 0.12.0 + - shapely + - cf_xarray >= 0.5.1 + - pytables + ############## + - pytest + - pip: + - codecov + - pytest-cov + - coverage[toml] + # - tables >= 3.7.0 # For exporting hd5 files through wm.to_file() (3.6.0 may have issues) diff --git a/ci/environment-py3.11.yml b/ci/environment-py3.11.yml new file mode 100644 index 0000000..252817c --- /dev/null +++ b/ci/environment-py3.11.yml @@ -0,0 +1,25 @@ +name: test_env_xagg_38 +channels: + - conda-forge +dependencies: + - python=3.11 + ############## These will have to be adjusted to your specific project + - numpy + - scipy + - xarray + - pandas + - netcdf4 + - geopandas >= 0.12.0 + - shapely + - xesmf >= 0.7.0 # These versions and explicit loads are to fix an issue in xesmf's call to cf_xarray (possibly through esmpy) + - cf_xarray >= 0.5.1 + - esmf >= 8.1.0 + - esmpy >= 8.1.0 + - pytables + ############## + - pytest + - pip: + - codecov + - pytest-cov + - coverage[toml] + # - tables >= 3.7.0 # For exporting hd5 files through wm.to_file() (3.6.0 may have issues) diff --git a/ci/environment-py3.12.yml b/ci/environment-py3.12.yml new file mode 100644 index 0000000..94c272d --- /dev/null +++ b/ci/environment-py3.12.yml @@ -0,0 +1,25 @@ +name: test_env_xagg_38 +channels: + - conda-forge +dependencies: + - python=3.12 + ############## These will have to be adjusted to your specific project + - numpy + - scipy + - xarray + - pandas + - netcdf4 + - geopandas >= 0.12.0 + - shapely + - xesmf >= 0.7.0 # These versions and explicit loads are to fix an issue in xesmf's call to cf_xarray (possibly through esmpy) + - cf_xarray >= 0.5.1 + - esmf >= 8.1.0 + - esmpy >= 8.1.0 + - pytables + ############## + - pytest + - pip: + - codecov + - pytest-cov + - coverage[toml] + # - tables >= 3.7.0 # For exporting hd5 files through wm.to_file() (3.6.0 may have issues) diff --git a/ci/environment-py3.8.yml b/ci/environment-py3.9.yml similarity index 97% rename from ci/environment-py3.8.yml rename to ci/environment-py3.9.yml index 880980a..1968ed8 100644 --- a/ci/environment-py3.8.yml +++ b/ci/environment-py3.9.yml @@ -2,7 +2,7 @@ name: test_env_xagg_38 channels: - conda-forge dependencies: - - python=3.8 + - python=3.9 ############## These will have to be adjusted to your specific project - numpy - scipy diff --git a/setup.py b/setup.py index 65d5d3a..9de00be 100644 --- a/setup.py +++ b/setup.py @@ -31,4 +31,11 @@ 'tables', 'cf_xarray>=0.5.1', ], + extras_require=[ + 'xesmf>=0.7.1', + 'esmpy>=8.1.0', + 'matplotlib', + 'cmocean', + 'cartopy', + ] ) diff --git a/tests/test_core.py b/tests/test_core.py index c05ed21..b9418d3 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -6,7 +6,12 @@ from geopandas import testing as gpdt from unittest import TestCase from shapely.geometry import Polygon -import xesmf as xe +try: + import xesmf as xe + _has_xesmf=True +except ImportError: + # To be able to test the rest with environments without xesmf + _has_xesmf=False from xagg.core import (process_weights,create_raster_polygons,get_pixel_overlaps,aggregate,NoOverlapError) @@ -46,32 +51,33 @@ def test_process_weights_basic(): xr.testing.assert_allclose(ds_compare,ds_t) # (weights_info isn't currently used by anything) -def test_process_weights_regrid_weights(): - # Now, test with a weights array twice the resolution as the - # ds, so it needs to be regridded - ds = xr.Dataset(coords={'lat':(['lat'],np.array([0,1])), - 'lon':(['lon'],np.array([0,1])), - }) - - # Synthetic weights grid, with double the resolution, and shifted - # by a half degree. Should regrid to the same weights grid as above - weights = xr.DataArray(data=np.array([[-0.5,0.5,0.5,1.5], - [0.5,-0.5,1.5,0.5], - [1.5,2.5,2.5,3.5], - [2.5,1.5,3.5,2.5]]), - dims=['lat','lon'], - coords=[np.array([-0.25,0.25,0.75,1.25]), - np.array([-0.25,0.25,0.75,1.25])]) +if _has_xesmf: + def test_process_weights_regrid_weights(): + # Now, test with a weights array twice the resolution as the + # ds, so it needs to be regridded + ds = xr.Dataset(coords={'lat':(['lat'],np.array([0,1])), + 'lon':(['lon'],np.array([0,1])), + }) + + # Synthetic weights grid, with double the resolution, and shifted + # by a half degree. Should regrid to the same weights grid as above + weights = xr.DataArray(data=np.array([[-0.5,0.5,0.5,1.5], + [0.5,-0.5,1.5,0.5], + [1.5,2.5,2.5,3.5], + [2.5,1.5,3.5,2.5]]), + dims=['lat','lon'], + coords=[np.array([-0.25,0.25,0.75,1.25]), + np.array([-0.25,0.25,0.75,1.25])]) - ds_t,weights_info = process_weights(ds,weights=weights) + ds_t,weights_info = process_weights(ds,weights=weights) - ds_compare = xr.Dataset({'weights':(('lat','lon'),np.array([[0,1],[2,3]]))}, - coords={'lat':(['lat'],np.array([0,1])), - 'lon':(['lon'],np.array([0,1])), - }) + ds_compare = xr.Dataset({'weights':(('lat','lon'),np.array([[0,1],[2,3]]))}, + coords={'lat':(['lat'],np.array([0,1])), + 'lon':(['lon'],np.array([0,1])), + }) - # Check if weights were correctly added to ds - xr.testing.assert_allclose(ds_compare,ds_t) + # Check if weights were correctly added to ds + xr.testing.assert_allclose(ds_compare,ds_t) def test_process_weights_close_weights(): # Make sure weights that are within `np.allclose` but not exactly diff --git a/xagg/core.py b/xagg/core.py index 1acff30..beacbc3 100644 --- a/xagg/core.py +++ b/xagg/core.py @@ -7,6 +7,11 @@ import warnings import re import os +try: + import xesmf as xe + _has_xesmf=True +except ImportError: + _has_xesmf=False from . auxfuncs import (find_rel_area,normalize,fix_ds,get_bnds,subset_find,list_or_first) from . classes import (weightmap,aggregated) @@ -155,9 +160,7 @@ def process_weights(ds,weights=None,target='ds',silent=False): # Import xesmf here to allow the code to work without it (it # often has dependency issues and isn't necessary for many # features of xagg) - try: - import xesmf as xe - except ImportError: + if not _has_xesmf: raise ImportError('If the `weights` grid and the `ds` grid are different, '+ '`xesmf` is needed for `xagg` to regrid them to match; however, '+ '`xesmf` is not installed. Either install `xesmf` or '+ From cf20ec5174c29021aa84aafed05b45b56e53aee6 Mon Sep 17 00:00:00 2001 From: Kevin Schwarzwald Date: Tue, 13 Feb 2024 17:36:11 -0500 Subject: [PATCH 2/6] fix in setup.py to be a dict --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 9de00be..362e8e7 100644 --- a/setup.py +++ b/setup.py @@ -31,11 +31,11 @@ 'tables', 'cf_xarray>=0.5.1', ], - extras_require=[ + extras_require={ 'xesmf>=0.7.1', 'esmpy>=8.1.0', 'matplotlib', 'cmocean', 'cartopy', - ] + } ) From e5bc8711c657b751480deafd336720c996734c42 Mon Sep 17 00:00:00 2001 From: Kevin Schwarzwald Date: Tue, 13 Feb 2024 17:39:22 -0500 Subject: [PATCH 3/6] fix in setup.py to be a functioning dict --- setup.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/setup.py b/setup.py index 362e8e7..d582ff8 100644 --- a/setup.py +++ b/setup.py @@ -32,10 +32,7 @@ 'cf_xarray>=0.5.1', ], extras_require={ - 'xesmf>=0.7.1', - 'esmpy>=8.1.0', - 'matplotlib', - 'cmocean', - 'cartopy', + 'regrid':['xesmf>=0.7.1','esmpy>=8.1.0'], + 'plots':['matplotlib','cmocean','cartopy'], } ) From 4d68e8b8a492849c9786fa6e1d301fb1401733aa Mon Sep 17 00:00:00 2001 From: Kevin Schwarzwald Date: Tue, 13 Feb 2024 17:43:20 -0500 Subject: [PATCH 4/6] Update for new python versions --- .github/workflows/test.yaml | 2 +- ci/{environment-noxesmf.yml => environment-py3.12.noxesmf.yml} | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename ci/{environment-noxesmf.yml => environment-py3.12.noxesmf.yml} (100%) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 4c8d188..cb48e0e 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -9,7 +9,7 @@ jobs: fail-fast: false matrix: os: ["ubuntu-latest"] - python-version: ["3.8", "3.10"] + python-version: ["3.9", "3.10", "3.11", "3.12", "3.12.noxesmf"] steps: - uses: actions/checkout@v4 - name: Create conda environment diff --git a/ci/environment-noxesmf.yml b/ci/environment-py3.12.noxesmf.yml similarity index 100% rename from ci/environment-noxesmf.yml rename to ci/environment-py3.12.noxesmf.yml From 405d9be6680a9619a29e5842a0def96516219a3b Mon Sep 17 00:00:00 2001 From: Kevin Schwarzwald Date: Tue, 13 Feb 2024 17:47:35 -0500 Subject: [PATCH 5/6] remove import of (not needed) --- tests/test_wrappers.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_wrappers.py b/tests/test_wrappers.py index f171b46..24666a2 100644 --- a/tests/test_wrappers.py +++ b/tests/test_wrappers.py @@ -7,7 +7,6 @@ from geopandas import testing as gpdt from unittest import TestCase from shapely.geometry import Polygon -import xesmf as xe from xagg.wrappers import (pixel_overlaps) From 83a3fc9d79de3aabb267def53427df346e1e6972 Mon Sep 17 00:00:00 2001 From: Kevin Schwarzwald Date: Tue, 13 Feb 2024 17:53:52 -0500 Subject: [PATCH 6/6] Make tests dynamically respond to xesmf presence in environment (checking for ImportError in the right places if no xesmf) --- tests/test_core.py | 64 +++++++++++++++++++++++++++------------------- 1 file changed, 37 insertions(+), 27 deletions(-) diff --git a/tests/test_core.py b/tests/test_core.py index b9418d3..84dc45b 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -51,24 +51,25 @@ def test_process_weights_basic(): xr.testing.assert_allclose(ds_compare,ds_t) # (weights_info isn't currently used by anything) -if _has_xesmf: - def test_process_weights_regrid_weights(): - # Now, test with a weights array twice the resolution as the - # ds, so it needs to be regridded - ds = xr.Dataset(coords={'lat':(['lat'],np.array([0,1])), - 'lon':(['lon'],np.array([0,1])), - }) - # Synthetic weights grid, with double the resolution, and shifted - # by a half degree. Should regrid to the same weights grid as above - weights = xr.DataArray(data=np.array([[-0.5,0.5,0.5,1.5], - [0.5,-0.5,1.5,0.5], - [1.5,2.5,2.5,3.5], - [2.5,1.5,3.5,2.5]]), - dims=['lat','lon'], - coords=[np.array([-0.25,0.25,0.75,1.25]), - np.array([-0.25,0.25,0.75,1.25])]) +def test_process_weights_regrid_weights(): + # Now, test with a weights array twice the resolution as the + # ds, so it needs to be regridded + ds = xr.Dataset(coords={'lat':(['lat'],np.array([0,1])), + 'lon':(['lon'],np.array([0,1])), + }) + + # Synthetic weights grid, with double the resolution, and shifted + # by a half degree. Should regrid to the same weights grid as above + weights = xr.DataArray(data=np.array([[-0.5,0.5,0.5,1.5], + [0.5,-0.5,1.5,0.5], + [1.5,2.5,2.5,3.5], + [2.5,1.5,3.5,2.5]]), + dims=['lat','lon'], + coords=[np.array([-0.25,0.25,0.75,1.25]), + np.array([-0.25,0.25,0.75,1.25])]) + if _has_xesmf: ds_t,weights_info = process_weights(ds,weights=weights) ds_compare = xr.Dataset({'weights':(('lat','lon'),np.array([[0,1],[2,3]]))}, @@ -78,6 +79,10 @@ def test_process_weights_regrid_weights(): # Check if weights were correctly added to ds xr.testing.assert_allclose(ds_compare,ds_t) + else: + # Should raise ImportError in the no-xesmf environment + with pytest.raises(ImportError): + ds_t,weights_info = process_weights(ds,weights=weights) def test_process_weights_close_weights(): # Make sure weights that are within `np.allclose` but not exactly @@ -142,7 +147,7 @@ def test_create_raster_polygons_with_weights(): 'lon':(['lon'],np.array([0,1])), 'bnds':(['bnds'],np.array([0,1]))}) - # Synethetic weights grid + # Synthetic weights grid that requires regridding weights = xr.DataArray(data=np.array([[-0.5,0.5,0.5,1.5], [0.5,-0.5,1.5,0.5], [1.5,2.5,2.5,3.5], @@ -151,19 +156,24 @@ def test_create_raster_polygons_with_weights(): coords=[np.array([-0.25,0.25,0.75,1.25]), np.array([-0.25,0.25,0.75,1.25])]) - pix_agg = create_raster_polygons(ds,weights=weights) + if _has_xesmf: + pix_agg = create_raster_polygons(ds,weights=weights) - compare_series = pd.Series(data=[np.array(v) for v in [0.,1.,2.,3.]], - index=[0,1,2,3], - name='weights') + compare_series = pd.Series(data=[np.array(v) for v in [0.,1.,2.,3.]], + index=[0,1,2,3], + name='weights') - # There's an issue here in pd.testing.assert_series_equal... - #pd.testing.assert_series_equal(pix_agg['gdf_pixels'].weights, - # compare_series) - #np.testing.assert_allclose(compare_series,pix_agg['gdf_pixels'].weights) - #assert np.allclose(compare_series,pix_agg['gdf_pixels'].weights) - assert np.allclose([float(v) for v in compare_series],[float(v) for v in pix_agg['gdf_pixels'].weights]) + # There's an issue here in pd.testing.assert_series_equal... + #pd.testing.assert_series_equal(pix_agg['gdf_pixels'].weights, + # compare_series) + #np.testing.assert_allclose(compare_series,pix_agg['gdf_pixels'].weights) + #assert np.allclose(compare_series,pix_agg['gdf_pixels'].weights) + assert np.allclose([float(v) for v in compare_series],[float(v) for v in pix_agg['gdf_pixels'].weights]) + else: + # Should raise ImportError in the no-xesmf environment + with pytest.raises(ImportError): + pix_agg = create_raster_polygons(ds,weights=weights) def test_create_raster_polygons_at180(): # Make sure raster polygons are correctly built at the 180/-180