From 583917fd22d5fcdb9091f60f00ea5fac2d8b98b0 Mon Sep 17 00:00:00 2001 From: Pascal Bourgault Date: Thu, 27 Jun 2024 19:20:43 -0400 Subject: [PATCH 01/33] Fix NaN and Inf --- environment.yml | 3 +-- pyproject.toml | 2 +- tests/test_calendar.py | 6 ++--- tests/test_ensembles.py | 2 +- tests/test_generic.py | 8 +++---- tests/test_indicators.py | 2 +- tests/test_indices.py | 38 +++++++++++++++--------------- tests/test_precip.py | 8 +++---- tests/test_run_length.py | 8 +++---- tests/test_sdba/test_processing.py | 10 ++++---- tests/test_sdba/test_sdba_utils.py | 4 ++-- tests/test_snow.py | 4 ++-- tests/test_temperature.py | 2 +- tests/test_utils.py | 4 ++-- xclim/analog.py | 2 +- xclim/core/bootstrapping.py | 2 +- xclim/core/utils.py | 4 ++-- xclim/ensembles/_reduce.py | 2 +- xclim/ensembles/_robustness.py | 6 ++--- xclim/indices/_agro.py | 8 +++---- xclim/indices/fire/_cffwis.py | 2 +- xclim/indices/run_length.py | 6 ++--- xclim/sdba/_adjustment.py | 8 +++---- xclim/sdba/adjustment.py | 6 ++--- xclim/sdba/loess.py | 2 +- xclim/sdba/nbutils.py | 8 +++---- xclim/sdba/utils.py | 8 +++---- 27 files changed, 82 insertions(+), 83 deletions(-) diff --git a/environment.yml b/environment.yml index 7e543d0a4..ae47afc66 100644 --- a/environment.yml +++ b/environment.yml @@ -1,6 +1,5 @@ name: xclim channels: - - numba # Added to gain access to Python3.12-compatible numba release candidates. - conda-forge - defaults dependencies: @@ -13,7 +12,7 @@ dependencies: - dask >=2.6.0 - jsonpickle - numba - - numpy >=1.20.0,<2.0.0 + - numpy >=1.20.0 - pandas >=2.2.0 - pint >=0.10,<0.24 - poppler >=0.67 diff --git a/pyproject.toml b/pyproject.toml index 8c101b404..97096807c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -40,7 +40,7 @@ dependencies = [ "dask[array]>=2.6", "jsonpickle", "numba", - "numpy>=1.20.0,<2.0.0", + "numpy>=1.20.0", "pandas>=2.2", "pint>=0.10,<0.24", "platformdirs >=3.2", diff --git a/tests/test_calendar.py b/tests/test_calendar.py index 31a3849a0..58b8d5857 100644 --- a/tests/test_calendar.py +++ b/tests/test_calendar.py @@ -281,7 +281,7 @@ def test_convert_calendar_and_doy(): out = convert_doy(doy, target_cal="360_day", align_on="date").convert_calendar( "360_day", align_on="date" ) - np.testing.assert_array_equal(out, [np.NaN, 31, 332, 360.23, np.NaN]) + np.testing.assert_array_equal(out, [np.nan, 31, 332, 360.23, np.nan]) assert out.time.dt.calendar == "360_day" @@ -456,7 +456,7 @@ def test_convert_doy(): ) out = convert_doy(doy, "360_day", align_on="date") - np.testing.assert_array_equal(out, [np.NaN, 31, 332, 360.23, np.NaN]) + np.testing.assert_array_equal(out, [np.nan, 31, 332, 360.23, np.nan]) assert out.time.dt.calendar == "noleap" out = convert_doy(doy, "360_day", align_on="year") np.testing.assert_allclose( @@ -475,7 +475,7 @@ def test_convert_doy(): ).expand_dims(lat=[10, 20, 30]) out = convert_doy(doy, "noleap", align_on="date") - np.testing.assert_array_equal(out.isel(lat=0), [31, 200.48, 190, np.NaN, 299.54]) + np.testing.assert_array_equal(out.isel(lat=0), [31, 200.48, 190, np.nan, 299.54]) out = convert_doy(doy, "noleap", align_on="year") np.testing.assert_allclose( out.isel(lat=0), [31.0, 200.48, 190.0, 59.83607, 299.71885] diff --git a/tests/test_ensembles.py b/tests/test_ensembles.py index 4db62df9b..cbe271a7b 100644 --- a/tests/test_ensembles.py +++ b/tests/test_ensembles.py @@ -311,7 +311,7 @@ def test_stats_min_members(self, ensemble_dataset_objects, open_dataset, aggfunc ds_all = [open_dataset(n) for n in ensemble_dataset_objects["nc_files_simple"]] ens = ensembles.create_ensemble(ds_all).isel(lat=0, lon=0) ens = ens.where(ens.realization > 0) - ens = xr.where((ens.realization == 1) & (ens.time.dt.year == 1950), np.NaN, ens) + ens = xr.where((ens.realization == 1) & (ens.time.dt.year == 1950), np.nan, ens) def first(ds): return ds[list(ds.data_vars.keys())[0]] diff --git a/tests/test_generic.py b/tests/test_generic.py index c1eab6bf2..1f6869937 100644 --- a/tests/test_generic.py +++ b/tests/test_generic.py @@ -393,13 +393,13 @@ def test_generic_forbidden_op(self, pr_series): class TestGetDailyEvents: def test_simple(self, tas_series): - arr = xr.DataArray(np.array([-10, 15, 20, np.NaN, 10]), name="Stuff") + arr = xr.DataArray(np.array([-10, 15, 20, np.nan, 10]), name="Stuff") out = generic.get_daily_events(arr, threshold=10, op=">=") assert out.name == "events" assert out.sum() == 3 - np.testing.assert_array_equal(out, [0, 1, 1, np.NaN, 1]) + np.testing.assert_array_equal(out, [0, 1, 1, np.nan, 1]) class TestGenericCountingIndices: @@ -470,7 +470,7 @@ def test_count_occurrences(self, tas_series, op, constrain, expected, should_fai @pytest.mark.parametrize( "op, constrain, expected, should_fail", [ - ("<", None, np.NaN, False), + ("<", None, np.nan, False), ("<=", None, 3, False), ("!=", ("!=",), 1, False), ("==", ("==", "!="), 3, False), @@ -497,7 +497,7 @@ def test_first_occurrence(self, tas_series, op, constrain, expected, should_fail @pytest.mark.parametrize( "op, constrain, expected, should_fail", [ - ("<", None, np.NaN, False), + ("<", None, np.nan, False), ("<=", None, 8, False), ("!=", ("!=",), 9, False), ("==", ("==", "!="), 8, False), diff --git a/tests/test_indicators.py b/tests/test_indicators.py index 1395f214c..4ca95ea2f 100644 --- a/tests/test_indicators.py +++ b/tests/test_indicators.py @@ -848,7 +848,7 @@ def test_resampling_indicator_with_indexing(tas_series): out = xclim.atmos.tx_days_above( tas, thresh="0 degC", freq="YS-JUL", doy_bounds=(1, 50) ) - np.testing.assert_allclose(out, [50, 50, np.NaN]) + np.testing.assert_allclose(out, [50, 50, np.nan]) out = xclim.atmos.tx_days_above( tas, thresh="0 degC", freq="YS", date_bounds=("02-29", "04-01") diff --git a/tests/test_indices.py b/tests/test_indices.py index 9629d22d5..e381897f7 100644 --- a/tests/test_indices.py +++ b/tests/test_indices.py @@ -454,7 +454,7 @@ def test_effective_growing_degree_days( tasmax=mx, tasmin=mn, method=method, freq="YS" ) - np.testing.assert_array_equal(out, np.array([np.NaN, expected])) + np.testing.assert_array_equal(out, np.array([np.nan, expected])) class TestStandardizedIndices: @@ -596,7 +596,7 @@ def test_standardized_precipitation_index( ): ds = open_dataset("sdba/CanESM2_1950-2100.nc").isel(location=1) if freq == "D": - ds = ds.convert_calendar("366_day", missing=np.NaN) + ds = ds.convert_calendar("366_day", missing=np.nan) # to compare with ``climate_indices`` pr = ds.pr.sel(time=slice("1998", "2000")) pr_cal = ds.pr.sel(time=slice("1950", "1980")) @@ -2833,7 +2833,7 @@ def test_simple(self, snd_series, snw_series): np.testing.assert_array_equal(out, [0.3, 0.01]) def test_nan_slices(self, snd_series, snw_series): - a = np.ones(366) * np.NaN + a = np.ones(366) * np.nan snd = snd_series(a) snw = snw_series(a) @@ -2858,7 +2858,7 @@ def test_simple(self, snd_series, snw_series): np.testing.assert_array_equal(out, [193, 182]) def test_nan_slices(self, snd_series, snw_series): - a = np.ones(366) * np.NaN + a = np.ones(366) * np.nan snd = snd_series(a) snw = snw_series(a) @@ -2954,7 +2954,7 @@ def test_snow_season_end(self, snd_series, snw_series): ["per_day", "total"], ) def test_rain_season(pr_series, result_type, method_dry_start): - pr = pr_series(np.arange(365) * np.NaN, start="2000-01-01", units="mm/d") + pr = pr_series(np.arange(365) * np.nan, start="2000-01-01", units="mm/d") # input values in mm (amount): a correcting factor is used below pr[{"time": slice(0, 0 + 3)}] = 10 # to satisfy cond1_start pr[{"time": slice(3, 3 + 30)}] = 5 # to satisfy cond2_start @@ -2963,13 +2963,13 @@ def test_rain_season(pr_series, result_type, method_dry_start): out_exp = [3, 100, 97] elif result_type == "start_cond1_fails": pr[{"time": 2}] = 0 - out_exp = [np.NaN, np.NaN, np.NaN] + out_exp = [np.nan, np.nan, np.nan] elif result_type == "start_cond2_fails": pr[{"time": slice(10, 10 + 7)}] = 0 - out_exp = [np.NaN, np.NaN, np.NaN] + out_exp = [np.nan, np.nan, np.nan] elif result_type == "end_cond_fails": pr[{"time": 99 + 20 - 1}] = 5 - out_exp = [3, np.NaN, 363] + out_exp = [3, np.nan, 363] else: raise ValueError(f"Unknown result_type: {result_type}") @@ -3099,7 +3099,7 @@ def test_wind_chill(tas_series, sfcWind_series): # The calculator was altered to remove the rounding of the output. np.testing.assert_allclose( out, - [-4.509267062481955, -22.619869069856854, -30.478945408950928, np.NaN, -16.443], + [-4.509267062481955, -22.619869069856854, -30.478945408950928, np.nan, -16.443], ) out = xci.wind_chill_index(tas=tas, sfcWind=sfcWind, method="US") @@ -3520,16 +3520,16 @@ def test_simple(self, pr_series, prc_series): [ 2, 0, - np.NaN, + np.nan, 2, - np.NaN, - np.NaN, - np.NaN, - np.NaN, - np.NaN, - np.NaN, - np.NaN, - np.NaN, + np.nan, + np.nan, + np.nan, + np.nan, + np.nan, + np.nan, + np.nan, + np.nan, ], ) @@ -3652,7 +3652,7 @@ def test_dryness_index(self, atmosds): (-47, "usda", 1), (-6, "anbg", 1), (19, "anbg", 6), - (-47, "anbg", np.NaN), + (-47, "anbg", np.nan), ], # There are 26 USDA zones: 1a -> 1, 1b -> 2, ... 13b -> 26 # There are 7 angb zones: 1,2, ..., 7 diff --git a/tests/test_precip.py b/tests/test_precip.py index ba470d985..e8243a8b7 100644 --- a/tests/test_precip.py +++ b/tests/test_precip.py @@ -674,7 +674,7 @@ def test_dry_spell(atmosds): def test_dry_spell_total_length_indexer(pr_series): pr = pr_series( - [np.NaN] + [1] * 4 + [0] * 10 + [1] * 350, start="1900-01-01", units="mm/d" + [np.nan] + [1] * 4 + [0] * 10 + [1] * 350, start="1900-01-01", units="mm/d" ) out = atmos.dry_spell_total_length( pr, @@ -683,7 +683,7 @@ def test_dry_spell_total_length_indexer(pr_series): thresh="3 mm", freq="MS", ) - np.testing.assert_allclose(out, [np.NaN] + [0] * 11) + np.testing.assert_allclose(out, [np.nan] + [0] * 11) out = atmos.dry_spell_total_length( pr, window=7, op="sum", thresh="3 mm", freq="MS", date_bounds=("01-10", "12-31") @@ -693,7 +693,7 @@ def test_dry_spell_total_length_indexer(pr_series): def test_dry_spell_max_length_indexer(pr_series): pr = pr_series( - [np.NaN] + [1] * 4 + [0] * 10 + [1] * 350, start="1900-01-01", units="mm/d" + [np.nan] + [1] * 4 + [0] * 10 + [1] * 350, start="1900-01-01", units="mm/d" ) out = atmos.dry_spell_max_length( pr, @@ -702,7 +702,7 @@ def test_dry_spell_max_length_indexer(pr_series): thresh="3 mm", freq="MS", ) - np.testing.assert_allclose(out, [np.NaN] + [0] * 11) + np.testing.assert_allclose(out, [np.nan] + [0] * 11) out = atmos.dry_spell_total_length( pr, window=7, op="sum", thresh="3 mm", freq="MS", date_bounds=("01-10", "12-31") diff --git a/tests/test_run_length.py b/tests/test_run_length.py index 7617c6f87..d0e10ae7e 100644 --- a/tests/test_run_length.py +++ b/tests/test_run_length.py @@ -470,7 +470,7 @@ class TestRunsWithDates: [ ("07-01", 210, 70), ("07-01", 190, 50), - ("04-01", 150, np.NaN), # date falls early + ("04-01", 150, np.nan), # date falls early ("11-01", 150, 165), # date ends late (None, 150, 10), # no date, real length ], @@ -501,7 +501,7 @@ def test_season_length(self, tas_series, date, end, expected, use_dask, ufunc): [ ("dayofyear", "07-01", 210, 211), (False, "07-01", 190, 190), - ("dayofyear", "04-01", 150, np.NaN), # date falls early + ("dayofyear", "04-01", 150, np.nan), # date falls early ("dayofyear", "11-01", 150, 306), # date ends late ], ) @@ -529,7 +529,7 @@ def test_run_end_after_date( [ ("dayofyear", "07-01", 210, 211), (False, "07-01", 190, 190), - ("dayofyear", "04-01", False, np.NaN), # no run + ("dayofyear", "04-01", False, np.nan), # no run ("dayofyear", "11-01", 150, 306), # run already started ], ) @@ -559,7 +559,7 @@ def test_first_run_after_date( [ ("dayofyear", "07-01", 210, 183), (False, "07-01", 190, 182), - ("dayofyear", "04-01", 150, np.NaN), # date falls early + ("dayofyear", "04-01", 150, np.nan), # date falls early ("dayofyear", "11-01", 150, 150), # date ends late ], ) diff --git a/tests/test_sdba/test_processing.py b/tests/test_sdba/test_processing.py index 7a8c21f80..efc1078e4 100644 --- a/tests/test_sdba/test_processing.py +++ b/tests/test_sdba/test_processing.py @@ -157,7 +157,7 @@ def test_escore(): def test_standardize(random): x = random.standard_normal((2, 10000)) - x[0, 50] = np.NaN + x[0, 50] = np.nan x = xr.DataArray(x, dims=("x", "y"), attrs={"units": "m"}) xp, avg, std = standardize(x, dim="y") @@ -216,7 +216,7 @@ def test_to_additive(pr_series, hurs_series): with units.context("hydro"): prlog = to_additive_space(pr, lower_bound="0 mm/d", trans="log") - np.testing.assert_allclose(prlog, [-np.Inf, -11.512925, 0, 10]) + np.testing.assert_allclose(prlog, [-np.inf, -11.512925, 0, 10]) assert prlog.attrs["sdba_transform"] == "log" assert prlog.attrs["sdba_transform_units"] == "kg m-2 s-1" @@ -224,7 +224,7 @@ def test_to_additive(pr_series, hurs_series): pr1 = pr + 1 with units.context("hydro"): prlog2 = to_additive_space(pr1, trans="log", lower_bound="1.0 kg m-2 s-1") - np.testing.assert_allclose(prlog2, [-np.Inf, -11.512925, 0, 10]) + np.testing.assert_allclose(prlog2, [-np.inf, -11.512925, 0, 10]) assert prlog2.attrs["sdba_transform_lower"] == 1.0 # logit @@ -234,7 +234,7 @@ def test_to_additive(pr_series, hurs_series): hurs, lower_bound="0 %", trans="logit", upper_bound="100 %" ) np.testing.assert_allclose( - hurslogit, [-np.Inf, -11.5129154649, 2.197224577, np.Inf] + hurslogit, [-np.inf, -11.5129154649, 2.197224577, np.inf] ) assert hurslogit.attrs["sdba_transform"] == "logit" assert hurslogit.attrs["sdba_transform_units"] == "%" @@ -245,7 +245,7 @@ def test_to_additive(pr_series, hurs_series): hursscl, trans="logit", lower_bound="2", upper_bound="6" ) np.testing.assert_allclose( - hurslogit2, [-np.Inf, -11.5129154649, 2.197224577, np.Inf] + hurslogit2, [-np.inf, -11.5129154649, 2.197224577, np.inf] ) assert hurslogit2.attrs["sdba_transform_lower"] == 200.0 assert hurslogit2.attrs["sdba_transform_upper"] == 600.0 diff --git a/tests/test_sdba/test_sdba_utils.py b/tests/test_sdba/test_sdba_utils.py index f415cf090..48fe0bdcc 100644 --- a/tests/test_sdba/test_sdba_utils.py +++ b/tests/test_sdba/test_sdba_utils.py @@ -68,7 +68,7 @@ def test_equally_spaced_nodes(): @pytest.mark.parametrize( "interp,expi", [("nearest", 2.9), ("linear", 2.95), ("cubic", 2.95)] ) -@pytest.mark.parametrize("extrap,expe", [("constant", 4.4), ("nan", np.NaN)]) +@pytest.mark.parametrize("extrap,expe", [("constant", 4.4), ("nan", np.nan)]) def test_interp_on_quantiles_constant(interp, expi, extrap, expe): quantiles = np.linspace(0, 1, num=25) xq = xr.DataArray( @@ -163,7 +163,7 @@ def test_interp_on_quantiles_monthly(random): @pytest.mark.parametrize( "interp,expi", [("nearest", 2.9), ("linear", 2.95), ("cubic", 2.95)] ) -@pytest.mark.parametrize("extrap,expe", [("constant", 4.4), ("nan", np.NaN)]) +@pytest.mark.parametrize("extrap,expe", [("constant", 4.4), ("nan", np.nan)]) def test_interp_on_quantiles_constant_with_nan(interp, expi, extrap, expe): quantiles = np.linspace(0, 1, num=30) xq = xr.DataArray( diff --git a/tests/test_snow.py b/tests/test_snow.py index 13b0ce5b0..9d6a1656b 100644 --- a/tests/test_snow.py +++ b/tests/test_snow.py @@ -26,7 +26,7 @@ def test_simple(self, snd_series): class TestSnowWaterCoverDuration: @pytest.mark.parametrize( - "factor,exp", ([1000, [31, 28, 31, np.NaN]], [0, [0, 0, 0, np.NaN]]) + "factor,exp", ([1000, [31, 28, 31, np.nan]], [0, [0, 0, 0, np.nan]]) ) def test_simple(self, snw_series, factor, exp): snw = snw_series(np.ones(110) * factor, start="2001-01-01") @@ -97,7 +97,7 @@ def test_no_snow(self, atmosds): # Put 0 on one row. snd = atmosds.snd.where(atmosds.location != "Victoria", 0) out = land.snd_max_doy(snd) - np.testing.assert_array_equal(out.isel(time=1), [16, 13, 91, 29, np.NaN]) + np.testing.assert_array_equal(out.isel(time=1), [16, 13, 91, 29, np.nan]) class TestSnwMax: diff --git a/tests/test_temperature.py b/tests/test_temperature.py index 5b069fd34..6b8f4eba0 100644 --- a/tests/test_temperature.py +++ b/tests/test_temperature.py @@ -1290,7 +1290,7 @@ def test_degree_days_exceedance_date(open_dataset): @pytest.mark.parametrize( - "never_reached,exp", [(None, np.NaN), (300, 300), ("12-01", 335)] + "never_reached,exp", [(None, np.nan), (300, 300), ("12-01", 335)] ) def test_degree_days_exceedance_date_never_reached(open_dataset, never_reached, exp): tas = open_dataset("FWI/GFWED_sample_2017.nc").tas diff --git a/tests/test_utils.py b/tests/test_utils.py index c5e614185..0d1e766f5 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -71,7 +71,7 @@ def test_calc_perc_2d(self): assert np.all(res[0][1] == 29) def test_calc_perc_nan(self): - arr = np.asarray([np.NAN]) + arr = np.asarray([np.nan]) res = nan_calc_percentiles(arr, percentiles=[50.0]) assert np.isnan(res) @@ -81,7 +81,7 @@ def test_calc_perc_empty(self): assert np.isnan(res) def test_calc_perc_partial_nan(self): - arr = np.asarray([np.NaN, 41.0, 41.0, 43.0, 43.0]) + arr = np.asarray([np.nan, 41.0, 41.0, 43.0, 43.0]) res = nan_calc_percentiles(arr, percentiles=[50.0], alpha=1 / 3.0, beta=1 / 3.0) # The expected is from R `quantile(arr, 0.5, type=8, na.rm = TRUE)` # Note that scipy mquantiles would give a different result here diff --git a/xclim/analog.py b/xclim/analog.py index bdc60f7c8..05d5157fc 100644 --- a/xclim/analog.py +++ b/xclim/analog.py @@ -139,7 +139,7 @@ def metric(func): @wraps(func) def _metric_overhead(x, y, **kwargs): if np.any(np.isnan(x)) or np.any(np.isnan(y)): - return np.NaN + return np.nan x = np.atleast_2d(x) y = np.atleast_2d(y) diff --git a/xclim/core/bootstrapping.py b/xclim/core/bootstrapping.py index 280ac316d..fd1dac8c1 100644 --- a/xclim/core/bootstrapping.py +++ b/xclim/core/bootstrapping.py @@ -264,7 +264,7 @@ def build_bootstrap_year_da( out_view.loc[{dim: bloc}] = source.convert_calendar("noleap").data elif len(bloc) == 366: out_view.loc[{dim: bloc}] = source.convert_calendar( - "366_day", missing=np.NAN + "366_day", missing=np.nan ).data elif len(bloc) < 365: # 360 days calendar case or anchored years for both source[dim] and bloc case diff --git a/xclim/core/utils.py b/xclim/core/utils.py index 25fc7b985..f56714894 100644 --- a/xclim/core/utils.py +++ b/xclim/core/utils.py @@ -438,7 +438,7 @@ def _nan_quantile( # --- Setup data_axis_length = arr.shape[axis] if data_axis_length == 0: - return np.NAN + return np.nan if data_axis_length == 1: result = np.take(arr, 0, axis=axis) return np.broadcast_to(result, (quantiles.size,) + result.shape) @@ -454,7 +454,7 @@ def _nan_quantile( too_few_values = valid_values_count < 2 if too_few_values.any(): # This will result in getting the only available value if it exists - valid_values_count[too_few_values] = np.NaN + valid_values_count[too_few_values] = np.nan # --- Computation of indexes # Add axis for quantiles valid_values_count = valid_values_count[..., np.newaxis] diff --git a/xclim/ensembles/_reduce.py b/xclim/ensembles/_reduce.py index 404188fd2..fdc9821bf 100644 --- a/xclim/ensembles/_reduce.py +++ b/xclim/ensembles/_reduce.py @@ -94,7 +94,7 @@ def _make_crit(da): ( da[crd].values if crd in da.coords - else [np.NaN] * da.criteria.size + else [np.nan] * da.criteria.size ) for da in stacked.values() ], diff --git a/xclim/ensembles/_robustness.py b/xclim/ensembles/_robustness.py index 322d85e53..fcfc81893 100644 --- a/xclim/ensembles/_robustness.py +++ b/xclim/ensembles/_robustness.py @@ -477,7 +477,7 @@ def _ttest(fut, ref, *, p_change=0.05): def _ttest_func(f, r): # scipy>=1.9: popmean.axis[-1] must equal 1 for both fut and ref if np.isnan(f).all() or np.isnan(r).all(): - return np.NaN + return np.nan return spstats.ttest_1samp(f, r[..., np.newaxis], axis=-1, nan_policy="omit")[1] @@ -508,7 +508,7 @@ def _welch_ttest(fut, ref, *, p_change=0.05): # equal_var=False -> Welch's T-test def wtt_wrapper(f, r): # This specific test can't manage an all-NaN slice if np.isnan(f).all() or np.isnan(r).all(): - return np.NaN + return np.nan return spstats.ttest_ind(f, r, axis=-1, equal_var=False, nan_policy="omit")[1] pvals = xr.apply_ufunc( @@ -534,7 +534,7 @@ def _mannwhitney_utest(ref, fut, *, p_change=0.05): def mwu_wrapper(f, r): # This specific test can't manage an all-NaN slice if np.isnan(f).all() or np.isnan(r).all(): - return np.NaN + return np.nan return spstats.mannwhitneyu(f, r, axis=-1, nan_policy="omit")[1] pvals = xr.apply_ufunc( diff --git a/xclim/indices/_agro.py b/xclim/indices/_agro.py index 9bfcb0cb4..619179d44 100644 --- a/xclim/indices/_agro.py +++ b/xclim/indices/_agro.py @@ -234,7 +234,7 @@ def huglin_index( if method.lower() == "smoothed": lat_mask = abs(lat) <= 50 lat_coefficient = ((abs(lat) - 40) / 10).clip(min=0) * 0.06 - k = 1 + xarray.where(lat_mask, lat_coefficient, np.NaN) + k = 1 + xarray.where(lat_mask, lat_coefficient, np.nan) k_aggregated = 1 elif method.lower() == "icclim": k_f = [0, 0.02, 0.03, 0.04, 0.05, 0.06] @@ -255,7 +255,7 @@ def huglin_index( (46 < abs(lat)) & (abs(lat) <= 48), k_f[4], xarray.where( - (48 < abs(lat)) & (abs(lat) <= 50), k_f[5], np.NaN + (48 < abs(lat)) & (abs(lat) <= 50), k_f[5], np.nan ), ), ), @@ -1010,7 +1010,7 @@ def _get_first_run(run_positions, start_date, end_date): run_positions = select_time(run_positions, date_bounds=(start_date, end_date)) first_start = run_positions.argmax("time") return xarray.where( - first_start != run_positions.argmin("time"), first_start, np.NaN + first_start != run_positions.argmin("time"), first_start, np.nan ) # Find the start of the rain season @@ -1065,7 +1065,7 @@ def _get_rain_season(_pram): # `start != NaN` only possible if a condition on next few time steps is respected. # Thus, `start+1` exists if `start != NaN` start_ind = (start + 1).fillna(-1).astype(int) - mask = _pram * np.NaN + mask = _pram * np.nan # Put "True" on the day of run start mask[{"time": start_ind}] = 1 # Mask back points without runs, propagate the True diff --git a/xclim/indices/fire/_cffwis.py b/xclim/indices/fire/_cffwis.py index 1a58b8448..869fa571d 100644 --- a/xclim/indices/fire/_cffwis.py +++ b/xclim/indices/fire/_cffwis.py @@ -430,7 +430,7 @@ def _drought_code( # pragma: no cover if dr > 0.0: dc = dr + pe elif np.isnan(dc0): - dc = np.NaN + dc = np.nan else: dc = pe else: # f p <= 2.8: diff --git a/xclim/indices/run_length.py b/xclim/indices/run_length.py index b3015230a..ebfe6323e 100644 --- a/xclim/indices/run_length.py +++ b/xclim/indices/run_length.py @@ -746,8 +746,8 @@ def extract_events( start_runs = _cumsum_reset_on_zero(da_start, dim=dim, index="first") stop_runs = _cumsum_reset_on_zero(da_stop, dim=dim, index="first") - start_positions = xr.where(start_runs >= window_start, 1, np.NaN) - stop_positions = xr.where(stop_runs >= window_stop, 0, np.NaN) + start_positions = xr.where(start_runs >= window_start, 1, np.nan) + stop_positions = xr.where(stop_runs >= window_stop, 0, np.nan) # start positions (1) are f-filled until a stop position (0) is met runs = stop_positions.combine_first(start_positions).ffill(dim=dim).fillna(0) @@ -1143,7 +1143,7 @@ def statistics_run_1d(arr: Sequence[bool], reducer: str, window: int) -> int: if not np.any(v) or np.all(v * rl < window): return 0 func = getattr(np, f"nan{reducer}") - return func(np.where(v * rl >= window, rl, np.NaN)) + return func(np.where(v * rl >= window, rl, np.nan)) def windowed_run_count_1d(arr: Sequence[bool], window: int) -> int: diff --git a/xclim/sdba/_adjustment.py b/xclim/sdba/_adjustment.py index 31b9cd874..6d0a67611 100644 --- a/xclim/sdba/_adjustment.py +++ b/xclim/sdba/_adjustment.py @@ -426,7 +426,7 @@ def npdf_transform(ds: xr.Dataset, **kwargs) -> xr.Dataset: # All NaN, but with the proper shape. escores = ( ref.isel({dim: 0, "time": 0}) * hist.isel({dim: 0, "time": 0}) - ).expand_dims(iterations=ds.iterations) * np.NaN + ).expand_dims(iterations=ds.iterations) * np.nan return xr.Dataset( data_vars={ @@ -480,8 +480,8 @@ def _extremes_train_1d(ref, hist, ref_params, *, q_thresh, cluster_thresh, dist, af = hist_in_ref / hist[Pcommon] # sort them in Px order, and pad to have N values. order = np.argsort(Px_hist) - px_hist = np.pad(Px_hist[order], ((0, N - af.size),), constant_values=np.NaN) - af = np.pad(af[order], ((0, N - af.size),), constant_values=np.NaN) + px_hist = np.pad(Px_hist[order], ((0, N - af.size),), constant_values=np.nan) + af = np.pad(af[order], ((0, N - af.size),), constant_values=np.nan) return px_hist, af, thresh @@ -524,7 +524,7 @@ def extremes_train( _extremes_train_1d, ds.ref, ds.hist, - ds.ref_params or np.NaN, + ds.ref_params or np.nan, input_core_dims=[("time",), ("time",), ()], output_core_dims=[("quantiles",), ("quantiles",), ()], vectorize=True, diff --git a/xclim/sdba/adjustment.py b/xclim/sdba/adjustment.py index a01fc81e5..d65a9d9be 100644 --- a/xclim/sdba/adjustment.py +++ b/xclim/sdba/adjustment.py @@ -682,7 +682,7 @@ def _train( { "ref": ref, "hist": hist, - "ref_params": ref_params or np.float32(np.NaN), + "ref_params": ref_params or np.float32(np.nan), } ), q_thresh=q_thresh, @@ -1171,8 +1171,8 @@ def _adjust( template = xr.Dataset( data_vars={ - "scenh": xr.full_like(hist, np.NaN).rename(time="time_hist"), - "scen": xr.full_like(sim, np.NaN), + "scenh": xr.full_like(hist, np.nan).rename(time="time_hist"), + "scen": xr.full_like(sim, np.nan), "escores": escores_tmpl, } ) diff --git a/xclim/sdba/loess.py b/xclim/sdba/loess.py index abb15480e..f20974efd 100644 --- a/xclim/sdba/loess.py +++ b/xclim/sdba/loess.py @@ -92,7 +92,7 @@ def _loess_nb( """ if skipna: nan = np.isnan(y) - out = np.full(x.size, np.NaN) + out = np.full(x.size, np.nan) y = y[~nan] x = x[~nan] if x.size == 0: diff --git a/xclim/sdba/nbutils.py b/xclim/sdba/nbutils.py index aa87fd0a3..45613dd9e 100644 --- a/xclim/sdba/nbutils.py +++ b/xclim/sdba/nbutils.py @@ -21,7 +21,7 @@ ) def _vecquantiles(arr, rnk, res): if np.isnan(rnk): - res[0] = np.NaN + res[0] = np.nan else: res[0] = np.nanquantile(arr, rnk) @@ -213,7 +213,7 @@ def _first_and_last_nonnull(arr): if idxs.size > 0: out[i] = arr[i][idxs[np.array([0, -1])]] else: - out[i] = np.array([np.NaN, np.NaN]) + out[i] = np.array([np.nan, np.nan]) return out @@ -234,8 +234,8 @@ def _extrapolate_on_quantiles(interp, oldx, oldg, oldy, newx, newg, method="cons interp[toolow] = cnstlow[toolow] interp[toohigh] = cnsthigh[toohigh] else: # 'nan' - interp[toolow] = np.NaN - interp[toohigh] = np.NaN + interp[toolow] = np.nan + interp[toohigh] = np.nan return interp diff --git a/xclim/sdba/utils.py b/xclim/sdba/utils.py index 89f451fba..e42958c55 100644 --- a/xclim/sdba/utils.py +++ b/xclim/sdba/utils.py @@ -313,7 +313,7 @@ def add_cyclic_bounds( def _interp_on_quantiles_1D(newx, oldx, oldy, method, extrap): # noqa: N802 mask_new = np.isnan(newx) mask_old = np.isnan(oldy) | np.isnan(oldx) - out = np.full_like(newx, np.NaN, dtype=f"float{oldy.dtype.itemsize * 8}") + out = np.full_like(newx, np.nan, dtype=f"float{oldy.dtype.itemsize * 8}") if np.all(mask_new) or np.all(mask_old): warn( "All-NaN slice encountered in interp_on_quantiles", @@ -327,7 +327,7 @@ def _interp_on_quantiles_1D(newx, oldx, oldy, method, extrap): # noqa: N802 oldy[~np.isnan(oldy)][-1], ) else: # extrap == 'nan' - fill_value = np.NaN + fill_value = np.nan out[~mask_new] = interp1d( oldx[~mask_old], @@ -342,7 +342,7 @@ def _interp_on_quantiles_1D(newx, oldx, oldy, method, extrap): # noqa: N802 def _interp_on_quantiles_2D(newx, newg, oldx, oldy, oldg, method, extrap): # noqa mask_new = np.isnan(newx) | np.isnan(newg) mask_old = np.isnan(oldy) | np.isnan(oldx) | np.isnan(oldg) - out = np.full_like(newx, np.NaN, dtype=f"float{oldy.dtype.itemsize * 8}") + out = np.full_like(newx, np.nan, dtype=f"float{oldy.dtype.itemsize * 8}") if np.all(mask_new) or np.all(mask_old): warn( "All-NaN slice encountered in interp_on_quantiles", @@ -766,7 +766,7 @@ def _get_clusters(arr, u1, u2, N): np.append(st, pad), np.append(ed, pad), np.append(mp, pad), - np.append(mv, [np.NaN] * (N - count)), + np.append(mv, [np.nan] * (N - count)), count, ) From 06d18ca1ae5a90b85c4725f104d7dd4b3b581b7a Mon Sep 17 00:00:00 2001 From: Pascal Bourgault Date: Fri, 28 Jun 2024 18:08:11 -0400 Subject: [PATCH 02/33] Fix tests for numpy 2 --- tests/test_ensembles.py | 8 +++++++- tests/test_indices.py | 2 +- tests/test_precip.py | 4 ++-- 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/tests/test_ensembles.py b/tests/test_ensembles.py index cbe271a7b..4af3db33d 100644 --- a/tests/test_ensembles.py +++ b/tests/test_ensembles.py @@ -288,8 +288,14 @@ def test_calc_mean_std_min_max(self, ensemble_dataset_objects, open_dataset): ) out2 = ensembles.ensemble_mean_std_max_min(ens, weights=weights) values = ens["tg_mean"][:, 0, 5, 5] + # Explicit float64 so numpy does the expected datatype promotion (change in numpy 2) np.testing.assert_array_equal( - (values[0] * 1 + values[1] * 0.1 + values[2] * 3.5 + values[3] * 5) + ( + values[0] * np.float64(1) + + values[1] * np.float64(0.1) + + values[2] * np.float64(3.5) + + values[3] * np.float64(5) + ) / np.sum(weights), out2.tg_mean_mean[0, 5, 5], ) diff --git a/tests/test_indices.py b/tests/test_indices.py index e381897f7..f054e1771 100644 --- a/tests/test_indices.py +++ b/tests/test_indices.py @@ -231,7 +231,7 @@ def test_no_cdd(self, tas_series): a = tas_series(np.array([10, 15, -5, 18]) + K2C) cdd = xci.cooling_degree_days(a) assert cdd == 0 - assert cdd.units == "K d" + assert cdd.units == "d K" def test_cdd(self, tas_series): a = tas_series(np.array([20, 25, -15, 19]) + K2C) diff --git a/tests/test_precip.py b/tests/test_precip.py index e8243a8b7..813f0f304 100644 --- a/tests/test_precip.py +++ b/tests/test_precip.py @@ -77,8 +77,8 @@ def test_3d_data_with_nans(self, open_dataset): pr.attrs["units"] = "kg m-2 s-1" out3 = atmos.precip_accumulation(pr, freq="MS") - np.testing.assert_array_almost_equal(out1, out2, 3) - np.testing.assert_array_almost_equal(out1, out3, 5) + np.testing.assert_allclose(out1, out2, rtol=1e-7, atol=1e-4) + np.testing.assert_allclose(out1, out3, rtol=1e-7, atol=1e-4) # check some vector with and without a nan x1 = prMM[:31, 0, 0].values From 7e9107eee9c61f37128569bab445f4882e5f2f64 Mon Sep 17 00:00:00 2001 From: Pascal Bourgault Date: Fri, 28 Jun 2024 18:09:57 -0400 Subject: [PATCH 03/33] remove pint pin --- environment.yml | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/environment.yml b/environment.yml index ae47afc66..bc527a6ad 100644 --- a/environment.yml +++ b/environment.yml @@ -14,7 +14,7 @@ dependencies: - numba - numpy >=1.20.0 - pandas >=2.2.0 - - pint >=0.10,<0.24 + - pint >=0.18 - poppler >=0.67 - pyarrow # Strongly encouraged for Pandas v2.2.0+ - pyyaml diff --git a/pyproject.toml b/pyproject.toml index 97096807c..d01985579 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -42,7 +42,7 @@ dependencies = [ "numba", "numpy>=1.20.0", "pandas>=2.2", - "pint>=0.10,<0.24", + "pint>=0.18", "platformdirs >=3.2", "pyarrow", # Strongly encouraged for pandas v2.2.0+ "pyyaml", From e3b98517b8d28375157d2aa38f05e50ee8cd0339 Mon Sep 17 00:00:00 2001 From: Pascal Bourgault Date: Fri, 28 Jun 2024 18:15:24 -0400 Subject: [PATCH 04/33] upd changes --- CHANGES.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGES.rst b/CHANGES.rst index 99c165ec3..a776936e1 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -14,6 +14,7 @@ New features and enhancements Bug fixes ^^^^^^^^^ * Clarified a typo in the docstring formula for `xclim.indices.growing_season_length`. (:pull:`1796`). +* ``nan`` and ``inf`` syntax adapted for numpy 2.0. (:pull:`1814`, :issue:`1785`). Internal changes ^^^^^^^^^^^^^^^^ From 73302216acb94e85e7b5cb8a9015d8097fd8899e Mon Sep 17 00:00:00 2001 From: Pascal Bourgault Date: Mon, 8 Jul 2024 14:37:02 -0400 Subject: [PATCH 05/33] upd pin for cf-xarray --- environment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/environment.yml b/environment.yml index bc527a6ad..6c66d45f4 100644 --- a/environment.yml +++ b/environment.yml @@ -6,7 +6,7 @@ dependencies: - python >=3.9 - boltons >=20.1 - bottleneck >=1.3.1 - - cf_xarray >=0.6.1 + - cf_xarray >=0.9.3 - cftime >=1.4.1 - click >=8.1 - dask >=2.6.0 From c14d0b944bdca09a34b80b8d34e4e336e193043b Mon Sep 17 00:00:00 2001 From: Pascal Bourgault Date: Mon, 8 Jul 2024 16:17:50 -0400 Subject: [PATCH 06/33] Pin cf_xarray, pint and numpy - adapt to "1" change --- CHANGELOG.rst | 7 ++++++- environment.yml | 4 ++-- tests/test_atmos.py | 2 +- tests/test_generic.py | 2 +- tests/test_hydrology.py | 2 +- tests/test_indices.py | 36 ++++++++++++++++++------------------ tests/test_land.py | 6 +++--- tests/test_snow.py | 8 ++++---- tests/test_units.py | 6 +++--- xclim/core/units.py | 14 +++----------- 10 files changed, 42 insertions(+), 45 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 4177b371b..db7e7dee6 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -4,7 +4,12 @@ Changelog v0.52.0 (unreleased) -------------------- -Contributors to this version: David Huard (:user:`huard`). +Contributors to this version: David Huard (:user:`huard`), Pascal Bourgault (:user:`aulemahal`). + +Breaking changes +^^^^^^^^^^^^^^^^ +* Dimensionless quantities now use the "1" units attribute as specified by the CF conventions, previously an empty string was returned. (:pull:`1814`). +* Updated minimum versions for some dependencies: ``numpy>=2``, ``cf-xarray>=0.9.3`` and ``pint>=0.24.1``. (:pull:`1814`). Internal changes ^^^^^^^^^^^^^^^^ diff --git a/environment.yml b/environment.yml index 6c66d45f4..453002aca 100644 --- a/environment.yml +++ b/environment.yml @@ -12,9 +12,9 @@ dependencies: - dask >=2.6.0 - jsonpickle - numba - - numpy >=1.20.0 + - numpy >=2 - pandas >=2.2.0 - - pint >=0.18 + - pint >=0.24.1 - poppler >=0.67 - pyarrow # Strongly encouraged for Pandas v2.2.0+ - pyyaml diff --git a/tests/test_atmos.py b/tests/test_atmos.py index adbd7b224..10d5d0efe 100644 --- a/tests/test_atmos.py +++ b/tests/test_atmos.py @@ -260,7 +260,7 @@ def test_wind_profile(atmosds): def test_wind_power_potential(atmosds): out = atmos.wind_power_potential(wind_speed=atmosds.sfcWind) - assert out.attrs["units"] == "" + assert out.attrs["units"] == "1" assert (out >= 0).all() assert (out <= 1).all() diff --git a/tests/test_generic.py b/tests/test_generic.py index 1f6869937..23301362d 100644 --- a/tests/test_generic.py +++ b/tests/test_generic.py @@ -101,7 +101,7 @@ def test_doyminmax(self, q_series): for da in [dmx, dmn]: for attr in ["units", "is_dayofyear", "calendar"]: assert attr in da.attrs.keys() - assert da.attrs["units"] == "" + assert da.attrs["units"] == "1" assert da.attrs["is_dayofyear"] == 1 diff --git a/tests/test_hydrology.py b/tests/test_hydrology.py index e31c64ffc..1a10534e3 100644 --- a/tests/test_hydrology.py +++ b/tests/test_hydrology.py @@ -40,7 +40,7 @@ def test_simple(self, snw_series): snw = snw_series(a, start="1999-01-01") out = xci.snw_max_doy(snw, freq="YS") np.testing.assert_array_equal(out, [11, np.nan]) - assert out.units == "" + assert out.units == "1" class TestSnowMeltWEMax: diff --git a/tests/test_indices.py b/tests/test_indices.py index f054e1771..bae179601 100644 --- a/tests/test_indices.py +++ b/tests/test_indices.py @@ -137,11 +137,11 @@ def test_simple(self, tas_series): out = xci.cold_spell_frequency(da, thresh="-10. C", freq="ME") np.testing.assert_array_equal(out, [1, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0]) - assert out.units == "" + assert out.units == "1" out = xci.cold_spell_frequency(da, thresh="-10. C", freq="YS") np.testing.assert_array_equal(out, 3) - assert out.units == "" + assert out.units == "1" class TestColdSpellMaxLength: @@ -885,7 +885,7 @@ def test_simple(self, tas_series): assert lsf == 180 for attr in ["units", "is_dayofyear", "calendar"]: assert attr in lsf.attrs.keys() - assert lsf.attrs["units"] == "" + assert lsf.attrs["units"] == "1" assert lsf.attrs["is_dayofyear"] == 1 assert lsf.attrs["is_dayofyear"].dtype == np.int32 @@ -906,7 +906,7 @@ def test_simple(self, tas_series): assert np.isnan(fdb) for attr in ["units", "is_dayofyear", "calendar"]: assert attr in fdb.attrs.keys() - assert fdb.attrs["units"] == "" + assert fdb.attrs["units"] == "1" assert fdb.attrs["is_dayofyear"] == 1 def test_below_forbidden(self, tasmax_series): @@ -937,7 +937,7 @@ def test_simple(self, tas_series): assert np.isnan(fda) for attr in ["units", "is_dayofyear", "calendar"]: assert attr in fda.attrs.keys() - assert fda.attrs["units"] == "" + assert fda.attrs["units"] == "1" assert fda.attrs["is_dayofyear"] == 1 def test_thresholds(self, tas_series): @@ -962,7 +962,7 @@ def test_thresholds(self, tas_series): assert out[0] == tg.indexes["time"][30].dayofyear for attr in ["units", "is_dayofyear", "calendar"]: assert attr in out.attrs.keys() - assert out.attrs["units"] == "" + assert out.attrs["units"] == "1" assert out.attrs["is_dayofyear"] == 1 def test_above_forbidden(self, tasmax_series): @@ -1049,7 +1049,7 @@ def test_simple(self, tas_series): assert out[0] == tg.indexes["time"][20].dayofyear for attr in ["units", "is_dayofyear", "calendar"]: assert attr in out.attrs.keys() - assert out.attrs["units"] == "" + assert out.attrs["units"] == "1" assert out.attrs["is_dayofyear"] == 1 def test_no_start(self, tas_series): @@ -1082,7 +1082,7 @@ def test_varying_mid_dates(self, tas_series, d1, d2, mid_date, expected): np.testing.assert_array_equal(gs_end, expected) for attr in ["units", "is_dayofyear", "calendar"]: assert attr in gs_end.attrs.keys() - assert gs_end.attrs["units"] == "" + assert gs_end.attrs["units"] == "1" assert gs_end.attrs["is_dayofyear"] == 1 @@ -1162,7 +1162,7 @@ def test_simple(self, tasmin_series): assert out[0] == tn.indexes["time"][20].dayofyear for attr in ["units", "is_dayofyear", "calendar"]: assert attr in out.attrs.keys() - assert out.attrs["units"] == "" + assert out.attrs["units"] == "1" assert out.attrs["is_dayofyear"] == 1 def test_no_start(self, tasmin_series): @@ -1195,7 +1195,7 @@ def test_varying_mid_dates(self, tasmin_series, d1, d2, mid_date, expected): np.testing.assert_array_equal(gs_end, expected) for attr in ["units", "is_dayofyear", "calendar"]: assert attr in gs_end.attrs.keys() - assert gs_end.attrs["units"] == "" + assert gs_end.attrs["units"] == "1" assert gs_end.attrs["is_dayofyear"] == 1 @@ -2704,7 +2704,7 @@ def test_degree_days_exceedance_date(tas_series): for attr in ["units", "is_dayofyear", "calendar"]: assert attr in out.attrs.keys() - assert out.attrs["units"] == "" + assert out.attrs["units"] == "1" assert out.attrs["is_dayofyear"] == 1 @@ -2744,7 +2744,7 @@ def test_first_snowfall(prsn_series, prsnd_series): assert out[0] == 166 for attr in ["units", "is_dayofyear", "calendar"]: assert attr in out.attrs.keys() - assert out.attrs["units"] == "" + assert out.attrs["units"] == "1" assert out.attrs["is_dayofyear"] == 1 # test with prsnd [m s-1] @@ -2753,7 +2753,7 @@ def test_first_snowfall(prsn_series, prsnd_series): assert out[0] == 166 for attr in ["units", "is_dayofyear", "calendar"]: assert attr in out.attrs.keys() - assert out.attrs["units"] == "" + assert out.attrs["units"] == "1" assert out.attrs["is_dayofyear"] == 1 # test with prsn [kg m-2 s-1] @@ -2765,7 +2765,7 @@ def test_first_snowfall(prsn_series, prsnd_series): assert out[0] == 166 for attr in ["units", "is_dayofyear", "calendar"]: assert attr in out.attrs.keys() - assert out.attrs["units"] == "" + assert out.attrs["units"] == "1" assert out.attrs["is_dayofyear"] == 1 @@ -2902,7 +2902,7 @@ def test_continous_snow_season_start(self, snd_series, snw_series): np.testing.assert_array_equal(out, [snd.time.dt.dayofyear[0].data + 2, np.nan]) for attr in ["units", "is_dayofyear", "calendar"]: assert attr in out.attrs.keys() - assert out.attrs["units"] == "" + assert out.attrs["units"] == "1" assert out.attrs["is_dayofyear"] == 1 out = xci.snw_season_start(snw) @@ -2910,7 +2910,7 @@ def test_continous_snow_season_start(self, snd_series, snw_series): np.testing.assert_array_equal(out, [snw.time.dt.dayofyear[0].data + 1, np.nan]) for attr in ["units", "is_dayofyear", "calendar"]: assert attr in out.attrs.keys() - assert out.attrs["units"] == "" + assert out.attrs["units"] == "1" assert out.attrs["is_dayofyear"] == 1 def test_snow_season_end(self, snd_series, snw_series): @@ -2932,7 +2932,7 @@ def test_snow_season_end(self, snd_series, snw_series): np.testing.assert_array_equal(out, [(doy + 219) % 366, np.nan]) for attr in ["units", "is_dayofyear", "calendar"]: assert attr in out.attrs.keys() - assert out.attrs["units"] == "" + assert out.attrs["units"] == "1" assert out.attrs["is_dayofyear"] == 1 out = xci.snw_season_end(snw) @@ -2941,7 +2941,7 @@ def test_snow_season_end(self, snd_series, snw_series): np.testing.assert_array_equal(out, [(doy + 219) % 366, np.nan]) for attr in ["units", "is_dayofyear", "calendar"]: assert attr in out.attrs.keys() - assert out.attrs["units"] == "" + assert out.attrs["units"] == "1" assert out.attrs["is_dayofyear"] == 1 diff --git a/tests/test_land.py b/tests/test_land.py index 4418aa2fc..a750146fa 100644 --- a/tests/test_land.py +++ b/tests/test_land.py @@ -10,19 +10,19 @@ def test_base_flow_index(ndq_series): out = land.base_flow_index(ndq_series, freq="YS") - assert out.attrs["units"] == "" + assert out.attrs["units"] == "1" assert isinstance(out, xr.DataArray) def test_rb_flashiness_index(ndq_series): out = land.base_flow_index(ndq_series, freq="YS") - assert out.attrs["units"] == "" + assert out.attrs["units"] == "1" assert isinstance(out, xr.DataArray) def test_qdoy_max(ndq_series, q_series): out = land.doy_qmax(ndq_series, freq="YS", season="JJA") - assert out.attrs["units"] == "" + assert out.attrs["units"] == "1" a = np.ones(450) a[100] = 2 diff --git a/tests/test_snow.py b/tests/test_snow.py index 9d6a1656b..e1174c357 100644 --- a/tests/test_snow.py +++ b/tests/test_snow.py @@ -45,11 +45,11 @@ def test_simple(self, snd_series): snd = snd.expand_dims(lat=[0, 1, 2]) out = land.snd_season_start(snd) - assert out.units == "" + assert out.units == "1" np.testing.assert_array_equal(out.isel(lat=0), snd.time.dt.dayofyear[100]) out = land.snd_season_end(snd) - assert out.units == "" + assert out.units == "1" np.testing.assert_array_equal(out.isel(lat=0), snd.time.dt.dayofyear[200]) out = land.snd_season_length(snd) @@ -67,11 +67,11 @@ def test_simple(self, snw_series): snw = snw.expand_dims(lat=[0, 1, 2]) out = land.snw_season_start(snw) - assert out.units == "" + assert out.units == "1" np.testing.assert_array_equal(out.isel(lat=0), snw.time.dt.dayofyear[100]) out = land.snw_season_end(snw) - assert out.units == "" + assert out.units == "1" np.testing.assert_array_equal(out.isel(lat=0), snw.time.dt.dayofyear[200]) out = land.snw_season_length(snw) diff --git a/tests/test_units.py b/tests/test_units.py index b59626fde..f585bf475 100644 --- a/tests/test_units.py +++ b/tests/test_units.py @@ -140,7 +140,7 @@ def test_units2pint(self, pr_series): assert pint2cfunits(u) == "%" u = units2pint("1") - assert pint2cfunits(u) == "" + assert pint2cfunits(u) == "1" def test_pint_multiply(self, pr_series): a = pr_series([1, 2, 3]) @@ -331,8 +331,8 @@ def index( ("", "sum", "count", 365, "d"), ("", "sum", "count", 365, "d"), ("kg m-2", "var", "var", 0, "kg2 m-4"), - ("°C", "argmax", "doymax", 0, ""), - ("°C", "sum", "integral", 365, "K d"), + ("°C", "argmax", "doymax", 0, "1"), + ("°C", "sum", "integral", 365, "d K"), ("°F", "sum", "integral", 365, "d °R"), # not sure why the order is different ], ) diff --git a/xclim/core/units.py b/xclim/core/units.py index 1c6ec4b62..245caae71 100644 --- a/xclim/core/units.py +++ b/xclim/core/units.py @@ -183,11 +183,7 @@ def pint2cfunits(value: units.Quantity | units.Unit) -> str: if isinstance(value, (pint.Quantity, units.Quantity)): value = value.units - # Issue originally introduced in https://github.com/hgrecco/pint/issues/1486 - # Should be resolved in pint v0.24. See: https://github.com/hgrecco/pint/issues/1913 - with warnings.catch_warnings(): - warnings.simplefilter("ignore", category=DeprecationWarning) - return f"{value:cf}".replace("dimensionless", "") + return f"{value:cf}" def ensure_cf_units(ustr: str) -> str: @@ -558,7 +554,7 @@ def to_agg_units( elif op in ["doymin", "doymax"]: out.attrs.update( - units="", is_dayofyear=np.int32(1), calendar=get_calendar(orig) + units="1", is_dayofyear=np.int32(1), calendar=get_calendar(orig) ) elif op in ["count", "integral"]: @@ -1219,11 +1215,7 @@ def wrapper(*args, **kwargs): context = None for ref, refvar in bound_args.arguments.items(): if f"<{ref}>" in dim: - # Issue originally introduced in https://github.com/hgrecco/pint/issues/1486 - # Should be resolved in pint v0.24. See: https://github.com/hgrecco/pint/issues/1913 - with warnings.catch_warnings(): - warnings.simplefilter("ignore", category=DeprecationWarning) - dim = dim.replace(f"<{ref}>", f"({units2pint(refvar)})") + dim = dim.replace(f"<{ref}>", f"({units2pint(refvar)})") # check_units will guess the hydro context if "precipitation" appears in dim, # but here we pass a real unit. It will also check the standard name of the arg, From da22473693f4d144487f17790bafa9d24397d050 Mon Sep 17 00:00:00 2001 From: Pascal Bourgault Date: Mon, 8 Jul 2024 16:22:34 -0400 Subject: [PATCH 07/33] Add ensure_absolute_tempetature to __all__ and refactor to put both similar functions together --- xclim/core/units.py | 57 ++++++++++++++++++++++++++------------------- 1 file changed, 33 insertions(+), 24 deletions(-) diff --git a/xclim/core/units.py b/xclim/core/units.py index 245caae71..f02c0f442 100644 --- a/xclim/core/units.py +++ b/xclim/core/units.py @@ -35,6 +35,7 @@ "convert_units_to", "declare_relative_units", "declare_units", + "ensure_absolute_temperature", "ensure_cf_units", "ensure_delta", "flux2rate", @@ -465,10 +466,14 @@ def infer_sampling_units( } -def ensure_absolute_temperature(units: str): +def ensure_absolute_temperature(units: str) -> str: """Convert temperature units to their absolute counterpart, assuming they represented a difference (delta). Celsius becomes Kelvin, Fahrenheit becomes Rankine. Does nothing for other units. + + See Also + -------- + :py:func:`ensure_delta` """ a = str2pint(units) # ensure a delta pint unit @@ -478,6 +483,33 @@ def ensure_absolute_temperature(units: str): return units +def ensure_delta(unit: str) -> str: + """Return delta units for temperature. + + For dimensions where delta exist in pint (Temperature), it replaces the temperature unit by delta_degC or + delta_degF based on the input unit. For other dimensionality, it just gives back the input units. + + Parameters + ---------- + unit : str + unit to transform in delta (or not) + + See Also + -------- + :py:func:`ensure_absolute_temperature` + """ + u = units2pint(unit) + d = 1 * u + # + delta_unit = pint2cfunits(d - d) + # replace kelvin/rankine by delta_degC/F + if "kelvin" in u._units: + delta_unit = pint2cfunits(u / units2pint("K") * units2pint("delta_degC")) + if "degree_Rankine" in u._units: + delta_unit = pint2cfunits(u / units2pint("°R") * units2pint("delta_degF")) + return delta_unit + + def to_agg_units( out: xr.DataArray, orig: xr.DataArray, op: str, dim: str = "time" ) -> xr.DataArray: @@ -1320,29 +1352,6 @@ def wrapper(*args, **kwargs): return dec -def ensure_delta(unit: str) -> str: - """Return delta units for temperature. - - For dimensions where delta exist in pint (Temperature), it replaces the temperature unit by delta_degC or - delta_degF based on the input unit. For other dimensionality, it just gives back the input units. - - Parameters - ---------- - unit : str - unit to transform in delta (or not) - """ - u = units2pint(unit) - d = 1 * u - # - delta_unit = pint2cfunits(d - d) - # replace kelvin/rankine by delta_degC/F - if "kelvin" in u._units: - delta_unit = pint2cfunits(u / units2pint("K") * units2pint("delta_degC")) - if "degree_Rankine" in u._units: - delta_unit = pint2cfunits(u / units2pint("°R") * units2pint("delta_degF")) - return delta_unit - - def infer_context( standard_name: str | None = None, dimension: str | None = None ) -> str: From 14c1f59dfdeed55fc9643582823442910fb77e79 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89ric=20Dupuis?= Date: Wed, 10 Jul 2024 11:20:14 -0400 Subject: [PATCH 08/33] new spi test values (due to np2's float32) --- tests/test_indices.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/test_indices.py b/tests/test_indices.py index bae179601..62e4aea3f 100644 --- a/tests/test_indices.py +++ b/tests/test_indices.py @@ -18,8 +18,8 @@ import numpy as np import pandas as pd import pytest -import xarray as xr +import xarray as xr from xclim import indices as xci from xclim.core.calendar import percentile_doy from xclim.core.options import set_options @@ -530,7 +530,7 @@ class TestStandardizedIndices: 1, "gamma", "ML", - [-0.011698, 1.597031, 0.969714, 0.265561, -0.132654], + [-0.083785, 1.457647, 0.993296, 0.271894, -0.449684], 2e-2, ), ( @@ -538,7 +538,7 @@ class TestStandardizedIndices: 12, "gamma", "ML", - [-0.158116, -0.049222, 0.672544, 1.08332, 0.660903], + [-0.158854, -0.049165, 0.675863, 0.960247, 0.660831], 2e-2, ), ( @@ -546,7 +546,7 @@ class TestStandardizedIndices: 1, "fisk", "ML", - [-0.158949, 1.308225, 0.449846, 0.146699, -0.502737], + [-0.194235, 1.308198, 0.530768, 0.22234, -0.502635], 2e-2, ), ( From 7be1d57f5d14879e7ed660872059ee4b7012442a Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 10 Jul 2024 15:28:47 +0000 Subject: [PATCH 09/33] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tests/test_indices.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_indices.py b/tests/test_indices.py index 62e4aea3f..c3768d344 100644 --- a/tests/test_indices.py +++ b/tests/test_indices.py @@ -18,8 +18,8 @@ import numpy as np import pandas as pd import pytest - import xarray as xr + from xclim import indices as xci from xclim.core.calendar import percentile_doy from xclim.core.options import set_options From d956c3f7e3cf7f247ff3de14a18f3d9e6e7494e8 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 26 Jul 2024 16:51:07 +0000 Subject: [PATCH 10/33] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- xclim/sdba/utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/xclim/sdba/utils.py b/xclim/sdba/utils.py index c4da2f844..09827f095 100644 --- a/xclim/sdba/utils.py +++ b/xclim/sdba/utils.py @@ -362,7 +362,7 @@ def _interp_on_quantiles_1D(newx, oldx, oldy, method, extrap): # noqa: N802 ) else: # extrap == 'nan' fill_value = np.nan - + out[~mask_new] = interp1d( oldx[~mask_old], oldy[~mask_old], @@ -370,7 +370,7 @@ def _interp_on_quantiles_1D(newx, oldx, oldy, method, extrap): # noqa: N802 bounds_error=False, fill_value=fill_value, )(newx[~mask_new]) - + return out From 97d653b1135053319a75645f88156ff23694bc6a Mon Sep 17 00:00:00 2001 From: Trevor James Smith <10819524+Zeitsperre@users.noreply.github.com> Date: Fri, 26 Jul 2024 14:21:02 -0400 Subject: [PATCH 11/33] fix changelog --- CHANGELOG.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index fe36e9b69..db69229b0 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -14,12 +14,13 @@ New features and enhancements Breaking changes ^^^^^^^^^^^^^^^^ * Dimensionless quantities now use the "1" units attribute as specified by the CF conventions, previously an empty string was returned. (:pull:`1814`). -* Updated minimum versions for some dependencies: ``numpy>=2``, ``cf-xarray>=0.9.3`` and ``pint>=0.24.1``. (:pull:`1814`). +* Updated minimum versions for some dependencies: ``numpy>=2.0.0``, ``cf-xarray>=0.9.3`` and ``pint>=0.24.1``. (:pull:`1814`). Bug fixes ^^^^^^^^^ * Fixed the indexer bug in the ``xclim.indices.standardized_index_fit_params`` when multiple or non-array indexers are specified and fitted parameters are reloaded from netCDF. (:issue:`1842`, :pull:`1843`). * Addressed a bug found in ``wet_spell_*`` indicators that was contributing to erroneous results. A new generic spell length statistic function ``xclim.indices.generic.spell_length_statistics`` is now used in wet and dry spells indicators. (:issue:`1834`, :pull:`1838`). +* Syntax for ``nan`` and ``inf`` was adapted to support ``numpy>=2.0.0``. (:pull:`1814`, :issue:`1785`). Internal changes ^^^^^^^^^^^^^^^^ @@ -44,7 +45,6 @@ Bug fixes ^^^^^^^^^ * Units of degree-days computations with Fahrenheit input fixed to yield "°R d". Added a new ``xclim.core.units.ensure_absolute_temperature`` method to convert from delta to absolute temperatures. (:issue:`1789`, :pull:`1804`). * Clarified a typo in the docstring formula for `xclim.indices.growing_season_length`. (:pull:`1796`). -* ``nan`` and ``inf`` syntax adapted for numpy 2.0. (:pull:`1814`, :issue:`1785`). Internal changes ^^^^^^^^^^^^^^^^ From 60a186cd6f82d24d495a84076c73f4834608dca0 Mon Sep 17 00:00:00 2001 From: Trevor James Smith <10819524+Zeitsperre@users.noreply.github.com> Date: Fri, 26 Jul 2024 14:21:28 -0400 Subject: [PATCH 12/33] revert pins --- environment.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/environment.yml b/environment.yml index 453002aca..ca2d6ce62 100644 --- a/environment.yml +++ b/environment.yml @@ -6,15 +6,15 @@ dependencies: - python >=3.9 - boltons >=20.1 - bottleneck >=1.3.1 - - cf_xarray >=0.9.3 + - cf_xarray >=0.6.1 - cftime >=1.4.1 - click >=8.1 - dask >=2.6.0 - jsonpickle - numba - - numpy >=2 + - numpy >=1.20.0 - pandas >=2.2.0 - - pint >=0.24.1 + - pint >=0.18.0 - poppler >=0.67 - pyarrow # Strongly encouraged for Pandas v2.2.0+ - pyyaml From 0f289e82f179642efb16a5bddddfdd852ca10d0b Mon Sep 17 00:00:00 2001 From: Trevor James Smith <10819524+Zeitsperre@users.noreply.github.com> Date: Fri, 26 Jul 2024 14:22:06 -0400 Subject: [PATCH 13/33] WIP - conditional tests --- tests/test_atmos.py | 8 ++- tests/test_generic.py | 8 ++- tests/test_hydrology.py | 8 ++- tests/test_indices.py | 121 ++++++++++++++++++++++++++++++++++------ tests/test_land.py | 25 ++++++++- tests/test_snow.py | 34 +++++++++-- tests/test_units.py | 27 ++++++++- 7 files changed, 202 insertions(+), 29 deletions(-) diff --git a/tests/test_atmos.py b/tests/test_atmos.py index 10d5d0efe..f7f99ee4a 100644 --- a/tests/test_atmos.py +++ b/tests/test_atmos.py @@ -4,6 +4,8 @@ import numpy as np import xarray as xr +from numpy import __version__ as __numpy_version__ +from packaging.version import Parse from xclim import atmos, set_options @@ -260,7 +262,11 @@ def test_wind_profile(atmosds): def test_wind_power_potential(atmosds): out = atmos.wind_power_potential(wind_speed=atmosds.sfcWind) - assert out.attrs["units"] == "1" + # FIXME: Confirm these expected outputs + if Parse(__numpy_version__) < Parse("2.0.0"): + assert out.attrs["units"] == "" + else: + assert out.attrs["units"] == "dimensionless" assert (out >= 0).all() assert (out <= 1).all() diff --git a/tests/test_generic.py b/tests/test_generic.py index 23301362d..d1b5e57cb 100644 --- a/tests/test_generic.py +++ b/tests/test_generic.py @@ -5,6 +5,8 @@ import numpy as np import pytest import xarray as xr +from numpy import __version__ as __numpy_version__ +from packaging.version import Parse from xclim.core.calendar import doy_to_days_since, select_time from xclim.indices import generic @@ -101,7 +103,11 @@ def test_doyminmax(self, q_series): for da in [dmx, dmn]: for attr in ["units", "is_dayofyear", "calendar"]: assert attr in da.attrs.keys() - assert da.attrs["units"] == "1" + # FIXME: Confirm these expected outputs + if Parse(__numpy_version__) < Parse("2.0.0"): + assert da.attrs["units"] == "" + else: + assert da.attrs["units"] == "1" assert da.attrs["is_dayofyear"] == 1 diff --git a/tests/test_hydrology.py b/tests/test_hydrology.py index 1a10534e3..c20322deb 100644 --- a/tests/test_hydrology.py +++ b/tests/test_hydrology.py @@ -1,6 +1,8 @@ from __future__ import annotations import numpy as np +from numpy import __version__ as __numpy_version__ +from packaging.version import Parse from xclim import indices as xci @@ -40,7 +42,11 @@ def test_simple(self, snw_series): snw = snw_series(a, start="1999-01-01") out = xci.snw_max_doy(snw, freq="YS") np.testing.assert_array_equal(out, [11, np.nan]) - assert out.units == "1" + # FIXME: Confirm these expected outputs + if Parse(__numpy_version__) < Parse("2.0.0"): + assert out.attrs["units"] == "" + else: + assert out.attrs["units"] == "1" class TestSnowMeltWEMax: diff --git a/tests/test_indices.py b/tests/test_indices.py index 80a70f476..d5e9583ec 100644 --- a/tests/test_indices.py +++ b/tests/test_indices.py @@ -19,6 +19,9 @@ import pandas as pd import pytest import xarray as xr +from numpy import __version__ as __numpy_version__ +from packaging.version import Parse +from pint import __versio__ as __pint_version__ from xclim import indices as xci from xclim.core.calendar import percentile_doy @@ -137,11 +140,19 @@ def test_simple(self, tas_series): out = xci.cold_spell_frequency(da, thresh="-10. C", freq="ME") np.testing.assert_array_equal(out, [1, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0]) - assert out.units == "1" + # FIXME: Confirm these expected outputs + if Parse(__numpy_version__) < Parse("2.0.0"): + assert out.units == "" + else: + assert out.units == "1" out = xci.cold_spell_frequency(da, thresh="-10. C", freq="YS") np.testing.assert_array_equal(out, 3) - assert out.units == "1" + # FIXME: Confirm these expected outputs + if Parse(__numpy_version__) < Parse("2.0.0"): + assert out.units == "" + else: + assert out.units == "1" class TestColdSpellMaxLength: @@ -231,7 +242,11 @@ def test_no_cdd(self, tas_series): a = tas_series(np.array([10, 15, -5, 18]) + K2C) cdd = xci.cooling_degree_days(a) assert cdd == 0 - assert cdd.units == "d K" + # FIXME: Confirm these expected outputs + if Parse(__pint_version__) < Parse("0.24.1"): + assert cdd.units == "K d" + else: + assert cdd.units == "d K" def test_cdd(self, tas_series): a = tas_series(np.array([20, 25, -15, 19]) + K2C) @@ -905,7 +920,13 @@ def test_simple(self, tas_series): assert lsf == 180 for attr in ["units", "is_dayofyear", "calendar"]: assert attr in lsf.attrs.keys() - assert lsf.attrs["units"] == "1" + + # FIXME: Confirm these expected outputs + if Parse(__numpy_version__) < Parse("2.0.0"): + assert lsf.attrs["units"] == "" + else: + assert lsf.attrs["units"] == "1" + assert lsf.attrs["is_dayofyear"] == 1 assert lsf.attrs["is_dayofyear"].dtype == np.int32 @@ -926,7 +947,13 @@ def test_simple(self, tas_series): assert np.isnan(fdb) for attr in ["units", "is_dayofyear", "calendar"]: assert attr in fdb.attrs.keys() - assert fdb.attrs["units"] == "1" + + # FIXME: Confirm these expected outputs + if Parse(__numpy_version__) < Parse("2.0.0"): + assert fdb.attrs["units"] == "" + else: + assert fdb.attrs["units"] == "1" + assert fdb.attrs["is_dayofyear"] == 1 def test_below_forbidden(self, tasmax_series): @@ -957,7 +984,13 @@ def test_simple(self, tas_series): assert np.isnan(fda) for attr in ["units", "is_dayofyear", "calendar"]: assert attr in fda.attrs.keys() - assert fda.attrs["units"] == "1" + + # FIXME: Confirm these expected outputs + if Parse(__numpy_version__) < Parse("2.0.0"): + assert fda.attrs["units"] == "" + else: + assert fda.attrs["units"] == "1" + assert fda.attrs["is_dayofyear"] == 1 def test_thresholds(self, tas_series): @@ -982,7 +1015,13 @@ def test_thresholds(self, tas_series): assert out[0] == tg.indexes["time"][30].dayofyear for attr in ["units", "is_dayofyear", "calendar"]: assert attr in out.attrs.keys() - assert out.attrs["units"] == "1" + + # FIXME: Confirm these expected outputs + if Parse(__numpy_version__) < Parse("2.0.0"): + assert out.attrs["units"] == "" + else: + assert out.attrs["units"] == "1" + assert out.attrs["is_dayofyear"] == 1 def test_above_forbidden(self, tasmax_series): @@ -1069,7 +1108,13 @@ def test_simple(self, tas_series): assert out[0] == tg.indexes["time"][20].dayofyear for attr in ["units", "is_dayofyear", "calendar"]: assert attr in out.attrs.keys() - assert out.attrs["units"] == "1" + + # FIXME: Confirm these expected outputs + if Parse(__numpy_version__) < Parse("2.0.0"): + assert out.attrs["units"] == "" + else: + assert out.attrs["units"] == "1" + assert out.attrs["is_dayofyear"] == 1 def test_no_start(self, tas_series): @@ -1182,7 +1227,13 @@ def test_simple(self, tasmin_series): assert out[0] == tn.indexes["time"][20].dayofyear for attr in ["units", "is_dayofyear", "calendar"]: assert attr in out.attrs.keys() - assert out.attrs["units"] == "1" + + # FIXME: Confirm these expected outputs + if Parse(__numpy_version__) < Parse("2.0.0"): + assert out.attrs["units"] == "" + else: + assert out.attrs["units"] == "1" + assert out.attrs["is_dayofyear"] == 1 def test_no_start(self, tasmin_series): @@ -1215,7 +1266,13 @@ def test_varying_mid_dates(self, tasmin_series, d1, d2, mid_date, expected): np.testing.assert_array_equal(gs_end, expected) for attr in ["units", "is_dayofyear", "calendar"]: assert attr in gs_end.attrs.keys() - assert gs_end.attrs["units"] == "1" + + # FIXME: Confirm these expected outputs + if Parse(__numpy_version__) < Parse("2.0.0"): + assert gs_end.attrs["units"] == "" + else: + assert gs_end.attrs["units"] == "1" + assert gs_end.attrs["is_dayofyear"] == 1 @@ -2724,7 +2781,13 @@ def test_degree_days_exceedance_date(tas_series): for attr in ["units", "is_dayofyear", "calendar"]: assert attr in out.attrs.keys() - assert out.attrs["units"] == "1" + + # FIXME: Confirm these expected outputs + if Parse(__numpy_version__) < Parse("2.0.0"): + assert out.attrs["units"] == "" + else: + assert out.attrs["units"] == "1" + assert out.attrs["is_dayofyear"] == 1 @@ -2764,7 +2827,11 @@ def test_first_snowfall(prsn_series, prsnd_series): assert out[0] == 166 for attr in ["units", "is_dayofyear", "calendar"]: assert attr in out.attrs.keys() - assert out.attrs["units"] == "1" + # FIXME: Confirm these expected outputs + if Parse(__numpy_version__) < Parse("2.0.0"): + assert out.attrs["units"] == "" + else: + assert out.attrs["units"] == "1" assert out.attrs["is_dayofyear"] == 1 # test with prsnd [m s-1] @@ -2785,7 +2852,11 @@ def test_first_snowfall(prsn_series, prsnd_series): assert out[0] == 166 for attr in ["units", "is_dayofyear", "calendar"]: assert attr in out.attrs.keys() - assert out.attrs["units"] == "1" + # FIXME: Confirm these expected outputs + if Parse(__numpy_version__) < Parse("2.0.0"): + assert out.attrs["units"] == "" + else: + assert out.attrs["units"] == "1" assert out.attrs["is_dayofyear"] == 1 @@ -2922,7 +2993,13 @@ def test_continous_snow_season_start(self, snd_series, snw_series): np.testing.assert_array_equal(out, [snd.time.dt.dayofyear[0].data + 2, np.nan]) for attr in ["units", "is_dayofyear", "calendar"]: assert attr in out.attrs.keys() - assert out.attrs["units"] == "1" + + # FIXME: Confirm these expected outputs + if Parse(__numpy_version__) < Parse("2.0.0"): + assert out.attrs["units"] == "" + else: + assert out.attrs["units"] == "1" + assert out.attrs["is_dayofyear"] == 1 out = xci.snw_season_start(snw) @@ -2930,7 +3007,13 @@ def test_continous_snow_season_start(self, snd_series, snw_series): np.testing.assert_array_equal(out, [snw.time.dt.dayofyear[0].data + 1, np.nan]) for attr in ["units", "is_dayofyear", "calendar"]: assert attr in out.attrs.keys() - assert out.attrs["units"] == "1" + + # FIXME: Confirm these expected outputs + if Parse(__numpy_version__) < Parse("2.0.0"): + assert out.attrs["units"] == "" + else: + assert out.attrs["units"] == "1" + assert out.attrs["is_dayofyear"] == 1 def test_snow_season_end(self, snd_series, snw_series): @@ -2952,7 +3035,13 @@ def test_snow_season_end(self, snd_series, snw_series): np.testing.assert_array_equal(out, [(doy + 219) % 366, np.nan]) for attr in ["units", "is_dayofyear", "calendar"]: assert attr in out.attrs.keys() - assert out.attrs["units"] == "1" + + # FIXME: Confirm these expected outputs + if Parse(__numpy_version__) < Parse("2.0.0"): + assert out.attrs["units"] == "" + else: + assert out.attrs["units"] == "1" + assert out.attrs["is_dayofyear"] == 1 out = xci.snw_season_end(snw) diff --git a/tests/test_land.py b/tests/test_land.py index a750146fa..4cf3d4dc9 100644 --- a/tests/test_land.py +++ b/tests/test_land.py @@ -4,25 +4,44 @@ import numpy as np import xarray as xr +from numpy import __version__ as __numpy_version__ +from packaging.version import Parse from xclim import land def test_base_flow_index(ndq_series): out = land.base_flow_index(ndq_series, freq="YS") - assert out.attrs["units"] == "1" + + # FIXME: Confirm these expected outputs + if Parse(__numpy_version__) < Parse("2.0.0"): + assert out.attrs["units"] == "" + else: + assert out.attrs["units"] == "1" + assert isinstance(out, xr.DataArray) def test_rb_flashiness_index(ndq_series): out = land.base_flow_index(ndq_series, freq="YS") - assert out.attrs["units"] == "1" + + # FIXME: Confirm these expected outputs + if Parse(__numpy_version__) < Parse("2.0.0"): + assert out.attrs["units"] == "" + else: + assert out.attrs["units"] == "1" + assert isinstance(out, xr.DataArray) def test_qdoy_max(ndq_series, q_series): out = land.doy_qmax(ndq_series, freq="YS", season="JJA") - assert out.attrs["units"] == "1" + + # FIXME: Confirm these expected outputs + if Parse(__numpy_version__) < Parse("2.0.0"): + assert out.attrs["units"] == "" + else: + assert out.attrs["units"] == "1" a = np.ones(450) a[100] = 2 diff --git a/tests/test_snow.py b/tests/test_snow.py index e1174c357..b135e315c 100644 --- a/tests/test_snow.py +++ b/tests/test_snow.py @@ -2,6 +2,8 @@ import numpy as np import pytest +from numpy import __version__ as __numpy_version__ +from packaging.version import Parse from xclim import land from xclim.core.utils import ValidationError @@ -45,11 +47,23 @@ def test_simple(self, snd_series): snd = snd.expand_dims(lat=[0, 1, 2]) out = land.snd_season_start(snd) - assert out.units == "1" + + # FIXME: Confirm these expected outputs + if Parse(__numpy_version__) < Parse("2.0.0"): + assert out.units == "" + else: + assert out.units == "1" + np.testing.assert_array_equal(out.isel(lat=0), snd.time.dt.dayofyear[100]) out = land.snd_season_end(snd) - assert out.units == "1" + + # FIXME: Confirm these expected outputs + if Parse(__numpy_version__) < Parse("2.0.0"): + assert out.units == "" + else: + assert out.units == "1" + np.testing.assert_array_equal(out.isel(lat=0), snd.time.dt.dayofyear[200]) out = land.snd_season_length(snd) @@ -67,11 +81,23 @@ def test_simple(self, snw_series): snw = snw.expand_dims(lat=[0, 1, 2]) out = land.snw_season_start(snw) - assert out.units == "1" + + # FIXME: Confirm these expected outputs + if Parse(__numpy_version__) < Parse("2.0.0"): + assert out.units == "" + else: + assert out.units == "1" + np.testing.assert_array_equal(out.isel(lat=0), snw.time.dt.dayofyear[100]) out = land.snw_season_end(snw) - assert out.units == "1" + + # FIXME: Confirm these expected outputs + if Parse(__numpy_version__) < Parse("2.0.0"): + assert out.units == "" + else: + assert out.units == "1" + np.testing.assert_array_equal(out.isel(lat=0), snw.time.dt.dayofyear[200]) out = land.snw_season_length(snw) diff --git a/tests/test_units.py b/tests/test_units.py index f585bf475..311c5d176 100644 --- a/tests/test_units.py +++ b/tests/test_units.py @@ -7,6 +7,8 @@ import pytest import xarray as xr from dask import array as dsk +from numpy import __version__ as __numpy_version__ +from packaging.version import Parse from xclim import indices, set_options from xclim.core.units import ( @@ -142,6 +144,12 @@ def test_units2pint(self, pr_series): u = units2pint("1") assert pint2cfunits(u) == "1" + # FIXME: Confirm these expected outputs + if Parse(__numpy_version__) < Parse("2.0.0"): + assert pint2cfunits(u) == "" + else: + assert pint2cfunits(u) == "1" + def test_pint_multiply(self, pr_series): a = pr_series([1, 2, 3]) out = pint_multiply(a, 1 * units.days) @@ -331,8 +339,14 @@ def index( ("", "sum", "count", 365, "d"), ("", "sum", "count", 365, "d"), ("kg m-2", "var", "var", 0, "kg2 m-4"), - ("°C", "argmax", "doymax", 0, "1"), - ("°C", "sum", "integral", 365, "d K"), + ("°C", "argmax", "doymax", 0, ("", "1")), # dependent on numpy/pint version + ( + "°C", + "sum", + "integral", + 365, + ("K d", "d K"), + ), # dependent on numpy/pint version ("°F", "sum", "integral", 365, "d °R"), # not sure why the order is different ], ) @@ -346,4 +360,11 @@ def test_to_agg_units(in_u, opfunc, op, exp, exp_u): out = to_agg_units(getattr(da, opfunc)(), da, op) np.testing.assert_allclose(out, exp) - assert out.attrs["units"] == exp_u + + if isinstance(exp_u, tuple): + if Parse(__numpy_version__) < Parse("2.0.0"): + assert out.attrs["units"] == exp_u[0] + else: + assert out.attrs["units"] == exp_u[1] + else: + assert out.attrs["units"] == exp_u From 9943914d6f3e8c29bcaf949dc9ce6551d45a023e Mon Sep 17 00:00:00 2001 From: Trevor James Smith <10819524+Zeitsperre@users.noreply.github.com> Date: Fri, 26 Jul 2024 14:49:23 -0400 Subject: [PATCH 14/33] add an older numpy build --- .github/workflows/main.yml | 2 +- tox.ini | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 3fcb61650..956a27f7e 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -123,7 +123,7 @@ jobs: markers: -m 'not slow' os: windows-latest # macOS builds - - tox-env: py310-coverage-lmoments-doctest + - tox-env: py310-coverage-lmoments-doctest-numpy python-version: "3.10" markers: -m 'not slow' os: macos-latest diff --git a/tox.ini b/tox.ini index ca18e6cb3..c03d4ada7 100644 --- a/tox.ini +++ b/tox.ini @@ -117,9 +117,10 @@ extras = dev extras: extras deps = - upstream: -r CI/requirements_upstream.txt - sbck: pybind11 lmoments: lmoments3 + numpy: numpy>=1.20,<2.0 + sbck: pybind11 + upstream: -r CI/requirements_upstream.txt install_command = python -m pip install --no-user {opts} {packages} download = True commands_pre = From 89675962bd9994f443efb837d732f0a4811abe03 Mon Sep 17 00:00:00 2001 From: Trevor James Smith <10819524+Zeitsperre@users.noreply.github.com> Date: Fri, 26 Jul 2024 14:49:35 -0400 Subject: [PATCH 15/33] fix imports --- tests/test_atmos.py | 4 ++-- tests/test_generic.py | 4 ++-- tests/test_hydrology.py | 4 ++-- tests/test_indices.py | 34 +++++++++++++++++----------------- tests/test_land.py | 8 ++++---- tests/test_snow.py | 10 +++++----- tests/test_units.py | 6 +++--- xclim/testing/utils.py | 1 + 8 files changed, 36 insertions(+), 35 deletions(-) diff --git a/tests/test_atmos.py b/tests/test_atmos.py index f7f99ee4a..88a072d16 100644 --- a/tests/test_atmos.py +++ b/tests/test_atmos.py @@ -5,7 +5,7 @@ import numpy as np import xarray as xr from numpy import __version__ as __numpy_version__ -from packaging.version import Parse +from packaging.version import Version from xclim import atmos, set_options @@ -263,7 +263,7 @@ def test_wind_profile(atmosds): def test_wind_power_potential(atmosds): out = atmos.wind_power_potential(wind_speed=atmosds.sfcWind) # FIXME: Confirm these expected outputs - if Parse(__numpy_version__) < Parse("2.0.0"): + if Version(__numpy_version__) < Version("2.0.0"): assert out.attrs["units"] == "" else: assert out.attrs["units"] == "dimensionless" diff --git a/tests/test_generic.py b/tests/test_generic.py index d1b5e57cb..ab7122584 100644 --- a/tests/test_generic.py +++ b/tests/test_generic.py @@ -6,7 +6,7 @@ import pytest import xarray as xr from numpy import __version__ as __numpy_version__ -from packaging.version import Parse +from packaging.version import Version from xclim.core.calendar import doy_to_days_since, select_time from xclim.indices import generic @@ -104,7 +104,7 @@ def test_doyminmax(self, q_series): for attr in ["units", "is_dayofyear", "calendar"]: assert attr in da.attrs.keys() # FIXME: Confirm these expected outputs - if Parse(__numpy_version__) < Parse("2.0.0"): + if Version(__numpy_version__) < Version("2.0.0"): assert da.attrs["units"] == "" else: assert da.attrs["units"] == "1" diff --git a/tests/test_hydrology.py b/tests/test_hydrology.py index c20322deb..7340874f1 100644 --- a/tests/test_hydrology.py +++ b/tests/test_hydrology.py @@ -2,7 +2,7 @@ import numpy as np from numpy import __version__ as __numpy_version__ -from packaging.version import Parse +from packaging.version import Version from xclim import indices as xci @@ -43,7 +43,7 @@ def test_simple(self, snw_series): out = xci.snw_max_doy(snw, freq="YS") np.testing.assert_array_equal(out, [11, np.nan]) # FIXME: Confirm these expected outputs - if Parse(__numpy_version__) < Parse("2.0.0"): + if Version(__numpy_version__) < Version("2.0.0"): assert out.attrs["units"] == "" else: assert out.attrs["units"] == "1" diff --git a/tests/test_indices.py b/tests/test_indices.py index d5e9583ec..2abed2e49 100644 --- a/tests/test_indices.py +++ b/tests/test_indices.py @@ -20,7 +20,7 @@ import pytest import xarray as xr from numpy import __version__ as __numpy_version__ -from packaging.version import Parse +from packaging.version import Version from pint import __versio__ as __pint_version__ from xclim import indices as xci @@ -141,7 +141,7 @@ def test_simple(self, tas_series): out = xci.cold_spell_frequency(da, thresh="-10. C", freq="ME") np.testing.assert_array_equal(out, [1, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0]) # FIXME: Confirm these expected outputs - if Parse(__numpy_version__) < Parse("2.0.0"): + if Version(__numpy_version__) < Version("2.0.0"): assert out.units == "" else: assert out.units == "1" @@ -149,7 +149,7 @@ def test_simple(self, tas_series): out = xci.cold_spell_frequency(da, thresh="-10. C", freq="YS") np.testing.assert_array_equal(out, 3) # FIXME: Confirm these expected outputs - if Parse(__numpy_version__) < Parse("2.0.0"): + if Version(__numpy_version__) < Version("2.0.0"): assert out.units == "" else: assert out.units == "1" @@ -243,7 +243,7 @@ def test_no_cdd(self, tas_series): cdd = xci.cooling_degree_days(a) assert cdd == 0 # FIXME: Confirm these expected outputs - if Parse(__pint_version__) < Parse("0.24.1"): + if Version(__pint_version__) < Version("0.24.1"): assert cdd.units == "K d" else: assert cdd.units == "d K" @@ -922,7 +922,7 @@ def test_simple(self, tas_series): assert attr in lsf.attrs.keys() # FIXME: Confirm these expected outputs - if Parse(__numpy_version__) < Parse("2.0.0"): + if Version(__numpy_version__) < Version("2.0.0"): assert lsf.attrs["units"] == "" else: assert lsf.attrs["units"] == "1" @@ -949,7 +949,7 @@ def test_simple(self, tas_series): assert attr in fdb.attrs.keys() # FIXME: Confirm these expected outputs - if Parse(__numpy_version__) < Parse("2.0.0"): + if Version(__numpy_version__) < Version("2.0.0"): assert fdb.attrs["units"] == "" else: assert fdb.attrs["units"] == "1" @@ -986,7 +986,7 @@ def test_simple(self, tas_series): assert attr in fda.attrs.keys() # FIXME: Confirm these expected outputs - if Parse(__numpy_version__) < Parse("2.0.0"): + if Version(__numpy_version__) < Version("2.0.0"): assert fda.attrs["units"] == "" else: assert fda.attrs["units"] == "1" @@ -1017,7 +1017,7 @@ def test_thresholds(self, tas_series): assert attr in out.attrs.keys() # FIXME: Confirm these expected outputs - if Parse(__numpy_version__) < Parse("2.0.0"): + if Version(__numpy_version__) < Version("2.0.0"): assert out.attrs["units"] == "" else: assert out.attrs["units"] == "1" @@ -1110,7 +1110,7 @@ def test_simple(self, tas_series): assert attr in out.attrs.keys() # FIXME: Confirm these expected outputs - if Parse(__numpy_version__) < Parse("2.0.0"): + if Version(__numpy_version__) < Version("2.0.0"): assert out.attrs["units"] == "" else: assert out.attrs["units"] == "1" @@ -1229,7 +1229,7 @@ def test_simple(self, tasmin_series): assert attr in out.attrs.keys() # FIXME: Confirm these expected outputs - if Parse(__numpy_version__) < Parse("2.0.0"): + if Version(__numpy_version__) < Version("2.0.0"): assert out.attrs["units"] == "" else: assert out.attrs["units"] == "1" @@ -1268,7 +1268,7 @@ def test_varying_mid_dates(self, tasmin_series, d1, d2, mid_date, expected): assert attr in gs_end.attrs.keys() # FIXME: Confirm these expected outputs - if Parse(__numpy_version__) < Parse("2.0.0"): + if Version(__numpy_version__) < Version("2.0.0"): assert gs_end.attrs["units"] == "" else: assert gs_end.attrs["units"] == "1" @@ -2783,7 +2783,7 @@ def test_degree_days_exceedance_date(tas_series): assert attr in out.attrs.keys() # FIXME: Confirm these expected outputs - if Parse(__numpy_version__) < Parse("2.0.0"): + if Version(__numpy_version__) < Version("2.0.0"): assert out.attrs["units"] == "" else: assert out.attrs["units"] == "1" @@ -2828,7 +2828,7 @@ def test_first_snowfall(prsn_series, prsnd_series): for attr in ["units", "is_dayofyear", "calendar"]: assert attr in out.attrs.keys() # FIXME: Confirm these expected outputs - if Parse(__numpy_version__) < Parse("2.0.0"): + if Version(__numpy_version__) < Version("2.0.0"): assert out.attrs["units"] == "" else: assert out.attrs["units"] == "1" @@ -2853,7 +2853,7 @@ def test_first_snowfall(prsn_series, prsnd_series): for attr in ["units", "is_dayofyear", "calendar"]: assert attr in out.attrs.keys() # FIXME: Confirm these expected outputs - if Parse(__numpy_version__) < Parse("2.0.0"): + if Version(__numpy_version__) < Version("2.0.0"): assert out.attrs["units"] == "" else: assert out.attrs["units"] == "1" @@ -2995,7 +2995,7 @@ def test_continous_snow_season_start(self, snd_series, snw_series): assert attr in out.attrs.keys() # FIXME: Confirm these expected outputs - if Parse(__numpy_version__) < Parse("2.0.0"): + if Version(__numpy_version__) < Version("2.0.0"): assert out.attrs["units"] == "" else: assert out.attrs["units"] == "1" @@ -3009,7 +3009,7 @@ def test_continous_snow_season_start(self, snd_series, snw_series): assert attr in out.attrs.keys() # FIXME: Confirm these expected outputs - if Parse(__numpy_version__) < Parse("2.0.0"): + if Version(__numpy_version__) < Version("2.0.0"): assert out.attrs["units"] == "" else: assert out.attrs["units"] == "1" @@ -3037,7 +3037,7 @@ def test_snow_season_end(self, snd_series, snw_series): assert attr in out.attrs.keys() # FIXME: Confirm these expected outputs - if Parse(__numpy_version__) < Parse("2.0.0"): + if Version(__numpy_version__) < Version("2.0.0"): assert out.attrs["units"] == "" else: assert out.attrs["units"] == "1" diff --git a/tests/test_land.py b/tests/test_land.py index 4cf3d4dc9..ef09127ec 100644 --- a/tests/test_land.py +++ b/tests/test_land.py @@ -5,7 +5,7 @@ import numpy as np import xarray as xr from numpy import __version__ as __numpy_version__ -from packaging.version import Parse +from packaging.version import Version from xclim import land @@ -14,7 +14,7 @@ def test_base_flow_index(ndq_series): out = land.base_flow_index(ndq_series, freq="YS") # FIXME: Confirm these expected outputs - if Parse(__numpy_version__) < Parse("2.0.0"): + if Version(__numpy_version__) < Version("2.0.0"): assert out.attrs["units"] == "" else: assert out.attrs["units"] == "1" @@ -26,7 +26,7 @@ def test_rb_flashiness_index(ndq_series): out = land.base_flow_index(ndq_series, freq="YS") # FIXME: Confirm these expected outputs - if Parse(__numpy_version__) < Parse("2.0.0"): + if Version(__numpy_version__) < Version("2.0.0"): assert out.attrs["units"] == "" else: assert out.attrs["units"] == "1" @@ -38,7 +38,7 @@ def test_qdoy_max(ndq_series, q_series): out = land.doy_qmax(ndq_series, freq="YS", season="JJA") # FIXME: Confirm these expected outputs - if Parse(__numpy_version__) < Parse("2.0.0"): + if Version(__numpy_version__) < Version("2.0.0"): assert out.attrs["units"] == "" else: assert out.attrs["units"] == "1" diff --git a/tests/test_snow.py b/tests/test_snow.py index b135e315c..b579b0ade 100644 --- a/tests/test_snow.py +++ b/tests/test_snow.py @@ -3,7 +3,7 @@ import numpy as np import pytest from numpy import __version__ as __numpy_version__ -from packaging.version import Parse +from packaging.version import Version from xclim import land from xclim.core.utils import ValidationError @@ -49,7 +49,7 @@ def test_simple(self, snd_series): out = land.snd_season_start(snd) # FIXME: Confirm these expected outputs - if Parse(__numpy_version__) < Parse("2.0.0"): + if Version(__numpy_version__) < Version("2.0.0"): assert out.units == "" else: assert out.units == "1" @@ -59,7 +59,7 @@ def test_simple(self, snd_series): out = land.snd_season_end(snd) # FIXME: Confirm these expected outputs - if Parse(__numpy_version__) < Parse("2.0.0"): + if Version(__numpy_version__) < Version("2.0.0"): assert out.units == "" else: assert out.units == "1" @@ -83,7 +83,7 @@ def test_simple(self, snw_series): out = land.snw_season_start(snw) # FIXME: Confirm these expected outputs - if Parse(__numpy_version__) < Parse("2.0.0"): + if Version(__numpy_version__) < Version("2.0.0"): assert out.units == "" else: assert out.units == "1" @@ -93,7 +93,7 @@ def test_simple(self, snw_series): out = land.snw_season_end(snw) # FIXME: Confirm these expected outputs - if Parse(__numpy_version__) < Parse("2.0.0"): + if Version(__numpy_version__) < Version("2.0.0"): assert out.units == "" else: assert out.units == "1" diff --git a/tests/test_units.py b/tests/test_units.py index 311c5d176..fd0d604e1 100644 --- a/tests/test_units.py +++ b/tests/test_units.py @@ -8,7 +8,7 @@ import xarray as xr from dask import array as dsk from numpy import __version__ as __numpy_version__ -from packaging.version import Parse +from packaging.version import Version from xclim import indices, set_options from xclim.core.units import ( @@ -145,7 +145,7 @@ def test_units2pint(self, pr_series): assert pint2cfunits(u) == "1" # FIXME: Confirm these expected outputs - if Parse(__numpy_version__) < Parse("2.0.0"): + if Version(__numpy_version__) < Version("2.0.0"): assert pint2cfunits(u) == "" else: assert pint2cfunits(u) == "1" @@ -362,7 +362,7 @@ def test_to_agg_units(in_u, opfunc, op, exp, exp_u): np.testing.assert_allclose(out, exp) if isinstance(exp_u, tuple): - if Parse(__numpy_version__) < Parse("2.0.0"): + if Version(__numpy_version__) < Version("2.0.0"): assert out.attrs["units"] == exp_u[0] else: assert out.attrs["units"] == exp_u[1] diff --git a/xclim/testing/utils.py b/xclim/testing/utils.py index 8a97a6bf2..746a2dc50 100644 --- a/xclim/testing/utils.py +++ b/xclim/testing/utils.py @@ -42,6 +42,7 @@ "scipy", "pint", "pandas", + "numpy", "numba", "lmoments3", "jsonpickle", From 82c6f20694162f3d5aa92d92c213fa1c787e8baf Mon Sep 17 00:00:00 2001 From: Trevor James Smith <10819524+Zeitsperre@users.noreply.github.com> Date: Fri, 26 Jul 2024 14:59:02 -0400 Subject: [PATCH 16/33] unpin netcdf4 --- environment.yml | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/environment.yml b/environment.yml index ca2d6ce62..bb1b0a294 100644 --- a/environment.yml +++ b/environment.yml @@ -52,7 +52,7 @@ dependencies: - nbsphinx - nbval >=0.11.0 - nc-time-axis - - netCDF4 >=1.4,<1.7 + - netCDF4 >=1.4 # Needs to be unpinned for numpy v2.0 (requires v1.7.0+) - notebook - pandas-stubs - platformdirs diff --git a/pyproject.toml b/pyproject.toml index 864533090..7eadc58b0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -72,7 +72,7 @@ dev = [ "nbconvert <7.14", # Pinned due to directive errors in sphinx. See: https://github.com/jupyter/nbconvert/issues/2092 "nbqa >=1.8.2", "nbval >=0.11.0", - "netCDF4 >=1.4,<1.7", + "netCDF4 >=1.4", # Needs to be unpinned for numpy v2.0 (requires v1.7.0+) "pandas-stubs >=2.2", "platformdirs >=3.2", "pooch", From 3577bbba1f0246dbc5f6ab1b68a006e76d29d049 Mon Sep 17 00:00:00 2001 From: Trevor James Smith <10819524+Zeitsperre@users.noreply.github.com> Date: Fri, 26 Jul 2024 15:07:56 -0400 Subject: [PATCH 17/33] woops --- tests/test_indices.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_indices.py b/tests/test_indices.py index 2abed2e49..7cceaff00 100644 --- a/tests/test_indices.py +++ b/tests/test_indices.py @@ -21,7 +21,7 @@ import xarray as xr from numpy import __version__ as __numpy_version__ from packaging.version import Version -from pint import __versio__ as __pint_version__ +from pint import __version__ as __pint_version__ from xclim import indices as xci from xclim.core.calendar import percentile_doy From 107d44c9e395aec18fab11721afa65d8a15e4b13 Mon Sep 17 00:00:00 2001 From: Trevor James Smith <10819524+Zeitsperre@users.noreply.github.com> Date: Fri, 26 Jul 2024 15:14:18 -0400 Subject: [PATCH 18/33] use h5netcdf --- xclim/testing/helpers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xclim/testing/helpers.py b/xclim/testing/helpers.py index c407323f3..4a5e80459 100644 --- a/xclim/testing/helpers.py +++ b/xclim/testing/helpers.py @@ -101,7 +101,7 @@ def generate_atmos(cache_dir: Path): # Create a file in session scoped temporary directory atmos_file = cache_dir.joinpath("atmosds.nc") - ds.to_netcdf(atmos_file) + ds.to_netcdf(atmos_file, engine="h5netcdf") def populate_testing_data( From fd952b2395612f0c2b19a5cdb51416abb2582ebb Mon Sep 17 00:00:00 2001 From: Trevor James Smith <10819524+Zeitsperre@users.noreply.github.com> Date: Fri, 26 Jul 2024 15:42:01 -0400 Subject: [PATCH 19/33] promote h5netcdf to dependency, address errors --- pyproject.toml | 2 +- tests/test_atmos.py | 2 +- tests/test_cli.py | 24 ++++++++++++------------ tests/test_sdba/test_adjustment.py | 4 ++-- tests/test_testing_utils.py | 2 +- xclim/cli.py | 8 +++++++- xclim/sdba/_adjustment.py | 6 +++--- 7 files changed, 27 insertions(+), 21 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 7eadc58b0..241ded18b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -38,6 +38,7 @@ dependencies = [ "cftime>=1.4.1", "click>=8.1", "dask[array]>=2.6", + "h5netcdf>=1.3.0", "jsonpickle", "numba", "numpy>=1.20.0", @@ -65,7 +66,6 @@ dev = [ "deptry ==0.17.0", "flake8 >=7.1.0", "flake8-rst-docstrings", - "h5netcdf>=1.3.0", "ipython", "isort ==5.13.2", "mypy", diff --git a/tests/test_atmos.py b/tests/test_atmos.py index 88a072d16..10c5e14db 100644 --- a/tests/test_atmos.py +++ b/tests/test_atmos.py @@ -266,7 +266,7 @@ def test_wind_power_potential(atmosds): if Version(__numpy_version__) < Version("2.0.0"): assert out.attrs["units"] == "" else: - assert out.attrs["units"] == "dimensionless" + assert out.attrs["units"] == "1" assert (out >= 0).all() assert (out <= 1).all() diff --git a/tests/test_cli.py b/tests/test_cli.py index 966308e74..089282686 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -89,7 +89,7 @@ def test_normal_computation( input_file = tmp_path / "in.nc" output_file = tmp_path / "out.nc" - ds.to_netcdf(input_file) + ds.to_netcdf(input_file, engine="h5netcdf") args = ["-i", str(input_file), "-o", str(output_file), "-v", indicator] runner = CliRunner() @@ -128,7 +128,7 @@ def test_multi_input(tas_series, pr_series, tmp_path): ) assert "Processing : solidprcptot" in results.output - out = xr.open_dataset(output_file) + out = xr.open_dataset(output_file, engine="h5netcdf") assert out.solidprcptot.sum() == 0 @@ -136,7 +136,7 @@ def test_multi_output(tmp_path, open_dataset): ds = open_dataset("ERA5/daily_surface_cancities_1990-1993.nc") input_file = tmp_path / "ws_in.nc" output_file = tmp_path / "out.nc" - ds.to_netcdf(input_file) + ds.to_netcdf(input_file, engine="h5netcdf") runner = CliRunner() results = runner.invoke( @@ -158,7 +158,7 @@ def test_renaming_variable(tas_series, tmp_path): input_file = tmp_path / "tas.nc" output_file = tmp_path / "out.nc" tas.name = "tas" - tas.to_netcdf(input_file) + tas.to_netcdf(input_file, engine="h5netcdf") with xclim.set_options(cf_compliance="warn"): runner = CliRunner() results = runner.invoke( @@ -177,7 +177,7 @@ def test_renaming_variable(tas_series, tmp_path): assert "Processing : tn_mean" in results.output assert "100% Completed" in results.output - out = xr.open_dataset(output_file) + out = xr.open_dataset(output_file, engine="h5netcdf") assert out.tn_mean[0] == 1.0 @@ -206,7 +206,7 @@ def test_indicator_chain(tas_series, tmp_path): assert "Processing : growing_degree_days" in results.output assert "100% Completed" in results.output - out = xr.open_dataset(output_file) + out = xr.open_dataset(output_file, engine="h5netcdf") assert out.tg_mean[0] == 1.0 assert out.growing_degree_days[0] == 0 @@ -216,7 +216,7 @@ def test_missing_variable(tas_series, tmp_path): input_file = tmp_path / "tas.nc" output_file = tmp_path / "out.nc" - tas.to_netcdf(input_file) + tas.to_netcdf(input_file, engine="h5netcdf") runner = CliRunner() results = runner.invoke( @@ -243,7 +243,7 @@ def test_global_options(tas_series, tmp_path, options, output): input_file = tmp_path / "tas.nc" output_file = tmp_path / "out.nc" - tas.to_netcdf(input_file) + tas.to_netcdf(input_file, engine="h5netcdf") runner = CliRunner() results = runner.invoke( @@ -264,13 +264,13 @@ def test_suspicious_precipitation_flags(pr_series, tmp_path): input_file = tmp_path / "bad_pr.nc" output_file = tmp_path / "out.nc" - bad_pr.to_netcdf(input_file) + bad_pr.to_netcdf(input_file, engine="h5netcdf") runner = CliRunner() runner.invoke( cli, ["-i", str(input_file), "-o", str(output_file), "dataflags", "pr"] ) - with xr.open_dataset(output_file) as ds: + with xr.open_dataset(output_file, engine="h5netcdf") as ds: for var in ds.data_vars: assert var @@ -283,7 +283,7 @@ def test_dataflags_output(tmp_path, tas_series, tasmax_series, tasmin_series): arr = series(vals, start="1971-01-01") ds = xr.merge([ds, arr]) input_file = tmp_path / "ws_in.nc" - ds.to_netcdf(input_file) + ds.to_netcdf(input_file, engine="h5netcdf") runner = CliRunner() results = runner.invoke( @@ -303,7 +303,7 @@ def test_bad_usage(tas_series, tmp_path): input_file = tmp_path / "tas.nc" output_file = tmp_path / "out.nc" - tas.to_netcdf(input_file) + tas.to_netcdf(input_file, engine="h5netcdf") runner = CliRunner() diff --git a/tests/test_sdba/test_adjustment.py b/tests/test_sdba/test_adjustment.py index d4ddfb865..d18423a5c 100644 --- a/tests/test_sdba/test_adjustment.py +++ b/tests/test_sdba/test_adjustment.py @@ -91,9 +91,9 @@ def test_time_and_from_ds(self, series, group, dec, tmp_path, random): assert "Bias-adjusted with LOCI(" in p.attrs["history"] file = tmp_path / "test_loci.nc" - loci.ds.to_netcdf(file) + loci.ds.to_netcdf(file, engine="h5netcdf") - ds = xr.open_dataset(file) + ds = xr.open_dataset(file, engine="h5netcdf") loci2 = LOCI.from_dataset(ds) xr.testing.assert_equal(loci.ds, loci2.ds) diff --git a/tests/test_testing_utils.py b/tests/test_testing_utils.py index 10a080cbb..cbf992533 100644 --- a/tests/test_testing_utils.py +++ b/tests/test_testing_utils.py @@ -78,7 +78,7 @@ def test_open_dataset_with_bad_file(self, tmp_path): @pytest.mark.requires_internet def test_open_testdata(self): ds = utilities.open_dataset( - Path("cmip5/tas_Amon_CanESM2_rcp85_r1i1p1_200701-200712") + Path("cmip5/tas_Amon_CanESM2_rcp85_r1i1p1_200701-200712"), engine="h5netcdf" ) assert ds.lon.size == 128 diff --git a/xclim/cli.py b/xclim/cli.py index fd4ddedbb..b9c11de0b 100644 --- a/xclim/cli.py +++ b/xclim/cli.py @@ -463,7 +463,13 @@ def cli(ctx, **kwargs): for dim, num in map(lambda x: x.split(":"), kwargs["chunks"].split(",")) } - kwargs["xr_kwargs"] = {"chunks": kwargs["chunks"] or {}} + if kwargs["engine"] is None: + kwargs["engine"] = "h5netcdf" + + kwargs["xr_kwargs"] = { + "chunks": kwargs["chunks"] or {}, + "engine": kwargs["engine"], + } ctx.obj = kwargs diff --git a/xclim/sdba/_adjustment.py b/xclim/sdba/_adjustment.py index 7e5fbfdaa..f6df322d6 100644 --- a/xclim/sdba/_adjustment.py +++ b/xclim/sdba/_adjustment.py @@ -175,7 +175,7 @@ def _npdft_train(ref, hist, rots, quantiles, method, extrap, n_escore, standardi np.nanstd(hist, axis=-1, keepdims=True) ) af_q = np.zeros((len(rots), ref.shape[0], len(quantiles))) - escores = np.zeros(len(rots)) * np.NaN + escores = np.zeros(len(rots)) * np.nan if n_escore > 0: ref_step, hist_step = ( int(np.ceil(arr.shape[1] / n_escore)) for arr in [ref, hist] @@ -708,7 +708,7 @@ def npdf_transform(ds: xr.Dataset, **kwargs) -> xr.Dataset: ------- xr.Dataset Dataset with `scenh`, `scens` and `escores` DataArrays, where `scenh` and `scens` are `hist` and `sim` - respectively after adjustment according to `ref`. If `n_escore` is negative, `escores` will be filled with NaNs. + respectively after adjustment according to `ref`. If `n_escore` is negative, `escores` will be filled with nans. """ ref = ds.ref.rename(time_hist="time") hist = ds.hist.rename(time_hist="time") @@ -752,7 +752,7 @@ def npdf_transform(ds: xr.Dataset, **kwargs) -> xr.Dataset: if kwargs["n_escore"] >= 0: escores = xr.concat(escores, "iterations") else: - # All NaN, but with the proper shape. + # All nan, but with the proper shape. escores = ( ref.isel({dim: 0, "time": 0}) * hist.isel({dim: 0, "time": 0}) ).expand_dims(iterations=ds.iterations) * np.nan From 0f6f7563565e9a6632833973f6d4e308c21d8cdc Mon Sep 17 00:00:00 2001 From: Trevor James Smith <10819524+Zeitsperre@users.noreply.github.com> Date: Fri, 26 Jul 2024 15:44:54 -0400 Subject: [PATCH 20/33] add h5netcdf to deptry --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 241ded18b..1236b4e15 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -179,7 +179,7 @@ pep621_dev_dependency_groups = ["all", "dev", "docs"] [tool.deptry.per_rule_ignores] DEP001 = ["SBCK"] -DEP002 = ["bottleneck", "pyarrow"] +DEP002 = ["bottleneck", "h5netcdf", "pyarrow"] DEP004 = ["matplotlib", "pytest_socket"] [tool.flit.sdist] From 4aeeb89d5f6059463f78804113dc6afc89d7b1a2 Mon Sep 17 00:00:00 2001 From: Trevor James Smith <10819524+Zeitsperre@users.noreply.github.com> Date: Fri, 26 Jul 2024 15:55:37 -0400 Subject: [PATCH 21/33] add xarray engine to cli --- xclim/cli.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/xclim/cli.py b/xclim/cli.py index b9c11de0b..1dd36e7c1 100644 --- a/xclim/cli.py +++ b/xclim/cli.py @@ -413,6 +413,11 @@ def get_command(self, ctx, name): help="Chunks to use when opening the input dataset(s). " "Given as :num,. Ex: time:365,lat:168,lon:150.", ) +@click.option( + "--engine", + help="Engine to use when opening the input dataset(s). " + "If not specified, 'h5netcdf' is used. Other options are 'scipy' and 'netcdf4' (requires python-netcdf4).", +) @click.pass_context def cli(ctx, **kwargs): """Entry point for the command line interface. From a2b0b42ba8422ebe65b73a384f02a9e9892c52c3 Mon Sep 17 00:00:00 2001 From: Trevor James Smith <10819524+Zeitsperre@users.noreply.github.com> Date: Fri, 26 Jul 2024 16:26:51 -0400 Subject: [PATCH 22/33] adjust more IO calls to use h5netcdf --- tests/test_cli.py | 4 ++-- tests/test_ensembles.py | 2 +- tests/test_indices.py | 4 ++-- tests/test_sdba/test_adjustment.py | 2 +- tests/test_sdba/test_detrending.py | 6 +++--- xclim/ensembles/_base.py | 1 + xclim/sdba/utils.py | 28 ++++++++++++++-------------- 7 files changed, 24 insertions(+), 23 deletions(-) diff --git a/tests/test_cli.py b/tests/test_cli.py index 089282686..d92f18994 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -111,8 +111,8 @@ def test_multi_input(tas_series, pr_series, tmp_path): pr_file = tmp_path / "multi_pr_in.nc" output_file = tmp_path / "out.nc" - tas.to_dataset().to_netcdf(tas_file) - pr.to_dataset().to_netcdf(pr_file) + tas.to_dataset().to_netcdf(tas_file, engine="h5netcdf") + pr.to_dataset().to_netcdf(pr_file, engine="h5netcdf") runner = CliRunner() results = runner.invoke( diff --git a/tests/test_ensembles.py b/tests/test_ensembles.py index 4af3db33d..79033b441 100644 --- a/tests/test_ensembles.py +++ b/tests/test_ensembles.py @@ -79,7 +79,7 @@ def test_no_time(self, tmp_path, ensemble_dataset_objects, open_dataset): ds["time"] = xr.decode_cf(ds).time ds_all.append(ds.groupby(ds.time.dt.month).mean("time", keep_attrs=True)) ds.groupby(ds.time.dt.month).mean("time", keep_attrs=True).to_netcdf( - f1.joinpath(Path(n).name) + f1.joinpath(Path(n).name), engine="h5netcdf" ) ens = ensembles.create_ensemble(ds_all) diff --git a/tests/test_indices.py b/tests/test_indices.py index 7cceaff00..1aa7e1e5b 100644 --- a/tests/test_indices.py +++ b/tests/test_indices.py @@ -784,9 +784,9 @@ def test_standardized_index_modularity(self, open_dataset, tmp_path, indexer): # Save the parameters to a file to test against that saving process may modify the netCDF file paramsfile = tmp_path / "params0.nc" - params.to_netcdf(paramsfile) + params.to_netcdf(paramsfile, engine="h5netcdf") params.close() - params = xr.open_dataset(paramsfile).__xarray_dataarray_variable__ + params = open_dataset(paramsfile).__xarray_dataarray_variable__ spei1 = xci.standardized_precipitation_evapotranspiration_index( wb.sel(time=slice("1998", "2000")), params=params diff --git a/tests/test_sdba/test_adjustment.py b/tests/test_sdba/test_adjustment.py index d18423a5c..cc6104429 100644 --- a/tests/test_sdba/test_adjustment.py +++ b/tests/test_sdba/test_adjustment.py @@ -293,7 +293,7 @@ def test_cannon_and_from_ds(self, cannon_2015_rvs, tmp_path, random): np.testing.assert_almost_equal(p.std(), 15.0, 0) file = tmp_path / "test_dqm.nc" - DQM.ds.to_netcdf(file) + DQM.ds.to_netcdf(file, engine="h5netcdf") ds = xr.open_dataset(file) DQM2 = DetrendedQuantileMapping.from_dataset(ds) diff --git a/tests/test_sdba/test_detrending.py b/tests/test_sdba/test_detrending.py index 8a237203a..99a7cca4f 100644 --- a/tests/test_sdba/test_detrending.py +++ b/tests/test_sdba/test_detrending.py @@ -15,7 +15,7 @@ ) -def test_poly_detrend_and_from_ds(series, tmp_path): +def test_poly_detrend_and_from_ds(series, tmp_path, open_dataset): x = series(np.arange(20 * 365.25), "tas") poly = PolyDetrend(degree=1) @@ -30,9 +30,9 @@ def test_poly_detrend_and_from_ds(series, tmp_path): np.testing.assert_array_almost_equal(xt, x) file = tmp_path / "test_polydetrend.nc" - fx.ds.to_netcdf(file) + fx.ds.to_netcdf(file, engine="h5netcdf") - ds = xr.open_dataset(file) + ds = open_dataset(file) fx2 = PolyDetrend.from_dataset(ds) xr.testing.assert_equal(fx.ds, fx2.ds) diff --git a/xclim/ensembles/_base.py b/xclim/ensembles/_base.py index 31988d559..39ddf7789 100644 --- a/xclim/ensembles/_base.py +++ b/xclim/ensembles/_base.py @@ -421,6 +421,7 @@ def _ens_align_datasets( """ xr_kwargs.setdefault("chunks", "auto") xr_kwargs.setdefault("decode_times", False) + xr_kwargs.setdefault("engine", "h5netcdf") if isinstance(datasets, str): datasets = glob(datasets) diff --git a/xclim/sdba/utils.py b/xclim/sdba/utils.py index 09827f095..153243ea4 100644 --- a/xclim/sdba/utils.py +++ b/xclim/sdba/utils.py @@ -325,7 +325,7 @@ def _interp_on_quantiles_1D_multi(newxs, oldx, oldy, method, extrap): # noqa: N oldy[~np.isnan(oldy)][-1], ) else: # extrap == 'nan' - fill_value = np.NaN + fill_value = np.nan finterp1d = interp1d( oldx[~mask_old], @@ -338,7 +338,7 @@ def _interp_on_quantiles_1D_multi(newxs, oldx, oldy, method, extrap): # noqa: N out = np.zeros_like(newxs) for ii in range(newxs.shape[0]): mask_new = np.isnan(newxs[ii, :]) - y1 = newxs[ii, :].copy() * np.NaN + y1 = newxs[ii, :].copy() * np.nan y1[~mask_new] = finterp1d(newxs[ii, ~mask_new]) out[ii, :] = y1.flatten() return out @@ -350,7 +350,7 @@ def _interp_on_quantiles_1D(newx, oldx, oldy, method, extrap): # noqa: N802 out = np.full_like(newx, np.nan, dtype=f"float{oldy.dtype.itemsize * 8}") if np.all(mask_new) or np.all(mask_old): warn( - "All-NaN slice encountered in interp_on_quantiles", + "All-nan slice encountered in interp_on_quantiles", category=RuntimeWarning, ) return out @@ -380,7 +380,7 @@ def _interp_on_quantiles_2D(newx, newg, oldx, oldy, oldg, method, extrap): # no out = np.full_like(newx, np.nan, dtype=f"float{oldy.dtype.itemsize * 8}") if np.all(mask_new) or np.all(mask_old): warn( - "All-NaN slice encountered in interp_on_quantiles", + "All-nan slice encountered in interp_on_quantiles", category=RuntimeWarning, ) return out @@ -415,8 +415,8 @@ def interp_on_quantiles( Interpolate in 2D with :py:func:`scipy.interpolate.griddata` if grouping is used, in 1D otherwise, with :py:class:`scipy.interpolate.interp1d`. - Any NaNs in `xq` or `yq` are removed from the input map. - Similarly, NaNs in newx are left NaNs. + Any nans in `xq` or `yq` are removed from the input map. + Similarly, nans in newx are left nans. Parameters ---------- @@ -441,7 +441,7 @@ def interp_on_quantiles( ----- Extrapolation methods: - - 'nan' : Any value of `newx` outside the range of `xq` is set to NaN. + - 'nan' : Any value of `newx` outside the range of `xq` is set to 'nan'. - 'constant' : Values of `newx` smaller than the minimum of `xq` are set to the first value of `yq` and those larger than the maximum, set to the last one (first and last non-nan values along the "quantiles" dimension). When the grouping is "time.month", @@ -538,7 +538,7 @@ def rank( Notes ----- - The `bottleneck` library is required. NaNs in the input array are returned as NaNs. + The `bottleneck` library is required. nans in the input array are returned as nans. See Also -------- @@ -579,8 +579,8 @@ def pc_matrix(arr: np.ndarray | dsk.Array) -> np.ndarray | dsk.Array: """Construct a Principal Component matrix. This matrix can be used to transform points in arr to principal components - coordinates. Note that this function does not manage NaNs; if a single observation is null, all elements - of the transformation matrix involving that variable will be NaN. + coordinates. Note that this function does not manage nans; if a single observation is null, all elements + of the transformation matrix involving that variable will be nan. Parameters ---------- @@ -797,7 +797,7 @@ def get_clusters(data: xr.DataArray, u1, u2, dim: str = "time") -> xr.Dataset: - `maxpos` : Index of the maximal value within the cluster (`dim` reduced, new `cluster`), int - `maximum` : Maximal value within the cluster (`dim` reduced, new `cluster`), same dtype as data. - For `start`, `end` and `maxpos`, -1 means NaN and should always correspond to a `NaN` in `maximum`. + For `start`, `end` and `maxpos`, -1 means nan and should always correspond to a `nan` in `maximum`. The length along `cluster` is half the size of "dim", the maximal theoretical number of clusters. """ @@ -919,7 +919,7 @@ def copy_all_attrs(ds: xr.Dataset | xr.DataArray, ref: xr.Dataset | xr.DataArray def _pairwise_spearman(da, dims): """Area-averaged pairwise temporal correlation. - With skipna-shortcuts for cases where all times or all points are NaN. + With skipna-shortcuts for cases where all times or all points are nan. """ da = da - da.mean(dims) da = ( @@ -930,7 +930,7 @@ def _pairwise_spearman(da, dims): def _skipna_correlation(data): nv, _nt = data.shape - # Mask of which variable are all NaN + # Mask of which variable are all nan mask_omit = np.isnan(data).all(axis=1) # Remove useless variables data_noallnan = data[~mask_omit, :] @@ -939,7 +939,7 @@ def _skipna_correlation(data): # Remove those times (they'll be omitted anyway) data_nonan = data_noallnan[:, ~mask_skip] - # We still have a possibility that a NaN was unique to a variable and time. + # We still have a possibility that a nan was unique to a variable and time. # If this is the case, it will be a lot longer, but what can we do. coef = spearmanr(data_nonan, axis=1, nan_policy="omit").correlation From cf286d248349dfce5da240fbfc14f9e1f419c325 Mon Sep 17 00:00:00 2001 From: Trevor James Smith <10819524+Zeitsperre@users.noreply.github.com> Date: Fri, 26 Jul 2024 16:42:00 -0400 Subject: [PATCH 23/33] fix cannon test --- tests/test_sdba/test_adjustment.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_sdba/test_adjustment.py b/tests/test_sdba/test_adjustment.py index cc6104429..d02463ec3 100644 --- a/tests/test_sdba/test_adjustment.py +++ b/tests/test_sdba/test_adjustment.py @@ -283,7 +283,7 @@ def test_mon_U(self, mon_series, series, kind, name, add_dims, random): np.testing.assert_array_almost_equal(mqm, int(kind == MULTIPLICATIVE), 1) np.testing.assert_allclose(p.transpose(..., "time"), ref_t, rtol=0.1, atol=0.5) - def test_cannon_and_from_ds(self, cannon_2015_rvs, tmp_path, random): + def test_cannon_and_from_ds(self, cannon_2015_rvs, tmp_path, open_dataset, random): ref, hist, sim = cannon_2015_rvs(15000, random=random) DQM = DetrendedQuantileMapping.train(ref, hist, kind="*", group="time") @@ -295,7 +295,7 @@ def test_cannon_and_from_ds(self, cannon_2015_rvs, tmp_path, random): file = tmp_path / "test_dqm.nc" DQM.ds.to_netcdf(file, engine="h5netcdf") - ds = xr.open_dataset(file) + ds = open_dataset(file) DQM2 = DetrendedQuantileMapping.from_dataset(ds) xr.testing.assert_equal(DQM.ds, DQM2.ds) From d557f4de40b03ec94211b31f65f3abd4bed3ead5 Mon Sep 17 00:00:00 2001 From: Trevor James Smith <10819524+Zeitsperre@users.noreply.github.com> Date: Fri, 26 Jul 2024 16:52:41 -0400 Subject: [PATCH 24/33] use h5netcdf --- tests/test_cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_cli.py b/tests/test_cli.py index d92f18994..c696db6f4 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -186,7 +186,7 @@ def test_indicator_chain(tas_series, tmp_path): input_file = tmp_path / "tas.nc" output_file = tmp_path / "out.nc" - tas.to_netcdf(input_file) + tas.to_netcdf(input_file, engine="h5netcdf") runner = CliRunner() results = runner.invoke( From 4d95e636576f3f694a929291d7ff8547edcc9797 Mon Sep 17 00:00:00 2001 From: Pascal Bourgault Date: Fri, 26 Jul 2024 17:57:24 -0400 Subject: [PATCH 25/33] Remove netCDF4 as a dep - put h5netcdf back to dev deps - fix cli tests --- environment.yml | 1 - pyproject.toml | 3 +-- tests/test_cli.py | 18 +++++++++--------- xclim/cli.py | 10 ++++------ xclim/ensembles/_base.py | 1 - 5 files changed, 14 insertions(+), 19 deletions(-) diff --git a/environment.yml b/environment.yml index bb1b0a294..60edd5f7a 100644 --- a/environment.yml +++ b/environment.yml @@ -52,7 +52,6 @@ dependencies: - nbsphinx - nbval >=0.11.0 - nc-time-axis - - netCDF4 >=1.4 # Needs to be unpinned for numpy v2.0 (requires v1.7.0+) - notebook - pandas-stubs - platformdirs diff --git a/pyproject.toml b/pyproject.toml index 1236b4e15..f94a889ee 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -38,7 +38,6 @@ dependencies = [ "cftime>=1.4.1", "click>=8.1", "dask[array]>=2.6", - "h5netcdf>=1.3.0", "jsonpickle", "numba", "numpy>=1.20.0", @@ -66,13 +65,13 @@ dev = [ "deptry ==0.17.0", "flake8 >=7.1.0", "flake8-rst-docstrings", + "h5netcdf>=1.3.0", "ipython", "isort ==5.13.2", "mypy", "nbconvert <7.14", # Pinned due to directive errors in sphinx. See: https://github.com/jupyter/nbconvert/issues/2092 "nbqa >=1.8.2", "nbval >=0.11.0", - "netCDF4 >=1.4", # Needs to be unpinned for numpy v2.0 (requires v1.7.0+) "pandas-stubs >=2.2", "platformdirs >=3.2", "pooch", diff --git a/tests/test_cli.py b/tests/test_cli.py index c696db6f4..7c18a04c5 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -111,8 +111,8 @@ def test_multi_input(tas_series, pr_series, tmp_path): pr_file = tmp_path / "multi_pr_in.nc" output_file = tmp_path / "out.nc" - tas.to_dataset().to_netcdf(tas_file, engine="h5netcdf") - pr.to_dataset().to_netcdf(pr_file, engine="h5netcdf") + tas.to_dataset().to_netcdf(tas_file) + pr.to_dataset().to_netcdf(pr_file) runner = CliRunner() results = runner.invoke( @@ -128,7 +128,7 @@ def test_multi_input(tas_series, pr_series, tmp_path): ) assert "Processing : solidprcptot" in results.output - out = xr.open_dataset(output_file, engine="h5netcdf") + out = xr.open_dataset(output_file) assert out.solidprcptot.sum() == 0 @@ -158,7 +158,7 @@ def test_renaming_variable(tas_series, tmp_path): input_file = tmp_path / "tas.nc" output_file = tmp_path / "out.nc" tas.name = "tas" - tas.to_netcdf(input_file, engine="h5netcdf") + tas.to_netcdf(input_file) with xclim.set_options(cf_compliance="warn"): runner = CliRunner() results = runner.invoke( @@ -177,7 +177,7 @@ def test_renaming_variable(tas_series, tmp_path): assert "Processing : tn_mean" in results.output assert "100% Completed" in results.output - out = xr.open_dataset(output_file, engine="h5netcdf") + out = xr.open_dataset(output_file) assert out.tn_mean[0] == 1.0 @@ -186,7 +186,7 @@ def test_indicator_chain(tas_series, tmp_path): input_file = tmp_path / "tas.nc" output_file = tmp_path / "out.nc" - tas.to_netcdf(input_file, engine="h5netcdf") + tas.to_netcdf(input_file) runner = CliRunner() results = runner.invoke( @@ -206,7 +206,7 @@ def test_indicator_chain(tas_series, tmp_path): assert "Processing : growing_degree_days" in results.output assert "100% Completed" in results.output - out = xr.open_dataset(output_file, engine="h5netcdf") + out = xr.open_dataset(output_file) assert out.tg_mean[0] == 1.0 assert out.growing_degree_days[0] == 0 @@ -264,13 +264,13 @@ def test_suspicious_precipitation_flags(pr_series, tmp_path): input_file = tmp_path / "bad_pr.nc" output_file = tmp_path / "out.nc" - bad_pr.to_netcdf(input_file, engine="h5netcdf") + bad_pr.to_netcdf(input_file) runner = CliRunner() runner.invoke( cli, ["-i", str(input_file), "-o", str(output_file), "dataflags", "pr"] ) - with xr.open_dataset(output_file, engine="h5netcdf") as ds: + with xr.open_dataset(output_file) as ds: for var in ds.data_vars: assert var diff --git a/xclim/cli.py b/xclim/cli.py index 1dd36e7c1..ebb35aba1 100644 --- a/xclim/cli.py +++ b/xclim/cli.py @@ -416,7 +416,7 @@ def get_command(self, ctx, name): @click.option( "--engine", help="Engine to use when opening the input dataset(s). " - "If not specified, 'h5netcdf' is used. Other options are 'scipy' and 'netcdf4' (requires python-netcdf4).", + "If not specified, xarrat decides.", ) @click.pass_context def cli(ctx, **kwargs): @@ -468,12 +468,8 @@ def cli(ctx, **kwargs): for dim, num in map(lambda x: x.split(":"), kwargs["chunks"].split(",")) } - if kwargs["engine"] is None: - kwargs["engine"] = "h5netcdf" - kwargs["xr_kwargs"] = { "chunks": kwargs["chunks"] or {}, - "engine": kwargs["engine"], } ctx.obj = kwargs @@ -486,7 +482,9 @@ def write_file(ctx, *args, **kwargs): if ctx.obj["verbose"]: click.echo(f"Writing to file {ctx.obj['output']}") with ProgressBar(): - r = ctx.obj["ds_out"].to_netcdf(ctx.obj["output"], compute=False) + r = ctx.obj["ds_out"].to_netcdf( + ctx.obj["output"], engine=kwargs["engine"], compute=False + ) if ctx.obj["dask_nthreads"] is not None: progress(r.data) r.compute() diff --git a/xclim/ensembles/_base.py b/xclim/ensembles/_base.py index 39ddf7789..31988d559 100644 --- a/xclim/ensembles/_base.py +++ b/xclim/ensembles/_base.py @@ -421,7 +421,6 @@ def _ens_align_datasets( """ xr_kwargs.setdefault("chunks", "auto") xr_kwargs.setdefault("decode_times", False) - xr_kwargs.setdefault("engine", "h5netcdf") if isinstance(datasets, str): datasets = glob(datasets) From 3312405df6a5a41e41cdd0f8950b8a1766371e7e Mon Sep 17 00:00:00 2001 From: Trevor James Smith <10819524+Zeitsperre@users.noreply.github.com> Date: Wed, 31 Jul 2024 11:59:28 -0400 Subject: [PATCH 26/33] replace open_dataset --- tests/test_sdba/test_adjustment.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_sdba/test_adjustment.py b/tests/test_sdba/test_adjustment.py index d02463ec3..aea505073 100644 --- a/tests/test_sdba/test_adjustment.py +++ b/tests/test_sdba/test_adjustment.py @@ -67,7 +67,7 @@ def test_harmonize_units_multivariate(self, series, random, use_dask): class TestLoci: @pytest.mark.parametrize("group,dec", (["time", 2], ["time.month", 1])) - def test_time_and_from_ds(self, series, group, dec, tmp_path, random): + def test_time_and_from_ds(self, series, group, dec, tmp_path, random, open_dataset): n = 10000 u = random.random(n) @@ -93,7 +93,7 @@ def test_time_and_from_ds(self, series, group, dec, tmp_path, random): file = tmp_path / "test_loci.nc" loci.ds.to_netcdf(file, engine="h5netcdf") - ds = xr.open_dataset(file, engine="h5netcdf") + ds = open_dataset(file) loci2 = LOCI.from_dataset(ds) xr.testing.assert_equal(loci.ds, loci2.ds) From 3f3ae71e234cf817245c0220ba484be0e77319e3 Mon Sep 17 00:00:00 2001 From: Pascal Bourgault Date: Wed, 31 Jul 2024 12:19:58 -0400 Subject: [PATCH 27/33] Better Version checks --- CHANGELOG.rst | 3 +-- tests/test_atmos.py | 6 ++--- tests/test_generic.py | 6 ++--- tests/test_hydrology.py | 6 ++--- tests/test_indices.py | 53 ++++++++++++++++------------------------- tests/test_land.py | 11 ++++----- tests/test_snow.py | 14 ++++------- tests/test_units.py | 7 +++--- 8 files changed, 43 insertions(+), 63 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index dc9f1b57d..85fcaaa1b 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -14,8 +14,7 @@ New features and enhancements Breaking changes ^^^^^^^^^^^^^^^^ -* Dimensionless quantities now use the "1" units attribute as specified by the CF conventions, previously an empty string was returned. (:pull:`1814`). -* Updated minimum versions for some dependencies: ``numpy>=2.0.0``, ``cf-xarray>=0.9.3`` and ``pint>=0.24.1``. (:pull:`1814`). +* As updated in ``cf_xarray>=0.9.4``, dimensionless quantities now use the "1" units attribute as specified by the CF conventions, previously an empty string was returned. (:pull:`1814`). Bug fixes ^^^^^^^^^ diff --git a/tests/test_atmos.py b/tests/test_atmos.py index 10c5e14db..e183a5362 100644 --- a/tests/test_atmos.py +++ b/tests/test_atmos.py @@ -4,7 +4,7 @@ import numpy as np import xarray as xr -from numpy import __version__ as __numpy_version__ +from cf_xarray import __version__ as __cfxr_version__ from packaging.version import Version from xclim import atmos, set_options @@ -262,8 +262,8 @@ def test_wind_profile(atmosds): def test_wind_power_potential(atmosds): out = atmos.wind_power_potential(wind_speed=atmosds.sfcWind) - # FIXME: Confirm these expected outputs - if Version(__numpy_version__) < Version("2.0.0"): + + if Version(__cfxr_version__) < Version("0.9.4"): assert out.attrs["units"] == "" else: assert out.attrs["units"] == "1" diff --git a/tests/test_generic.py b/tests/test_generic.py index ab7122584..7e2b2dc21 100644 --- a/tests/test_generic.py +++ b/tests/test_generic.py @@ -5,7 +5,7 @@ import numpy as np import pytest import xarray as xr -from numpy import __version__ as __numpy_version__ +from cf_xarray import __version__ as __cfxr_version__ from packaging.version import Version from xclim.core.calendar import doy_to_days_since, select_time @@ -103,8 +103,8 @@ def test_doyminmax(self, q_series): for da in [dmx, dmn]: for attr in ["units", "is_dayofyear", "calendar"]: assert attr in da.attrs.keys() - # FIXME: Confirm these expected outputs - if Version(__numpy_version__) < Version("2.0.0"): + + if Version(__cfxr_version__) < Version("0.9.4"): assert da.attrs["units"] == "" else: assert da.attrs["units"] == "1" diff --git a/tests/test_hydrology.py b/tests/test_hydrology.py index 7340874f1..2ad29bd35 100644 --- a/tests/test_hydrology.py +++ b/tests/test_hydrology.py @@ -1,7 +1,7 @@ from __future__ import annotations import numpy as np -from numpy import __version__ as __numpy_version__ +from cf_xarray import __version__ as __cfxr_version__ from packaging.version import Version from xclim import indices as xci @@ -42,8 +42,8 @@ def test_simple(self, snw_series): snw = snw_series(a, start="1999-01-01") out = xci.snw_max_doy(snw, freq="YS") np.testing.assert_array_equal(out, [11, np.nan]) - # FIXME: Confirm these expected outputs - if Version(__numpy_version__) < Version("2.0.0"): + + if Version(__cfxr_version__) < Version("0.9.4"): assert out.attrs["units"] == "" else: assert out.attrs["units"] == "1" diff --git a/tests/test_indices.py b/tests/test_indices.py index 1aa7e1e5b..e735ea360 100644 --- a/tests/test_indices.py +++ b/tests/test_indices.py @@ -19,7 +19,7 @@ import pandas as pd import pytest import xarray as xr -from numpy import __version__ as __numpy_version__ +from cf_xarray import __version__ as __cfxr_version__ from packaging.version import Version from pint import __version__ as __pint_version__ @@ -140,16 +140,16 @@ def test_simple(self, tas_series): out = xci.cold_spell_frequency(da, thresh="-10. C", freq="ME") np.testing.assert_array_equal(out, [1, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0]) - # FIXME: Confirm these expected outputs - if Version(__numpy_version__) < Version("2.0.0"): + + if Version(__cfxr_version__) < Version("0.9.4"): assert out.units == "" else: assert out.units == "1" out = xci.cold_spell_frequency(da, thresh="-10. C", freq="YS") np.testing.assert_array_equal(out, 3) - # FIXME: Confirm these expected outputs - if Version(__numpy_version__) < Version("2.0.0"): + + if Version(__cfxr_version__) < Version("0.9.4"): assert out.units == "" else: assert out.units == "1" @@ -242,7 +242,7 @@ def test_no_cdd(self, tas_series): a = tas_series(np.array([10, 15, -5, 18]) + K2C) cdd = xci.cooling_degree_days(a) assert cdd == 0 - # FIXME: Confirm these expected outputs + if Version(__pint_version__) < Version("0.24.1"): assert cdd.units == "K d" else: @@ -921,8 +921,7 @@ def test_simple(self, tas_series): for attr in ["units", "is_dayofyear", "calendar"]: assert attr in lsf.attrs.keys() - # FIXME: Confirm these expected outputs - if Version(__numpy_version__) < Version("2.0.0"): + if Version(__cfxr_version__) < Version("0.9.4"): assert lsf.attrs["units"] == "" else: assert lsf.attrs["units"] == "1" @@ -948,8 +947,7 @@ def test_simple(self, tas_series): for attr in ["units", "is_dayofyear", "calendar"]: assert attr in fdb.attrs.keys() - # FIXME: Confirm these expected outputs - if Version(__numpy_version__) < Version("2.0.0"): + if Version(__cfxr_version__) < Version("0.9.4"): assert fdb.attrs["units"] == "" else: assert fdb.attrs["units"] == "1" @@ -985,8 +983,7 @@ def test_simple(self, tas_series): for attr in ["units", "is_dayofyear", "calendar"]: assert attr in fda.attrs.keys() - # FIXME: Confirm these expected outputs - if Version(__numpy_version__) < Version("2.0.0"): + if Version(__cfxr_version__) < Version("0.9.4"): assert fda.attrs["units"] == "" else: assert fda.attrs["units"] == "1" @@ -1016,8 +1013,7 @@ def test_thresholds(self, tas_series): for attr in ["units", "is_dayofyear", "calendar"]: assert attr in out.attrs.keys() - # FIXME: Confirm these expected outputs - if Version(__numpy_version__) < Version("2.0.0"): + if Version(__cfxr_version__) < Version("0.9.4"): assert out.attrs["units"] == "" else: assert out.attrs["units"] == "1" @@ -1109,8 +1105,7 @@ def test_simple(self, tas_series): for attr in ["units", "is_dayofyear", "calendar"]: assert attr in out.attrs.keys() - # FIXME: Confirm these expected outputs - if Version(__numpy_version__) < Version("2.0.0"): + if Version(__cfxr_version__) < Version("0.9.4"): assert out.attrs["units"] == "" else: assert out.attrs["units"] == "1" @@ -1228,8 +1223,7 @@ def test_simple(self, tasmin_series): for attr in ["units", "is_dayofyear", "calendar"]: assert attr in out.attrs.keys() - # FIXME: Confirm these expected outputs - if Version(__numpy_version__) < Version("2.0.0"): + if Version(__cfxr_version__) < Version("0.9.4"): assert out.attrs["units"] == "" else: assert out.attrs["units"] == "1" @@ -1267,8 +1261,7 @@ def test_varying_mid_dates(self, tasmin_series, d1, d2, mid_date, expected): for attr in ["units", "is_dayofyear", "calendar"]: assert attr in gs_end.attrs.keys() - # FIXME: Confirm these expected outputs - if Version(__numpy_version__) < Version("2.0.0"): + if Version(__cfxr_version__) < Version("0.9.4"): assert gs_end.attrs["units"] == "" else: assert gs_end.attrs["units"] == "1" @@ -2782,8 +2775,7 @@ def test_degree_days_exceedance_date(tas_series): for attr in ["units", "is_dayofyear", "calendar"]: assert attr in out.attrs.keys() - # FIXME: Confirm these expected outputs - if Version(__numpy_version__) < Version("2.0.0"): + if Version(__cfxr_version__) < Version("0.9.4"): assert out.attrs["units"] == "" else: assert out.attrs["units"] == "1" @@ -2827,8 +2819,8 @@ def test_first_snowfall(prsn_series, prsnd_series): assert out[0] == 166 for attr in ["units", "is_dayofyear", "calendar"]: assert attr in out.attrs.keys() - # FIXME: Confirm these expected outputs - if Version(__numpy_version__) < Version("2.0.0"): + + if Version(__cfxr_version__) < Version("0.9.4"): assert out.attrs["units"] == "" else: assert out.attrs["units"] == "1" @@ -2852,8 +2844,8 @@ def test_first_snowfall(prsn_series, prsnd_series): assert out[0] == 166 for attr in ["units", "is_dayofyear", "calendar"]: assert attr in out.attrs.keys() - # FIXME: Confirm these expected outputs - if Version(__numpy_version__) < Version("2.0.0"): + + if Version(__cfxr_version__) < Version("0.9.4"): assert out.attrs["units"] == "" else: assert out.attrs["units"] == "1" @@ -2994,8 +2986,7 @@ def test_continous_snow_season_start(self, snd_series, snw_series): for attr in ["units", "is_dayofyear", "calendar"]: assert attr in out.attrs.keys() - # FIXME: Confirm these expected outputs - if Version(__numpy_version__) < Version("2.0.0"): + if Version(__cfxr_version__) < Version("0.9.4"): assert out.attrs["units"] == "" else: assert out.attrs["units"] == "1" @@ -3008,8 +2999,7 @@ def test_continous_snow_season_start(self, snd_series, snw_series): for attr in ["units", "is_dayofyear", "calendar"]: assert attr in out.attrs.keys() - # FIXME: Confirm these expected outputs - if Version(__numpy_version__) < Version("2.0.0"): + if Version(__cfxr_version__) < Version("0.9.4"): assert out.attrs["units"] == "" else: assert out.attrs["units"] == "1" @@ -3036,8 +3026,7 @@ def test_snow_season_end(self, snd_series, snw_series): for attr in ["units", "is_dayofyear", "calendar"]: assert attr in out.attrs.keys() - # FIXME: Confirm these expected outputs - if Version(__numpy_version__) < Version("2.0.0"): + if Version(__cfxr_version__) < Version("0.9.4"): assert out.attrs["units"] == "" else: assert out.attrs["units"] == "1" diff --git a/tests/test_land.py b/tests/test_land.py index ef09127ec..7c1057f25 100644 --- a/tests/test_land.py +++ b/tests/test_land.py @@ -4,7 +4,7 @@ import numpy as np import xarray as xr -from numpy import __version__ as __numpy_version__ +from cf_xarray import __version__ as __cfxr_version__ from packaging.version import Version from xclim import land @@ -13,8 +13,7 @@ def test_base_flow_index(ndq_series): out = land.base_flow_index(ndq_series, freq="YS") - # FIXME: Confirm these expected outputs - if Version(__numpy_version__) < Version("2.0.0"): + if Version(__cfxr_version__) < Version("0.9.4"): assert out.attrs["units"] == "" else: assert out.attrs["units"] == "1" @@ -25,8 +24,7 @@ def test_base_flow_index(ndq_series): def test_rb_flashiness_index(ndq_series): out = land.base_flow_index(ndq_series, freq="YS") - # FIXME: Confirm these expected outputs - if Version(__numpy_version__) < Version("2.0.0"): + if Version(__cfxr_version__) < Version("0.9.4"): assert out.attrs["units"] == "" else: assert out.attrs["units"] == "1" @@ -37,8 +35,7 @@ def test_rb_flashiness_index(ndq_series): def test_qdoy_max(ndq_series, q_series): out = land.doy_qmax(ndq_series, freq="YS", season="JJA") - # FIXME: Confirm these expected outputs - if Version(__numpy_version__) < Version("2.0.0"): + if Version(__cfxr_version__) < Version("0.9.4"): assert out.attrs["units"] == "" else: assert out.attrs["units"] == "1" diff --git a/tests/test_snow.py b/tests/test_snow.py index b579b0ade..38889cdb5 100644 --- a/tests/test_snow.py +++ b/tests/test_snow.py @@ -2,7 +2,7 @@ import numpy as np import pytest -from numpy import __version__ as __numpy_version__ +from cf_xarray import __version__ as __cfxr_version__ from packaging.version import Version from xclim import land @@ -48,8 +48,7 @@ def test_simple(self, snd_series): out = land.snd_season_start(snd) - # FIXME: Confirm these expected outputs - if Version(__numpy_version__) < Version("2.0.0"): + if Version(__cfxr_version__) < Version("0.9.4"): assert out.units == "" else: assert out.units == "1" @@ -58,8 +57,7 @@ def test_simple(self, snd_series): out = land.snd_season_end(snd) - # FIXME: Confirm these expected outputs - if Version(__numpy_version__) < Version("2.0.0"): + if Version(__cfxr_version__) < Version("0.9.4"): assert out.units == "" else: assert out.units == "1" @@ -82,8 +80,7 @@ def test_simple(self, snw_series): out = land.snw_season_start(snw) - # FIXME: Confirm these expected outputs - if Version(__numpy_version__) < Version("2.0.0"): + if Version(__cfxr_version__) < Version("0.9.4"): assert out.units == "" else: assert out.units == "1" @@ -92,8 +89,7 @@ def test_simple(self, snw_series): out = land.snw_season_end(snw) - # FIXME: Confirm these expected outputs - if Version(__numpy_version__) < Version("2.0.0"): + if Version(__cfxr_version__) < Version("0.9.4"): assert out.units == "" else: assert out.units == "1" diff --git a/tests/test_units.py b/tests/test_units.py index fd0d604e1..a178e58b9 100644 --- a/tests/test_units.py +++ b/tests/test_units.py @@ -6,8 +6,8 @@ import pint.errors import pytest import xarray as xr +from cf_xarray import __version__ as __cfxr_version__ from dask import array as dsk -from numpy import __version__ as __numpy_version__ from packaging.version import Version from xclim import indices, set_options @@ -144,8 +144,7 @@ def test_units2pint(self, pr_series): u = units2pint("1") assert pint2cfunits(u) == "1" - # FIXME: Confirm these expected outputs - if Version(__numpy_version__) < Version("2.0.0"): + if Version(__cfxr_version__) < Version("0.9.4"): assert pint2cfunits(u) == "" else: assert pint2cfunits(u) == "1" @@ -362,7 +361,7 @@ def test_to_agg_units(in_u, opfunc, op, exp, exp_u): np.testing.assert_allclose(out, exp) if isinstance(exp_u, tuple): - if Version(__numpy_version__) < Version("2.0.0"): + if Version(__cfxr_version__) < Version("0.9.4"): assert out.attrs["units"] == exp_u[0] else: assert out.attrs["units"] == exp_u[1] From 967ddbb15b20decc839a8b5c6833aa3eb09b7651 Mon Sep 17 00:00:00 2001 From: Pascal Bourgault Date: Wed, 31 Jul 2024 12:23:32 -0400 Subject: [PATCH 28/33] oups wrong version --- tests/test_atmos.py | 2 +- tests/test_generic.py | 2 +- tests/test_hydrology.py | 2 +- tests/test_indices.py | 30 +++++++++++++++--------------- tests/test_land.py | 6 +++--- tests/test_snow.py | 8 ++++---- tests/test_units.py | 4 ++-- 7 files changed, 27 insertions(+), 27 deletions(-) diff --git a/tests/test_atmos.py b/tests/test_atmos.py index e183a5362..d705e7493 100644 --- a/tests/test_atmos.py +++ b/tests/test_atmos.py @@ -263,7 +263,7 @@ def test_wind_profile(atmosds): def test_wind_power_potential(atmosds): out = atmos.wind_power_potential(wind_speed=atmosds.sfcWind) - if Version(__cfxr_version__) < Version("0.9.4"): + if Version(__cfxr_version__) < Version("0.9.3"): assert out.attrs["units"] == "" else: assert out.attrs["units"] == "1" diff --git a/tests/test_generic.py b/tests/test_generic.py index 7e2b2dc21..ef129121a 100644 --- a/tests/test_generic.py +++ b/tests/test_generic.py @@ -104,7 +104,7 @@ def test_doyminmax(self, q_series): for attr in ["units", "is_dayofyear", "calendar"]: assert attr in da.attrs.keys() - if Version(__cfxr_version__) < Version("0.9.4"): + if Version(__cfxr_version__) < Version("0.9.3"): assert da.attrs["units"] == "" else: assert da.attrs["units"] == "1" diff --git a/tests/test_hydrology.py b/tests/test_hydrology.py index 2ad29bd35..b2276bda7 100644 --- a/tests/test_hydrology.py +++ b/tests/test_hydrology.py @@ -43,7 +43,7 @@ def test_simple(self, snw_series): out = xci.snw_max_doy(snw, freq="YS") np.testing.assert_array_equal(out, [11, np.nan]) - if Version(__cfxr_version__) < Version("0.9.4"): + if Version(__cfxr_version__) < Version("0.9.3"): assert out.attrs["units"] == "" else: assert out.attrs["units"] == "1" diff --git a/tests/test_indices.py b/tests/test_indices.py index e735ea360..b635b27db 100644 --- a/tests/test_indices.py +++ b/tests/test_indices.py @@ -141,7 +141,7 @@ def test_simple(self, tas_series): out = xci.cold_spell_frequency(da, thresh="-10. C", freq="ME") np.testing.assert_array_equal(out, [1, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0]) - if Version(__cfxr_version__) < Version("0.9.4"): + if Version(__cfxr_version__) < Version("0.9.3"): assert out.units == "" else: assert out.units == "1" @@ -149,7 +149,7 @@ def test_simple(self, tas_series): out = xci.cold_spell_frequency(da, thresh="-10. C", freq="YS") np.testing.assert_array_equal(out, 3) - if Version(__cfxr_version__) < Version("0.9.4"): + if Version(__cfxr_version__) < Version("0.9.3"): assert out.units == "" else: assert out.units == "1" @@ -921,7 +921,7 @@ def test_simple(self, tas_series): for attr in ["units", "is_dayofyear", "calendar"]: assert attr in lsf.attrs.keys() - if Version(__cfxr_version__) < Version("0.9.4"): + if Version(__cfxr_version__) < Version("0.9.3"): assert lsf.attrs["units"] == "" else: assert lsf.attrs["units"] == "1" @@ -947,7 +947,7 @@ def test_simple(self, tas_series): for attr in ["units", "is_dayofyear", "calendar"]: assert attr in fdb.attrs.keys() - if Version(__cfxr_version__) < Version("0.9.4"): + if Version(__cfxr_version__) < Version("0.9.3"): assert fdb.attrs["units"] == "" else: assert fdb.attrs["units"] == "1" @@ -983,7 +983,7 @@ def test_simple(self, tas_series): for attr in ["units", "is_dayofyear", "calendar"]: assert attr in fda.attrs.keys() - if Version(__cfxr_version__) < Version("0.9.4"): + if Version(__cfxr_version__) < Version("0.9.3"): assert fda.attrs["units"] == "" else: assert fda.attrs["units"] == "1" @@ -1013,7 +1013,7 @@ def test_thresholds(self, tas_series): for attr in ["units", "is_dayofyear", "calendar"]: assert attr in out.attrs.keys() - if Version(__cfxr_version__) < Version("0.9.4"): + if Version(__cfxr_version__) < Version("0.9.3"): assert out.attrs["units"] == "" else: assert out.attrs["units"] == "1" @@ -1105,7 +1105,7 @@ def test_simple(self, tas_series): for attr in ["units", "is_dayofyear", "calendar"]: assert attr in out.attrs.keys() - if Version(__cfxr_version__) < Version("0.9.4"): + if Version(__cfxr_version__) < Version("0.9.3"): assert out.attrs["units"] == "" else: assert out.attrs["units"] == "1" @@ -1223,7 +1223,7 @@ def test_simple(self, tasmin_series): for attr in ["units", "is_dayofyear", "calendar"]: assert attr in out.attrs.keys() - if Version(__cfxr_version__) < Version("0.9.4"): + if Version(__cfxr_version__) < Version("0.9.3"): assert out.attrs["units"] == "" else: assert out.attrs["units"] == "1" @@ -1261,7 +1261,7 @@ def test_varying_mid_dates(self, tasmin_series, d1, d2, mid_date, expected): for attr in ["units", "is_dayofyear", "calendar"]: assert attr in gs_end.attrs.keys() - if Version(__cfxr_version__) < Version("0.9.4"): + if Version(__cfxr_version__) < Version("0.9.3"): assert gs_end.attrs["units"] == "" else: assert gs_end.attrs["units"] == "1" @@ -2775,7 +2775,7 @@ def test_degree_days_exceedance_date(tas_series): for attr in ["units", "is_dayofyear", "calendar"]: assert attr in out.attrs.keys() - if Version(__cfxr_version__) < Version("0.9.4"): + if Version(__cfxr_version__) < Version("0.9.3"): assert out.attrs["units"] == "" else: assert out.attrs["units"] == "1" @@ -2820,7 +2820,7 @@ def test_first_snowfall(prsn_series, prsnd_series): for attr in ["units", "is_dayofyear", "calendar"]: assert attr in out.attrs.keys() - if Version(__cfxr_version__) < Version("0.9.4"): + if Version(__cfxr_version__) < Version("0.9.3"): assert out.attrs["units"] == "" else: assert out.attrs["units"] == "1" @@ -2845,7 +2845,7 @@ def test_first_snowfall(prsn_series, prsnd_series): for attr in ["units", "is_dayofyear", "calendar"]: assert attr in out.attrs.keys() - if Version(__cfxr_version__) < Version("0.9.4"): + if Version(__cfxr_version__) < Version("0.9.3"): assert out.attrs["units"] == "" else: assert out.attrs["units"] == "1" @@ -2986,7 +2986,7 @@ def test_continous_snow_season_start(self, snd_series, snw_series): for attr in ["units", "is_dayofyear", "calendar"]: assert attr in out.attrs.keys() - if Version(__cfxr_version__) < Version("0.9.4"): + if Version(__cfxr_version__) < Version("0.9.3"): assert out.attrs["units"] == "" else: assert out.attrs["units"] == "1" @@ -2999,7 +2999,7 @@ def test_continous_snow_season_start(self, snd_series, snw_series): for attr in ["units", "is_dayofyear", "calendar"]: assert attr in out.attrs.keys() - if Version(__cfxr_version__) < Version("0.9.4"): + if Version(__cfxr_version__) < Version("0.9.3"): assert out.attrs["units"] == "" else: assert out.attrs["units"] == "1" @@ -3026,7 +3026,7 @@ def test_snow_season_end(self, snd_series, snw_series): for attr in ["units", "is_dayofyear", "calendar"]: assert attr in out.attrs.keys() - if Version(__cfxr_version__) < Version("0.9.4"): + if Version(__cfxr_version__) < Version("0.9.3"): assert out.attrs["units"] == "" else: assert out.attrs["units"] == "1" diff --git a/tests/test_land.py b/tests/test_land.py index 7c1057f25..07841cd87 100644 --- a/tests/test_land.py +++ b/tests/test_land.py @@ -13,7 +13,7 @@ def test_base_flow_index(ndq_series): out = land.base_flow_index(ndq_series, freq="YS") - if Version(__cfxr_version__) < Version("0.9.4"): + if Version(__cfxr_version__) < Version("0.9.3"): assert out.attrs["units"] == "" else: assert out.attrs["units"] == "1" @@ -24,7 +24,7 @@ def test_base_flow_index(ndq_series): def test_rb_flashiness_index(ndq_series): out = land.base_flow_index(ndq_series, freq="YS") - if Version(__cfxr_version__) < Version("0.9.4"): + if Version(__cfxr_version__) < Version("0.9.3"): assert out.attrs["units"] == "" else: assert out.attrs["units"] == "1" @@ -35,7 +35,7 @@ def test_rb_flashiness_index(ndq_series): def test_qdoy_max(ndq_series, q_series): out = land.doy_qmax(ndq_series, freq="YS", season="JJA") - if Version(__cfxr_version__) < Version("0.9.4"): + if Version(__cfxr_version__) < Version("0.9.3"): assert out.attrs["units"] == "" else: assert out.attrs["units"] == "1" diff --git a/tests/test_snow.py b/tests/test_snow.py index 38889cdb5..6cbb20c65 100644 --- a/tests/test_snow.py +++ b/tests/test_snow.py @@ -48,7 +48,7 @@ def test_simple(self, snd_series): out = land.snd_season_start(snd) - if Version(__cfxr_version__) < Version("0.9.4"): + if Version(__cfxr_version__) < Version("0.9.3"): assert out.units == "" else: assert out.units == "1" @@ -57,7 +57,7 @@ def test_simple(self, snd_series): out = land.snd_season_end(snd) - if Version(__cfxr_version__) < Version("0.9.4"): + if Version(__cfxr_version__) < Version("0.9.3"): assert out.units == "" else: assert out.units == "1" @@ -80,7 +80,7 @@ def test_simple(self, snw_series): out = land.snw_season_start(snw) - if Version(__cfxr_version__) < Version("0.9.4"): + if Version(__cfxr_version__) < Version("0.9.3"): assert out.units == "" else: assert out.units == "1" @@ -89,7 +89,7 @@ def test_simple(self, snw_series): out = land.snw_season_end(snw) - if Version(__cfxr_version__) < Version("0.9.4"): + if Version(__cfxr_version__) < Version("0.9.3"): assert out.units == "" else: assert out.units == "1" diff --git a/tests/test_units.py b/tests/test_units.py index a178e58b9..040d50dd0 100644 --- a/tests/test_units.py +++ b/tests/test_units.py @@ -144,7 +144,7 @@ def test_units2pint(self, pr_series): u = units2pint("1") assert pint2cfunits(u) == "1" - if Version(__cfxr_version__) < Version("0.9.4"): + if Version(__cfxr_version__) < Version("0.9.3"): assert pint2cfunits(u) == "" else: assert pint2cfunits(u) == "1" @@ -361,7 +361,7 @@ def test_to_agg_units(in_u, opfunc, op, exp, exp_u): np.testing.assert_allclose(out, exp) if isinstance(exp_u, tuple): - if Version(__cfxr_version__) < Version("0.9.4"): + if Version(__cfxr_version__) < Version("0.9.3"): assert out.attrs["units"] == exp_u[0] else: assert out.attrs["units"] == exp_u[1] From 14a62d500588019fd5f21f57bbf8f5ce667af8dd Mon Sep 17 00:00:00 2001 From: Pascal Bourgault Date: Wed, 31 Jul 2024 13:56:45 -0400 Subject: [PATCH 29/33] Update CHANGELOG.rst --- CHANGELOG.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 85fcaaa1b..3f700e367 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -14,7 +14,7 @@ New features and enhancements Breaking changes ^^^^^^^^^^^^^^^^ -* As updated in ``cf_xarray>=0.9.4``, dimensionless quantities now use the "1" units attribute as specified by the CF conventions, previously an empty string was returned. (:pull:`1814`). +* As updated in ``cf_xarray>=0.9.3``, dimensionless quantities now use the "1" units attribute as specified by the CF conventions, previously an empty string was returned. (:pull:`1814`). Bug fixes ^^^^^^^^^ From 05243865b9bb4e34146a72144e40312afeed1988 Mon Sep 17 00:00:00 2001 From: Pascal Bourgault Date: Wed, 31 Jul 2024 14:08:42 -0400 Subject: [PATCH 30/33] fix doctests --- xclim/core/calendar.py | 4 ++-- xclim/core/units.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/xclim/core/calendar.py b/xclim/core/calendar.py index 98e01e283..00bafc83a 100644 --- a/xclim/core/calendar.py +++ b/xclim/core/calendar.py @@ -693,8 +693,8 @@ def is_offset_divisor(divisor: str, offset: str): # Simple length comparison is sufficient for submonthly freqs # In case one of bA or bB is > W, we test many to be sure. tA = pd.date_range("1970-01-01T00:00:00", freq=offAs, periods=13) - return np.all( - (np.diff(tB)[:, np.newaxis] / np.diff(tA)[np.newaxis, :]) % 1 == 0 + return bool( + np.all((np.diff(tB)[:, np.newaxis] / np.diff(tA)[np.newaxis, :]) % 1 == 0) ) # else, we test alignment with some real dates diff --git a/xclim/core/units.py b/xclim/core/units.py index f02c0f442..183c66d4d 100644 --- a/xclim/core/units.py +++ b/xclim/core/units.py @@ -571,7 +571,7 @@ def to_agg_units( >>> degdays = convert_units_to(degdays, "K days") >>> degdays.units - 'K d' + 'd K' """ if op in ["amin", "min", "amax", "max", "mean", "sum"]: out.attrs["units"] = orig.attrs["units"] From 3d4beab4f4895b8818975be331973501a7fe03ba Mon Sep 17 00:00:00 2001 From: Pascal Bourgault Date: Wed, 31 Jul 2024 14:28:10 -0400 Subject: [PATCH 31/33] Remove opendap test as the CI doesnt have opendap support anymore --- tests/test_testing_utils.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/tests/test_testing_utils.py b/tests/test_testing_utils.py index cbf992533..3bbc044e3 100644 --- a/tests/test_testing_utils.py +++ b/tests/test_testing_utils.py @@ -146,15 +146,6 @@ def test_unsafe_urls(self): "doesnt_exist.nc", github_url="ftp://domain.does.not.exist/" ) - with pytest.raises( - OSError, - match="OPeNDAP file not read. Verify that the service is available: " - "'https://seemingly.trustworthy.com/doesnt_exist.nc'", - ): - utilities.open_dataset( - "doesnt_exist.nc", dap_url="https://seemingly.trustworthy.com/" - ) - def test_malicious_urls(self): with pytest.raises( URLError, From 1bc53fd19e7a457744f2ebe12ab7940b28284909 Mon Sep 17 00:00:00 2001 From: Pascal Bourgault Date: Wed, 31 Jul 2024 14:51:33 -0400 Subject: [PATCH 32/33] Fix notebooks for netCDF4 absence --- docs/notebooks/sdba-advanced.ipynb | 8 +++++--- docs/notebooks/sdba.ipynb | 2 +- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/docs/notebooks/sdba-advanced.ipynb b/docs/notebooks/sdba-advanced.ipynb index a14e07402..97456b6bd 100644 --- a/docs/notebooks/sdba-advanced.ipynb +++ b/docs/notebooks/sdba-advanced.ipynb @@ -271,7 +271,9 @@ "metadata": {}, "outputs": [], "source": [ - "QDM.ds.to_netcdf(\"QDM_training.nc\")\n", + "# The engine keyword is only needed if netCDF4 is not available\n", + "# The training dataset contains some attributes with types not supported by scipy/netCDF3\n", + "QDM.ds.to_netcdf(\"QDM_training.nc\", engine=\"h5netcdf\")\n", "ds = xr.open_dataset(\"QDM_training.nc\")\n", "QDM2 = QuantileDeltaMapping.from_dataset(ds)\n", "QDM2" @@ -292,7 +294,7 @@ "metadata": {}, "outputs": [], "source": [ - "QDM.ds.to_netcdf(\"QDM_training2.nc\")\n", + "QDM.ds.to_netcdf(\"QDM_training2.nc\", engine=\"h5netcdf\")\n", "ds = xr.open_dataset(\"QDM_training2.nc\")\n", "ds.attrs[\"title\"] = \"This is the dataset, but read from disk.\"\n", "QDM.set_dataset(ds)\n", @@ -878,7 +880,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.11.4" + "version": "3.12.4" }, "toc": { "base_numbering": 1, diff --git a/docs/notebooks/sdba.ipynb b/docs/notebooks/sdba.ipynb index d1030788d..73305818c 100644 --- a/docs/notebooks/sdba.ipynb +++ b/docs/notebooks/sdba.ipynb @@ -598,7 +598,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.12.3" + "version": "3.12.4" }, "toc": { "base_numbering": 1, From 3ded3ed06c1e9b306747aff3f70c4f2b2fdbf704 Mon Sep 17 00:00:00 2001 From: Pascal Bourgault Date: Wed, 31 Jul 2024 15:28:13 -0400 Subject: [PATCH 33/33] Fix xarray 2024.7.0 regression (?) issue 9300 --- docs/notebooks/sdba.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/notebooks/sdba.ipynb b/docs/notebooks/sdba.ipynb index 73305818c..6bcc5c4f8 100644 --- a/docs/notebooks/sdba.ipynb +++ b/docs/notebooks/sdba.ipynb @@ -360,7 +360,7 @@ "outputs": [], "source": [ "# We are using xarray's \"air_temperature\" dataset\n", - "ds = xr.tutorial.open_dataset(\"air_temperature\")" + "ds = xr.tutorial.load_dataset(\"air_temperature\")" ] }, {