Merge pull request #52 from dataiku/fix/resampling-last-timestamp-not…

…-round-unit-sc-113991

 fix resampling extra date

CHANGELOG.md

@@ -1,5 +1,8 @@
 # Changelog
 
+## Version 2.0.2 - Bugfix release - 2023-01
+- 🪲 Fix the bug that was adding an extra date at the end after resampling when the last input timestamp was exactly at the end of a period (week, month, half-year, year)
+
 ## Version 2.0.1 - Bugfix release - 2021-06
 - :bug: Keep the empty values rather than filtering them with the extrapolation method "Don't extrapolate (impute nulls)"
 - :scissors: Add the extrapolation method "Don't extrapolate (no imputation)" to filter missing values

plugin.json

@@ -1,6 +1,6 @@
 {
  "id": "timeseries-preparation",
- "version": "2.0.1",
+ "version": "2.0.2",
  "meta": {
  "supportLevel": "SUPPORTED",
  "label": "Time Series Preparation",
@@ -15,4 +15,4 @@
  "Time Series"
  ]
  }
-}
+}

python-lib/dku_timeseries/timeseries_helpers.py

@@ -5,6 +5,7 @@
 import pandas as pd
 from pandas.tseries.frequencies import to_offset
 from pandas.tseries.offsets import BDay
+from pandas.tseries.offsets import Day
 
 logger = logging.getLogger(__name__)
 
@@ -77,17 +78,35 @@ def get_date_offset(time_unit, offset_value):
 
 
 def generate_date_range(start_time, end_time, clip_start, clip_end, shift, frequency, time_step, time_unit):
- rounding_freq_string = FREQUENCY_STRINGS.get(time_unit)
  clip_start_value = get_date_offset(time_unit, clip_start)
  clip_end_value = get_date_offset(time_unit, clip_end)
  shift_value = get_date_offset(time_unit, shift)
- if time_unit in ROUND_COMPATIBLE_TIME_UNIT:
- start_index = start_time.round(rounding_freq_string) + clip_start_value + shift_value
- end_index = end_time.round(rounding_freq_string) - clip_end_value + shift_value
- else: # for week, month, year we round up to closest day
- start_index = start_time.round("D") + clip_start_value + shift_value
- # for some reason date_range omit the last entry when dealing with months, years
- end_index = end_time.round("D") - clip_end_value + get_date_offset(time_unit, time_step) + shift_value
+
+ # for business day, week, month, year we round up to closest day
+ rounding_freq_string = FREQUENCY_STRINGS.get(time_unit) if time_unit in ROUND_COMPATIBLE_TIME_UNIT else "D"
+ start_index = start_time.round(rounding_freq_string)
+ end_index = end_time.round(rounding_freq_string)
+
+ if time_unit not in ROUND_COMPATIBLE_TIME_UNIT:
+ # pd.date_range omits the end index when frequency is business day, week, month or year, 
+ # unless the end index is exactly at the end of the period.
+ # so we need to offset the end index to make sure it falls between the last time step and the following one
+ if time_unit == "business_days":
+ # if start index is not a business day, then we want to start the range on the next Monday
+ # adding BDay(0) does nothing if the timestamp is already a business day and converts it into the first next business day otherwise
+ start_index = start_index + BDay(0)
+
+ # if end index is not a business day, then we want to end the range on the previous Friday
+ # adding Day(1) then subtracting BDay(1) does nothing if the timestamp is already a business day and converts it into the last previous business day otherwise
+ end_index = (end_index + Day(1)) - BDay(1)
+ else:
+ # we add one less Day to the end index to make sure we do not include the following time stamp
+ # if the end index is exactly at the end of the period
+ end_index = end_index + get_date_offset(time_unit, time_step) - Day(1)
+
+ start_index = start_index + clip_start_value + shift_value
+ end_index = end_index - clip_end_value + shift_value
+
  return pd.date_range(start=start_index, end=end_index, freq=frequency)
 
 

tests/python/unit/dku_timeseries/resampling/test_resampler_helpers.py

@@ -91,7 +91,7 @@ def test_generate_date_range_week(self, config):
 
  end_time = pd.Timestamp('2021-01-24 00:00:00')
  date_range = generate_date_range(start_time, end_time, 0, 0, 0, frequency, time_step, time_unit)
- np.testing.assert_array_equal(date_range, pd.DatetimeIndex(['2020-12-27', '2021-01-10', '2021-01-24', '2021-02-07']))
+ np.testing.assert_array_equal(date_range, pd.DatetimeIndex(['2020-12-27', '2021-01-10', '2021-01-24']))
 
  date_range = generate_date_range(start_time, end_time, 1, 0, 1, frequency, time_step, time_unit)
  np.testing.assert_array_equal(date_range, pd.DatetimeIndex(['2021-01-10', '2021-01-24', '2021-02-07']))
@@ -145,19 +145,19 @@ def test_generate_date_range_b_days(self, config):
  time_step = params.time_step
 
  date_range = generate_date_range(start_time, end_time, 0, 0, 0, frequency, time_step, time_unit)
- np.testing.assert_array_equal(date_range, pd.DatetimeIndex(['2021-01-04', '2021-01-05', '2021-01-06', '2021-01-07', '2021-01-08', '2021-01-11']))
+ np.testing.assert_array_equal(date_range, pd.DatetimeIndex(['2021-01-04', '2021-01-05', '2021-01-06', '2021-01-07', '2021-01-08']))
 
  clip_start = 1
  clip_end = 1
  shift = 0
  date_range = generate_date_range(start_time, end_time, clip_start, clip_end, shift, frequency, time_step, time_unit)
- np.testing.assert_array_equal(date_range, pd.DatetimeIndex(['2021-01-04', '2021-01-05', '2021-01-06', '2021-01-07', '2021-01-08', '2021-01-11']))
+ np.testing.assert_array_equal(date_range, pd.DatetimeIndex(['2021-01-05', '2021-01-06', '2021-01-07']))
 
  clip_start = 2
  clip_end = 2
  shift = 0
  date_range = generate_date_range(start_time, end_time, clip_start, clip_end, shift, frequency, time_step, time_unit)
- np.testing.assert_array_equal(date_range, pd.DatetimeIndex(['2021-01-05', '2021-01-06', '2021-01-07', '2021-01-08']))
+ np.testing.assert_array_equal(date_range, pd.DatetimeIndex(['2021-01-06']))
 
  def test_generate_date_range_days(self, config):
  config["time_unit"] = "days"

tests/python/unit/dku_timeseries/resampling/test_resampling_frequencies.py

@@ -44,7 +44,6 @@ def test_month(self, config, columns):
  df = get_df("Y", columns)
  output_df = resampler.transform(df, columns.date)
 
- assert np.mean(output_df[columns.data]) == 316.32550000000003
  expected_dates = pd.DatetimeIndex(['1959-12-31T00:00:00.000000000', '1960-02-29T00:00:00.000000000',
  '1960-04-30T00:00:00.000000000', '1960-06-30T00:00:00.000000000',
  '1960-08-31T00:00:00.000000000', '1960-10-31T00:00:00.000000000',
@@ -54,7 +53,7 @@ def test_month(self, config, columns):
  '1961-12-31T00:00:00.000000000', '1962-02-28T00:00:00.000000000',
  '1962-04-30T00:00:00.000000000', '1962-06-30T00:00:00.000000000',
  '1962-08-31T00:00:00.000000000', '1962-10-31T00:00:00.000000000',
- '1962-12-31T00:00:00.000000000', '1963-02-28T00:00:00.000000000'])
+ '1962-12-31T00:00:00.000000000'])
  np.testing.assert_array_equal(output_df[columns.date].values, expected_dates)
 
  def test_weeks_sunday_end(self, config, columns):