From ed2ca81e0a528c46b0a5c81cbb62ca1359b0e102 Mon Sep 17 00:00:00 2001 From: Daniel Wolfensberger Date: Thu, 22 Feb 2024 19:46:33 +0100 Subject: [PATCH 1/9] added rainforest version to docs/conf.py --- docs/conf.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/conf.py b/docs/conf.py index 56fc545..b447708 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -10,6 +10,7 @@ # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. # +import rainforest import os import sys sys.path.insert(0, os.path.abspath('.')) @@ -31,7 +32,7 @@ # ones. extensions = ['sphinx.ext.napoleon'] master_doc = 'index' - +version = rainforest.__version__ # Add any paths that contain templates here, relative to this directory. templates_path = ['_templates'] From c45f060f0c1d09ce223ba7255f3909bcf8e220e7 Mon Sep 17 00:00:00 2001 From: Daniel Wolfensberger Date: Thu, 22 Feb 2024 19:51:18 +0100 Subject: [PATCH 2/9] updated version nb --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 8b9d14d..b25c94d 100644 --- a/setup.py +++ b/setup.py @@ -32,7 +32,7 @@ s = setup(name = "rainforest_mch", description = "RandomForest QPE python library", long_description = "A library to compute a gauge/radar database, train a RF QPE predictor and apply it to generate QPE fields", - version = "1.4.0", + version = "1.4.1", author = 'Rebecca Gugerli, Daniel Wolfensberger', author_email = ' rebecca.gugerli@epfl.ch, daniel.wolfensberger@meteoswiss.ch', license = 'GPL-3.0', From f84db720844615926abf1819d3c2e5a68780d685 Mon Sep 17 00:00:00 2001 From: gugerlir Date: Fri, 23 Feb 2024 09:51:40 +0000 Subject: [PATCH 3/9] make all datetime object timezone aware for real-time application --- rainforest/common/retrieve_data.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/rainforest/common/retrieve_data.py b/rainforest/common/retrieve_data.py index d4b9632..6b6121a 100644 --- a/rainforest/common/retrieve_data.py +++ b/rainforest/common/retrieve_data.py @@ -124,7 +124,9 @@ def retrieve_hzt_RT(tstep): # Sort filelist to most recent prediction content_filt = np.array([c for c in content_zip if c.endswith('800')]) times_filt = np.array([datetime.datetime.strptime(c[3:12], - '%y%j%H%M')+datetime.timedelta(hours=int(c[-2::])) for c in content_filt]) + '%y%j%H%M').replace(tzinfo=datetime.timezone.utc)+\ + datetime.timedelta(hours=int(c[-2::])) \ + for c in content_filt]) conditions = np.array([np.logical_and((t >= start_time), (t <= end_time)) for t in times_filt]) content_filt = content_filt[conditions] @@ -139,7 +141,8 @@ def retrieve_hzt_RT(tstep): if len(all_hours) != len(times_filt): content_times = np.array([datetime.datetime.strptime(c[3:12], - '%y%j%H%M')+datetime.timedelta(hours=int(c[-2::])) for c in content_zip]) + '%y%j%H%M').replace(tzinfo=datetime.timezone.utc)+\ + datetime.timedelta(hours=int(c[-2::])) for c in content_zip]) # Find time that is missing: for hh in all_hours: if not hh in times_filt: @@ -571,7 +574,8 @@ def retrieve_prod_RT(time, product_name, # Derive datetime of each file times_zip = np.array([datetime.datetime.strptime(c[3:12], - '%y%j%H%M') for c in content_zip]) + '%y%j%H%M').replace(tzinfo=datetime.timezone.utc) + for c in content_zip]) # Get a list of all files to retrieve conditions = (times_zip == time) From aa01ad02f989c764f54cdb843a2d3c1e2c4afda3 Mon Sep 17 00:00:00 2001 From: gugerlir Date: Fri, 23 Feb 2024 13:01:57 +0000 Subject: [PATCH 4/9] default metranet reader set to c-reader, adapted metadata of h5 output --- rainforest/common/radarprocessing.py | 2 +- rainforest/qpe/qpe.py | 40 +++++++++++++++++++++++----- 2 files changed, 34 insertions(+), 8 deletions(-) diff --git a/rainforest/common/radarprocessing.py b/rainforest/common/radarprocessing.py index 18f32c0..7391b4b 100644 --- a/rainforest/common/radarprocessing.py +++ b/rainforest/common/radarprocessing.py @@ -36,7 +36,7 @@ class Radar(object): single pyart radar instance as this was found to be faster in practice ''' def __init__(self, radname, polfiles, statusfile=None, vprfile=None, - temp_ref='TAIR', metranet_reader = 'python'): + temp_ref='TAIR', metranet_reader = 'C'): """ Creates an Radar class instance diff --git a/rainforest/qpe/qpe.py b/rainforest/qpe/qpe.py index 3cf1b0a..9781bbd 100644 --- a/rainforest/qpe/qpe.py +++ b/rainforest/qpe/qpe.py @@ -236,7 +236,17 @@ def _qpe_to_chgrid(qpe, time, missing_files, precision=2): qual_new = qual_new.replace(rad, '-') quality = qual_new grid.metadata['radar'] = quality.encode() - grid.metadata['nodes'] = 'WMO:06661,WMO:06699,WMO:06768,WMO:06726,WMO:06776' + + if '-' not in quality: + grid.metadata['nodes'] = 'WMO:06661,WMO:06699,WMO:06768,WMO:06726,WMO:06776' + else: + # 06661: Albis; 06699: Dôle; 06768: Lema; 06726: Plaine Morte; 06776: Weissfluh + all_wmo = ['WMO:06661','WMO:06699','WMO:06768','WMO:06726','WMO:06776'] + rad_wmo = [] + for ir, rad in enumerate(['A', 'D', 'L', 'P', 'W']): + if rad in quality: + rad_wmo.append(all_wmo[ir]) + grid.metadata['nodes'] = ','.join(rad_wmo) return grid @@ -423,8 +433,8 @@ def fetch_data(self, t0, t1 = None): self.radar_files[rad] = split_by_time(radfiles) self.status_files[rad] = split_by_time(statfiles) - except: - logger.error('Failed to retrieve data for radar {:s}'.format(rad)) + except Exception as error: + logger.error('Failed to retrieve data for radar {:s}. Error: {:s}'.format(rad, error)) # Retrieve iso0 height files if 'ISO0_HEIGHT' in self.cosmo_var: @@ -664,12 +674,28 @@ def compute(self, output_folder, t0, t1, timestep = 5, # Read raw radar file and create a RADAR object radobjects[rad] = Radar(rad, self.radar_files[rad][t], - self.status_files[rad][t]) + self.status_files[rad][t], + metranet_reader='C') + # try: + # radobjects[rad] = Radar(rad, self.radar_files[rad][t], + # self.status_files[rad][t], + # metranet_reader='C') + # except: + # try: + # wrnmsg = 'Could not read polar radar data for radar {:s}'.format(rad)+\ + # ' with c-reader (default), trying with python-reader' + # logger.warning(wrnmsg) + # radobjects[rad] = Radar(rad, self.radar_files[rad][t], + # self.status_files[rad][t], + # metranet_reader='python') + # except: + # logger.error('Could not read polar radar data of {:s}'.format(rad)) + # if problem with radar file, exclude it and add to missing files list if len(radobjects[rad].radarfields) == 0: self.missing_files[rad] = t - logger.info('Removing timestep {:s} of radar {:s}'.format(str(t), rad)) + logger.warning('Removing timestep {:s} of radar {:s}'.format(str(t), rad)) continue # Process the radar data @@ -680,8 +706,8 @@ def compute(self, output_folder, t0, t1, timestep = 5, radobjects[rad].snr_mask(self.config['SNR_THRESHOLD']) except Exception as e: self.missing_files[rad] = t - logger.info(e) - logger.info('Removing timestep {:s} of radar {:s}'.format(str(t), rad)) + logger.warning(e) + logger.warning('Removing timestep {:s} of radar {:s}'.format(str(t), rad)) continue radobjects[rad].compute_kdp(self.config['KDP_PARAMETERS']) From 33eafac266060f5438cd57ade508f44fa8ae6f5e Mon Sep 17 00:00:00 2001 From: gugerlir Date: Mon, 26 Feb 2024 10:14:15 +0000 Subject: [PATCH 5/9] modified error message when file retrieval does not work --- rainforest/qpe/qpe.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/rainforest/qpe/qpe.py b/rainforest/qpe/qpe.py index 9781bbd..fe6ff64 100644 --- a/rainforest/qpe/qpe.py +++ b/rainforest/qpe/qpe.py @@ -434,7 +434,8 @@ def fetch_data(self, t0, t1 = None): self.radar_files[rad] = split_by_time(radfiles) self.status_files[rad] = split_by_time(statfiles) except Exception as error: - logger.error('Failed to retrieve data for radar {:s}. Error: {:s}'.format(rad, error)) + error_msg = 'Failed to retrieve data for radar {:s}.\n'.format(rad)+ str(error) + logger.error(error_msg) # Retrieve iso0 height files if 'ISO0_HEIGHT' in self.cosmo_var: From 821900908db9a2c65eda3b075827bb0caae64359 Mon Sep 17 00:00:00 2001 From: gugerlir Date: Mon, 26 Feb 2024 13:50:54 +0000 Subject: [PATCH 6/9] modified logs for real-time --- rainforest/qpe/qpe.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/rainforest/qpe/qpe.py b/rainforest/qpe/qpe.py index fe6ff64..2acbefa 100644 --- a/rainforest/qpe/qpe.py +++ b/rainforest/qpe/qpe.py @@ -661,6 +661,7 @@ def compute(self, output_folder, t0, t1, timestep = 5, """Part one - compute radar variables and mask""" # Begin compilation of radarobject + logger.info('Preparing all polar radar data') radobjects = {} for rad in self.config['RADARS']: # if radar does not exist, add to missing file list @@ -749,9 +750,11 @@ def compute(self, output_folder, t0, t1, timestep = 5, except: logger.error('No cleanup was defined, unzipped files remain in temp-folder') + logger.info('Processing all sweeps of all radars') for sweep in self.config['SWEEPS']: # Loop on sweeps - logger.info('---') - logger.info('Processing sweep ' + str(sweep)) + if not self.rt: + logger.info('---') + logger.info('Processing sweep ' + str(sweep)) for rad in self.config['RADARS']: # Loop on radars, A,D,L,P,W # If there is no radar file for the specific radar, continue to next radar @@ -762,7 +765,8 @@ def compute(self, output_folder, t0, t1, timestep = 5, logger.info('Processing sweep {} of {} - no data for this timestep!'.format(sweep, rad)) continue else: - logger.info('Processing radar ' + str(rad)) + if not self.rt: + logger.info('Processing radar ' + str(rad) + '; sweep '+str(sweep)) try: """Part two - retrieve radar data at every sweep""" @@ -849,6 +853,7 @@ def compute(self, output_folder, t0, t1, timestep = 5, """Part four - RF prediction""" + logger.info('Applying RF model to retrieve predictions') # Get QPE estimate # X: current time step; X_prev: previous timestep (t-5min) for k in self.models.keys(): From 7c1e4809e4d1b69da60f9def9c2298085823911e Mon Sep 17 00:00:00 2001 From: gugerlir Date: Mon, 26 Feb 2024 13:56:15 +0000 Subject: [PATCH 7/9] new release number --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index b25c94d..38f4044 100644 --- a/setup.py +++ b/setup.py @@ -32,7 +32,7 @@ s = setup(name = "rainforest_mch", description = "RandomForest QPE python library", long_description = "A library to compute a gauge/radar database, train a RF QPE predictor and apply it to generate QPE fields", - version = "1.4.1", + version = "1.4.2", author = 'Rebecca Gugerli, Daniel Wolfensberger', author_email = ' rebecca.gugerli@epfl.ch, daniel.wolfensberger@meteoswiss.ch', license = 'GPL-3.0', From 1ffc3a42dfea552d3fda56afaf6c1fe858fc0d96 Mon Sep 17 00:00:00 2001 From: Daniel Wolfensberger Date: Wed, 28 Feb 2024 12:47:59 +0100 Subject: [PATCH 8/9] small change to qpe grid for pyart ODIM writer consistency --- rainforest/qpe/qpe.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rainforest/qpe/qpe.py b/rainforest/qpe/qpe.py index 2acbefa..086d604 100644 --- a/rainforest/qpe/qpe.py +++ b/rainforest/qpe/qpe.py @@ -132,7 +132,7 @@ def _features_to_chgrid(features, features_labels, time, missing_files): 1000 * np.max(Y_QPE_CENTERS)]]) - time_start = time - datetime.timedelta(seconds = 5 * 60) + time_start = time - datetime.timedelta(seconds = 5 * 60 + 1) grid.time['units'] = 'seconds since {:s}'.format( datetime.datetime.strftime(time_start, '%Y-%m-%dT%H:%M:%SZ')) @@ -203,7 +203,7 @@ def _qpe_to_chgrid(qpe, time, missing_files, precision=2): 1000 * np.max(Y_QPE_CENTERS)]]) - time_start = time - datetime.timedelta(seconds = 5 * 60) + time_start = time - datetime.timedelta(seconds = 5 * 60 + 1) grid.time['units'] = 'seconds since {:s}'.format( datetime.datetime.strftime(time_start, '%Y-%m-%dT%H:%M:%SZ')) From adcfe2b510fc20b04e3fd0f8f509d6f25cb47f49 Mon Sep 17 00:00:00 2001 From: Daniel Wolfensberger Date: Wed, 28 Feb 2024 20:50:09 +0100 Subject: [PATCH 9/9] Update setup.py --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 38f4044..178e398 100644 --- a/setup.py +++ b/setup.py @@ -32,7 +32,7 @@ s = setup(name = "rainforest_mch", description = "RandomForest QPE python library", long_description = "A library to compute a gauge/radar database, train a RF QPE predictor and apply it to generate QPE fields", - version = "1.4.2", + version = "1.4.3", author = 'Rebecca Gugerli, Daniel Wolfensberger', author_email = ' rebecca.gugerli@epfl.ch, daniel.wolfensberger@meteoswiss.ch', license = 'GPL-3.0',