From 454177fe38f2ea8b15b8b245a60399977cca8e60 Mon Sep 17 00:00:00 2001 From: sit23 Date: Fri, 31 Jan 2020 12:57:10 +0000 Subject: [PATCH 1/5] This fix was made on the pre-ictp branch in august 2018, but only now being merged into master. Need to make sure that mkmf template used to compile plev script is the same as used for the model. --- .../plevel_interpolation/compile_plev_interpolation.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/postprocessing/plevel_interpolation/compile_plev_interpolation.sh b/postprocessing/plevel_interpolation/compile_plev_interpolation.sh index c9c668813..6942bca3a 100755 --- a/postprocessing/plevel_interpolation/compile_plev_interpolation.sh +++ b/postprocessing/plevel_interpolation/compile_plev_interpolation.sh @@ -5,6 +5,6 @@ cd ./exec source $GFDL_BASE/src/extra/env/$GFDL_ENV -../bin/mkmf -p plev.x -t ../bin/mkmf.template.ia64 -c "-Duse_netCDF" -a ../src ../src/path_names ../src/shared/mpp/include ../src/shared/include +../bin/mkmf -p plev.x -t $GFDL_BASE/src/extra/python/isca/templates/mkmf.template.ia64 -c "-Duse_netCDF" -a ../src ../src/path_names ../src/shared/mpp/include ../src/shared/include make -f Makefile From 6ec540ad01c57a7af653375c74ba1367f4f5e8af Mon Sep 17 00:00:00 2001 From: sit23 Date: Fri, 31 Jan 2020 12:58:41 +0000 Subject: [PATCH 2/5] Added simple run-plevel script from pre-ictp mods branch too. --- .../scripts/run_plevel_simple.py | 51 +++++++++++++++++++ 1 file changed, 51 insertions(+) create mode 100644 postprocessing/plevel_interpolation/scripts/run_plevel_simple.py diff --git a/postprocessing/plevel_interpolation/scripts/run_plevel_simple.py b/postprocessing/plevel_interpolation/scripts/run_plevel_simple.py new file mode 100644 index 000000000..4ad6bd567 --- /dev/null +++ b/postprocessing/plevel_interpolation/scripts/run_plevel_simple.py @@ -0,0 +1,51 @@ +from netCDF4 import Dataset +from plevel_fn import plevel_call, daily_average, join_files, two_daily_average, monthly_average +import sys +import os +import time +import pdb +import subprocess + +start_time=time.time() +base_dir='/home/sthomson/datadir_isca/' +exp_name_list = ['project_7_ice_albedo_False_t42'] +avg_or_daily_list=['monthly'] +start_file=1 +end_file=7 +nfiles=(end_file-start_file)+1 + +level_set='standard' #Default is the standard levels used previously. ssw_diagnostics are the ones blanca requested for MiMa validation +mask_below_surface_set=' ' #Default is to mask values that lie below the surface pressure when interpolated. For some applications, you want to have values interpolated below ground, i.e. as if the ground wasn't there. To use this option, this value should be set to '-x '. + +try: + out_dir +except: + out_dir = base_dir + +plevs={} +var_names={} + +if level_set=='standard': + + plevs['monthly']=' -p "1000 10000 25000 50000 85000 92500"' + plevs['6hourly']=' -p "1000 10000 25000 50000 85000 92500"' + plevs['daily'] =' -p "1000 10000 25000 50000 85000 92500"' + + var_names['monthly']='-a slp height' + var_names['6hourly']='-a slp height' + var_names['daily']='-a slp height' + file_suffix='_interp' + +for exp_name in exp_name_list: + for n in range(nfiles): + for avg_or_daily in avg_or_daily_list: + print(n+start_file) + + nc_file_in = base_dir+'/'+exp_name+'/run%04d'%(n+start_file)+'/atmos_'+avg_or_daily+'.nc' + nc_file_out = out_dir+'/'+exp_name+'/run%04d'%(n+start_file)+'/atmos_'+avg_or_daily+file_suffix+'.nc' + + if not os.path.isfile(nc_file_out): + plevel_call(nc_file_in,nc_file_out, var_names = var_names[avg_or_daily], p_levels = plevs[avg_or_daily], mask_below_surface_option=mask_below_surface_set) + +print('execution time', time.time()-start_time) + From c87172e049c0f891f476b93ddd03fbb4ba48b504 Mon Sep 17 00:00:00 2001 From: sit23 Date: Fri, 31 Jan 2020 13:01:26 +0000 Subject: [PATCH 3/5] Added gfdl data definition. --- .../plevel_interpolation/scripts/run_plevel_simple.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/postprocessing/plevel_interpolation/scripts/run_plevel_simple.py b/postprocessing/plevel_interpolation/scripts/run_plevel_simple.py index 4ad6bd567..48ac0d366 100644 --- a/postprocessing/plevel_interpolation/scripts/run_plevel_simple.py +++ b/postprocessing/plevel_interpolation/scripts/run_plevel_simple.py @@ -7,11 +7,11 @@ import subprocess start_time=time.time() -base_dir='/home/sthomson/datadir_isca/' -exp_name_list = ['project_7_ice_albedo_False_t42'] -avg_or_daily_list=['monthly'] +base_dir=os.environ['GFDL_DATA'] +exp_name_list = ['exp_name_here'] +avg_or_daily_list=['daily'] start_file=1 -end_file=7 +end_file=12 nfiles=(end_file-start_file)+1 level_set='standard' #Default is the standard levels used previously. ssw_diagnostics are the ones blanca requested for MiMa validation From 7a552690f76e2eae98e56e0656a5075d95e9c387 Mon Sep 17 00:00:00 2001 From: sit23 Date: Fri, 31 Jan 2020 13:05:37 +0000 Subject: [PATCH 4/5] Added option to make mkmf template the same as that used by compile.sh. --- .../plevel_interpolation/compile_plev_interpolation.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/postprocessing/plevel_interpolation/compile_plev_interpolation.sh b/postprocessing/plevel_interpolation/compile_plev_interpolation.sh index 6942bca3a..fa2790dde 100755 --- a/postprocessing/plevel_interpolation/compile_plev_interpolation.sh +++ b/postprocessing/plevel_interpolation/compile_plev_interpolation.sh @@ -5,6 +5,9 @@ cd ./exec source $GFDL_BASE/src/extra/env/$GFDL_ENV -../bin/mkmf -p plev.x -t $GFDL_BASE/src/extra/python/isca/templates/mkmf.template.ia64 -c "-Duse_netCDF" -a ../src ../src/path_names ../src/shared/mpp/include ../src/shared/include +compiler=${GFDL_MKMF_TEMPLATE:-ia64} +template=mkmf.template.${compiler} + +../bin/mkmf -p plev.x -t $GFDL_BASE/src/extra/python/isca/templates/$template -c "-Duse_netCDF" -a ../src ../src/path_names ../src/shared/mpp/include ../src/shared/include make -f Makefile From d3ae624396d55cc82bb9aea223666d7b98496acc Mon Sep 17 00:00:00 2001 From: sit23 Date: Fri, 7 Feb 2020 08:49:02 +0000 Subject: [PATCH 5/5] Made some upgrades so that you can specify a particular folder and it will remove the restarts from all experiments within that folder. --- .../remove_certain_restart_and_data_files.py | 111 ++++++++++-------- 1 file changed, 60 insertions(+), 51 deletions(-) diff --git a/src/extra/python/scripts/remove_certain_restart_and_data_files.py b/src/extra/python/scripts/remove_certain_restart_and_data_files.py index 98d658a95..6e8f005ca 100644 --- a/src/extra/python/scripts/remove_certain_restart_and_data_files.py +++ b/src/extra/python/scripts/remove_certain_restart_and_data_files.py @@ -1,6 +1,7 @@ import sh import os import pdb +from glob import glob P = os.path.join @@ -12,11 +13,15 @@ def __init__(self, basedir, workdir, datadir, exp_name): self.expname = exp_name -def create_exp_object(exp_name): +def create_exp_object(exp_name, data_directory=None): + + if data_directory is None: + datadir = os.environ['GFDL_DATA'] + else: + datadir = data_directory - basedir = os.environ['GFDL_BASE'] workdir = os.environ['GFDL_WORK'] - datadir = os.environ['GFDL_DATA'] + basedir = os.environ['GFDL_BASE'] expname = '/'+exp_name+'/' exp_object = temporary_exp_object(basedir, workdir, datadir, exp_name) @@ -24,34 +29,42 @@ def create_exp_object(exp_name): return exp_object -def keep_only_certain_restart_files(exp_object, max_num_files, interval=12): +# def keep_only_certain_restart_files(exp_object, max_num_files, interval=12): - # sh.ls(sh.glob(P(self.workdir,'restarts','res_*.cpio'))) #TODO get max_num_files calculated in line, rather than a variable to pass. +# # sh.ls(sh.glob(P(self.workdir,'restarts','res_*.cpio'))) #TODO get max_num_files calculated in line, rather than a variable to pass. - #First defines a list of ALL the restart file numbers - files_to_remove=list(range(0,max_num_files)) +# #First defines a list of ALL the restart file numbers +# files_to_remove=list(range(0,max_num_files)) - #Then defines a list of the ones we want to KEEP - files_to_keep =list(range(0,max_num_files,interval)) +# #Then defines a list of the ones we want to KEEP +# files_to_keep =list(range(0,max_num_files,interval)) - #Then we remove the files we want to keep from the list of all files, giving a list of those we wish to remove - for x in files_to_keep: - files_to_remove.remove(x) +# #Then we remove the files we want to keep from the list of all files, giving a list of those we wish to remove +# for x in files_to_keep: +# files_to_remove.remove(x) - #Then we remove them. - for entry in files_to_remove: - try: - sh.rm(P(exp_object.workdir,exp_object.expname,'restarts','res_'+str(entry)+'.cpio')) -# print P(exp_object.workdir,exp_object.expname,'restarts','res_'+str(entry)+'.cpio') +# #Then we remove them. +# for entry in files_to_remove: +# try: +# sh.rm(P(exp_object.workdir,exp_object.expname,'restarts','res_'+str(entry)+'.cpio')) +# # print P(exp_object.workdir,exp_object.expname,'restarts','res_'+str(entry)+'.cpio') - except sh.ErrorReturnCode_1: - pass -# print 'Tried to remove some restart files, but number '+str(entry)+' does not exist' +# except sh.ErrorReturnCode_1: +# pass +# # print 'Tried to remove some restart files, but number '+str(entry)+' does not exist' -def keep_only_certain_restart_files_data_dir(exp_object, max_num_files, interval=12): +def keep_only_certain_restart_files_data_dir(exp_object, max_num_files=None, interval=12): # sh.ls(sh.glob(P(self.workdir,'restarts','res_*.cpio'))) #TODO get max_num_files calculated in line, rather than a variable to pass. + if max_num_files is None: + month_list = glob(P(exp_object.datadir,exp_object.expname, 'restarts')+'/res*.tar.gz') + if len(month_list)==0: + return + else: + final_month = month_list[-1].split('/res') + max_num_files = int(final_month[-1].split('.tar.gz')[0]) + #First defines a list of ALL the restart file numbers files_to_remove=list(range(0,max_num_files)) @@ -62,13 +75,27 @@ def keep_only_certain_restart_files_data_dir(exp_object, max_num_files, interval for x in files_to_keep: files_to_remove.remove(x) + first_to_be_removed = True + number_removed = 0 + number_not_removed = 0 #Then we remove them. - for entry in files_to_remove: + for entry in files_to_remove[1:-1]: try: - sh.rm(P(exp_object.datadir,exp_object.expname,'run%03d' % entry,'INPUT','res')) -# print 'would be removing ' + P(exp_object.datadir,exp_object.expname,'run'+str(entry),'INPUT','res') + file_to_remove = P(exp_object.datadir,exp_object.expname, 'restarts', 'res%04d.tar.gz' % entry) + if os.path.isfile(file_to_remove) and first_to_be_removed: + first_to_be_removed=False + number_not_removed+=1 + # print('would have removed '+file_to_remove+' but wanted to make sure not to delete the first restart') + else: + sh.rm(file_to_remove) + number_removed+=1 + # print('have removed ' + file_to_remove) except sh.ErrorReturnCode_1: + number_not_removed+=1 + # print('could not remove ' + file_to_remove) pass + print(P(exp_object.datadir,exp_object.expname), 'number removed '+str(number_removed), 'number not removed '+str(number_not_removed)) + # print 'Tried to remove some restart files, but number '+str(entry)+' does not exist' def keep_only_certain_daily_data_uninterp(exp_object, max_num_files, interval=None, file_name = 'atmos_daily.nc'): @@ -88,8 +115,8 @@ def keep_only_certain_daily_data_uninterp(exp_object, max_num_files, interval=No #Then we remove them. for entry in files_to_remove: try: - sh.rm(P(exp_object.datadir,exp_object.expname,'run%03d' % entry,file_name)) - print(('Removed '+P(exp_object.datadir,exp_object.expname,'run%03d' % entry,file_name))) + sh.rm(P(exp_object.datadir,exp_object.expname,'run%04d' % entry,file_name)) + print(('Removed '+P(exp_object.datadir,exp_object.expname,'run%04d' % entry,file_name))) except sh.ErrorReturnCode_1: pass # print 'Tried to remove some atmos_daily files, but number '+str(entry)+' does not exist' @@ -98,36 +125,18 @@ def keep_only_certain_daily_data_uninterp(exp_object, max_num_files, interval=No if __name__=="__main__": - max_num_files_input = 325 + max_num_files_input = None -# exp_name_list=['simple_continents_post_princeton_qflux_anoms_'+str(x) for x in range(31,32)] - -# exp_name_list=['aquaplanet_qflux_anoms_'+str(x) for x in [12,18,23,32,8]] - -# exp_name_list = ['simple_continents_post_princeton_qflux_control_1','simple_continents_post_princeton_fixed_sst_1', 'simple_continents_post_princeton_qflux_control_nod_1', 'simple_continents_post_princeton_qflux_control_scf_1'] -# -# exp_name_list = ['annual_mean_ice_princeton_qflux_control_matrix_qflux_2017_code_1', 'annual_mean_ice_post_princeton_fixed_sst_1', 'annual_mean_ice_princeton_fixed_sst_1'] -# -# exp_name_list.extend(['annual_mean_ice_post_princeton_fixed_sst_el_nino_1']) - -# exp_name_list = ['simple_continents_post_princeton_qflux_control_1'] - -# exp_name_list = ['annual_mean_ice_princeton_qflux_control_1']#, 'annual_mean_ice_post_princeton_qflux_control_1'] - -# exp_name_list = ['annual_mean_ice_post_princeton_fixed_sst_TEST_1', 'annual_mean_ice_princeton_qflux_control_matrix_qflux_1'] - -# exp_name_list.extend(['simple_continents_post_princeton_fixed_sst_1']) - -# exp_name_list = ['giant_drag_exp_chai_values_without_dc_bug_latest_1'] -# exp_name_list = ['aquaplanet_qflux_control_1'] + # exp_name_list = [''] + exp_name_list = glob('/disca/share/sit204/data_from_isca_cpu/*/') - exp_name_list = ['giant_drag_exp_chai_values_with_dc_bug_latest_start_to_finish_1', 'giant_drag_exp_chai_values_without_dc_bug_latest_start_to_finish_1'] for exp_name_input in exp_name_list: - temp_obj = create_exp_object(exp_name_input) - keep_only_certain_restart_files(temp_obj, max_num_files_input) + print('Percentage progress through list:'+str(exp_name_list.index(exp_name_input)/len(exp_name_list))) + temp_obj = create_exp_object(exp_name_input, data_directory='/disca/share/sit204/data_from_isca_cpu/') + # keep_only_certain_restart_files(temp_obj, max_num_files_input) keep_only_certain_restart_files_data_dir(temp_obj, max_num_files_input) - keep_only_certain_daily_data_uninterp(temp_obj, max_num_files_input, file_name = 'fms_moist.x') + # keep_only_certain_daily_data_uninterp(temp_obj, max_num_files_input, file_name = 'fms_moist.x') # keep_only_certain_daily_data_uninterp(temp_obj, max_num_files_input) \ No newline at end of file