From 6010effc7141ca1bca561ab82787dacdd7465ebf Mon Sep 17 00:00:00 2001 From: Christina Holt <56881914+christinaholtNOAA@users.noreply.github.com> Date: Wed, 20 Mar 2024 07:15:12 -0600 Subject: [PATCH] [develop] Use uwtools instead of set_namelist (#1054) Continues the integration of the uwtools package. In this PR, I've done the following: * Call the UW config tool instead of set_namelist using the uwtools CLI in bash scripts and API in Python scripts * Lint the ush/set_fv3nml*.py files * Update uwtools to the latest release version --------- Co-authored-by: michael.lueken --- .github/workflows/python_tests.yaml | 2 + environment.yml | 2 +- modulefiles/build_hera_gnu.lua | 5 +- modulefiles/build_hera_intel.lua | 5 +- parm/FV3.input.yml | 123 +++--- parm/fixed_files_mapping.yaml | 29 -- scripts/exregional_make_grid.sh | 50 +-- scripts/exregional_make_ics.sh | 101 +++-- scripts/exregional_make_lbcs.sh | 80 ++-- scripts/exregional_run_fcst.sh | 8 +- ...onal_run_met_genensprod_or_ensemblestat.sh | 2 +- ...gional_run_met_gridstat_or_pointstat_vx.sh | 2 +- ...un_met_gridstat_or_pointstat_vx_ensmean.sh | 2 +- ...un_met_gridstat_or_pointstat_vx_ensprob.sh | 2 +- scripts/exregional_run_met_pb2nc_obs.sh | 2 +- scripts/exregional_run_met_pcpcombine.sh | 2 +- ....py => test_set_fv3nml_ens_stoch_seeds.py} | 34 +- ...=> test_set_fv3nml_sfc_climo_filenames.py} | 24 +- ush/generate_FV3LAM_wflow.py | 51 ++- ...seeds.py => set_fv3nml_ens_stoch_seeds.py} | 94 ++--- ...s.py => set_fv3nml_sfc_climo_filenames.py} | 106 +++--- ush/set_namelist.py | 355 ------------------ ush/update_input_nml.py | 142 +++---- 23 files changed, 369 insertions(+), 854 deletions(-) rename tests/test_python/{test_set_FV3nml_ens_stoch_seeds.py => test_set_fv3nml_ens_stoch_seeds.py} (68%) rename tests/test_python/{test_set_FV3nml_sfc_climo_filenames.py => test_set_fv3nml_sfc_climo_filenames.py} (69%) rename ush/{set_FV3nml_ens_stoch_seeds.py => set_fv3nml_ens_stoch_seeds.py} (64%) rename ush/{set_FV3nml_sfc_climo_filenames.py => set_fv3nml_sfc_climo_filenames.py} (53%) delete mode 100755 ush/set_namelist.py diff --git a/.github/workflows/python_tests.yaml b/.github/workflows/python_tests.yaml index 0e71f8d72d..fb0de16910 100644 --- a/.github/workflows/python_tests.yaml +++ b/.github/workflows/python_tests.yaml @@ -41,6 +41,8 @@ jobs: pylint --ignore-imports=yes tests/test_python/ pylint ush/create_*.py pylint ush/generate_FV3LAM_wflow.py + pylint ush/set_fv3nml*.py + pylint ush/update_input_nml.py - name: Run python unittests run: | diff --git a/environment.yml b/environment.yml index c574df5e23..faeb19d466 100644 --- a/environment.yml +++ b/environment.yml @@ -5,4 +5,4 @@ channels: dependencies: - pylint=2.17* - pytest=7.2* - - uwtools=1.0.0 + - uwtools=2.1* diff --git a/modulefiles/build_hera_gnu.lua b/modulefiles/build_hera_gnu.lua index 90bd671b5a..d5f78f397b 100644 --- a/modulefiles/build_hera_gnu.lua +++ b/modulefiles/build_hera_gnu.lua @@ -5,10 +5,7 @@ the NOAA RDHPC machine Hera using GNU 9.2.0 whatis([===[Loads libraries needed for building the UFS SRW App on Hera using GNU 9.2.0 ]===]) --- When Hera switches from CentOS to Rocky, replace line withh correct path to spack-stack --- If you want to use Rocky OS now, use line below ---prepend_path("MODULEPATH", "/scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-1.5.0/envs/unified-env-rocky8/install/modulefiles/Core") -prepend_path("MODULEPATH", "/scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-1.5.0/envs/unified-env-noavx512/install/modulefiles/Core") +prepend_path("MODULEPATH", "/scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-1.5.0/envs/unified-env-rocky8/install/modulefiles/Core") prepend_path("MODULEPATH", "/scratch1/NCEPDEV/jcsda/jedipara/spack-stack/modulefiles") load("stack-gcc/9.2.0") diff --git a/modulefiles/build_hera_intel.lua b/modulefiles/build_hera_intel.lua index 314fd89183..2121d303dc 100644 --- a/modulefiles/build_hera_intel.lua +++ b/modulefiles/build_hera_intel.lua @@ -8,10 +8,7 @@ whatis([===[Loads libraries needed for building the UFS SRW App on Hera ]===]) prepend_path("MODULEPATH","/contrib/sutils/modulefiles") load("sutils") --- When Hera switches from CentOS to Rocky, replace line withh correct path to spack-stack --- If you want to use Rocky OS now, use line below ---prepend_path("MODULEPATH", "/scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-1.5.0/envs/unified-env-rocky8/install/modulefiles/Core") -prepend_path("MODULEPATH", "/scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-1.5.0/envs/unified-env-noavx512/install/modulefiles/Core") +prepend_path("MODULEPATH", "/scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-1.5.0/envs/unified-env-rocky8/install/modulefiles/Core") prepend_path("MODULEPATH", "/scratch1/NCEPDEV/jcsda/jedipara/spack-stack/modulefiles") stack_intel_ver=os.getenv("stack_intel_ver") or "2021.5.0" diff --git a/parm/FV3.input.yml b/parm/FV3.input.yml index aa15625532..efb6c85f5b 100644 --- a/parm/FV3.input.yml +++ b/parm/FV3.input.yml @@ -4,12 +4,7 @@ # parm/input.nml.FV3 # # to obtain the namelist for each physics suite that the SRW App can -# run with. To build a namelist for one of these configurations, use -# the Python helper script -# -# ush/set_namelist.py -# -# and provide this file and the desired section via the -c option. +# run with. FV3_RRFS_v1beta: @@ -83,11 +78,11 @@ FV3_HRRR: <<: *RRFS_v1beta_phys cdmbgwd: [3.5, 1.0] do_mynnsfclay: True - do_sfcperts: !!python/none + do_sfcperts: null gwd_opt: 3 do_gsl_drag_ss: True do_gsl_drag_tofd: True - do_gsl_drag_ls_bl: True + do_gsl_drag_ls_bl: True iaer: 5111 icliq_sw: 2 iovr: 3 @@ -102,8 +97,8 @@ FV3_HRRR: mosaic_lu: 0 mosaic_soil: 0 thsfc_loc: False - nst_anl: - nstf_name: + nst_anl: null + nstf_name: null FV3_RAP: fv_core_nml: @@ -112,7 +107,7 @@ FV3_RAP: <<: *RRFS_v1beta_phys cdmbgwd: [3.5, 1.0] do_mynnsfclay: True - do_sfcperts: !!python/none + do_sfcperts: null gwd_opt: 3 do_gsl_drag_ss: True do_gsl_drag_tofd: True @@ -140,40 +135,40 @@ FV3_GFS_2017_gfdlmp: k_split: 6 n_split: 6 nord: 2 - nord_zs_filter: !!python/none + nord_zs_filter: null range_warn: False vtdm4: 0.075 gfs_physics_nml: &gfs_2017_gfdlmp_phys avg_max_length: 3600.0 - bl_mynn_tkeadvect: !!python/none - bl_mynn_edmf: !!python/none - bl_mynn_edmf_mom: !!python/none + bl_mynn_tkeadvect: null + bl_mynn_edmf: null + bl_mynn_edmf_mom: null cdmbgwd: [3.5, 0.01] - cplflx: !!python/none + cplflx: null do_deep: False - do_mynnedmf: !!python/none - do_mynnsfclay: !!python/none + do_mynnedmf: null + do_mynnsfclay: null fhcyc: 0.0 fhlwr: 3600.0 fhswr: 3600.0 hybedmf: True - icloud_bl: !!python/none + icloud_bl: null imfdeepcnv: 2 imfshalcnv: 2 imp_physics: 11 lgfdlmprad: True - lheatstrg: !!python/none - lndp_type: !!python/none - lsm: !!python/none - lsoil: !!python/none - lsoil_lsm: !!python/none - ltaerosol: !!python/none - n_var_lndp: !!python/none + lheatstrg: null + lndp_type: null + lsm: null + lsoil: null + lsoil_lsm: null + ltaerosol: null + n_var_lndp: null oz_phys: True oz_phys_2015: False - satmedmf: !!python/none + satmedmf: null shal_cnv: True - ttendlim: !!python/none + ttendlim: null gfdl_cloud_microphysics_nml: &gfs_gfdl_cloud_mp c_cracw: 0.8 c_paut: 0.5 @@ -272,7 +267,7 @@ FV3_GFS_v15p2: kord_wz: 9 n_split: 8 n_sponge: 30 - nord_zs_filter: !!python/none + nord_zs_filter: null nudge_qv: True range_warn: False rf_cutoff: 750.0 @@ -283,16 +278,16 @@ FV3_GFS_v15p2: tau_l2v: 225.0 tau_v2l: 150.0 gfs_physics_nml: &gfs_v15_gfs_physics - bl_mynn_edmf: !!python/none - bl_mynn_edmf_mom: !!python/none - bl_mynn_tkeadvect: !!python/none + bl_mynn_edmf: null + bl_mynn_edmf_mom: null + bl_mynn_tkeadvect: null cnvcld: True cnvgwd: True - cplflx: !!python/none + cplflx: null do_myjpbl: False do_myjsfc: False - do_mynnedmf: !!python/none - do_mynnsfclay: !!python/none + do_mynnedmf: null + do_mynnsfclay: null do_tofd: False do_ugwp: False do_ysu: False @@ -300,12 +295,12 @@ FV3_GFS_v15p2: fhlwr: 3600.0 fhswr: 3600.0 hybedmf: True - iau_delthrs: !!python/none - iaufhrs: !!python/none + iau_delthrs: null + iaufhrs: null imfdeepcnv: 2 imfshalcnv: 2 imp_physics: 11 - icloud_bl: !!python/none + icloud_bl: null iopt_alb: 2 iopt_btr: 1 iopt_crs: 1 @@ -321,28 +316,28 @@ FV3_GFS_v15p2: iopt_trs: 2 ldiag_ugwp: False lgfdlmprad: True - lradar: !!python/none + lradar: null lsm: 1 - lsoil: !!python/none - lsoil_lsm: !!python/none - ltaerosol: !!python/none + lsoil: null + lsoil_lsm: null + ltaerosol: null shal_cnv: True shinhong: False - ttendlim: !!python/none + ttendlim: null xkzm_h: 1.0 xkzm_m: 1.0 xkzminv: 0.3 namsfc: landice: True ldebug: False - surf_map_nml: + surf_map_nml: null FV3_GFS_v15_thompson_mynn_lam3km: atmos_model_nml: avg_max_length: 3600.0 fv_core_nml: agrid_vel_rst: True - full_zs_filter: !!python/none + full_zs_filter: null n_sponge: 9 npz_type: '' rf_fast: False @@ -382,20 +377,20 @@ FV3_GFS_v15_thompson_mynn_lam3km: iopt_snf: 4 iopt_stc: 1 iopt_tbot: 2 - iopt_trs: !!python/none + iopt_trs: null iovr: 3 ldiag_ugwp: False lgfdlmprad: False lsm: 1 - lsoil: !!python/none - lsoil_lsm: !!python/none + lsoil: null + lsoil_lsm: null ltaerosol: False print_diff_pgr: True - sfclay_compute_flux: !!python/none + sfclay_compute_flux: null xkzminv: 0.3 xkzm_m: 1.0 xkzm_h: 1.0 - surf_map_nml: !!python/none + surf_map_nml: null FV3_GFS_v16: cires_ugwp_nml: @@ -419,7 +414,7 @@ FV3_GFS_v16: na_init: 0 nudge_dz: False res_latlon_dynamics: '' - rf_fast: !!python/none + rf_fast: null tau: 10.0 gfdl_cloud_microphysics_nml: <<: *gfs_gfdl_cloud_mp @@ -431,10 +426,10 @@ FV3_GFS_v16: gfs_physics_nml: <<: *gfs_v15_gfs_physics cdmbgwd: [4.0, 0.15, 1.0, 1.0] - do_myjpbl: !!python/none - do_myjsfc: !!python/none + do_myjpbl: null + do_myjsfc: null do_tofd: True - do_ysu: !!python/none + do_ysu: null hybedmf: False iaer: 5111 icliq_sw: 2 @@ -443,23 +438,23 @@ FV3_GFS_v16: isatmedmf: 1 lgfdlmprad: True lheatstrg: True - lndp_type: !!python/none + lndp_type: null lsoil: 4 - n_var_lndp: !!python/none + n_var_lndp: null prautco: [0.00015, 0.00015] psautco: [0.0008, 0.0005] satmedmf: True - shinhong: !!python/none - xkzminv: !!python/none - xkzm_m: !!python/none - xkzm_h: !!python/none + shinhong: null + xkzminv: null + xkzm_m: null + xkzm_h: null mpp_io_nml: deflate_level: 1 shuffle: 1 namsfc: landice: True ldebug: False - surf_map_nml: !!python/none + surf_map_nml: null FV3_GFS_v17_p8: cires_ugwp_nml: @@ -485,7 +480,7 @@ FV3_GFS_v17_p8: nord: 1 nudge_dz: False res_latlon_dynamics: '' - rf_fast: !!python/none + rf_fast: null tau: 10.0 gfs_physics_nml: cdmbgwd: [4.0, 0.05, 1.0, 1.0] @@ -548,6 +543,4 @@ FV3_GFS_v17_p8: mpp_io_nml: deflate_level: 1 shuffle: 1 - namsfc: - surf_map_nml: !!python/none - + surf_map_nml: null diff --git a/parm/fixed_files_mapping.yaml b/parm/fixed_files_mapping.yaml index 90fd1870a4..54ddd41a81 100644 --- a/parm/fixed_files_mapping.yaml +++ b/parm/fixed_files_mapping.yaml @@ -139,35 +139,6 @@ fixed_files: !join_str ["FNSMCC | ",*FNSMCC], !join_str ["FNMSKH | ",*FNMSKH] ] - #"FNZORC | $FNZORC", - - # - #----------------------------------------------------------------------- - # - # FV3_NML_VARNAME_TO_SFC_CLIMO_FIELD_MAPPING: - # This array is used to set some of the namelist variables in the forecast - # model's namelist file that represent the relative or absolute paths of - # various fixed files (the first column of the array, where columns are - # delineated by the pipe symbol "|") to the full paths to surface climatology - # files (on the native FV3-LAM grid) in the FIXlam directory derived from - # the corresponding surface climatology fields (the second column of the - # array). - # - #----------------------------------------------------------------------- - # - FV3_NML_VARNAME_TO_SFC_CLIMO_FIELD_MAPPING: [ - "FNALBC | snowfree_albedo", - "FNALBC2 | facsf", - "FNTG3C | substrate_temperature", - "FNVEGC | vegetation_greenness", - "FNVETC | vegetation_type", - "FNSOTC | soil_type", - "FNVMNC | vegetation_greenness", - "FNVMXC | vegetation_greenness", - "FNSLPC | slope_type", - "FNABSC | maximum_snow_albedo" - ] - # #----------------------------------------------------------------------- diff --git a/scripts/exregional_make_grid.sh b/scripts/exregional_make_grid.sh index 755e1c95c4..1f95ea8f91 100755 --- a/scripts/exregional_make_grid.sh +++ b/scripts/exregional_make_grid.sh @@ -266,29 +266,31 @@ generation executable (exec_fp): # namelist file. # settings=" -'regional_grid_nml': { - 'plon': ${LON_CTR}, - 'plat': ${LAT_CTR}, - 'delx': ${DEL_ANGLE_X_SG}, - 'dely': ${DEL_ANGLE_Y_SG}, - 'lx': ${NEG_NX_OF_DOM_WITH_WIDE_HALO}, - 'ly': ${NEG_NY_OF_DOM_WITH_WIDE_HALO}, - 'pazi': ${PAZI}, - } +'regional_grid_nml': + 'plon': ${LON_CTR} + 'plat': ${LAT_CTR} + 'delx': ${DEL_ANGLE_X_SG} + 'dely': ${DEL_ANGLE_Y_SG} + 'lx': ${NEG_NX_OF_DOM_WITH_WIDE_HALO} + 'ly': ${NEG_NY_OF_DOM_WITH_WIDE_HALO} + 'pazi': ${PAZI} " -# -# Call the python script to create the namelist file. -# - ${USHdir}/set_namelist.py -q -u "$settings" -o ${rgnl_grid_nml_fp} || \ - print_err_msg_exit "\ -Call to python script set_namelist.py to set the variables in the -regional_esg_grid namelist file failed. Parameters passed to this script -are: - Full path to output namelist file: - rgnl_grid_nml_fp = \"${rgnl_grid_nml_fp}\" - Namelist settings specified on command line (these have highest precedence): - settings = -$settings" + + (cat << EOF +$settings +EOF +) | uw config realize \ + --input-format yaml \ + -o ${rgnl_grid_nml_fp} \ + -v \ + + err=$? + if [ $err -ne 0 ]; then + print_err_msg_exit "\ + Error creating regional_esg_grid namelist. + Settings for input are: + $settings" + fi # # Call the executable that generates the grid file. # @@ -611,7 +613,7 @@ failed." # #----------------------------------------------------------------------- # -# Call a function (set_FV3nml_sfc_climo_filenames) to set the values of +# Call a function (set_fv3nml_sfc_climo_filenames) to set the values of # those variables in the forecast model's namelist file that specify the # paths to the surface climatology files. These files will either already # be avaialable in a user-specified directory (SFC_CLIMO_DIR) or will be @@ -620,7 +622,7 @@ failed." # #----------------------------------------------------------------------- # -python3 $USHdir/set_FV3nml_sfc_climo_filenames.py \ +python3 $USHdir/set_fv3nml_sfc_climo_filenames.py \ --path-to-defns ${GLOBAL_VAR_DEFNS_FP} \ || print_err_msg_exit "\ Call to function to set surface climatology file names in the FV3 namelist diff --git a/scripts/exregional_make_ics.sh b/scripts/exregional_make_ics.sh index b42c086624..60852095ee 100755 --- a/scripts/exregional_make_ics.sh +++ b/scripts/exregional_make_ics.sh @@ -546,64 +546,60 @@ fi # IMPORTANT: # If we want a namelist variable to be removed from the namelist file, # in the "settings" variable below, we need to set its value to the -# string "null". This is equivalent to setting its value to -# !!python/none -# in the base namelist file specified by FV3_NML_BASE_SUITE_FP or the -# suite-specific yaml settings file specified by FV3_NML_YAML_CONFIG_FP. -# -# It turns out that setting the variable to an empty string also works -# to remove it from the namelist! Which is better to use?? +# string "null". # settings=" -'config': { - 'fix_dir_target_grid': ${FIXlam}, - 'mosaic_file_target_grid': ${FIXlam}/${CRES}${DOT_OR_USCORE}mosaic.halo$((10#${NH4})).nc, - 'orog_dir_target_grid': ${FIXlam}, - 'orog_files_target_grid': ${CRES}${DOT_OR_USCORE}oro_data.tile${TILE_RGNL}.halo$((10#${NH4})).nc, - 'vcoord_file_target_grid': ${VCOORD_FILE}, - 'varmap_file': ${PARMdir}/ufs_utils/varmap_tables/${varmap_file}, - 'data_dir_input_grid': ${extrn_mdl_staging_dir}, - 'atm_files_input_grid': ${fn_atm}, - 'sfc_files_input_grid': ${fn_sfc}, - 'grib2_file_input_grid': \"${fn_grib2}\", - 'cycle_mon': $((10#${mm})), - 'cycle_day': $((10#${dd})), - 'cycle_hour': $((10#${hh})), - 'convert_atm': True, - 'convert_sfc': True, - 'convert_nst': ${convert_nst}, - 'regional': 1, - 'halo_bndy': $((10#${NH4})), - 'halo_blend': $((10#${HALO_BLEND})), - 'input_type': ${input_type}, - 'external_model': ${external_model}, - 'tracers_input': ${tracers_input}, - 'tracers': ${tracers}, - 'nsoill_out': $((10#${nsoill_out})), - 'geogrid_file_input_grid': ${geogrid_file_input_grid}, - 'vgtyp_from_climo': ${vgtyp_from_climo}, - 'sotyp_from_climo': ${sotyp_from_climo}, - 'vgfrc_from_climo': ${vgfrc_from_climo}, - 'minmax_vgfrc_from_climo': ${minmax_vgfrc_from_climo}, - 'lai_from_climo': ${lai_from_climo}, - 'tg3_from_soil': ${tg3_from_soil}, - 'thomp_mp_climo_file': ${thomp_mp_climo_file}, -} +'config': + 'fix_dir_target_grid': ${FIXlam} + 'mosaic_file_target_grid': ${FIXlam}/${CRES}${DOT_OR_USCORE}mosaic.halo$((10#${NH4})).nc + 'orog_dir_target_grid': ${FIXlam} + 'orog_files_target_grid': ${CRES}${DOT_OR_USCORE}oro_data.tile${TILE_RGNL}.halo$((10#${NH4})).nc + 'vcoord_file_target_grid': ${VCOORD_FILE} + 'varmap_file': ${PARMdir}/ufs_utils/varmap_tables/${varmap_file} + 'data_dir_input_grid': ${extrn_mdl_staging_dir} + 'atm_files_input_grid': ${fn_atm} + 'sfc_files_input_grid': ${fn_sfc} + 'grib2_file_input_grid': \"${fn_grib2}\" + 'cycle_mon': $((10#${mm})) + 'cycle_day': $((10#${dd})) + 'cycle_hour': $((10#${hh})) + 'convert_atm': True + 'convert_sfc': True + 'convert_nst': ${convert_nst} + 'regional': 1 + 'halo_bndy': $((10#${NH4})) + 'halo_blend': $((10#${HALO_BLEND})) + 'input_type': ${input_type} + 'external_model': ${external_model} + 'tracers_input': ${tracers_input} + 'tracers': ${tracers} + 'nsoill_out': $((10#${nsoill_out})) + 'geogrid_file_input_grid': ${geogrid_file_input_grid} + 'vgtyp_from_climo': ${vgtyp_from_climo} + 'sotyp_from_climo': ${sotyp_from_climo} + 'vgfrc_from_climo': ${vgfrc_from_climo} + 'minmax_vgfrc_from_climo': ${minmax_vgfrc_from_climo} + 'lai_from_climo': ${lai_from_climo} + 'tg3_from_soil': ${tg3_from_soil} + 'thomp_mp_climo_file': ${thomp_mp_climo_file} " -# -# Call the python script to create the namelist file. -# + + nml_fn="fort.41" -${USHdir}/set_namelist.py -q -u "$settings" -o ${nml_fn} + +(cat << EOF +$settings +EOF +) | uw config realize \ + --input-format yaml \ + -o ${nml_fn} \ + --output-format nml\ + -v \ + err=$? if [ $err -ne 0 ]; then - message_txt="Call to python script set_namelist.py to set the variables -in the namelist file read in by the ${exec_fn} executable failed. Parameters -passed to this script are: - Name of output namelist file: - nml_fn = \"${nml_fn}\" - Namelist settings specified on command line (these have highest precedence): - settings = + message_txt="Error creating namelist read by ${exec_fn} failed. + Settings for input are: $settings" if [ "${RUN_ENVIR}" = "nco" ] && [ "${MACHINE}" = "WCOSS2" ]; then err_exit "${message_txt}" @@ -611,6 +607,7 @@ $settings" print_err_msg_exit "${message_txt}" fi fi + # #----------------------------------------------------------------------- # diff --git a/scripts/exregional_make_lbcs.sh b/scripts/exregional_make_lbcs.sh index 695af1b409..fcde8e6f46 100755 --- a/scripts/exregional_make_lbcs.sh +++ b/scripts/exregional_make_lbcs.sh @@ -467,53 +467,47 @@ FORTRAN namelist file has not specified for this external LBC model (EXTRN_MDL_N # IMPORTANT: # If we want a namelist variable to be removed from the namelist file, # in the "settings" variable below, we need to set its value to the -# string "null". This is equivalent to setting its value to -# !!python/none -# in the base namelist file specified by FV3_NML_BASE_SUITE_FP or the -# suite-specific yaml settings file specified by FV3_NML_YAML_CONFIG_FP. -# -# It turns out that setting the variable to an empty string also works -# to remove it from the namelist! Which is better to use?? -# -settings=" -'config': { - 'fix_dir_target_grid': ${FIXlam}, - 'mosaic_file_target_grid': ${FIXlam}/${CRES}${DOT_OR_USCORE}mosaic.halo$((10#${NH4})).nc, - 'orog_dir_target_grid': ${FIXlam}, - 'orog_files_target_grid': ${CRES}${DOT_OR_USCORE}oro_data.tile${TILE_RGNL}.halo$((10#${NH4})).nc, - 'vcoord_file_target_grid': ${VCOORD_FILE}, - 'varmap_file': ${PARMdir}/ufs_utils/varmap_tables/${varmap_file}, - 'data_dir_input_grid': ${extrn_mdl_staging_dir}, - 'atm_files_input_grid': ${fn_atm}, - 'grib2_file_input_grid': \"${fn_grib2}\", - 'cycle_mon': $((10#${mm})), - 'cycle_day': $((10#${dd})), - 'cycle_hour': $((10#${hh})), - 'convert_atm': True, - 'regional': 2, - 'halo_bndy': $((10#${NH4})), - 'halo_blend': $((10#${HALO_BLEND})), - 'input_type': ${input_type}, - 'external_model': ${external_model}, - 'tracers_input': ${tracers_input}, - 'tracers': ${tracers}, - 'thomp_mp_climo_file': ${thomp_mp_climo_file}, -} +# string "null". +# + settings=" +'config': + 'fix_dir_target_grid': ${FIXlam} + 'mosaic_file_target_grid': ${FIXlam}/${CRES}${DOT_OR_USCORE}mosaic.halo$((10#${NH4})).nc + 'orog_dir_target_grid': ${FIXlam} + 'orog_files_target_grid': ${CRES}${DOT_OR_USCORE}oro_data.tile${TILE_RGNL}.halo$((10#${NH4})).nc + 'vcoord_file_target_grid': ${VCOORD_FILE} + 'varmap_file': ${PARMdir}/ufs_utils/varmap_tables/${varmap_file} + 'data_dir_input_grid': ${extrn_mdl_staging_dir} + 'atm_files_input_grid': ${fn_atm} + 'grib2_file_input_grid': \"${fn_grib2}\" + 'cycle_mon': $((10#${mm})) + 'cycle_day': $((10#${dd})) + 'cycle_hour': $((10#${hh})) + 'convert_atm': True + 'regional': 2 + 'halo_bndy': $((10#${NH4})) + 'halo_blend': $((10#${HALO_BLEND})) + 'input_type': ${input_type} + 'external_model': ${external_model} + 'tracers_input': ${tracers_input} + 'tracers': ${tracers} + 'thomp_mp_climo_file': ${thomp_mp_climo_file} " -# -# Call the python script to create the namelist file. -# + nml_fn="fort.41" - ${USHdir}/set_namelist.py -q -u "$settings" -o ${nml_fn} + (cat << EOF +$settings +EOF +) | uw config realize \ + --input-format yaml \ + -o ${nml_fn} \ + --output-format nml \ + -v \ + export err=$? if [ $err -ne 0 ]; then - message_txt="Call to python script set_namelist.py to set the variables -in the namelist file read in by the ${exec_fn} executable failed. Parameters -passed to this script are: - Name of output namelist file: - nml_fn = \"${nml_fn}\" - Namelist settings specified on command line (these have highest precedence): - settings = + message_txt="Error creating namelist read by ${exec_fn} failed. + Settings for input are: $settings" if [ "${RUN_ENVIR}" = "nco" ] && [ "${MACHINE}" = "WCOSS2" ]; then err_exit "${message_txt}" diff --git a/scripts/exregional_run_fcst.sh b/scripts/exregional_run_fcst.sh index c5519d923c..723086b077 100755 --- a/scripts/exregional_run_fcst.sh +++ b/scripts/exregional_run_fcst.sh @@ -474,7 +474,7 @@ fi #----------------------------------------------------------------------- # if ([ "$STOCH" == "TRUE" ] && [ "${DO_ENSEMBLE}" = "TRUE" ]); then - python3 $USHdir/set_FV3nml_ens_stoch_seeds.py \ + python3 $USHdir/set_fv3nml_ens_stoch_seeds.py \ --path-to-defns ${GLOBAL_VAR_DEFNS_FP} \ --cdate "$CDATE" || print_err_msg_exit "\ Call to function to create the ensemble-based namelist for the current @@ -492,8 +492,7 @@ fi # if [ "${CPL_AQM}" = "TRUE" ] && [ "${PREDEF_GRID_NAME}" = "AQM_NA_13km" ]; then python3 $USHdir/update_input_nml.py \ - --path-to-defns ${GLOBAL_VAR_DEFNS_FP} \ - --run_dir "${DATA}" \ + --namelist "${DATA}/${FV3_NML_FN}" \ --aqm_na_13km || print_err_msg_exit "\ Call to function to update the FV3 input.nml file for air quality modeling using AQM_NA_13km for the current cycle's (cdate) run directory (DATA) failed: @@ -520,8 +519,7 @@ if [ "${DO_FCST_RESTART}" = "TRUE" ] && [ "$(ls -A ${DATA}/RESTART )" ]; then # Update FV3 input.nml for restart python3 $USHdir/update_input_nml.py \ - --path-to-defns ${GLOBAL_VAR_DEFNS_FP} \ - --run_dir "${DATA}" \ + --namelist "${DATA}/${FV3_NML_FN}" \ --restart export err=$? if [ $err -ne 0 ]; then diff --git a/scripts/exregional_run_met_genensprod_or_ensemblestat.sh b/scripts/exregional_run_met_genensprod_or_ensemblestat.sh index 5bbe61f530..fe0e119b19 100755 --- a/scripts/exregional_run_met_genensprod_or_ensemblestat.sh +++ b/scripts/exregional_run_met_genensprod_or_ensemblestat.sh @@ -399,7 +399,7 @@ EOF uw template render \ -i ${metplus_config_tmpl_fp} \ -o ${metplus_config_fp} \ - -v \ + --verbose \ --values-file "${tmpfile}" err=$? diff --git a/scripts/exregional_run_met_gridstat_or_pointstat_vx.sh b/scripts/exregional_run_met_gridstat_or_pointstat_vx.sh index 1fa249ecf8..7eb1ce4605 100755 --- a/scripts/exregional_run_met_gridstat_or_pointstat_vx.sh +++ b/scripts/exregional_run_met_gridstat_or_pointstat_vx.sh @@ -400,7 +400,7 @@ EOF uw template render \ -i ${metplus_config_tmpl_fp} \ -o ${metplus_config_fp} \ - -v \ + --verbose \ --values-file "${tmpfile}" err=$? diff --git a/scripts/exregional_run_met_gridstat_or_pointstat_vx_ensmean.sh b/scripts/exregional_run_met_gridstat_or_pointstat_vx_ensmean.sh index 067c24ec07..458dcec33f 100755 --- a/scripts/exregional_run_met_gridstat_or_pointstat_vx_ensmean.sh +++ b/scripts/exregional_run_met_gridstat_or_pointstat_vx_ensmean.sh @@ -358,7 +358,7 @@ EOF uw template render \ -i ${metplus_config_tmpl_fp} \ -o ${metplus_config_fp} \ - -v \ + --verbose \ --values-file "${tmpfile}" err=$? diff --git a/scripts/exregional_run_met_gridstat_or_pointstat_vx_ensprob.sh b/scripts/exregional_run_met_gridstat_or_pointstat_vx_ensprob.sh index e042b68bfe..fc735845c9 100755 --- a/scripts/exregional_run_met_gridstat_or_pointstat_vx_ensprob.sh +++ b/scripts/exregional_run_met_gridstat_or_pointstat_vx_ensprob.sh @@ -310,7 +310,7 @@ EOF uw template render \ -i ${metplus_config_tmpl_fp} \ -o ${metplus_config_fp} \ - -v \ + --verbose \ --values-file "${tmpfile}" err=$? diff --git a/scripts/exregional_run_met_pb2nc_obs.sh b/scripts/exregional_run_met_pb2nc_obs.sh index 10d1beba4d..92d39102fc 100755 --- a/scripts/exregional_run_met_pb2nc_obs.sh +++ b/scripts/exregional_run_met_pb2nc_obs.sh @@ -284,7 +284,7 @@ EOF uw template render \ -i ${metplus_config_tmpl_fp} \ -o ${metplus_config_fp} \ - -v \ + --verbose \ --values-file "${tmpfile}" err=$? diff --git a/scripts/exregional_run_met_pcpcombine.sh b/scripts/exregional_run_met_pcpcombine.sh index 4a9222707a..7eabe02901 100755 --- a/scripts/exregional_run_met_pcpcombine.sh +++ b/scripts/exregional_run_met_pcpcombine.sh @@ -362,7 +362,7 @@ EOF uw template render \ -i ${metplus_config_tmpl_fp} \ -o ${metplus_config_fp} \ - -v \ + --verbose \ --values-file "${tmpfile}" err=$? diff --git a/tests/test_python/test_set_FV3nml_ens_stoch_seeds.py b/tests/test_python/test_set_fv3nml_ens_stoch_seeds.py similarity index 68% rename from tests/test_python/test_set_FV3nml_ens_stoch_seeds.py rename to tests/test_python/test_set_fv3nml_ens_stoch_seeds.py index f87d57d53b..17bf74c04b 100644 --- a/tests/test_python/test_set_FV3nml_ens_stoch_seeds.py +++ b/tests/test_python/test_set_fv3nml_ens_stoch_seeds.py @@ -1,4 +1,4 @@ -""" Tests for set_FV3nml_ens_stoch_seeds.py """ +""" Tests for set_fv3nml_ens_stoch_seeds.py """ #pylint: disable=invalid-name @@ -15,18 +15,17 @@ set_env_var, ) -from set_FV3nml_ens_stoch_seeds import set_FV3nml_ens_stoch_seeds +from set_fv3nml_ens_stoch_seeds import set_fv3nml_ens_stoch_seeds class Testing(unittest.TestCase): """ Define the tests """ - def test_set_FV3nml_ens_stoch_seeds(self): + def test_set_fv3nml_ens_stoch_seeds(self): """ Call the function and make sure it doesn't fail""" os.chdir(self.mem_dir) - set_FV3nml_ens_stoch_seeds(cdate=self.cdate) + set_fv3nml_ens_stoch_seeds(cdate=self.cdate, expt_config=self.config) def setUp(self): define_macos_utilities() - set_env_var("DEBUG", True) set_env_var("VERBOSE", True) self.cdate = datetime(2021, 1, 1) test_dir = os.path.dirname(os.path.abspath(__file__)) @@ -55,17 +54,22 @@ def setUp(self): ) - set_env_var("USHdir", USHdir) set_env_var("ENSMEM_INDX", 2) - set_env_var("FV3_NML_FN", "input.nml") - set_env_var("FV3_NML_FP", os.path.join(self.mem_dir, "input.nml")) - set_env_var("DO_SHUM", True) - set_env_var("DO_SKEB", True) - set_env_var("DO_SPPT", True) - set_env_var("DO_SPP", True) - set_env_var("DO_LSM_SPP", True) - ISEED_SPP = [4, 5, 6, 7, 8] - set_env_var("ISEED_SPP", ISEED_SPP) + + self.config = { + "workflow": { + "VERBOSE": True, + "FV3_NML_FN": "input.nml", + }, + "global": { + "DO_SHUM": True, + "DO_SKEB": True, + "DO_SPPT": True, + "DO_SPP": True, + "DO_LSM_SPP": True, + "ISEED_SPP": [4, 5, 6, 7, 8], + }, + } def tearDown(self): self.tmp_dir.cleanup() diff --git a/tests/test_python/test_set_FV3nml_sfc_climo_filenames.py b/tests/test_python/test_set_fv3nml_sfc_climo_filenames.py similarity index 69% rename from tests/test_python/test_set_FV3nml_sfc_climo_filenames.py rename to tests/test_python/test_set_fv3nml_sfc_climo_filenames.py index 131af70506..b0daf50fea 100644 --- a/tests/test_python/test_set_FV3nml_sfc_climo_filenames.py +++ b/tests/test_python/test_set_fv3nml_sfc_climo_filenames.py @@ -1,4 +1,4 @@ -""" Tests for set_FV3nml_sfc_climo_filenames.py """ +""" Tests for set_fv3nml_sfc_climo_filenames.py """ #pylint: disable=invalid-name @@ -12,13 +12,13 @@ mkdir_vrfy, set_env_var, ) -from set_FV3nml_sfc_climo_filenames import set_FV3nml_sfc_climo_filenames +from set_fv3nml_sfc_climo_filenames import set_fv3nml_sfc_climo_filenames class Testing(unittest.TestCase): """ Define the tests """ - def test_set_FV3nml_sfc_climo_filenames(self): + def test_set_fv3nml_sfc_climo_filenames(self): """ Call the function and don't raise an Exception. """ - set_FV3nml_sfc_climo_filenames() + set_fv3nml_sfc_climo_filenames(config=self.config) def setUp(self): define_macos_utilities() @@ -42,13 +42,15 @@ def setUp(self): os.path.join(PARMdir, "input.nml.FV3"), os.path.join(EXPTDIR, "input.nml"), ) - set_env_var("PARMdir", PARMdir) - set_env_var("EXPTDIR", EXPTDIR) - set_env_var("FIXlam", FIXlam) - set_env_var("DO_ENSEMBLE", False) - set_env_var("CRES", "C3357") - set_env_var("RUN_ENVIR", "nco") - set_env_var("FV3_NML_FP", os.path.join(EXPTDIR, "input.nml")) + self.config = { + "CRES": "C3357", + "DO_ENSEMBLE": False, + "EXPTDIR": EXPTDIR, + "FIXlam": FIXlam, + "FV3_NML_FP": os.path.join(EXPTDIR, "input.nml"), + "PARMdir": PARMdir, + "RUN_ENVIR": "nco", + } def tearDown(self): self.tmp_dir.cleanup() diff --git a/ush/generate_FV3LAM_wflow.py b/ush/generate_FV3LAM_wflow.py index a678a61132..ec2b95c3f3 100755 --- a/ush/generate_FV3LAM_wflow.py +++ b/ush/generate_FV3LAM_wflow.py @@ -14,6 +14,8 @@ from subprocess import STDOUT, CalledProcessError, check_output from textwrap import dedent +from uwtools.api.config import get_nml_config, get_yaml_config, realize + from python_utils import ( log_info, import_vars, @@ -30,9 +32,8 @@ ) from setup import setup -from set_FV3nml_sfc_climo_filenames import set_FV3nml_sfc_climo_filenames +from set_fv3nml_sfc_climo_filenames import set_fv3nml_sfc_climo_filenames from get_crontab_contents import add_crontab_line -from set_namelist import set_namelist from check_python_version import check_python_version # pylint: disable=too-many-locals,too-many-branches, too-many-statements @@ -506,24 +507,23 @@ def generate_FV3LAM_wflow( # # ----------------------------------------------------------------------- # - # Call the set_namelist.py script to create a new FV3 namelist file (full - # path specified by FV3_NML_FP) using the file FV3_NML_BASE_SUITE_FP as - # the base (i.e. starting) namelist file, with physics-suite-dependent - # modifications to the base file specified in the yaml configuration file - # FV3_NML_YAML_CONFIG_FP (for the physics suite specified by CCPP_PHYS_SUITE), - # and with additional physics-suite-independent modifications specified - # in the variable "settings" set above. + # Create a new FV3 namelist file # # ----------------------------------------------------------------------- # - args=[ "-n", FV3_NML_BASE_SUITE_FP, - "-c", FV3_NML_YAML_CONFIG_FP, CCPP_PHYS_SUITE, - "-u", settings_str, - "-o", FV3_NML_FP, - ] - if not debug: - args.append("-q") - set_namelist(args) + + physics_cfg = get_yaml_config(FV3_NML_YAML_CONFIG_FP) + base_namelist = get_nml_config(FV3_NML_BASE_SUITE_FP) + base_namelist.update_values(physics_cfg[CCPP_PHYS_SUITE]) + base_namelist.update_values(settings) + for sect, values in base_namelist.copy().items(): + if not values: + del base_namelist[sect] + continue + for k, v in values.copy().items(): + if v is None: + del base_namelist[sect][k] + base_namelist.dump(FV3_NML_FP) # # If not running the TN_MAKE_GRID task (which implies the workflow will # use pregenerated grid files), set the namelist variables specifying @@ -538,7 +538,7 @@ def generate_FV3LAM_wflow( # if not expt_config['rocoto']['tasks'].get('task_make_grid'): - set_FV3nml_sfc_climo_filenames(debug) + set_fv3nml_sfc_climo_filenames(flatten_dict(expt_config), debug) # # ----------------------------------------------------------------------- @@ -652,14 +652,13 @@ def generate_FV3LAM_wflow( #----------------------------------------------------------------------- # if any((DO_SPP, DO_SPPT, DO_SHUM, DO_SKEB, DO_LSM_SPP)): - - args=[ "-n", FV3_NML_FP, - "-u", settings_str, - "-o", FV3_NML_STOCH_FP, - ] - if not debug: - args.append("-q") - set_namelist(args) + realize( + input_config=FV3_NML_FP, + input_format="nml", + output_file=FV3_NML_STOCH_FP, + output_format="nml", + supplemental_configs=[settings], + ) # # ----------------------------------------------------------------------- diff --git a/ush/set_FV3nml_ens_stoch_seeds.py b/ush/set_fv3nml_ens_stoch_seeds.py similarity index 64% rename from ush/set_FV3nml_ens_stoch_seeds.py rename to ush/set_fv3nml_ens_stoch_seeds.py index c8a90e2797..3459fa8707 100644 --- a/ush/set_FV3nml_ens_stoch_seeds.py +++ b/ush/set_fv3nml_ens_stoch_seeds.py @@ -1,32 +1,27 @@ #!/usr/bin/env python3 +""" +Updates stochastic physics parameters in the namelist based on user configuration settings. +""" + +import argparse +import datetime as dt import os import sys -import argparse from textwrap import dedent -from datetime import datetime + +from uwtools.api.config import realize from python_utils import ( - print_input_args, - print_info_msg, - print_err_msg_exit, - date_to_str, - mkdir_vrfy, - cp_vrfy, - cd_vrfy, - str_to_type, - import_vars, - set_env_var, - define_macos_utilities, cfg_to_yaml_str, + import_vars, load_shell_config, - flatten_dict, + print_input_args, + print_info_msg, ) -from set_namelist import set_namelist - -def set_FV3nml_ens_stoch_seeds(cdate): +def set_fv3nml_ens_stoch_seeds(cdate, expt_config): """ This function, for an ensemble-enabled experiment (i.e. for an experiment for which the workflow configuration variable @@ -39,15 +34,20 @@ def set_FV3nml_ens_stoch_seeds(cdate): called as part of the TN_RUN_FCST task. Args: - cdate + cdate the cycle + expt_config the in-memory dict representing the experiment configuration Returns: None """ print_input_args(locals()) - # import all environment variables - import_vars() + fv3_nml_fn = expt_config["workflow"]["FV3_NML_FN"] + verbose = expt_config["workflow"]["VERBOSE"] + + # set variables important to this function from the experiment definition + import_vars(dictionary=expt_config["global"]) + # pylint: disable=undefined-variable # # ----------------------------------------------------------------------- @@ -57,9 +57,9 @@ def set_FV3nml_ens_stoch_seeds(cdate): # # ----------------------------------------------------------------------- # - fv3_nml_ensmem_fp = f"{os.getcwd()}{os.sep}{FV3_NML_FN}" + fv3_nml_ensmem_fp = f"{os.getcwd()}{os.sep}{fv3_nml_fn}" - ensmem_num = int(ENSMEM_INDX) + ensmem_num = int(os.environ["ENSMEM_INDX"]) cdate_i = int(cdate.strftime("%Y%m%d%H")) @@ -95,49 +95,39 @@ def set_FV3nml_ens_stoch_seeds(cdate): settings["nam_sfcperts"] = {"iseed_lndp": [iseed_lsm_spp]} - settings_str = cfg_to_yaml_str(settings) - print_info_msg( dedent( f""" The variable 'settings' specifying seeds in '{fv3_nml_ensmem_fp}' has been set as follows: - settings =\n\n""" - ) - + settings_str, - verbose=VERBOSE, - ) + settings =\n\n - try: - set_namelist( - ["-q", "-n", fv3_nml_ensmem_fp, "-u", settings_str, "-o", fv3_nml_ensmem_fp] - ) - except: - print_err_msg_exit( - dedent( - f""" - Call to python script set_namelist.py to set the variables in the FV3 - namelist file that specify the paths to the surface climatology files - failed. Parameters passed to this script are: - Full path to base namelist file: - FV3_NML_FP = '{FV3_NML_FP}' - Full path to output namelist file: - fv3_nml_ensmem_fp = '{fv3_nml_ensmem_fp}' - Namelist settings specified on command line (these have highest precedence):\n - settings =\n\n""" - ) - + settings_str + {cfg_to_yaml_str(settings)}""" + ), + verbose=verbose, + ) + realize( + input_config=fv3_nml_ensmem_fp, + input_format="nml", + output_file=fv3_nml_ensmem_fp, + output_format="nml", + supplemental_configs=[settings], ) - def parse_args(argv): """Parse command line arguments""" parser = argparse.ArgumentParser( description="Creates stochastic seeds for an ensemble experiment." ) - parser.add_argument("-c", "--cdate", dest="cdate", required=True, help="Date.") + parser.add_argument( + "-c", "--cdate", + dest="cdate", + required=True, + type=lambda d: dt.datetime.strptime(d, '%Y%m%d%H'), + help="Date.", + ) parser.add_argument( "-p", @@ -153,6 +143,4 @@ def parse_args(argv): if __name__ == "__main__": args = parse_args(sys.argv[1:]) cfg = load_shell_config(args.path_to_defns) - cfg = flatten_dict(cfg) - import_vars(dictionary=cfg) - set_FV3nml_ens_stoch_seeds(str_to_type(args.cdate)) + set_fv3nml_ens_stoch_seeds(args.cdate, cfg) diff --git a/ush/set_FV3nml_sfc_climo_filenames.py b/ush/set_fv3nml_sfc_climo_filenames.py similarity index 53% rename from ush/set_FV3nml_sfc_climo_filenames.py rename to ush/set_fv3nml_sfc_climo_filenames.py index a1ffaa57ef..417aa0b5ee 100644 --- a/ush/set_FV3nml_sfc_climo_filenames.py +++ b/ush/set_fv3nml_sfc_climo_filenames.py @@ -1,33 +1,42 @@ #!/usr/bin/env python3 +""" +Update filenames for surface climotology files in the namelist. +""" + +import argparse import os +import re import sys -import argparse from textwrap import dedent +from uwtools.api.config import get_yaml_config, realize + from python_utils import ( - print_input_args, - print_info_msg, - print_err_msg_exit, + cfg_to_yaml_str, check_var_valid_value, - mv_vrfy, - mkdir_vrfy, - cp_vrfy, - rm_vrfy, + flatten_dict, import_vars, - set_env_var, - load_config_file, load_shell_config, - flatten_dict, - define_macos_utilities, - find_pattern_in_str, - cfg_to_yaml_str, + print_info_msg, ) -from set_namelist import set_namelist +VERBOSE = os.environ.get("VERBOSE", "true") + +NEEDED_VARS = [ + "CRES", + "DO_ENSEMBLE", + "EXPTDIR", + "FIXlam", + "FV3_NML_FP", + "PARMdir", + "RUN_ENVIR", + ] + +# pylint: disable=undefined-variable -def set_FV3nml_sfc_climo_filenames(debug=False): +def set_fv3nml_sfc_climo_filenames(config, debug=False): """ This function sets the values of the variables in the forecast model's namelist file that specify the paths to the surface @@ -43,13 +52,9 @@ def set_FV3nml_sfc_climo_filenames(debug=False): None """ - # import all environment variables - import_vars() + import_vars(dictionary=config, env_vars=NEEDED_VARS) - # fixed file mapping variables - fixed_cfg = load_config_file(os.path.join(PARMdir, "fixed_files_mapping.yaml")) - IMPORTS = ["SFC_CLIMO_FIELDS", "FV3_NML_VARNAME_TO_SFC_CLIMO_FIELD_MAPPING"] - import_vars(dictionary=flatten_dict(fixed_cfg), env_vars=IMPORTS) + fixed_cfg = get_yaml_config(os.path.join(PARMdir, "fixed_files_mapping.yaml"))["fixed_files"] # The regular expression regex_search set below will be used to extract # from the elements of the array FV3_NML_VARNAME_TO_SFC_CLIMO_FIELD_MAPPING @@ -68,18 +73,16 @@ def set_FV3nml_sfc_climo_filenames(debug=False): dummy_run_dir += os.sep + "any_ensmem" namsfc_dict = {} - for mapping in FV3_NML_VARNAME_TO_SFC_CLIMO_FIELD_MAPPING: - tup = find_pattern_in_str(regex_search, mapping) - nml_var_name = tup[0] - sfc_climo_field_name = tup[1] + for mapping in fixed_cfg["FV3_NML_VARNAME_TO_SFC_CLIMO_FIELD_MAPPING"]: + nml_var_name, sfc_climo_field_name = re.search(regex_search, mapping).groups() - check_var_valid_value(sfc_climo_field_name, SFC_CLIMO_FIELDS) + check_var_valid_value(sfc_climo_field_name, fixed_cfg["SFC_CLIMO_FIELDS"]) - fp = os.path.join(FIXlam, f"{CRES}.{sfc_climo_field_name}.{suffix}") + file_path = os.path.join(FIXlam, f"{CRES}.{sfc_climo_field_name}.{suffix}") if RUN_ENVIR != "nco": - fp = os.path.relpath(os.path.realpath(fp), start=dummy_run_dir) + file_path = os.path.relpath(os.path.realpath(file_path), start=dummy_run_dir) - namsfc_dict[nml_var_name] = fp + namsfc_dict[nml_var_name] = file_path settings["namsfc_dict"] = namsfc_dict settings_str = cfg_to_yaml_str(settings) @@ -89,40 +92,22 @@ def set_FV3nml_sfc_climo_filenames(debug=False): f""" The variable 'settings' specifying values of the namelist variables has been set as follows:\n - settings =\n\n""" - ) - + settings_str, + settings = + + {settings_str} + """ + ), verbose=debug, ) - # Rename the FV3 namelist and call set_namelist - fv3_nml_base_fp = f"{FV3_NML_FP}.base" - mv_vrfy(f"{FV3_NML_FP} {fv3_nml_base_fp}") - - try: - set_namelist( - ["-q", "-n", fv3_nml_base_fp, "-u", settings_str, "-o", FV3_NML_FP] - ) - except: - print_err_msg_exit( - dedent( - f""" - Call to python script set_namelist.py to set the variables in the FV3 - namelist file that specify the paths to the surface climatology files - failed. Parameters passed to this script are: - Full path to base namelist file: - fv3_nml_base_fp = '{fv3_nml_base_fp}' - Full path to output namelist file: - FV3_NML_FP = '{FV3_NML_FP}' - Namelist settings specified on command line (these have highest precedence):\n - settings =\n\n""" - ) - + settings_str + realize( + input_config=FV3_NML_FP, + input_format="nml", + output_file=FV3_NML_FP, + output_format="nml", + supplemental_configs=[settings], ) - rm_vrfy(f"{fv3_nml_base_fp}") - - def parse_args(argv): """Parse command line arguments""" parser = argparse.ArgumentParser(description="Set surface climatology fields.") @@ -144,5 +129,4 @@ def parse_args(argv): args = parse_args(sys.argv[1:]) cfg = load_shell_config(args.path_to_defns) cfg = flatten_dict(cfg) - import_vars(dictionary=cfg) - set_FV3nml_sfc_climo_filenames(args.debug) + set_fv3nml_sfc_climo_filenames(cfg, args.debug) diff --git a/ush/set_namelist.py b/ush/set_namelist.py deleted file mode 100755 index e578d3201f..0000000000 --- a/ush/set_namelist.py +++ /dev/null @@ -1,355 +0,0 @@ -#!/usr/bin/env python3 - -""" -This utility updates a Fortran namelist file using the f90nml package. The -settings that are modified are supplied via command line YAML-formatted string -and/or YAML configuration files. - -Additionally, the tool can be used to create a YAML file from an input namelist, -or the difference between two namelists. - -The user configuration file should contain a heirarchy that follows the -heirarchy for the Fortran namelist. An example of modifying an FV3 namelist: - - Configuration file contains: - - fv_core_nml: - k_split: 4 - n_split: 5 - - gfs_physics_nml: - do_sppt: True - -The output namelist will differ from the input namelist by only these three -settings. If one of these sections and/or variables did not previously exist, it -will be automatically created. It is up to the user to ensure that configuration -settings are provided under the correct sections and variable names. - -The optional base configuration file (provided via the -c command line argument) -contains the known set of configurations used and supported by the community, if -using the one provided in parm/FV3.input.yml. If maintaining this file -for a different set of configurations, ensure that the heirarchy is such that it -names the configuration at the top level (section), and the subsequent sections -match those in the F90 namelist that will be updated. - -Examples - - To show help options: - - set_namelist.py -h - - To produce a namelist (fv3_expt.nml) by specifying a physics package: - - set_namelist.py -n ../parm/input.nml.FV3 -c ../parm/FV3.input.yml FV3_HRRR - -o fv3_expt.nml - - To produce a YAML file (fv3_namelist.yml) from a user namelist: - - set_namelist.py -i my_namelist.nml -o fv3_namelist.nml -t yaml - - To produce a YAML file (fv3_my_namelist.yml) with differences from base nml: - - set_namelist.py -n ../parm/input.nml.FV3 -i my_namelist.nml -t yaml - -o fv3_my_namelist.nml - -Expected behavior: - - - A Fortran namelist that contains only user-defined settings will be - generated if no input namelist is provided. - - An unmodified copy of an input namelist will be generated in the - designated output location if no user-settings are provided. - - Command-line-entered settings over-ride settings in YAML configuration - file. - - Given a user namelist, the script can dump a YAML file. - - Given a user namelist and a base namelist, the script can dump the - difference in the two to a YAML file that can be included as a section - in the supported configs. -""" - -import argparse -import collections -import os -import sys - -import f90nml -import yaml - - -def config_exists(arg): - - """ - Checks whether the config file exists and if it contains the input - section. Returns the arg as provided if checks are passed. - """ - - # Agument is expected to be a 2-item list of file name and internal section - # name. - file_name = arg[0] - section_name = arg[1] - - file_exists(file_name) - - # Load the YAML file into a dictionary - with open(file_name, "r") as fn: - cfg = yaml.load(fn, Loader=yaml.Loader) - - # Grab only the section that is specified by the user - try: - cfg = cfg[section_name] - except KeyError: - msg = f"Section {section_name} does not exist in top level of {file_name}" - raise argparse.ArgumentTypeError(msg) - - return [cfg, section_name] - - -def file_exists(arg): - - """Check for existence of file""" - - if not os.path.exists(arg): - msg = f"{arg} does not exist!" - raise argparse.ArgumentTypeError(msg) - - return arg - - -def load_config(arg): - - """ - Check to ensure that the provided config file exists. If it does, load it - with YAML's safe loader and return the resulting dict. - """ - - return yaml.safe_load(arg) - - -def path_ok(arg): - - """ - Check whether the path to the file exists, and is writeable. Return the path - if it passes all checks, otherwise raise an error. - """ - - # Get the absolute path provided by arg - dir_name = os.path.abspath(os.path.dirname(arg)) - - # Ensure the arg path exists, and is writable. Raise error if not. - if os.path.lexists(dir_name) and os.access(dir_name, os.W_OK): - return arg - - msg = f"{arg} is not a writable path!" - raise argparse.ArgumentTypeError(msg) - - -def parse_args(argv): - - """ - Function maintains the arguments accepted by this script. Please see - Python's argparse documenation for more information about settings of each - argument. - """ - - parser = argparse.ArgumentParser( - description="Update a Fortran namelist with user-defined settings." - ) - - # Required - parser.add_argument( - "-o", - "--outfile", - help="Required: Full path to output file. This is a \ - namelist by default.", - required=True, - type=path_ok, - ) - - # Optional - parser.add_argument( - "-c", - "--config", - help="Full path to a YAML config file containing multiple \ - configurations, and the top-level section to use. Optional.", - metavar=("[FILE,", "SECTION]"), - nargs=2, - ) - parser.add_argument( - "-i", - "--input_nml", - help="Path to a user namelist. Use with -n and \ - -t yaml to get a YAML file to use with workflow.", - type=file_exists, - ) - parser.add_argument( - "-n", - "--basenml", - dest="nml", - help="Full path to the input Fortran namelist. Optional.", - type=file_exists, - ) - parser.add_argument( - "-t", - "--type", - choices=["nml", "yaml"], - default="nml", - help="Output file type.", - ) - parser.add_argument( - "-u", - "--user_config", - help="Command-line user config options in YAML-formatted \ - string. These options will override any provided in an \ - input file. Optional.", - metavar="YAML STRING", - type=load_config, - ) - - # Flags - parser.add_argument( - "-q", - "--quiet", - action="store_true", - help="If provided, suppress all output.", - ) - return parser.parse_args(argv) - - -def dict_diff(dict1, dict2): - - """ - Produces a dictionary of how dict2 differs from dict1 - """ - - diffs = {} - - # Loop through dict1 sections and key/value pairs - for sect, items in dict1.items(): - for key, val in items.items(): - - # If dict 2 has a different value, record the dict2 value - if val != dict2.get(sect, {}).get(key, ""): - if not diffs.get(sect): - diffs[sect] = {} - diffs[sect][key] = dict2.get(sect, {}).get(key) - - # Loop through dict2 sections and key/value pairs to catch any settings that - # may be present in the 2nd dict that weren't in the first. - for sect, items in dict2.items(): - for key, val in items.items(): - - # If dict1 has a diffent value than dict2, record the dict2 value - if val != dict1.get(sect, {}).get(key, ""): - - # Check to make sure it hasn't already been recorded - if diffs.get(sect, {}).get(key, "DNE") == "DNE": - if not diffs.get(sect): - diffs[sect] = {} - diffs[sect][key] = val - return diffs - - -def to_dict(odict): - - """Recursively convert OrderedDict to Python dict.""" - - if not isinstance(odict, collections.OrderedDict): - return odict - - ret = dict(odict) - for key, value in ret.items(): - if isinstance(value, collections.OrderedDict): - ret[key] = to_dict(value) - return ret - - -def update_dict(dest, newdict, quiet=False): - - """ - Overwrites all values in dest dictionary with values from newdict. Turn off - print statements with queit=True. - - Input: - - dest A dict that is to be updated. - newdict A dict containing sections and keys corresponding to - those in dest and potentially additional ones, that will be used to - update the dest dict. - quiet An optional boolean flag to turn off output. - - Output: - - None - - Result: - - The dest dict is updated in place. - """ - - for sect, values in newdict: - # If section is set to None, remove all contents from namelist - if values is None: - dest[sect] = {} - else: - for key, value in values.items(): - if not quiet: - print(f"Setting {sect}.{key} = {value}") - - # Remove key from dict if config is set to None - if value is None: - _ = dest[sect].pop(key, None) - else: - - try: - dest[sect][key] = value - except KeyError: - # Namelist section did not exist. Create it and update the value. - dest[sect] = {} - dest[sect][key] = value - - -def set_namelist(argv): - - """Using input command line arguments (cla), update a Fortran namelist file.""" - - # parse argumetns - cla = parse_args(argv) - if cla.config: - cla.config, _ = config_exists(cla.config) - - # Load base namelist into dict - nml = f90nml.Namelist() - if cla.nml is not None: - nml = f90nml.read(cla.nml) - - # Update namelist settings (nml) with config file settings (cfg) - cfg = {} - if cla.config is not None: - cfg = cla.config - update_dict(nml, cfg.items(), quiet=cla.quiet) - - # Update nml, overriding YAML if needed, with any command-line entries - if cla.user_config: - update_dict(nml, cla.user_config.items(), quiet=cla.quiet) - - # Write the resulting file - with open(cla.outfile, "w") as fn: - if cla.type == "nml": - nml.write(fn, sort=True) - - if cla.type == "yaml": - if cla.input_nml: - input_nml = f90nml.read(cla.input_nml) - - # Determine how input_nml differs from the configured namelist - diff = dict_diff(nml, input_nml) - - # Write diffs to YAML file - yaml.dump(diff, fn) - - else: - # Write the namelist to YAML file - yaml.dump(to_dict(nml.todict()), fn) - - -if __name__ == "__main__": - set_namelist(sys.argv[1:]) diff --git a/ush/update_input_nml.py b/ush/update_input_nml.py index 0f10c675b2..e975d9bc08 100644 --- a/ush/update_input_nml.py +++ b/ush/update_input_nml.py @@ -1,62 +1,41 @@ #!/usr/bin/env python3 +""" +Update the model namelist for a variety of different settings. +""" + +import argparse import os import sys -import argparse -import logging from textwrap import dedent +from uwtools.api.config import realize + from python_utils import ( - import_vars, print_input_args, print_info_msg, - print_err_msg_exit, cfg_to_yaml_str, - load_shell_config, - flatten_dict, ) -from set_namelist import set_namelist +VERBOSE = os.environ.get("VERBOSE", "true") - -def update_input_nml(run_dir): +def update_input_nml(namelist, restart, aqm_na_13km): """Update the FV3 input.nml file in the specified run directory Args: - run_dir: run directory + namelist: path to the namelist + restart: should forecast start from restart? + aqm_na_13km: should the 13km AQM config be used? + Returns: Boolean """ print_input_args(locals()) - - # import all environment variables - import_vars() - - # - # ----------------------------------------------------------------------- - # - # Update the FV3 input.nml file in the specified run directory. - # - # ----------------------------------------------------------------------- - # - print_info_msg( - f""" - Updating the FV3 input.nml file in the specified run directory (run_dir): - run_dir = '{run_dir}'""", - verbose=VERBOSE, - ) - # - # ----------------------------------------------------------------------- - # - # Set new values of the specific parameters to be updated. - # - # ----------------------------------------------------------------------- - # settings = {} # For restart run - if args.restart: + if restart: settings["fv_core_nml"] = { "external_ic": False, "make_nh": False, @@ -69,105 +48,68 @@ def update_input_nml(run_dir): settings["gfs_physics_nml"] = { "nstf_name": [2, 0, 0, 0, 0], } - + # For AQM_NA_13km domain for air quality modeling - if args.aqm_na_13km: + if aqm_na_13km: settings["fv_core_nml"] = { "k_split": 1, "n_split": 8, } - settings_str = cfg_to_yaml_str(settings) - print_info_msg( dedent( f""" - The variable 'settings' specifying values to be used in the FV3 'input.nml' - file for restart has been set as follows:\n - settings =\n\n""" - ) - + settings_str, + Updating {namelist} + + The updated values are: + + {cfg_to_yaml_str(settings)} + + """ + ), verbose=VERBOSE, ) - # - # ----------------------------------------------------------------------- - # - # Call a python script to update the experiment's actual FV3 INPUT.NML - # file for restart. - # - # ----------------------------------------------------------------------- - # - fv3_input_nml_fp = os.path.join(run_dir, FV3_NML_FN) - - try: - set_namelist( - [ - "-q", - "-n", - fv3_input_nml_fp, - "-u", - settings_str, - "-o", - fv3_input_nml_fp, - ] - ) - except: - logging.exception( - dedent( - f""" - Call to python script set_namelist.py to generate an FV3 namelist file - failed. Parameters passed to this script are: - Full path to base namelist file: - fv3_input_nml_fp = '{fv3_input_nml_fp}' - Full path to output namelist file: - fv3_input_nml_fp = '{fv3_input_nml_fp}' - Namelist settings specified on command line:\n - settings =\n\n""" - ) - + settings_str - ) - return False - - return True + # Update the experiment's FV3 INPUT.NML file + realize( + input_config=namelist, + input_format="nml", + output_file=namelist, + output_format="nml", + supplemental_configs=[settings], + ) def parse_args(argv): """Parse command line arguments""" parser = argparse.ArgumentParser(description="Update FV3 input.nml file for restart.") parser.add_argument( - "-r", "--run_dir", - dest="run_dir", + "-n", "--namelist", + dest="namelist", required=True, - help="Run directory." - ) - - parser.add_argument( - "-p", "--path-to-defns", - dest="path_to_defns", - required=True, - help="Path to var_defns file.", + help="Path to namelist to update.", ) parser.add_argument( "--restart", action='store_true', - help='Update for restart') + help='Update for restart', + ) parser.add_argument( "--aqm_na_13km", action='store_true', - help='Update for AQM_NA_13km in air quality modeling') + help='Update for AQM_NA_13km in air quality modeling', + ) return parser.parse_args(argv) if __name__ == "__main__": args = parse_args(sys.argv[1:]) - cfg = load_shell_config(args.path_to_defns) - cfg = flatten_dict(cfg) - import_vars(dictionary=cfg) update_input_nml( - run_dir=args.run_dir, + namelist=args.namelist, + restart=args.restart, + aqm_na_13km=args.aqm_na_13km, )