From 845681b42b348c0e1e6a47f9da50b85f1bce3b8d Mon Sep 17 00:00:00 2001 From: Tracy Date: Mon, 13 Jun 2022 08:07:19 -0600 Subject: [PATCH 1/5] Update community run script to add numx option --- scripts/run_upp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/scripts/run_upp b/scripts/run_upp index 39cc9a986..68d29b0ec 100755 --- a/scripts/run_upp +++ b/scripts/run_upp @@ -96,6 +96,9 @@ export RUN_COMMAND="mpirun -np 1 ${POSTEXEC}/upp.x " #export RUN_COMMAND="mpirun.lsf ${POSTEXEC}/upp.x " #export RUN_COMMAND="mpiexec_mpt ${POSTEXEC}/upp.x " +# The number of subdomains used for 2d decomposition +export numx=2 + # Shouldn't need to edit these. # tmmark is an variable used as the file extention of the output # filename .GrbF is used if this variable is not set @@ -330,6 +333,9 @@ MODELNAME='${tag}' fileNameFlux='${flxFileName}' fileNameFlat='postxconfig-NT.txt' / +&nampgb +numx=${numx} +/ EOF fi fi From b524103de90c732ed5434b6018f23ecbcd8f8616 Mon Sep 17 00:00:00 2001 From: Tracy Date: Mon, 13 Jun 2022 13:14:44 -0600 Subject: [PATCH 2/5] Documentation updates for 2D decomp --- docs/AddNewVariable.rst | 29 ++++++++++++++++++----------- docs/CodeOverview.rst | 3 +++ docs/InputsOutputs.rst | 29 ++++++++++++++++------------- docs/Running.rst | 9 ++++++++- scripts/run_upp | 6 +++++- 5 files changed, 50 insertions(+), 26 deletions(-) diff --git a/docs/AddNewVariable.rst b/docs/AddNewVariable.rst index a502d4b9c..f244f3d46 100644 --- a/docs/AddNewVariable.rst +++ b/docs/AddNewVariable.rst @@ -79,7 +79,7 @@ with examples in the sections below. This flat file (instead of the xml file) is read in by UPP as it was much faster to read a text file than an xml file. -2. Allocate the field: ALLOCATE.f +2. Allocate and initialize the field: ALLOCATE.f This file is the instantiation or allocation of the variable. Note that the variables are defined based on the parallel processing capability of UPP - use an example from the file. @@ -171,7 +171,8 @@ with examples in the sections below. e) Add the new variable to the UPP/parm/post_avblflds.xml, which lists all fields available for output in GRIB2 format. This file is generally not modified unless adding a new field or modifying an existing one. - - Post_avblfldidx: the unique array index number used to store this variable. + - Post_avblfldidx: the unique array index number used to store this variable. The number chosen here + is just an example and it is important to pick one that is not yet in use. - Shortname: name describing the variable and level type - Pname: the abbreviation for your variable (should match what is used in params_grib2_tbl_new) - Table info: table used if not standard WMO @@ -185,7 +186,7 @@ with examples in the sections below. :: - 999 + 1003 DEEP_TSOIL_ON_DEPTH_BEL_LAND_SFC TG3 depth_bel_land_sfc @@ -220,16 +221,21 @@ with examples in the sections below. This flat file (instead of the xml file) is read in by UPP as it was much faster to read a text file than an xml file. -2. Allocate the new variable in ALLOCATE_ALL.f +2. Allocate and initialize the new variable in ALLOCATE_ALL.f This file is the instantiation or allocation of the variable. Note that the variables are defined based on the parallel processing capability of UPP - use an example from the file. User Procedure - - Add in VRBLS2D GFS section as: + - Allocate in VRBLS2D GFS section as: + + :: + + allocate(tg3(ista_2l:iend_2u,jsta_2l:jend_2u)) + - Initialize in the initialization section that comes after the allocations section you added to. :: - allocate(tg3(im,jsta_2l:jend_2u)) + tg3(i,j)=spval 3. De-allocate the variable to give the resources back in DEALLOCATE.f All good programmers give back their resources when they are done. Please update this @@ -270,7 +276,7 @@ with examples in the sections below. ! deep soil temperature VarName='tg3' - call read_netcdf_2d_para(ncid2d,im,jsta,jsta_2l,jend,jend_2u, & + call read_netcdf_2d_para(ncid2d,ista,ista_2l,iend,iend_2u,jsta,jsta_2l,jend,jend_2u, & spval,VarName,tg3) 6. Determine the appropriate routine to add the new variable to (e.g. SURFCE.f, MDLFLD.f, @@ -299,16 +305,17 @@ with examples in the sections below. :: ! DEEP SOIL TEMPERATURE - IF ( IGET(999).GT.0 ) THEN + IF ( IGET(1003).GT.0 ) THEN ID(1:25) = 0 If(grib=='grib2') then cfld=cfld+1 - fld_info(cfld)%ifld=IAVBLFLD(IGET(999)) + fld_info(cfld)%ifld=IAVBLFLD(IGET(1003)) !$omp parallel do private(i,j,jj) do j=1,jend-jsta+1 jj = jsta+j-1 - do i=1,im - datapd(i,j,cfld) = TG3(i,jj) + do i=1,iend-ista+1 + ii = ista+i-1 + datapd(i,j,cfld) = TG3(ii,jj) enddo enddo endiF diff --git a/docs/CodeOverview.rst b/docs/CodeOverview.rst index b06efdb08..b2bd85b3f 100644 --- a/docs/CodeOverview.rst +++ b/docs/CodeOverview.rst @@ -30,4 +30,7 @@ Examples of UPP products include: A full list of fields that can be generated by the UPP is provided in :doc:`UPP_GRIB2_Table`. +The standalone offline UPP v11.0.0 can be run using 1D or 2D decomposition, with the 2D decomposition +demonstrating run time improvments, especially for larger domains. + Support for the community UPP is provided through the `UFS Forum `_. diff --git a/docs/InputsOutputs.rst b/docs/InputsOutputs.rst index ebc1626fc..c70bc831f 100644 --- a/docs/InputsOutputs.rst +++ b/docs/InputsOutputs.rst @@ -36,26 +36,29 @@ UFS Unified Model Variables ITAG ---- -The file called :bolditalic:`itag` is a text file that contains the fortran namelist &model_inputs. It is -read in by the :bolditalic:`upp.x` executable from stdin (unit 5) and is -generated automatically within the UFS application workflow or stand-alone run script based on -user-defined options. It should not be necessary to edit this. For description purposes, the namelist -&model_inputs (:bolditalic:`itag` file) contains 7 lines for FV3: +The file called :bolditalic:`itag` is a text file that contains the fortran namelist &model_inputs as +well as the 2D decomposition specification in &nampgb. It is read in by the :bolditalic:`upp.x` executable +from stdin (unit 5) and is generated automatically within the UFS application workflow or stand-alone run script based on user-defined options. It should not be necessary to edit this. For description purposes, +the namelist &model_inputs (:bolditalic:`itag` file) contains 7 lines for FV3 and one line for &nampgb: -#. Name of the FV3 (pressure level) output file to be post-processed. +&model_inputs +#. filName: Name of the FV3 (pressure level) output file to be post-processed. -#. Format of FV3 model output (netcdfpara). +#. IOFORM: Format of FV3 model output (netcdfpara). -#. Format of UPP output (GRIB2) +#. grib: Format of UPP output (grib2) -#. Forecast valid time (not model start time) in YYYY-MM-DD_HH:00:00 format (the forecast time desired - to be post-processed). +#. DateStr: Forecast valid time (not model start time) in YYYY-MM-DD_HH:00:00 format (the forecast time + desired to be post-processed). -#. Model used (GFS, FV3R - regional FV3 also known as the LAM - Limited Area Model). +#. MODELNAME: Model used (GFS, FV3R - regional FV3 also known as the LAM - Limited Area Model). -#. Name of the FV3 (surface) output file to be post-processed. +#. fileNameFlux: Name of the FV3 (surface) output file to be post-processed. -#. Name of configuration file (postxconfig-NT.txt) +#. fileNameFlat: Name of configuration file (postxconfig-NT.txt) + +&nampgb +#. numx: Number of subdomains in the x-direction used for 2D decomposition. ------------ Control File diff --git a/docs/Running.rst b/docs/Running.rst index c52db193f..cc4153fae 100644 --- a/docs/Running.rst +++ b/docs/Running.rst @@ -107,7 +107,14 @@ Run Script Overview | >> IBM: ``mpirun.lsf upp.x < itag > outpost`` | >> SGI MPT: ``mpiexec_mpt upp.x < itag > outpost`` -6. Set naming convention for prefix and extension of output file name +6. Set the value for numx. + + | **numx**: The number of subdomains in the x-direction used for decomposition. + + | - For 1D decomposition, set numx=1 (default) + | - For 2D decomposition, set numx>1 + +7. Set naming convention for prefix and extension of output file name - **comsp** is the initial string of the output file name. By default it is not set and the prefix of the output file will be the string set in the XML file DATSET parameter. If set it will concatenate the setting to the front of the string specified in the XML file DATSET parameter. diff --git a/scripts/run_upp b/scripts/run_upp index 68d29b0ec..3748f0e32 100755 --- a/scripts/run_upp +++ b/scripts/run_upp @@ -26,6 +26,7 @@ set -x # Added netcdfpara; Removed netcdf; # Changed UPP directory name and path # +# June 2022: Add 2D decomposition capabilities, TH (DTC) #-------------------------------------------------------- # # This script runs the stand-alone community version of UPP @@ -59,6 +60,9 @@ set -x # * Do not set to 0 or the script will loop continuously * # RUN_COMMAND : System run command for serial or parallel runs, examples below. # +# numx : The number of subdomains in the x-direction for 2D decomposition. +# (default is 1) +# #---------------------------------------------------------------------------------- #--- BEGIN USER EDIT HERE --------------------------------------------------------- #---------------------------------------------------------------------------------- @@ -96,7 +100,7 @@ export RUN_COMMAND="mpirun -np 1 ${POSTEXEC}/upp.x " #export RUN_COMMAND="mpirun.lsf ${POSTEXEC}/upp.x " #export RUN_COMMAND="mpiexec_mpt ${POSTEXEC}/upp.x " -# The number of subdomains used for 2d decomposition +# The number of subdomains in the x-direction for 2d decomposition export numx=2 # Shouldn't need to edit these. From bd9171a19d33b4cc2b02099073708bef84df72fc Mon Sep 17 00:00:00 2001 From: Tracy Date: Mon, 13 Jun 2022 13:51:18 -0600 Subject: [PATCH 3/5] Formatting --- docs/InputsOutputs.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/InputsOutputs.rst b/docs/InputsOutputs.rst index c70bc831f..9e017b34d 100644 --- a/docs/InputsOutputs.rst +++ b/docs/InputsOutputs.rst @@ -42,6 +42,7 @@ from stdin (unit 5) and is generated automatically within the UFS application wo the namelist &model_inputs (:bolditalic:`itag` file) contains 7 lines for FV3 and one line for &nampgb: &model_inputs + #. filName: Name of the FV3 (pressure level) output file to be post-processed. #. IOFORM: Format of FV3 model output (netcdfpara). @@ -58,6 +59,7 @@ the namelist &model_inputs (:bolditalic:`itag` file) contains 7 lines for FV3 an #. fileNameFlat: Name of configuration file (postxconfig-NT.txt) &nampgb + #. numx: Number of subdomains in the x-direction used for 2D decomposition. ------------ From 368a5abd27804567a42ba62277c913e00286716b Mon Sep 17 00:00:00 2001 From: Tracy Date: Mon, 13 Jun 2022 14:04:32 -0600 Subject: [PATCH 4/5] typo --- docs/InputsOutputs.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/InputsOutputs.rst b/docs/InputsOutputs.rst index 9e017b34d..d91a2ae1b 100644 --- a/docs/InputsOutputs.rst +++ b/docs/InputsOutputs.rst @@ -43,7 +43,7 @@ the namelist &model_inputs (:bolditalic:`itag` file) contains 7 lines for FV3 an &model_inputs -#. filName: Name of the FV3 (pressure level) output file to be post-processed. +#. fileName: Name of the FV3 (pressure level) output file to be post-processed. #. IOFORM: Format of FV3 model output (netcdfpara). From 85cc97115033a453613c38edff89e94f48926740 Mon Sep 17 00:00:00 2001 From: Tracy Date: Mon, 13 Jun 2022 14:18:24 -0600 Subject: [PATCH 5/5] text adjustments --- docs/CodeOverview.rst | 5 +++-- docs/InputsOutputs.rst | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/CodeOverview.rst b/docs/CodeOverview.rst index b2bd85b3f..a00be61dc 100644 --- a/docs/CodeOverview.rst +++ b/docs/CodeOverview.rst @@ -30,7 +30,8 @@ Examples of UPP products include: A full list of fields that can be generated by the UPP is provided in :doc:`UPP_GRIB2_Table`. -The standalone offline UPP v11.0.0 can be run using 1D or 2D decomposition, with the 2D decomposition -demonstrating run time improvments, especially for larger domains. +The UPP has 2D decomposition capabilities (V11.0.0) and is also backwards compatible for 1D decomposition. +The functionality demonstrates run-time improvements, especially for larger domains. Support for this +feature is available for standalone UPP applications. Support for the community UPP is provided through the `UFS Forum `_. diff --git a/docs/InputsOutputs.rst b/docs/InputsOutputs.rst index d91a2ae1b..a4c669fb3 100644 --- a/docs/InputsOutputs.rst +++ b/docs/InputsOutputs.rst @@ -39,7 +39,7 @@ ITAG The file called :bolditalic:`itag` is a text file that contains the fortran namelist &model_inputs as well as the 2D decomposition specification in &nampgb. It is read in by the :bolditalic:`upp.x` executable from stdin (unit 5) and is generated automatically within the UFS application workflow or stand-alone run script based on user-defined options. It should not be necessary to edit this. For description purposes, -the namelist &model_inputs (:bolditalic:`itag` file) contains 7 lines for FV3 and one line for &nampgb: +the namelist &model_inputs and &nampgb (:bolditalic:`itag` file) contain the following lines for FV3: &model_inputs