Sync with ESCOMP/CDEPS (2024-12-10)

.github/workflows/extbuild.yml

@@ -19,11 +19,11 @@ jobs:
       CPPFLAGS: "-I/usr/include -I/usr/local/include "
       LDFLAGS: "-L/usr/lib/x86_64-linux-gnu "
       # Versions of all dependencies can be updated here - these match tag names in the github repo
-      ESMF_VERSION: v8.5.0
-      ParallelIO_VERSION: pio2_6_0
+      ESMF_VERSION: v8.6.1
+      ParallelIO_VERSION: pio2_6_2
     steps:
       - id: checkout-CDEPS
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           submodules: recursive
       - id: load-env
@@ -37,7 +37,7 @@ jobs:
           sudo apt-get install autotools-dev autoconf
       - name: Cache PARALLELIO
         id: cache-PARALLELIO
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         with:
           path: ${GITHUB_WORKSPACE}/pio
           key: ${{ runner.os }}-${{ env.ParallelIO_VERSION }}-parallelio2

.gitmodules

@@ -0,0 +1,3 @@
+# This is a git-fleximod adapted .gitmodules file.  Any field with a name starting in fx is a git-fleximod
+# specific field.  See https://github.com/ESMCI/git-fleximod for details.
+

CMakeLists.txt

@@ -71,23 +71,6 @@ add_subdirectory(streams)
 add_subdirectory(dshr)
 
 if(NOT DISABLE_FoX)
-  if(IS_DIRECTORY "${FOX_ROOT}")
-    message(STATUS "FoX library is already checked out!")
-    message(STATUS "FoX source dir: ${FOX_ROOT}")
-  else()
-    FetchContent_Declare(fox
-      GIT_REPOSITORY https://github.com/ESMCI/fox.git
-      GIT_TAG 4.1.2.1
-      SOURCE_DIR ${FOX_ROOT}
-      BINARY_DIR ${FOX_ROOT}/..
-    )
-    FetchContent_GetProperties(fox)
-    if(NOT fox_POPULATED)
-      FetchContent_Populate(fox)
-      message(STATUS "FoX source dir: ${fox_SOURCE_DIR}")
-      message(STATUS "FoX binary dir: ${fox_BINARY_DIR}")
-    endif()
-  endif()
   add_subdirectory(fox)
 
   target_include_directories(streams PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}/fox/include>
@@ -99,7 +82,7 @@ endif()
 target_include_directories(dshr PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}/streams>
                                        $<INSTALL_INTERFACE:mod>)
 
-foreach(COMP datm dice dlnd docn drof dwav)
+foreach(COMP datm dice dglc dlnd docn drof dwav)
   add_subdirectory("${COMP}")
   if(BLD_STANDALONE)
     target_include_directories(${COMP} PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}/share>

cime_config/buildlib

@@ -73,8 +73,10 @@ def buildlib(bldroot, libroot, case):
         strthread = "nothreads"
     mpilib = case.get_value("MPILIB")
     compiler = case.get_value("COMPILER")
-    sharedpath = os.path.join(compiler, mpilib, strdebug, strthread, "nuopc")
-
+    sharedpath = os.path.join(compiler, mpilib, strdebug, strthread)
+    sharedroot = case.get_value("SHAREDLIBROOT")
+    cdepsblddir = os.path.join(sharedroot, sharedpath, "CDEPS")
+
     logger.info("Running cmake for CDEPS")
     srcpath = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir))
     cmake_flags = get_standard_cmake_args(case, os.path.join(sharedpath, "cdeps"))
@@ -147,31 +149,47 @@ def buildlib(bldroot, libroot, case):
     else:
         bld_time = src_time - 1
 
-    # if any file in src is newer than CmakeFiles in the build directory, rerun cmake
+    # Make sure that no other process is currently trying to build this library, done with a simple lockfile
+    if os.path.exists(cdepsblddir):
+        logger.info("{} already exists, checking for lockfile".format(cdepsblddir))
+        while os.path.exists(os.path.join(cdepsblddir,"lockfile")):
+            logger.info("Waiting for lockfile in {}".format(cdepsblddir))
+            time.sleep(10)
+    else:
+        logger.info("{} does not exist, creating lockfile".format(cdepsblddir))
+        os.makedirs(cdepsblddir)
+        with open(os.path.join(cdepsblddir,"lockfile"),"w") as fd:
+            fd.write(str(os.getpid()))
+
+    try:
+        # if any file in src is newer than CmakeFiles in the build directory, rerun cmake
+        if src_time > bld_time:
+            logger.info("cmake_flags {}".format(cmake_flags))
+            s, o, e = run_cmd(
+                "cmake {} ".format(cmake_flags), from_dir=bldroot, verbose=True
+            )
+            expect(not s, "ERROR from cmake output={}, error={}".format(o, e))
+        else:
+            # The dwav_lib is the last file built in cdeps, wait for it to be built
+            dwav_lib = os.path.join(bldroot, "dwav", "libdwav.a")
+            time_to_wait = 600
+            time_counter = 0
+            while not os.path.exists(dwav_lib):
+                time.sleep(1)
+                time_counter += 1
+                if time_counter > time_to_wait:
+                    break
+            expect(time_counter <= time_to_wait, " Timeout waiting for {}".format(dwav_lib))
 
-    if src_time > bld_time:
-        logger.info("cmake_flags {}".format(cmake_flags))
         s, o, e = run_cmd(
-            "cmake {} ".format(cmake_flags), from_dir=bldroot, verbose=True
+            "make install VERBOSE=1 DESTDIR={}".format(libroot),
+            from_dir=bldroot,
+            verbose=True,
         )
-        expect(not s, "ERROR from cmake output={}, error={}".format(o, e))
-    else:
-        # The dwav_lib is the last file built in cdeps, wait for it to be built
-        dwav_lib = os.path.join(bldroot, "dwav", "libdwav.a")
-        time_to_wait = 300
-        time_counter = 0
-        while not os.path.exists(dwav_lib):
-            time.sleep(1)
-            time_counter += 1
-            if time_counter > time_to_wait:
-                break
-        expect(time_counter <= time_to_wait, " Timeout waiting for {}".format(dwav_lib))
-
-    s, o, e = run_cmd(
-        "make install VERBOSE=1 DESTDIR={}".format(libroot),
-        from_dir=bldroot,
-        verbose=True,
-    )
+    finally:
+        if os.path.exists(os.path.join(cdepsblddir,"lockfile")):
+            os.remove(os.path.join(cdepsblddir,"lockfile"))
+
     expect(not s, "ERROR from make output={}, error={}".format(o, e))
     logger.info("make output={}\nerror={}".format(o, e))
     if compiler == "gnu" and case.get_value("DEBUG"):
@@ -184,7 +202,7 @@ def buildlib(bldroot, libroot, case):
             expect(False, nextline)
 
     # Link the CDEPS component directories to the location expected by cime
-    for comp in ("atm", "lnd", "ice", "ocn", "rof", "wav"):
+    for comp in ("atm", "glc", "lnd", "ice", "ocn", "rof", "wav"):
         compname = case.get_value("COMP_{}".format(comp.upper()))
         comppath = os.path.join(case.get_value("EXEROOT"), comp, "obj")
         if compname == "d" + comp:

cime_config/stream_cdeps.py

@@ -76,6 +76,7 @@ def create_stream_xml(
         data_list_file,
         user_mods_file,
         available_neon_data=None,
+        available_plumber_data=None
     ):
         """
         Create the stream xml file and append the required stream input data to the input data list file
@@ -187,7 +188,17 @@ def create_stream_xml(
                     {"name": "NEON.NEON_PRECIP.$NEONSITE"},
                     err_msg="No stream_entry {} found".format(stream_name),
                 )
+            elif stream_name.startswith("PLUMBER2"):
+                self.stream_nodes = super(StreamCDEPS, self).get_child(
+                    "stream_entry",
+                    {"name": "PLUMBER2.$PLUMBER2SITE"},
+                    err_msg="No stream_entry {} found".format(stream_name),
+                )
             elif stream_name.startswith("CLM_USRDAT."):
+                if 'PLUMBER2' in stream_name:
+                    # if PLUMBER2 is in the stream name
+                    # we want to use PLUMBER2.PLUMBER2SITE instead of CLM_USRDAT.PLUMBER2
+                    continue
                 self.stream_nodes = super(StreamCDEPS, self).get_child(
                     "stream_entry",
                     {"name": "CLM_USRDAT.$CLM_USRDAT_NAME"},
@@ -231,6 +242,7 @@ def create_stream_xml(
                 elif node_name == "stream_datafiles":
                     # Get the resolved stream data files
                     stream_vars[node_name] = ""
+                    stream_datafiles_list = [] # to join stream_datafiles if multiple entries are present
                     stream_datafiles = ""
                     for child in self.get_children(root=node):
                         if (
@@ -244,6 +256,13 @@ def create_stream_xml(
                                     os.path.join(rundir, "inputdata", "atm", neon)
                                     + "\n"
                                 )
+                        elif available_plumber_data and stream_name.startswith("PLUMBER2"):
+                            rundir = case.get_value("RUNDIR")
+                            for plumber in available_plumber_data:
+                                stream_datafiles += (
+                                    os.path.join(rundir, "inputdata", "atm", plumber)
+                                    + "\n"
+                                )
                         else:
                             stream_datafiles = child.xml_element.text
                             stream_datafiles = self._resolve_values(
@@ -288,6 +307,8 @@ def create_stream_xml(
                                 stream_datafiles.split("\n"), "file"
                             )
                         # endif
+                        stream_datafiles_list.append(stream_datafiles)
+                    stream_datafiles = "\n".join(stream_datafiles_list)
                 elif node_name in xml_scalar_names:
                     attributes["model_grid"] = case.get_value("GRID")
                     attributes["compset"] = case.get_value("COMPSET")

cime_config/stream_definition_v2.0.xsd

@@ -93,7 +93,8 @@
       <xs:simpleContent>
         <xs:extension base="xs:string">
           <xs:attribute name="model_grid" type="xs:string" />
-          <xs:attribute name="compset" type="xs:string" />
+	  <xs:attribute name="compset" type="xs:string" />
+	  <xs:attribute name="CLM_USRDAT_NAME" type="xs:string" />
         </xs:extension>
       </xs:simpleContent>
     </xs:complexType>