Merge pull request #16218 from YaqiWang/split_mesh_meta_data

Make split mesh work with mesh meta data without re-run mesh generators

framework/doc/content/source/meshgenerators/FileMeshGenerator.md

@@ -24,6 +24,7 @@ supports reading and writing a large number of formats and could be extended to
 | .unv        | I-deas Universal format |
 | .xda, .xdr  | libMesh formats |
 | .vtk, .pvtu | Visualization Toolkit |
+| .cpr        | Checkpoint file |
 
 When reading a mesh file in Sandia's ExodusII format, users can use parameter `exodus_extra_element_integers` to load elemental variables for setting extra element integers of the mesh.
 The names of the extra element integers will be the same as the names of the element variables in the mesh file.

framework/doc/content/syntax/Mesh/index.md

@@ -190,3 +190,11 @@ Based on this ID, one can proceed with any particular operations, for example, c
 
 IDs can be assigned to the mesh elements with `MeshGenerators` in a similar way to assigning subdomain IDs.
 We note that the element IDs are part of the mesh and will be initialized properly for restart/recover.
+
+## Mesh meta data
+
+Mesh generators can declare mesh meta data, which can be obtained later in Actions or in UserObjects.
+Mesh meta data can only be declared in the constructors of mesh generators so that they can be restarted without re-running mesh generators.
+Mesh meta data can be useful for setting up specific postprocessors, kernels, etc. that require certain geometry information.
+Mesh meta data are not possible or extremely hard to be derived directly from libMesh mesh object.
+A simple example of mesh meta data is the `num_elements_x` provided by [GeneratedMeshGenerator](GeneratedMeshGenerator.md), which can be used as an indicator for a mesh regular in x direction.

framework/doc/content/syntax/Mesh/splitting.md

@@ -48,6 +48,10 @@ Splitting 42 ways...
 This will generate the same split configuration as the 42 chunk split generated by the first
 command but just generates it in parallel.
 
+It is noted that if there are mesh meta data generated by mesh generators, these meta data
+will be written to a binary file under the generated directory that can be loaded when using
+split meshes.
+
 ## Using Split Meshes
 
 To use a mesh split configuration use the `--use-split` flag (which takes no arguments):

framework/include/base/MooseApp.h

@@ -662,6 +662,11 @@ class MooseApp : public ConsoleStreamInterface, public libMesh::ParallelObject
    */
   void executeMeshGenerators();
 
+  /**
+   * Whether this app is executing mesh generators
+   */
+  bool executingMeshGenerators() const { return _executing_mesh_generators; }
+
   /**
    * Get the generated mesh generated by executeMeshGenerators();
    */
@@ -1080,6 +1085,9 @@ class MooseApp : public ConsoleStreamInterface, public libMesh::ParallelObject
   /// Whether to turn on automatic scaling by default
   const bool _automatic_automatic_scaling;
 
+  /// Whether the app is executing all mesh generators
+  bool _executing_mesh_generators;
+
   /// Whether the mesh generator MeshBase has been popped off its storage container and is no
   /// longer accessible
   bool _popped_final_mesh_generator;

framework/include/meshgenerators/MeshGenerator.h

@@ -129,6 +129,10 @@ template <typename T>
 T &
 MeshGenerator::declareMeshProperty(const std::string & data_name)
 {
+  if (_app.executingMeshGenerators())
+    mooseError(
+        "Declaration of mesh meta data can only happen within the constructor of mesh generators");
+
   std::string full_name =
       std::string(MeshMetaDataInterface::SYSTEM) + "/" + name() + "/" + data_name;
 

framework/src/actions/ExecuteMeshGenerators.C

@@ -29,7 +29,7 @@ ExecuteMeshGenerators::act()
   // Don't do mesh generators when recovering as the master app or using master mesh! We do need
   // to run MeshGenerators for sub-apps because we don't currently have checkpoint/restart
   // information for the sub-app meshes; e.g. we just need to re-build them
-  if ((_app.isRecovering() && _app.isUltimateMaster()) || _app.masterMesh())
+  if ((_app.isRecovering() && _app.isUltimateMaster()) || _app.masterMesh() || _app.isUseSplit())
     return;
 
   _app.executeMeshGenerators();

framework/src/actions/SetupMeshAction.C

@@ -285,7 +285,8 @@ SetupMeshAction::act()
       // 1. We have mesh generators
       // 2. We are not: recovering/restarting and we are the master application
       if (!_app.getMeshGeneratorNames().empty() &&
-          !((_app.isRecovering() || _app.isRestarting()) && _app.isUltimateMaster()))
+          !((_app.isUseSplit() || _app.isRecovering() || _app.isRestarting()) &&
+            _app.isUltimateMaster()))
       {
         auto mesh_base = _app.getMeshGeneratorMesh();
         if (_mesh->allowRemoteElementRemoval() != mesh_base->allow_remote_element_removal())

framework/src/actions/SetupRecoverFileBaseAction.C

@@ -60,7 +60,9 @@ SetupRecoverFileBaseAction::act()
     {
       const RestartableDataMap & meta_data = map_iter->second.first;
       const std::string & suffix = map_iter->second.second;
-      if (restartable.readRestartableDataHeader(false, suffix))
+      std::string meta_suffix =
+          "_mesh." + _app.getRestartRecoverFileSuffix() + "/meta_data" + suffix;
+      if (restartable.readRestartableDataHeader(false, meta_suffix))
         restartable.readRestartableData(meta_data, DataNames());
     }
   }

framework/src/actions/SplitMeshAction.C

@@ -12,6 +12,8 @@
 #include "MooseApp.h"
 #include "MooseUtils.h"
 #include "MooseMesh.h"
+#include "RestartableDataIO.h"
+
 #include "libmesh/checkpoint_io.h"
 
 registerMooseAction("MooseApp", SplitMeshAction, "split_mesh");
@@ -72,6 +74,15 @@ SplitMeshAction::act()
                  ", must not end in a file extension other than .cpr or .cpa");
   }
 
+  // To name the split files, we start with the given mesh filename
+  // (if set) or the argument to --split-file, strip any existing
+  // extension, and then append either .cpr or .cpa depending on the
+  // checkpoint_binary_flag.
+  auto fname = mesh->getFileName();
+  if (fname == "")
+    fname = split_file_arg;
+  fname = MooseUtils::stripExtension(fname) + (checkpoint_binary_flag ? ".cpr" : ".cpa");
+
   for (std::size_t i = 0; i < splits.size(); i++)
   {
     processor_id_type n = splits[i];
@@ -82,14 +93,21 @@ SplitMeshAction::act()
                << std::endl;
     cp->binary() = checkpoint_binary_flag;
 
-    // To name the split files, we start with the given mesh filename
-    // (if set) or the argument to --split-file, strip any existing
-    // extension, and then append either .cpr or .cpa depending on the
-    // checkpoint_binary_flag.
-    auto fname = mesh->getFileName();
-    if (fname == "")
-      fname = split_file_arg;
-    fname = MooseUtils::stripExtension(fname) + (checkpoint_binary_flag ? ".cpr" : ".cpa");
+    // different splits will be written into subfolders with n being the folder name
     cp->write(fname);
   }
+
+  if (processor_id() == 0)
+  {
+    RestartableDataIO restartable_data_io(_app);
+    auto & meta_data = _app.getRestartableDataMap(MooseApp::MESH_META_DATA);
+    if (!meta_data.empty())
+    {
+      const std::string filename(fname + "/meta_data_mesh" +
+                                 restartable_data_io.getRestartableDataExt());
+      Moose::out << "Meta data are written into " << filename << "." << std::endl;
+
+      restartable_data_io.writeRestartableData(filename, meta_data);
+    }
+  }
 }

framework/src/base/MooseApp.C

@@ -342,6 +342,7 @@ MooseApp::MooseApp(InputParameters parameters)
     _restore_cached_backup_timer(_perf_graph.registerSection("MooseApp::restoreCachedBackup", 2)),
     _create_minimal_app_timer(_perf_graph.registerSection("MooseApp::createMinimalApp", 3)),
     _automatic_automatic_scaling(getParam<bool>("automatic_automatic_scaling")),
+    _executing_mesh_generators(false),
     _popped_final_mesh_generator(false)
 {
 #ifdef HAVE_GPERFTOOLS
@@ -1677,6 +1678,8 @@ MooseApp::executeMeshGenerators()
   if (_mesh_generators.empty())
     return;
 
+  _executing_mesh_generators = true;
+
   createMeshGeneratorOrder();
 
   // set the final generator name
@@ -1721,9 +1724,14 @@ MooseApp::executeMeshGenerators()
       // Once we hit the generator we want, we'll terminate the loops (this might be the last
       // iteration anyway)
       if (_final_generator_name == name)
+      {
+        _executing_mesh_generators = false;
         return;
+      }
     }
   }
+
+  _executing_mesh_generators = false;
 }
 
 void
@@ -2209,7 +2217,7 @@ MooseApp::getRestartableDataMap(const RestartableDataMapName & name) const
 void
 MooseApp::registerRestartableDataMapName(const RestartableDataMapName & name, std::string suffix)
 {
-  if (suffix.empty())
+  if (!suffix.empty())
     std::transform(suffix.begin(), suffix.end(), suffix.begin(), ::tolower);
   suffix.insert(0, "_");
   _restartable_meta_data.emplace(

framework/src/mesh/FileMesh.C

@@ -12,6 +12,7 @@
 #include "MooseUtils.h"
 #include "Moose.h"
 #include "MooseApp.h"
+#include "RestartableDataIO.h"
 
 #include "libmesh/exodusII_io.h"
 #include "libmesh/nemesis_io.h"
@@ -118,6 +119,18 @@ FileMesh::buildMesh()
         mooseError("cannot locate mesh file '", _file_name, "'");
       getMesh().read(_file_name);
 
+      // we also read declared mesh meta data here if there is meta data file
+      RestartableDataIO restartable(_app);
+      std::string fname = _file_name + "/meta_data_mesh" + restartable.getRestartableDataExt();
+      if (MooseUtils::pathExists(fname))
+      {
+        restartable.setErrorOnLoadWithDifferentNumberOfProcessors(false);
+        // get reference to mesh meta data (created by MooseApp)
+        auto & meta_data = _app.getRestartableDataMap(MooseApp::MESH_META_DATA);
+        if (restartable.readRestartableDataHeaderFromFile(fname, false))
+          restartable.readRestartableData(meta_data, DataNames());
+      }
+
       if (restarting)
       {
         getMesh().allow_renumbering(allow_renumbering_later);

framework/src/meshgenerators/FileMeshGenerator.C

@@ -9,6 +9,7 @@
 
 #include "FileMeshGenerator.h"
 #include "CastUniquePointer.h"
+#include "RestartableDataIO.h"
 
 #include "libmesh/replicated_mesh.h"
 #include "libmesh/face_quad4.h"
@@ -80,7 +81,22 @@ FileMeshGenerator::generate()
     if (_pars.isParamSetByUser("use_for_exodus_restart"))
       mooseError("\"use_for_exodus_restart\" should be given only for Exodus mesh files");
 
-    mesh->read(_file_name);
+    // to support LATEST word for loading checkpoint files
+    std::string file_name = MooseUtils::convertLatestCheckpoint(_file_name, false);
+
+    mesh->read(file_name);
+
+    // we also read declared mesh meta data here if there is meta data file
+    RestartableDataIO restartable(_app);
+    std::string fname = file_name + "/meta_data_mesh" + restartable.getRestartableDataExt();
+    if (MooseUtils::pathExists(fname))
+    {
+      restartable.setErrorOnLoadWithDifferentNumberOfProcessors(false);
+      // get reference to mesh meta data (created by MooseApp)
+      auto & meta_data = _app.getRestartableDataMap(MooseApp::MESH_META_DATA);
+      if (restartable.readRestartableDataHeaderFromFile(fname, false))
+        restartable.readRestartableData(meta_data, DataNames());
+    }
   }
 
   return dynamic_pointer_cast<MeshBase>(mesh);

framework/src/outputs/Checkpoint.C

@@ -104,15 +104,6 @@ Checkpoint::output(const ExecFlagType & /*type*/)
   // Write the checkpoint file
   io.write(curr_file_struct.checkpoint);
 
-  // Write the system data, using ENCODE vs WRITE based on ascii vs binary format
-  _es_ptr->write(curr_file_struct.system,
-                 EquationSystems::WRITE_DATA | EquationSystems::WRITE_ADDITIONAL_DATA |
-                     EquationSystems::WRITE_PARALLEL_FILES,
-                 renumber);
-
-  // Write the restartable data
-  _restartable_data_io.writeRestartableDataPerProc(curr_file_struct.restart, _restartable_data);
-
   // Write out the restartable mesh meta data if there is any (only on processor zero)
   if (processor_id() == 0)
   {
@@ -121,14 +112,23 @@ Checkpoint::output(const ExecFlagType & /*type*/)
     {
       const RestartableDataMap & meta_data = map_pair.second.first;
       const std::string & suffix = map_pair.second.second;
-      const std::string filename(current_file + suffix +
+      const std::string filename(curr_file_struct.checkpoint + "/meta_data" + suffix +
                                  _restartable_data_io.getRestartableDataExt());
 
       curr_file_struct.restart_meta_data.emplace(filename);
       _restartable_data_io.writeRestartableData(filename, meta_data);
     }
   }
 
+  // Write the system data, using ENCODE vs WRITE based on ascii vs binary format
+  _es_ptr->write(curr_file_struct.system,
+                 EquationSystems::WRITE_DATA | EquationSystems::WRITE_ADDITIONAL_DATA |
+                     EquationSystems::WRITE_PARALLEL_FILES,
+                 renumber);
+
+  // Write the restartable data
+  _restartable_data_io.writeRestartableDataPerProc(curr_file_struct.restart, _restartable_data);
+
   // Remove old checkpoint files
   updateCheckpointFiles(curr_file_struct);
 }
@@ -154,6 +154,10 @@ Checkpoint::updateCheckpointFiles(CheckpointFileNames file_struct)
     // Delete checkpoint files (_mesh.cpr)
     if (proc_id == 0)
     {
+      for (const auto & file_name : delete_files.restart_meta_data)
+        remove(file_name.c_str());
+      // This file may not exist so don't worry about checking for success
+
       CheckpointIO::cleanup(delete_files.checkpoint, _parallel_mesh ? comm().size() : 1);
 
       // Delete the system files (xdr and xdr.0000, ...)
@@ -173,13 +177,6 @@ Checkpoint::updateCheckpointFiles(CheckpointFileNames file_struct)
         mooseWarning("Error during the deletion of file '", file_name, "': ", std::strerror(ret));
     }
 
-    if (proc_id == 0)
-    {
-      for (const auto & file_name : delete_files.restart_meta_data)
-        remove(file_name.c_str());
-      // This file may not exist so don't worry about checking for success
-    }
-
     // Remove the restart files (rd)
     unsigned int n_threads = libMesh::n_threads();
     for (THREAD_ID tid = 0; tid < n_threads; tid++)

test/tests/meshgenerators/file_mesh_generator/2d_diffusion_test.i

@@ -0,0 +1,44 @@
+[Mesh]
+  [square]
+    type = GeneratedMeshGenerator
+    nx = 4
+    ny = 4
+    dim = 2
+  []
+[]
+
+[Variables]
+  [u]
+  []
+[]
+
+[Kernels]
+  [diff]
+    type = Diffusion
+    variable = u
+  []
+[]
+
+[BCs]
+  [left]
+    type = DirichletBC
+    variable = u
+    boundary = 3
+    value = 0
+  []
+
+  [right]
+    type = DirichletBC
+    variable = u
+    boundary = 1
+    value = 1
+  []
+[]
+
+[Executioner]
+  type = Steady
+[]
+
+[Outputs]
+  exodus = true
+[]

test/tests/meshgenerators/file_mesh_generator/tests

@@ -21,4 +21,27 @@
     design = 'meshgenerators/FileMeshGenerator.md'
     issues = '#14916'
   [../]
+
+  [./pre_checkpoint_load_test]
+    type = 'RunApp'
+    input = '2d_diffusion_test.i'
+    expect_out = 'Solve Converged'
+    cli_args = 'Outputs/checkpoint=true'
+    recover = false
+    requirement = 'The system shall have the ability to output checkpoint files along with the mesh meta data.'
+    design = 'meshgenerators/FileMeshGenerator.md'
+    issues = '#16192'
+  []
+
+  [./checkpoint_load_test]
+    type = 'Exodiff'
+    input = '2d_diffusion_test.i'
+    exodiff = '2d_diffusion_test_out.e'
+    cli_args = 'Mesh/inactive=square Mesh/fmg/type=FileMeshGenerator Mesh/fmg/file=2d_diffusion_test_out_cp/0001_mesh.cpr'
+    recover = false
+    prereq = pre_checkpoint_load_test
+    requirement = 'The system shall have the ability to load the mesh from checkpoint files.'
+    design = 'meshgenerators/FileMeshGenerator.md'
+    issues = '#16192'
+  []
 []

test/tests/meshgenerators/meta_data_store/mesh_meta_data_store.i

@@ -9,6 +9,10 @@
   []
 []
 
+[Debug]
+  show_mesh_meta_data = true
+[]
+
 [Variables]
   [./u]
   [../]