From ab9d16714d31e853d000a0757f3487bcb9804023 Mon Sep 17 00:00:00 2001
From: Patrick Mullen <33095288+pdmullen@users.noreply.github.com>
Date: Fri, 11 Aug 2023 10:15:38 -0600
Subject: [PATCH 01/36] Update Iterative Tasking Infrastructure (#917)

* Add lroberts36/merge-sparse-with-jdolence-sparse's changes to iterative tasking

* Update copyrights

* Add undordered map include

* Update changelog

* Fix function name typos in task_list.hpp

---------

Co-authored-by: Patrick Mullen <pdmullen@pn2293621.lanl.gov>
Co-authored-by: Patrick Mullen <pdmullen@mbp175.local>
---
 CHANGELOG.md            |   1 +
 src/basic_types.hpp     |   4 +-
 src/tasks/task_list.hpp | 191 +++++++++++++++++++++++++++-------------
 3 files changed, 134 insertions(+), 62 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index d841a9231061..29feeb213f33 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -21,6 +21,7 @@
 - [[PR 885]](https://github.com/parthenon-hpc-lab/parthenon/pull/885) Expose PackDescriptor and use uids in SparsePacks
 
 ### Fixed (not changing behavior/API/variables/...)
+- [[PR 917]](https://github.com/parthenon-hpc-lab/parthenon/pull/917) Update Iterative Tasking Infrastructure
 - [[PR 890]](https://github.com/parthenon-hpc-lab/parthenon/pull/890) Fix bugs in sparse communication and prolongation
 
 ### Infrastructure (changes irrelevant to downstream codes)
diff --git a/src/basic_types.hpp b/src/basic_types.hpp
index ad599373e465..4a5fa41a6f24 100644
--- a/src/basic_types.hpp
+++ b/src/basic_types.hpp
@@ -1,5 +1,5 @@
 //========================================================================================
-// (C) (or copyright) 2021-2022. Triad National Security, LLC. All rights reserved.
+// (C) (or copyright) 2021-2023. Triad National Security, LLC. All rights reserved.
 //
 // This program was produced under U.S. Government contract 89233218CNA000001 for Los
 // Alamos National Laboratory (LANL), which is operated by Triad National Security, LLC
@@ -36,7 +36,7 @@ using Real = double;
 #endif
 #endif
 
-enum class TaskStatus { fail, complete, incomplete, iterate, skip };
+enum class TaskStatus { fail, complete, incomplete, iterate, skip, waiting };
 enum class AmrTag : int { derefine = -1, same = 0, refine = 1 };
 enum class RefinementOp_t { Prolongation, Restriction, None };
 
diff --git a/src/tasks/task_list.hpp b/src/tasks/task_list.hpp
index 961e2e123f16..322d1a788d70 100644
--- a/src/tasks/task_list.hpp
+++ b/src/tasks/task_list.hpp
@@ -1,5 +1,5 @@
 //========================================================================================
-// (C) (or copyright) 2021. Triad National Security, LLC. All rights reserved.
+// (C) (or copyright) 2023. Triad National Security, LLC. All rights reserved.
 //
 // This program was produced under U.S. Government contract 89233218CNA000001 for Los
 // Alamos National Laboratory (LANL), which is operated by Triad National Security, LLC
@@ -24,6 +24,7 @@
 #include <stdexcept>
 #include <string>
 #include <tuple>
+#include <unordered_map>
 #include <utility>
 #include <vector>
 
@@ -44,17 +45,18 @@ TaskID AddTaskHelper(TaskList *, Task);
 
 class IterativeTasks {
  public:
+  IterativeTasks() = default;
   IterativeTasks(TaskList *tl, int key) : tl_(tl), key_(key) {
     max_iterations_ = std::numeric_limits<int>::max();
   }
 
   // overload to add member functions of class T to task list
   // NOTE: we must capture the object pointer
-  template <class T, class U, class... Args>
-  TaskID AddTask(TaskID const &dep, TaskStatus (T::*func)(Args...), U *obj,
-                 Args &&...args) {
+  template <class T, class U, class... Args1, class... Args2>
+  TaskID AddTask(TaskID const &dep, TaskStatus (T::*func)(Args1...), U *obj,
+                 Args2 &&...args) {
     return this->AddTask_(TaskType::iterative, 1, dep, [=]() mutable -> TaskStatus {
-      return (obj->*func)(std::forward<Args>(args)...);
+      return (obj->*func)(std::forward<Args2>(args)...);
     });
   }
 
@@ -94,6 +96,8 @@ class IterativeTasks {
   int GetMaxIterations() const { return max_iterations_; }
   int GetIterationCount() const { return count_; }
   void IncrementCount() { count_++; }
+  void ResetCount() { count_ = 0; }
+  void PrintList() { std::cout << "tl_ = " << tl_ << std::endl; }
 
  private:
   template <class F, class... Args>
@@ -139,7 +143,8 @@ class TaskList {
     for (auto &task : task_list_) {
       if (task.GetID() == id) {
         return (task.GetStatus() != TaskStatus::incomplete &&
-                task.GetStatus() != TaskStatus::skip);
+                task.GetStatus() != TaskStatus::skip &&
+                task.GetStatus() != TaskStatus::waiting);
       }
     }
     return false;
@@ -184,6 +189,7 @@ class TaskList {
         ++task;
       }
     }
+    iter_tasks[key].ResetCount();
   }
   void ResetIteration(const int key) {
     PARTHENON_REQUIRE_THROWS(key < iter_tasks.size(), "Invalid iteration key");
@@ -197,6 +203,11 @@ class TaskList {
         PARTHENON_WARN("Iteration " + iter_labels[key] +
                        " reached maximum allowed cycles without convergence.");
       }
+      for (auto &task : task_list_) {
+        if (task.GetKey() == key && task.GetType() == TaskType::completion_criteria) {
+          MarkTaskComplete(task.GetID());
+        }
+      }
       ClearIteration(key);
       return;
     }
@@ -219,36 +230,47 @@ class TaskList {
       }
     }
   }
-  void CompleteIfNeeded(const TaskID &id) {
+  bool CompleteIfNeeded(const TaskID &id) {
     MarkTaskComplete(id);
     auto task = task_list_.begin();
     while (task != task_list_.end()) {
       if (task->GetID() == id) {
         if (task->GetType() == TaskType::completion_criteria) {
           ClearIteration(task->GetKey());
+          return true;
         } else if (task->GetType() == TaskType::single) {
-          task_list_.erase(task);
+          task = task_list_.erase(task);
+        } else {
+          task->SetStatus(TaskStatus::waiting);
         }
         break;
       } else {
         ++task;
       }
     }
+    return false;
   }
   void DoAvailable() {
-    for (auto &task : task_list_) {
+    auto task = task_list_.begin();
+    while (task != task_list_.end()) {
       // first skip task if it's complete.  Possible for iterative tasks
-      if (task.GetStatus() != TaskStatus::incomplete) continue;
-      auto dep = task.GetDependency();
+      if (task->GetStatus() != TaskStatus::incomplete) {
+        ++task;
+        continue;
+      }
+      auto dep = task->GetDependency();
       if (CheckDependencies(dep)) {
-        task();
-        if (task.GetStatus() == TaskStatus::complete && !task.IsRegional()) {
-          MarkTaskComplete(task.GetID());
-        } else if (task.GetStatus() == TaskStatus::skip &&
-                   task.GetType() == TaskType::completion_criteria) {
-          ResetIteration(task.GetKey());
+        (*task)();
+        if (task->GetStatus() == TaskStatus::complete && !task->IsRegional()) {
+          MarkTaskComplete(task->GetID());
+        } else if (task->GetStatus() == TaskStatus::skip &&
+                   task->GetType() == TaskType::completion_criteria) {
+          ResetIteration(task->GetKey());
+        } else if (task->GetStatus() == TaskStatus::iterate && !task->IsRegional()) {
+          ResetIteration(task->GetKey());
         }
       }
+      ++task;
     }
     ClearComplete();
   }
@@ -274,21 +296,21 @@ class TaskList {
     return valid;
   }
 
-  TaskID AddTask(Task tsk) {
+  TaskID AddTask(Task &tsk) {
     TaskID id(tasks_added_ + 1);
     tsk.SetID(id);
-    task_list_.push_back(tsk);
+    task_list_.push_back(std::move(tsk));
     tasks_added_++;
     return id;
   }
 
   // overload to add member functions of class T to task list
   // NOTE: we must capture the object pointer
-  template <class T, class U, class... Args>
-  TaskID AddTask(TaskID const &dep, TaskStatus (T::*func)(Args...), U *obj,
-                 Args &&...args) {
+  template <class T, class U, class... Args1, class... Args2>
+  TaskID AddTask(TaskID const &dep, TaskStatus (T::*func)(Args1...), U *obj,
+                 Args2 &&...args) {
     return this->AddTask(dep, [=]() mutable -> TaskStatus {
-      return (obj->*func)(std::forward<Args>(args)...);
+      return (obj->*func)(std::forward<Args2>(args)...);
     });
   }
 
@@ -305,9 +327,9 @@ class TaskList {
 
   IterativeTasks &AddIteration(const std::string &label) {
     int key = iter_tasks.size();
-    iter_tasks.push_back(IterativeTasks(this, key));
-    iter_labels.push_back(label);
-    return iter_tasks.back();
+    iter_tasks[key] = IterativeTasks(this, key);
+    iter_labels[key] = label;
+    return iter_tasks[key];
   }
 
   void Print() {
@@ -315,8 +337,8 @@ class TaskList {
     std::cout << "TaskList::Print():" << std::endl;
     for (auto &t : task_list_) {
       std::cout << "  " << i << "  " << t.GetID().to_string() << "  "
-                << t.GetDependency().to_string() << " "
-                << (t.GetStatus() == TaskStatus::incomplete)
+                << t.GetDependency().to_string() << " " << tasks_completed_.to_string()
+                << " " << (t.GetStatus() == TaskStatus::incomplete)
                 << (t.GetStatus() == TaskStatus::complete)
                 << (t.GetStatus() == TaskStatus::skip)
                 << (t.GetStatus() == TaskStatus::iterate)
@@ -327,8 +349,8 @@ class TaskList {
   }
 
  protected:
-  std::vector<IterativeTasks> iter_tasks;
-  std::vector<std::string> iter_labels;
+  std::map<int, IterativeTasks> iter_tasks;
+  std::map<int, std::string> iter_labels;
   std::list<Task> task_list_;
   int tasks_added_ = 0;
   TaskID tasks_completed_;
@@ -340,41 +362,92 @@ namespace task_list_impl {
 inline TaskID AddTaskHelper(TaskList *tl, Task tsk) { return tl->AddTask(tsk); }
 } // namespace task_list_impl
 
+class RegionCounter {
+ public:
+  explicit RegionCounter(const std::string &base) : base_(base), cnt_(0) {}
+  std::string ID() { return base_ + std::to_string(cnt_++); }
+
+ private:
+  const std::string base_;
+  int cnt_;
+};
+
 class TaskRegion {
  public:
   explicit TaskRegion(const int size) : lists(size) {}
-  void AddRegionalDependencies(const int reg_dep_id, const int list_index, TaskID id) {
-    auto task_pair = std::make_pair(list_index, id);
-    id_for_reg[reg_dep_id].push_back(task_pair);
-    lists[list_index].MarkRegional(id);
-    all_done[reg_dep_id].val = 0;
+  void AddRegionalDependencies(const int reg_dep_id, const int list_index,
+                               const TaskID &id) {
+    AddRegionalDependencies(std::to_string(reg_dep_id), list_index, id);
+  }
+  void AddRegionalDependencies(const std::string &reg_dep_id, const int list_index,
+                               const TaskID &id) {
+    AddDependencies(reg_dep_id, list_index, id);
+    global[reg_dep_id] = false;
+  }
+  void AddGlobalDependencies(const int reg_dep_id, const int list_index,
+                             const TaskID &id) {
+    AddGlobalDependencies(std::to_string(reg_dep_id), list_index, id);
+  }
+  void AddGlobalDependencies(const std::string &reg_dep_id, const int list_index,
+                             const TaskID &id) {
+    AddDependencies(reg_dep_id, list_index, id);
+    global[reg_dep_id] = true;
   }
 
   TaskList &operator[](int i) { return lists[i]; }
 
   int size() const { return lists.size(); }
 
+  bool Execute() {
+    for (auto i = 0; i < lists.size(); ++i) {
+      if (!lists[i].IsComplete()) {
+        lists[i].DoAvailable();
+      }
+    }
+    return CheckAndUpdate();
+  }
+
   bool CheckAndUpdate() {
-    for (auto &reg_dep : id_for_reg) {
-      auto reg_id = reg_dep.first;
+    auto it = id_for_reg.begin();
+    while (it != id_for_reg.end()) {
+      auto &reg_id = it->first;
+      bool check = false;
       if (HasRun(reg_id) && !all_done[reg_id].active) {
         all_done[reg_id].val = IsComplete(reg_id);
-        all_done[reg_id].StartReduce(MPI_MIN);
+        if (global[reg_id]) {
+          all_done[reg_id].StartReduce(MPI_MIN);
+        } else {
+          check = true;
+        }
       }
-      if (all_done[reg_id].active) {
+      if (global[reg_id] && all_done[reg_id].active) {
         auto status = all_done[reg_id].CheckReduce();
         if (status == TaskStatus::complete) {
-          if (all_done[reg_id].val) {
-            for (auto &lst : reg_dep.second) {
-              lists[lst.first].CompleteIfNeeded(lst.second);
-            }
+          check = true;
+        }
+      }
+      if (check) {
+        if (all_done[reg_id].val) {
+          bool clear = false;
+          for (auto &lst : it->second) {
+            clear = lists[lst.first].CompleteIfNeeded(lst.second);
+          }
+          if (clear) {
+            all_done.erase(reg_id);
+            global.erase(reg_id);
+            it = id_for_reg.erase(it);
           } else {
-            for (auto &lst : reg_dep.second) {
-              lists[lst.first].ResetIfNeeded(lst.second);
-            }
-            all_done[reg_id].val = 0;
+            ++it;
+          }
+        } else {
+          for (auto &lst : it->second) {
+            lists[lst.first].ResetIfNeeded(lst.second);
           }
+          all_done[reg_id].val = 0;
+          ++it;
         }
+      } else {
+        ++it;
       }
     }
     int complete_cnt = 0;
@@ -393,7 +466,12 @@ class TaskRegion {
   }
 
  private:
-  bool HasRun(const int reg_id) {
+  void AddDependencies(const std::string &label, const int list_id, const TaskID &tid) {
+    id_for_reg[label][list_id] = tid;
+    lists[list_id].MarkRegional(tid);
+    all_done[label].val = 0;
+  }
+  bool HasRun(const std::string &reg_id) {
     auto &lvec = id_for_reg[reg_id];
     int n_to_run = lvec.size();
     int n_ran = 0;
@@ -406,7 +484,7 @@ class TaskRegion {
     }
     return n_ran == n_to_run;
   }
-  bool IsComplete(const int reg_id) {
+  bool IsComplete(const std::string &reg_id) {
     auto &lvec = id_for_reg[reg_id];
     int n_to_finish = lvec.size();
     int n_finished = 0;
@@ -420,10 +498,10 @@ class TaskRegion {
     return n_finished == n_to_finish;
   }
 
-  // id_for_reg[region_id] = std::pair<>(task_list_index, task_id_of_regional_task)
-  std::map<int, std::vector<std::pair<int, TaskID>>> id_for_reg;
+  std::unordered_map<std::string, std::map<int, TaskID>> id_for_reg;
   std::vector<TaskList> lists;
-  std::map<int, AllReduce<int>> all_done;
+  std::unordered_map<std::string, AllReduce<int>> all_done;
+  std::unordered_map<std::string, bool> global;
 };
 
 class TaskCollection {
@@ -437,15 +515,8 @@ class TaskCollection {
     assert(Validate());
     for (auto &region : regions) {
       bool complete = false;
-      auto num_lists = region.size();
       while (!complete) {
-        // TODO(pgrete): need to let Kokkos::PartitionManager handle this
-        for (auto i = 0; i < num_lists; ++i) {
-          if (!region[i].IsComplete()) {
-            region[i].DoAvailable();
-          }
-        }
-        complete = region.CheckAndUpdate();
+        complete = region.Execute();
       }
     }
     return TaskListStatus::complete;

From fd6d56549ddf7d4322c5fc0b258c01f9e4916c63 Mon Sep 17 00:00:00 2001
From: jdolence <jcdolence@gmail.com>
Date: Fri, 11 Aug 2023 10:25:57 -0600
Subject: [PATCH 02/36] add the ability to get a list of vars given constraints

---
 src/interface/data_collection.cpp  |  8 +--
 src/interface/metadata.cpp         | 15 ++++++
 src/interface/metadata.hpp         | 12 +++++
 src/interface/state_descriptor.cpp | 80 ++++++++++++++++++++++++++++++
 src/interface/state_descriptor.hpp | 27 +++++++++-
 tst/unit/test_metadata.cpp         |  5 +-
 tst/unit/test_state_descriptor.cpp | 74 +++++++++++++++++++++++++++
 7 files changed, 213 insertions(+), 8 deletions(-)

diff --git a/src/interface/data_collection.cpp b/src/interface/data_collection.cpp
index 6a1d72c9ce0b..7d8cb08258ee 100644
--- a/src/interface/data_collection.cpp
+++ b/src/interface/data_collection.cpp
@@ -24,17 +24,17 @@ namespace parthenon {
 template <typename T>
 std::shared_ptr<T> DataCollection<T>::Add(const std::string &name,
                                           const std::shared_ptr<T> &src,
-                                          const std::vector<std::string> &flags) {
+                                          const std::vector<std::string> &field_names) {
   auto it = containers_.find(name);
   if (it != containers_.end()) {
-    if (!(it->second)->Contains(flags)) {
-      PARTHENON_THROW(name + "already exists in collection but does not contain flags");
+    if (!(it->second)->Contains(field_names)) {
+      PARTHENON_THROW(name + "already exists in collection but does not contain field names");
     }
     return it->second;
   }
 
   auto c = std::make_shared<T>();
-  c->Copy(src, flags);
+  c->Copy(src, field_names);
 
   containers_[name] = c;
   return containers_[name];
diff --git a/src/interface/metadata.cpp b/src/interface/metadata.cpp
index ee355c1773ed..38466b2f4ed1 100644
--- a/src/interface/metadata.cpp
+++ b/src/interface/metadata.cpp
@@ -37,6 +37,8 @@ namespace parthenon {
 PARTHENON_INTERNAL_FOREACH_BUILTIN_FLAG
 #undef PARTHENON_INTERNAL_FOR_FLAG
 
+int Metadata::num_flags = static_cast<int>(internal::MetadataInternal::Max);
+
 namespace internal {
 
 class UserMetadataState {
@@ -85,6 +87,7 @@ class UserMetadataState {
 parthenon::internal::UserMetadataState metadata_state;
 
 MetadataFlag Metadata::AddUserFlag(const std::string &name) {
+  num_flags++;
   return metadata_state.AllocateNewFlag(name);
 }
 
@@ -256,4 +259,16 @@ Metadata::GetArrayDims(std::weak_ptr<MeshBlock> wpmb, bool coarse) const {
   return arrDims;
 }
 
+namespace MetadataUtils {
+bool MatchFlags(const Metadata::FlagCollection &flags, Metadata m) {
+  const auto &intersections = flags.GetIntersections();
+  const auto &unions = flags.GetUnions();
+  const auto &exclusions = flags.GetExclusions();
+
+  return m.AllFlagsSet(intersections) &&
+         (unions.empty() || m.AnyFlagsSet(unions)) &&
+         m.NoFlagsSet(exclusions);
+}
+} // namespace MetadataUtils
+
 } // namespace parthenon
diff --git a/src/interface/metadata.hpp b/src/interface/metadata.hpp
index fabfc083cbe8..755a0778d237 100644
--- a/src/interface/metadata.hpp
+++ b/src/interface/metadata.hpp
@@ -331,6 +331,7 @@ class Metadata {
   static MetadataFlag AddUserFlag(const std::string &name);
   static bool FlagNameExists(const std::string &flagname);
   static MetadataFlag GetUserFlag(const std::string &flagname);
+  static int num_flags;
 
   // Sparse threshold routines
   void SetSparseThresholds(parthenon::Real alloc, parthenon::Real dealloc,
@@ -510,6 +511,16 @@ class Metadata {
     return AllFlagsSet(FlagVec{flag, std::forward<Args>(args)...});
   }
 
+  template <template <class...> class Container_t, class... extra>
+  bool NoFlagsSet(const Container_t<MetadataFlag, extra...> &flags) const {
+    return std::none_of(flags.begin(), flags.end(),
+                       [this](MetadataFlag const &f) { return IsSet(f); });
+  }
+  template <typename... Args>
+  bool NoFlagsSet(const MetadataFlag &flag, Args... args) const {
+    return AllFlagsSet(FlagVec{flag, std::forward<Args>(args)...});
+  }
+
   template <template <class...> class Container_t, class... extra>
   bool FlagsSet(const Container_t<MetadataFlag, extra...> &flags,
                 bool matchAny = false) const {
@@ -617,6 +628,7 @@ inline TopologicalType GetTopologicalType(const Metadata &md) {
 }
 
 namespace MetadataUtils {
+bool MatchFlags(const Metadata::FlagCollection &flags, Metadata m);
 // From a given container, extract all variables whose Metadata matchs the all of the
 // given flags (if the list of flags is empty, extract all variables), optionally only
 // extracting sparse fields with an index from the given list of sparse indices
diff --git a/src/interface/state_descriptor.cpp b/src/interface/state_descriptor.cpp
index c043be34d990..9ed5e96f39a4 100644
--- a/src/interface/state_descriptor.cpp
+++ b/src/interface/state_descriptor.cpp
@@ -443,4 +443,84 @@ StateDescriptor::CreateResolvedStateDescriptor(Packages_t &packages) {
   return state;
 }
 
+std::vector<std::string>
+StateDescriptor::GetVariableNames(const std::vector<std::string> &req_names,
+                                  const Metadata::FlagCollection &flags,
+                                  const std::vector<int> &sparse_ids) {
+  std::unordered_set<int> sparse_ids_set(sparse_ids.begin(), sparse_ids.end());
+  std::unordered_set<std::string> names;
+  std::vector<std::string> names_vec;
+  // first add names that are present
+  for (const auto &name : req_names) {
+    if (FieldPresent(name)) {
+      if (metadataMap_[VarID(name)].IsSet(Metadata::Sparse)) {
+        PARTHENON_THROW("Cannot ask for variables by name with specific sparse ids");
+      }
+      if (names.count(name) == 0) names_vec.push_back(name);
+      names.insert(name);
+    } else if (SparseBaseNamePresent(name)) {
+      const auto &sparse_pool = GetSparsePool(name);
+      for (const auto &s : sparse_pool.pool()) {
+        if (sparse_ids_set.empty() || sparse_ids_set.count(s.first) > 0) {
+          if (names.count(MakeVarLabel(name, s.first)) == 0)
+            names_vec.push_back(MakeVarLabel(name, s.first));
+          names.insert(MakeVarLabel(name, s.first));
+        }
+      }
+    }
+  }
+
+  // now add by metadata flags
+  if (!flags.Empty()) {
+    // first handle non-sparse fields
+    for (const auto &v : metadataMap_) {
+      auto name = (v.first).label();
+      auto &meta = v.second;
+      if (MetadataUtils::MatchFlags(flags, meta)) {
+        if (!meta.IsSet(Metadata::Sparse)) {
+          if (names.count(name) == 0) names_vec.push_back(name);
+          names.insert(name);
+        }
+      }
+    }
+    // now handle sparse fields
+    for (const auto &p : sparsePoolMap_) {
+      auto &name = p.first;
+      auto &pool = p.second;
+      auto &meta = pool.shared_metadata();
+      printf("checking %s\n", name.c_str());
+      if (MetadataUtils::MatchFlags(flags, meta)) {
+        printf("matched flag\n");
+        for (const auto &s : pool.pool()) {
+          if (sparse_ids_set.empty() || sparse_ids_set.count(s.first) > 0) {
+            if (names.count(MakeVarLabel(name, s.first)) == 0)
+              names_vec.push_back(MakeVarLabel(name, s.first));
+            names.insert(MakeVarLabel(name, s.first));
+          }
+        }
+      }
+    }
+  }
+
+  return names_vec;
+}
+std::vector<std::string>
+StateDescriptor::GetVariableNames(const std::vector<std::string> &req_names,
+                                  const std::vector<int> &sparse_ids) {
+  return GetVariableNames(req_names, Metadata::FlagCollection(), sparse_ids);
+}
+std::vector<std::string>
+StateDescriptor::GetVariableNames(const Metadata::FlagCollection &flags,
+                                  const std::vector<int> &sparse_ids) {
+  return GetVariableNames({}, flags, sparse_ids);
+}
+std::vector<std::string>
+StateDescriptor::GetVariableNames(const std::vector<std::string> &req_names) {
+  return GetVariableNames(req_names, Metadata::FlagCollection(), {});
+}
+std::vector<std::string>
+StateDescriptor::GetVariableNames(const Metadata::FlagCollection &flags) {
+  return GetVariableNames({}, flags, {});
+}
+
 } // namespace parthenon
diff --git a/src/interface/state_descriptor.hpp b/src/interface/state_descriptor.hpp
index ac9a93dcb58a..8966535a4f27 100644
--- a/src/interface/state_descriptor.hpp
+++ b/src/interface/state_descriptor.hpp
@@ -100,11 +100,21 @@ class StateDescriptor {
   StateDescriptor(const StateDescriptor &s) = delete;
 
   // Preferred constructor
-  explicit StateDescriptor(std::string const &label) : label_(label) {}
+  explicit StateDescriptor(std::string const &label) : label_(label) {
+    if (Metadata::FlagNameExists(label)) {
+      AddParam("MetadataFlag", Metadata::GetUserFlag(label));
+    } else {
+      AddParam("MetadataFlag", Metadata::AddUserFlag(label));
+    }
+  }
 
   static std::shared_ptr<StateDescriptor>
   CreateResolvedStateDescriptor(Packages_t &packages);
 
+  MetadataFlag GetMetadataFlag() {
+    return params_.Get<MetadataFlag>("MetadataFlag");
+  }
+
   template <typename T>
   void AddParam(const std::string &key, T value, Params::Mutability mutability) {
     params_.Add<T>(key, value, mutability);
@@ -223,6 +233,21 @@ class StateDescriptor {
     // automatically have x, y, z.
     return swarmValueMetadataMap_[swarm_name];
   }
+  std::vector<std::string>
+  GetVariableNames(const std::vector<std::string> &req_names,
+                   const Metadata::FlagCollection &flags,
+                   const std::vector<int> &sparse_ids);
+  std::vector<std::string>
+  GetVariableNames(const std::vector<std::string> &req_names,
+                   const std::vector<int> &sparse_ids);
+  std::vector<std::string>
+  GetVariableNames(const Metadata::FlagCollection &flags,
+                   const std::vector<int> &sparse_ids);
+  std::vector<std::string>
+  GetVariableNames(const std::vector<std::string> &req_names);
+  std::vector<std::string>
+  GetVariableNames(const Metadata::FlagCollection &flags);
+
   std::size_t
   RefinementFuncID(const refinement::RefinementFunctions_t &funcs) const noexcept {
     return refinementFuncMaps_.funcs_to_ids.at(funcs);
diff --git a/tst/unit/test_metadata.cpp b/tst/unit/test_metadata.cpp
index 9ea5d5b67459..417fed077a1c 100644
--- a/tst/unit/test_metadata.cpp
+++ b/tst/unit/test_metadata.cpp
@@ -81,10 +81,9 @@ TEST_CASE("A Metadata flag is allocated", "[Metadata]") {
     // Note: `parthenon::internal` is subject to change, and so this test may
     // rightfully break later - this test needn't be maintained if so.
     //
-    // Checks that the first allocated flag is equal to `Max` - the final built-in
-    // flag + 1.
+    // Checks that an allocated flag was ended to the end of existing flags
     REQUIRE(f.InternalFlagValue() ==
-            static_cast<int>(parthenon::internal::MetadataInternal::Max));
+            static_cast<int>(Metadata::num_flags)-1);
     REQUIRE(name == f.Name());
 
     // Metadata should be able to report that this flag exists and
diff --git a/tst/unit/test_state_descriptor.cpp b/tst/unit/test_state_descriptor.cpp
index 8f290836c0fa..6eb97aa12564 100644
--- a/tst/unit/test_state_descriptor.cpp
+++ b/tst/unit/test_state_descriptor.cpp
@@ -35,6 +35,7 @@ using parthenon::Coordinates_t;
 using parthenon::IndexRange;
 using parthenon::Metadata;
 using parthenon::MetadataFlag;
+using FC_t = parthenon::Metadata::FlagCollection;
 using parthenon::Packages_t;
 using parthenon::ParArrayND;
 using parthenon::Real;
@@ -518,3 +519,76 @@ TEST_CASE("Test SparsePool interface", "[StateDescriptor]") {
     }
   }
 }
+
+TEST_CASE("Test getting a vector of variable names given criteria", "[StateDescriptor]") {
+  FlagVec m_indc = {Metadata::Independent, Metadata::FillGhost, Metadata::Cell};
+  FlagVec m_indf = {Metadata::Independent, Metadata::FillGhost, Metadata::Face};
+  FlagVec m_derc = {Metadata::Derived, Metadata::OneCopy, Metadata::Cell};
+  FlagVec m_sparse = {Metadata::Sparse, Metadata::OneCopy};
+
+  StateDescriptor state("state");
+  state.AddField("indc", m_indc);
+  state.AddField("indf", m_indf);
+  state.AddField("derc", m_derc);
+  state.AddSparsePool("sp", m_sparse, std::vector<int>{3,7,12});
+  GIVEN("A state descriptor with some fields with differenet metadata") {
+    WHEN("We ask for fields by name") {
+      std::vector<std::string> req_names({"indc", "derc"});
+      auto names = state.GetVariableNames(req_names);
+      THEN("The vector contains the requested fields") {
+        REQUIRE(names == req_names);
+      }
+    }
+    WHEN("We ask for fields by metadata") {
+      FC_t fc = FC_t({Metadata::Independent, Metadata::FillGhost}) - FC_t({Metadata::Face});
+      auto names = state.GetVariableNames(fc);
+      THEN("The vector contains the request fields") {
+        REQUIRE(names == std::vector<std::string>({"indc"}));
+      }
+    }
+    WHEN("We try to pull out sparse fields by base name") {
+      std::vector<std::string> bname({"sp"});
+      auto names = state.GetVariableNames(bname);
+      THEN("We should get all the sparse variables tied to that base name") {
+        REQUIRE(names == std::vector<std::string>({"sp_3", "sp_7", "sp_12"}));
+      }
+    }
+    WHEN("We try to filter based on sparse ids") {
+      std::vector<std::string> bname({"sp"});
+      std::vector<int> sids({3,12});
+      auto names = state.GetVariableNames(bname, sids);
+      THEN("We should get only the ids we asked for") {
+        REQUIRE(names == std::vector<std::string>({"sp_3", "sp_12"}));
+      }
+    }
+    WHEN("We use metadata and filter on sparse ids") {
+      FC_t fc({Metadata::OneCopy});
+      std::vector<int> sids({3,12});
+      auto names = state.GetVariableNames(fc, sids);
+      THEN("We should get fields that satisfy the metadata criteria including only "
+           "specific sparse ids") {
+        REQUIRE(names == std::vector<std::string>({"derc", "sp_3", "sp_12"}));
+      }
+    }
+    WHEN("We use all possible filters") {
+      std::vector<std::string> req_names({"indc", "indf"});
+      FC_t fc = FC_t({Metadata::OneCopy}) - FC_t({Metadata::Cell});
+      std::vector<int> sids({3,7});
+      auto names = state.GetVariableNames(req_names, fc, sids);
+      THEN("We should get just what we asked for") {
+        REQUIRE(names == std::vector<std::string>({"indc", "indf", "sp_3", "sp_7"}));
+      }
+    }
+    WHEN("We ask for a specific sparse id by name") {
+      std::vector<std::string> req_names({"indc", "sp_7"});
+      REQUIRE_THROWS(state.GetVariableNames(req_names));
+    }
+    WHEN("We ask for metadata that are not satisfied for any variable") {
+      FC_t fc = FC_t({Metadata::Independent,Metadata::Sparse});
+      auto names = state.GetVariableNames(fc);
+      THEN("We should get nothing back") {
+        REQUIRE(names == std::vector<std::string>());
+      }
+    }
+  }
+}
\ No newline at end of file

From 033baec70e57355596d51c9b9d4f8165df319955 Mon Sep 17 00:00:00 2001
From: jdolence <jcdolence@gmail.com>
Date: Fri, 11 Aug 2023 11:39:15 -0600
Subject: [PATCH 03/36] expose a mesh-level interface that uses
 resolved_packages

---
 src/mesh/mesh.hpp | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/src/mesh/mesh.hpp b/src/mesh/mesh.hpp
index cda8cc0d5272..17e2e314aa2e 100644
--- a/src/mesh/mesh.hpp
+++ b/src/mesh/mesh.hpp
@@ -209,6 +209,12 @@ class Mesh {
     return buffer_memory;
   }
 
+  // expose a mesh-level call to get lists of variables from resolved_packages
+  template <typename... Args>
+  std::vector<std::string> GetVariablesNames(Args &&... args) {
+    return resolved_packages->GetVariableNames(std::forward<Args>(args)...);
+  }
+
  private:
   // data
   int root_level, max_level, current_level;

From ae64275d083aa86262131f9d7e5e8adcc2f22d2d Mon Sep 17 00:00:00 2001
From: jdolence <jcdolence@gmail.com>
Date: Fri, 11 Aug 2023 13:34:57 -0600
Subject: [PATCH 04/36] add package metadata to all fields, sparse fields, and
 swarms

---
 src/interface/state_descriptor.cpp |  2 ++
 src/interface/state_descriptor.hpp | 17 +++++++++++++++--
 tst/unit/test_state_descriptor.cpp | 17 ++++++++++++++---
 3 files changed, 31 insertions(+), 5 deletions(-)

diff --git a/src/interface/state_descriptor.cpp b/src/interface/state_descriptor.cpp
index 9ed5e96f39a4..60b8f81558d6 100644
--- a/src/interface/state_descriptor.cpp
+++ b/src/interface/state_descriptor.cpp
@@ -247,6 +247,8 @@ bool StateDescriptor::AddSwarmValue(const std::string &value_name,
   // Swarm variables are always Metadata::Particle
   Metadata newm(m);
   newm.Set(Metadata::Particle);
+  if (!newm.IsSet(GetMetadataFlag()))
+    newm.Set(GetMetadataFlag());
 
   if (swarmMetadataMap_.count(swarm_name) == 0) {
     throw std::invalid_argument("Swarm " + swarm_name + " does not exist!");
diff --git a/src/interface/state_descriptor.hpp b/src/interface/state_descriptor.hpp
index 8966535a4f27..589b6218f1b4 100644
--- a/src/interface/state_descriptor.hpp
+++ b/src/interface/state_descriptor.hpp
@@ -156,7 +156,7 @@ class StateDescriptor {
   // retrieve label
   const std::string &label() const noexcept { return label_; }
 
-  bool AddSwarm(const std::string &swarm_name, const Metadata &m) {
+  bool AddSwarm(const std::string &swarm_name, const Metadata &m_in) {
     PARTHENON_REQUIRE(
         swarm_name != "swarm",
         "A swarm may not be named \"swarm\", as this may cause name collisions.");
@@ -165,6 +165,9 @@ class StateDescriptor {
     if (swarmMetadataMap_.count(swarm_name) > 0) {
       throw std::invalid_argument("Swarm " + swarm_name + " already exists!");
     }
+    Metadata m = m_in; // so we can modify it
+    if (!m.IsSet(GetMetadataFlag()))
+      m.Set(GetMetadataFlag());
     swarmMetadataMap_[swarm_name] = m;
 
     return true;
@@ -183,12 +186,15 @@ class StateDescriptor {
   bool AddSparsePoolImpl(const SparsePool &pool);
 
  public:
-  bool AddField(const std::string &field_name, const Metadata &m,
+  bool AddField(const std::string &field_name, const Metadata &m_in,
                 const std::string &controlling_field = "") {
+    Metadata m = m_in; // so we can modify it
     if (m.IsSet(Metadata::Sparse)) {
       PARTHENON_THROW(
           "Tried to add a sparse field with AddField, use AddSparsePool instead");
     }
+    if (!m.IsSet(GetMetadataFlag()))
+      m.Set(GetMetadataFlag());
     VarID controller = VarID(controlling_field);
     if (controlling_field == "") controller = VarID(field_name);
     return AddFieldImpl(VarID(field_name), m, controller);
@@ -201,6 +207,13 @@ class StateDescriptor {
   bool AddSparsePool(Args &&...args) {
     return AddSparsePoolImpl(SparsePool(std::forward<Args>(args)...));
   }
+  template <typename... Args>
+  bool AddSparsePool(const std::string &base_name, const Metadata &m_in, Args &&... args) {
+    Metadata m = m_in; // so we can modify it
+    if (!m.IsSet(GetMetadataFlag()))
+      m.Set(GetMetadataFlag());
+    return AddSparsePoolImpl(SparsePool(base_name, m, std::forward<Args>(args)...));
+  }
 
   // retrieve number of fields
   int size() const noexcept { return metadataMap_.size(); }
diff --git a/tst/unit/test_state_descriptor.cpp b/tst/unit/test_state_descriptor.cpp
index 6eb97aa12564..c383e6d20c15 100644
--- a/tst/unit/test_state_descriptor.cpp
+++ b/tst/unit/test_state_descriptor.cpp
@@ -278,7 +278,11 @@ TEST_CASE("Test dependency resolution in StateDescriptor", "[StateDescriptor]")
         auto pkg3 = ResolvePackages(packages);
         AND_THEN("The provides package is available") {
           REQUIRE(pkg3->FieldPresent("dense"));
-          REQUIRE(pkg3->FieldMetadata("dense") == m_provides);
+          // add in package Metadata before checking equality
+          Metadata m_provides_local = m_provides;
+          m_provides_local.Set(Metadata::GetUserFlag("package2"));
+          m_provides_local.Set(Metadata::GetUserFlag("parthenon::resolved_state"));
+          REQUIRE(pkg3->FieldMetadata("dense") == m_provides_local);
         }
       }
     }
@@ -335,9 +339,16 @@ TEST_CASE("Test dependency resolution in StateDescriptor", "[StateDescriptor]")
         auto pkg4 = ResolvePackages(packages);
         AND_THEN("The provides variables take precedence.") {
           REQUIRE(pkg4->FieldPresent("dense"));
-          REQUIRE(pkg4->FieldMetadata("dense") == m_provides);
+          // add in package metadata before equality check
+          Metadata m_provides_local = m_provides;
+          m_provides_local.Set(Metadata::GetUserFlag("package1"));
+          m_provides_local.Set(Metadata::GetUserFlag("parthenon::resolved_state"));
+          REQUIRE(pkg4->FieldMetadata("dense") == m_provides_local);
           REQUIRE(pkg4->SwarmPresent("myswarm"));
-          REQUIRE(pkg4->SwarmMetadata("myswarm") == m_provides_swarm);
+          Metadata m_provides_swarm_local = m_provides_swarm;
+          m_provides_swarm_local.Set(Metadata::GetUserFlag("package2"));
+          m_provides_swarm_local.Set(Metadata::GetUserFlag("parthenon::resolved_state"));
+          REQUIRE(pkg4->SwarmMetadata("myswarm") == m_provides_swarm_local);
           REQUIRE(pkg4->SwarmValuePresent("provides", "myswarm"));
           REQUIRE(!(pkg4->SwarmValuePresent("overridable", "myswarm")));
           REQUIRE(pkg4->SparseBaseNamePresent("sparse"));

From e85c7f92956f70236bcdfd17eedffa8057cdbbb6 Mon Sep 17 00:00:00 2001
From: jdolence <jcdolence@gmail.com>
Date: Fri, 11 Aug 2023 14:28:58 -0600
Subject: [PATCH 05/36] refactor Copy and make accessible from DataCollection
 Add functions

---
 src/interface/data_collection.cpp         | 40 ++++++------
 src/interface/data_collection.hpp         |  5 ++
 src/interface/mesh_data.hpp               |  6 +-
 src/interface/meshblock_data.cpp          | 79 ++++++++---------------
 src/interface/meshblock_data.hpp          | 38 ++---------
 tst/unit/test_meshblock_data_iterator.cpp | 10 ---
 6 files changed, 64 insertions(+), 114 deletions(-)

diff --git a/src/interface/data_collection.cpp b/src/interface/data_collection.cpp
index 7d8cb08258ee..57925af68a68 100644
--- a/src/interface/data_collection.cpp
+++ b/src/interface/data_collection.cpp
@@ -24,7 +24,8 @@ namespace parthenon {
 template <typename T>
 std::shared_ptr<T> DataCollection<T>::Add(const std::string &name,
                                           const std::shared_ptr<T> &src,
-                                          const std::vector<std::string> &field_names) {
+                                          const std::vector<std::string> &field_names,
+                                          const bool shallow) {
   auto it = containers_.find(name);
   if (it != containers_.end()) {
     if (!(it->second)->Contains(field_names)) {
@@ -34,30 +35,33 @@ std::shared_ptr<T> DataCollection<T>::Add(const std::string &name,
   }
 
   auto c = std::make_shared<T>();
-  c->Copy(src, field_names);
+  c->Copy(src.get(), field_names, shallow);
 
   containers_[name] = c;
   return containers_[name];
 }
-
+template <typename T>
+std::shared_ptr<T> DataCollection<T>::Add(const std::string &name,
+                                          const std::shared_ptr<T> &src,
+                                          const std::vector<std::string> &field_names) {
+  return Add(name, src, field_names, false);
+}
+template <typename T>
+std::shared_ptr<T>
+DataCollection<T>::AddShallow(const std::string &name,
+                              const std::shared_ptr<T> &src,
+                              const std::vector<std::string> &field_names) {
+  return Add(name, src, field_names, true);
+}
 template <typename T>
 std::shared_ptr<T> DataCollection<T>::Add(const std::string &name,
                                           const std::shared_ptr<T> &src) {
-  // error check for duplicate names
-  auto it = containers_.find(name);
-  if (it != containers_.end()) {
-    // check to make sure they are the same
-    if (!(*src == *(it->second))) {
-      PARTHENON_THROW("Error attempting to add a Container to a Collection");
-    }
-    return it->second;
-  }
-
-  auto c = std::make_shared<T>();
-  c->Copy(src);
-
-  containers_[name] = c;
-  return containers_[name];
+  return Add(name, src, {}, false);
+}
+template <typename T>
+std::shared_ptr<T> DataCollection<T>::AddShallow(const std::string &name,
+                                                 const std::shared_ptr<T> &src) {
+  return Add(name, src, {}, true);
 }
 
 template <>
diff --git a/src/interface/data_collection.hpp b/src/interface/data_collection.hpp
index 67589b15f85a..039146afceac 100644
--- a/src/interface/data_collection.hpp
+++ b/src/interface/data_collection.hpp
@@ -38,9 +38,14 @@ class DataCollection {
 
   void SetMeshPointer(Mesh *pmesh) { pmy_mesh_ = pmesh; }
 
+  std::shared_ptr<T> Add(const std::string &label, const std::shared_ptr<T> &src,
+                         const std::vector<std::string> &flags, const bool shallow);
   std::shared_ptr<T> Add(const std::string &label, const std::shared_ptr<T> &src,
                          const std::vector<std::string> &flags);
+  std::shared_ptr<T> AddShallow(const std::string &label, const std::shared_ptr<T> &src,
+                                const std::vector<std::string> &flags);
   std::shared_ptr<T> Add(const std::string &label, const std::shared_ptr<T> &src);
+  std::shared_ptr<T> AddShallow(const std::string &label, const std::shared_ptr<T> &src);
   std::shared_ptr<T> Add(const std::string &label) {
     // error check for duplicate names
     auto it = containers_.find(label);
diff --git a/src/interface/mesh_data.hpp b/src/interface/mesh_data.hpp
index 8d45767c9a25..7aee69801fe6 100644
--- a/src/interface/mesh_data.hpp
+++ b/src/interface/mesh_data.hpp
@@ -236,14 +236,14 @@ class MeshData {
   }
 
   template <typename... Args>
-  void Copy(const std::shared_ptr<MeshData<T>> src, Args &&...args) {
-    if (src.get() == nullptr) {
+  void Copy(const MeshData<T> *src, Args &&...args) {
+    if (src == nullptr) {
       PARTHENON_THROW("src points at null");
     }
     const int nblocks = src->NumBlocks();
     block_data_.resize(nblocks);
     for (int i = 0; i < nblocks; i++) {
-      block_data_[i]->Copy(src->GetBlockData(i), std::forward<Args>(args)...);
+      block_data_[i]->Copy(src->GetBlockData(i).get(), std::forward<Args>(args)...);
     }
   }
 
diff --git a/src/interface/meshblock_data.cpp b/src/interface/meshblock_data.cpp
index 8862e02173a3..365726c883f2 100644
--- a/src/interface/meshblock_data.cpp
+++ b/src/interface/meshblock_data.cpp
@@ -75,24 +75,13 @@ void MeshBlockData<T>::AddField(const std::string &base_name, const Metadata &me
 
 // TODO(JMM): Add CopyFrom that takes unique IDs
 template <typename T>
-void MeshBlockData<T>::CopyFrom(const MeshBlockData<T> &src, bool shallow_copy,
-                                const std::vector<std::string> &names,
-                                const std::vector<MetadataFlag> &flags,
-                                const std::vector<int> &sparse_ids) {
+void MeshBlockData<T>::Copy(const MeshBlockData<T> *src,
+                            const std::vector<std::string> &names,
+                            const bool shallow_copy) {
   SetBlockPointer(src);
-  resolved_packages_ = src.resolved_packages_;
-  std::unordered_set<int> sparse_ids_set(sparse_ids.begin(), sparse_ids.end());
-
-  auto add_var = [=, &flags, &sparse_ids](auto var) {
-    if (!flags.empty() && !var->metadata().AnyFlagsSet(flags)) {
-      return;
-    }
-
-    if (!sparse_ids.empty() && var->IsSparse() &&
-        (sparse_ids_set.count(var->GetSparseID()) == 0)) {
-      return;
-    }
+  resolved_packages_ = src->resolved_packages_;
 
+  auto add_var = [=](auto var) {
     if (shallow_copy || var->IsSet(Metadata::OneCopy)) {
       Add(var);
     } else {
@@ -100,12 +89,13 @@ void MeshBlockData<T>::CopyFrom(const MeshBlockData<T> &src, bool shallow_copy,
     }
   };
 
+  // special case when the list of names is empty, copy everything
   if (names.empty()) {
-    for (auto v : src.GetVariableVector()) {
+    for (auto v : src->GetVariableVector()) {
       add_var(v);
     }
   } else {
-    auto var_map = src.GetVariableMap();
+    auto var_map = src->GetVariableMap();
 
     for (const auto &name : names) {
       bool found = false;
@@ -114,21 +104,6 @@ void MeshBlockData<T>::CopyFrom(const MeshBlockData<T> &src, bool shallow_copy,
         found = true;
         add_var(v->second);
       }
-
-      if (!found && (resolved_packages_ != nullptr)) {
-        // check if this is a sparse base name, if so we get its pool of sparse_ids,
-        // otherwise we get an empty pool
-        const auto &sparse_pool = resolved_packages_->GetSparsePool(name);
-
-        // add all sparse ids of the pool
-        for (const auto iter : sparse_pool.pool()) {
-          // this variable must exist, if it doesn't something is very wrong
-          const auto &v = varMap_.at(MakeVarLabel(name, iter.first));
-          add_var(v);
-          found = true;
-        }
-      }
-
       PARTHENON_REQUIRE_THROWS(found, "MeshBlockData::CopyFrom: Variable '" + name +
                                           "' not found");
     }
@@ -138,29 +113,29 @@ void MeshBlockData<T>::CopyFrom(const MeshBlockData<T> &src, bool shallow_copy,
 // Constructor for getting sub-containers
 // the variables returned are all shallow copies of the src container.
 // Optionally extract only some of the sparse ids of src variable.
-template <typename T>
-MeshBlockData<T>::MeshBlockData(const MeshBlockData<T> &src,
-                                const std::vector<std::string> &names,
-                                const std::vector<int> &sparse_ids) {
-  CopyFrom(src, true, names, {}, sparse_ids);
-}
+//template <typename T>
+//MeshBlockData<T>::MeshBlockData(const MeshBlockData<T> &src,
+                                //const std::vector<std::string> &names,
+                                //const std::vector<int> &sparse_ids) {
+  //CopyFrom(src, true, names, {}, sparse_ids);
+//}
 
 // TODO(JMM): Add constructor that takes unique IDs
-template <typename T>
-MeshBlockData<T>::MeshBlockData(const MeshBlockData<T> &src,
-                                const std::vector<MetadataFlag> &flags,
-                                const std::vector<int> &sparse_ids) {
-  CopyFrom(src, true, {}, flags, sparse_ids);
-}
+//template <typename T>
+//MeshBlockData<T>::MeshBlockData(const MeshBlockData<T> &src,
+                                //const std::vector<MetadataFlag> &flags,
+                                //const std::vector<int> &sparse_ids) {
+  //CopyFrom(src, true, {}, flags, sparse_ids);
+//}
 
 // provides a container that has a single sparse slice
-template <typename T>
-std::shared_ptr<MeshBlockData<T>>
-MeshBlockData<T>::SparseSlice(const std::vector<int> &sparse_ids) const {
-  auto c = std::make_shared<MeshBlockData<T>>();
-  c->CopyFrom(*this, true, {}, {}, sparse_ids);
-  return c;
-}
+//template <typename T>
+//std::shared_ptr<MeshBlockData<T>>
+//MeshBlockData<T>::SparseSlice(const std::vector<int> &sparse_ids) const {
+  //auto c = std::make_shared<MeshBlockData<T>>();
+  //c->CopyFrom(*this, true, {}, {}, sparse_ids);
+  //return c;
+//}
 
 /// Queries related to variable packs
 /// This is a helper function that queries the cache for the given pack.
diff --git a/src/interface/meshblock_data.hpp b/src/interface/meshblock_data.hpp
index 366d5066dfda..8fea14612a22 100644
--- a/src/interface/meshblock_data.hpp
+++ b/src/interface/meshblock_data.hpp
@@ -58,17 +58,6 @@ class MeshBlockData {
   /// Constructor
   MeshBlockData<T>() = default;
 
-  /// Copies variables from src, optionally only copying names given and/or variables
-  /// matching any of the flags given. If sparse_ids is not empty, only sparse fields with
-  /// listed sparse ids will be copied, dense fields will always be copied. If both names
-  /// and flags are provided, only variables that show up in names AND have metadata in
-  /// FLAGS are copied. If shallow_copy is true, no copies of variables will be allocated
-  /// regardless whether they are flagged as OneCopy or not
-  void CopyFrom(const MeshBlockData<T> &src, bool shallow_copy,
-                const std::vector<std::string> &names = {},
-                const std::vector<MetadataFlag> &flags = {},
-                const std::vector<int> &sparse_ids = {});
-
   // Constructors for getting sub-containers
   // the variables returned are all shallow copies of the src container.
   MeshBlockData<T>(const MeshBlockData<T> &src, const std::vector<std::string> &names,
@@ -97,33 +86,20 @@ class MeshBlockData {
     return GetBlockPointer()->cellbounds.GetBoundsK(domain);
   }
 
-  /// Create non-shallow copy of MeshBlockData, but only include named variables
-  void Copy(const std::shared_ptr<MeshBlockData<T>> &src,
-            const std::vector<std::string> &names) {
-    CopyFrom(*src, false, names);
-  }
-  /// Create non-shallow copy of MeshBlockData
-  void Copy(const std::shared_ptr<MeshBlockData<T>> &src) { CopyFrom(*src, false); }
-
-  /// Get a container containing only dense fields and the sparse fields with a sparse id
-  /// from the given list of sparse ids.
-  ///
-  /// @param sparse_ids The list of sparse ids to include
-  /// @return New container with slices from all variables
-  std::shared_ptr<MeshBlockData<T>> SparseSlice(const std::vector<int> &sparse_ids) const;
-
-  /// As above but for just one sparse id
-  std::shared_ptr<MeshBlockData<T>> SparseSlice(int sparse_id) const {
-    return SparseSlice({sparse_id});
-  }
+  /// Create copy of MeshBlockData, possibly with a subset of named fields,
+  /// and possibly shallow.  Note when shallow=false, new storage is allocated
+  /// for non-OneCopy vars, but the data from src is not actually deep copied
+  void Copy(const MeshBlockData<T> *src,
+            const std::vector<std::string> &names,
+            const bool shallow);
 
-  ///
   /// Set the pointer to the mesh block for this container
   void SetBlockPointer(std::weak_ptr<MeshBlock> pmb) { pmy_block = pmb; }
   void SetBlockPointer(const std::shared_ptr<MeshBlockData<T>> &other) {
     SetBlockPointer(*other);
   }
   void SetBlockPointer(const MeshBlockData<T> &other) { pmy_block = other.pmy_block; }
+  void SetBlockPointer(const MeshBlockData<T> *other) { pmy_block = other->pmy_block; }
 
   void Initialize(const std::shared_ptr<StateDescriptor> resolved_packages,
                   const std::shared_ptr<MeshBlock> pmb);
diff --git a/tst/unit/test_meshblock_data_iterator.cpp b/tst/unit/test_meshblock_data_iterator.cpp
index 4d8d81bae586..1ed919f042b7 100644
--- a/tst/unit/test_meshblock_data_iterator.cpp
+++ b/tst/unit/test_meshblock_data_iterator.cpp
@@ -135,16 +135,6 @@ TEST_CASE("Can pull variables from containers based on Metadata",
       }
     }
 
-    WHEN("We extract a subcontainer") {
-      auto subcontainer = MeshBlockData<Real>(mbd, {"v1", "v3", "v5"});
-      THEN("The container has the names in the right order") {
-        auto vars = subcontainer.GetVariableVector();
-        REQUIRE(vars[0]->label() == "v1");
-        REQUIRE(vars[1]->label() == "v3");
-        REQUIRE(vars[2]->label() == "v5");
-      }
-    }
-
     auto v = mbd.PackVariables();
     par_for(
         DEFAULT_LOOP_PATTERN, "Initialize variables", DevExecSpace(), 0, v.GetDim(4) - 1,

From 7d14eb84690b9f7eeb64483197c8469430578ae6 Mon Sep 17 00:00:00 2001
From: jdolence <jcdolence@gmail.com>
Date: Fri, 11 Aug 2023 14:31:41 -0600
Subject: [PATCH 06/36] formatting

---
 src/interface/data_collection.cpp  | 13 +++++-----
 src/interface/meshblock_data.cpp   | 32 ++++++++++++-------------
 src/interface/meshblock_data.hpp   |  3 +--
 src/interface/metadata.cpp         |  3 +--
 src/interface/metadata.hpp         |  2 +-
 src/interface/state_descriptor.cpp |  3 +--
 src/interface/state_descriptor.hpp | 38 +++++++++++-------------------
 src/mesh/mesh.hpp                  |  2 +-
 tst/unit/test_metadata.cpp         |  3 +--
 tst/unit/test_state_descriptor.cpp | 21 +++++++----------
 10 files changed, 51 insertions(+), 69 deletions(-)

diff --git a/src/interface/data_collection.cpp b/src/interface/data_collection.cpp
index 57925af68a68..ad8c28efbd0e 100644
--- a/src/interface/data_collection.cpp
+++ b/src/interface/data_collection.cpp
@@ -22,14 +22,14 @@
 namespace parthenon {
 
 template <typename T>
-std::shared_ptr<T> DataCollection<T>::Add(const std::string &name,
-                                          const std::shared_ptr<T> &src,
-                                          const std::vector<std::string> &field_names,
-                                          const bool shallow) {
+std::shared_ptr<T>
+DataCollection<T>::Add(const std::string &name, const std::shared_ptr<T> &src,
+                       const std::vector<std::string> &field_names, const bool shallow) {
   auto it = containers_.find(name);
   if (it != containers_.end()) {
     if (!(it->second)->Contains(field_names)) {
-      PARTHENON_THROW(name + "already exists in collection but does not contain field names");
+      PARTHENON_THROW(name +
+                      "already exists in collection but does not contain field names");
     }
     return it->second;
   }
@@ -48,8 +48,7 @@ std::shared_ptr<T> DataCollection<T>::Add(const std::string &name,
 }
 template <typename T>
 std::shared_ptr<T>
-DataCollection<T>::AddShallow(const std::string &name,
-                              const std::shared_ptr<T> &src,
+DataCollection<T>::AddShallow(const std::string &name, const std::shared_ptr<T> &src,
                               const std::vector<std::string> &field_names) {
   return Add(name, src, field_names, true);
 }
diff --git a/src/interface/meshblock_data.cpp b/src/interface/meshblock_data.cpp
index 365726c883f2..874699f00316 100644
--- a/src/interface/meshblock_data.cpp
+++ b/src/interface/meshblock_data.cpp
@@ -113,28 +113,28 @@ void MeshBlockData<T>::Copy(const MeshBlockData<T> *src,
 // Constructor for getting sub-containers
 // the variables returned are all shallow copies of the src container.
 // Optionally extract only some of the sparse ids of src variable.
-//template <typename T>
-//MeshBlockData<T>::MeshBlockData(const MeshBlockData<T> &src,
-                                //const std::vector<std::string> &names,
-                                //const std::vector<int> &sparse_ids) {
-  //CopyFrom(src, true, names, {}, sparse_ids);
+// template <typename T>
+// MeshBlockData<T>::MeshBlockData(const MeshBlockData<T> &src,
+// const std::vector<std::string> &names,
+// const std::vector<int> &sparse_ids) {
+// CopyFrom(src, true, names, {}, sparse_ids);
 //}
 
 // TODO(JMM): Add constructor that takes unique IDs
-//template <typename T>
-//MeshBlockData<T>::MeshBlockData(const MeshBlockData<T> &src,
-                                //const std::vector<MetadataFlag> &flags,
-                                //const std::vector<int> &sparse_ids) {
-  //CopyFrom(src, true, {}, flags, sparse_ids);
+// template <typename T>
+// MeshBlockData<T>::MeshBlockData(const MeshBlockData<T> &src,
+// const std::vector<MetadataFlag> &flags,
+// const std::vector<int> &sparse_ids) {
+// CopyFrom(src, true, {}, flags, sparse_ids);
 //}
 
 // provides a container that has a single sparse slice
-//template <typename T>
-//std::shared_ptr<MeshBlockData<T>>
-//MeshBlockData<T>::SparseSlice(const std::vector<int> &sparse_ids) const {
-  //auto c = std::make_shared<MeshBlockData<T>>();
-  //c->CopyFrom(*this, true, {}, {}, sparse_ids);
-  //return c;
+// template <typename T>
+// std::shared_ptr<MeshBlockData<T>>
+// MeshBlockData<T>::SparseSlice(const std::vector<int> &sparse_ids) const {
+// auto c = std::make_shared<MeshBlockData<T>>();
+// c->CopyFrom(*this, true, {}, {}, sparse_ids);
+// return c;
 //}
 
 /// Queries related to variable packs
diff --git a/src/interface/meshblock_data.hpp b/src/interface/meshblock_data.hpp
index 8fea14612a22..429aea08a003 100644
--- a/src/interface/meshblock_data.hpp
+++ b/src/interface/meshblock_data.hpp
@@ -89,8 +89,7 @@ class MeshBlockData {
   /// Create copy of MeshBlockData, possibly with a subset of named fields,
   /// and possibly shallow.  Note when shallow=false, new storage is allocated
   /// for non-OneCopy vars, but the data from src is not actually deep copied
-  void Copy(const MeshBlockData<T> *src,
-            const std::vector<std::string> &names,
+  void Copy(const MeshBlockData<T> *src, const std::vector<std::string> &names,
             const bool shallow);
 
   /// Set the pointer to the mesh block for this container
diff --git a/src/interface/metadata.cpp b/src/interface/metadata.cpp
index 38466b2f4ed1..89a70d7468a8 100644
--- a/src/interface/metadata.cpp
+++ b/src/interface/metadata.cpp
@@ -265,8 +265,7 @@ bool MatchFlags(const Metadata::FlagCollection &flags, Metadata m) {
   const auto &unions = flags.GetUnions();
   const auto &exclusions = flags.GetExclusions();
 
-  return m.AllFlagsSet(intersections) &&
-         (unions.empty() || m.AnyFlagsSet(unions)) &&
+  return m.AllFlagsSet(intersections) && (unions.empty() || m.AnyFlagsSet(unions)) &&
          m.NoFlagsSet(exclusions);
 }
 } // namespace MetadataUtils
diff --git a/src/interface/metadata.hpp b/src/interface/metadata.hpp
index 755a0778d237..f5ba4807e6cc 100644
--- a/src/interface/metadata.hpp
+++ b/src/interface/metadata.hpp
@@ -514,7 +514,7 @@ class Metadata {
   template <template <class...> class Container_t, class... extra>
   bool NoFlagsSet(const Container_t<MetadataFlag, extra...> &flags) const {
     return std::none_of(flags.begin(), flags.end(),
-                       [this](MetadataFlag const &f) { return IsSet(f); });
+                        [this](MetadataFlag const &f) { return IsSet(f); });
   }
   template <typename... Args>
   bool NoFlagsSet(const MetadataFlag &flag, Args... args) const {
diff --git a/src/interface/state_descriptor.cpp b/src/interface/state_descriptor.cpp
index 60b8f81558d6..2e0dcc64ee32 100644
--- a/src/interface/state_descriptor.cpp
+++ b/src/interface/state_descriptor.cpp
@@ -247,8 +247,7 @@ bool StateDescriptor::AddSwarmValue(const std::string &value_name,
   // Swarm variables are always Metadata::Particle
   Metadata newm(m);
   newm.Set(Metadata::Particle);
-  if (!newm.IsSet(GetMetadataFlag()))
-    newm.Set(GetMetadataFlag());
+  if (!newm.IsSet(GetMetadataFlag())) newm.Set(GetMetadataFlag());
 
   if (swarmMetadataMap_.count(swarm_name) == 0) {
     throw std::invalid_argument("Swarm " + swarm_name + " does not exist!");
diff --git a/src/interface/state_descriptor.hpp b/src/interface/state_descriptor.hpp
index 589b6218f1b4..6d295c573ded 100644
--- a/src/interface/state_descriptor.hpp
+++ b/src/interface/state_descriptor.hpp
@@ -111,9 +111,7 @@ class StateDescriptor {
   static std::shared_ptr<StateDescriptor>
   CreateResolvedStateDescriptor(Packages_t &packages);
 
-  MetadataFlag GetMetadataFlag() {
-    return params_.Get<MetadataFlag>("MetadataFlag");
-  }
+  MetadataFlag GetMetadataFlag() { return params_.Get<MetadataFlag>("MetadataFlag"); }
 
   template <typename T>
   void AddParam(const std::string &key, T value, Params::Mutability mutability) {
@@ -166,8 +164,7 @@ class StateDescriptor {
       throw std::invalid_argument("Swarm " + swarm_name + " already exists!");
     }
     Metadata m = m_in; // so we can modify it
-    if (!m.IsSet(GetMetadataFlag()))
-      m.Set(GetMetadataFlag());
+    if (!m.IsSet(GetMetadataFlag())) m.Set(GetMetadataFlag());
     swarmMetadataMap_[swarm_name] = m;
 
     return true;
@@ -193,8 +190,7 @@ class StateDescriptor {
       PARTHENON_THROW(
           "Tried to add a sparse field with AddField, use AddSparsePool instead");
     }
-    if (!m.IsSet(GetMetadataFlag()))
-      m.Set(GetMetadataFlag());
+    if (!m.IsSet(GetMetadataFlag())) m.Set(GetMetadataFlag());
     VarID controller = VarID(controlling_field);
     if (controlling_field == "") controller = VarID(field_name);
     return AddFieldImpl(VarID(field_name), m, controller);
@@ -208,10 +204,9 @@ class StateDescriptor {
     return AddSparsePoolImpl(SparsePool(std::forward<Args>(args)...));
   }
   template <typename... Args>
-  bool AddSparsePool(const std::string &base_name, const Metadata &m_in, Args &&... args) {
+  bool AddSparsePool(const std::string &base_name, const Metadata &m_in, Args &&...args) {
     Metadata m = m_in; // so we can modify it
-    if (!m.IsSet(GetMetadataFlag()))
-      m.Set(GetMetadataFlag());
+    if (!m.IsSet(GetMetadataFlag())) m.Set(GetMetadataFlag());
     return AddSparsePoolImpl(SparsePool(base_name, m, std::forward<Args>(args)...));
   }
 
@@ -246,20 +241,15 @@ class StateDescriptor {
     // automatically have x, y, z.
     return swarmValueMetadataMap_[swarm_name];
   }
-  std::vector<std::string>
-  GetVariableNames(const std::vector<std::string> &req_names,
-                   const Metadata::FlagCollection &flags,
-                   const std::vector<int> &sparse_ids);
-  std::vector<std::string>
-  GetVariableNames(const std::vector<std::string> &req_names,
-                   const std::vector<int> &sparse_ids);
-  std::vector<std::string>
-  GetVariableNames(const Metadata::FlagCollection &flags,
-                   const std::vector<int> &sparse_ids);
-  std::vector<std::string>
-  GetVariableNames(const std::vector<std::string> &req_names);
-  std::vector<std::string>
-  GetVariableNames(const Metadata::FlagCollection &flags);
+  std::vector<std::string> GetVariableNames(const std::vector<std::string> &req_names,
+                                            const Metadata::FlagCollection &flags,
+                                            const std::vector<int> &sparse_ids);
+  std::vector<std::string> GetVariableNames(const std::vector<std::string> &req_names,
+                                            const std::vector<int> &sparse_ids);
+  std::vector<std::string> GetVariableNames(const Metadata::FlagCollection &flags,
+                                            const std::vector<int> &sparse_ids);
+  std::vector<std::string> GetVariableNames(const std::vector<std::string> &req_names);
+  std::vector<std::string> GetVariableNames(const Metadata::FlagCollection &flags);
 
   std::size_t
   RefinementFuncID(const refinement::RefinementFunctions_t &funcs) const noexcept {
diff --git a/src/mesh/mesh.hpp b/src/mesh/mesh.hpp
index 17e2e314aa2e..d32b389b93ed 100644
--- a/src/mesh/mesh.hpp
+++ b/src/mesh/mesh.hpp
@@ -211,7 +211,7 @@ class Mesh {
 
   // expose a mesh-level call to get lists of variables from resolved_packages
   template <typename... Args>
-  std::vector<std::string> GetVariablesNames(Args &&... args) {
+  std::vector<std::string> GetVariablesNames(Args &&...args) {
     return resolved_packages->GetVariableNames(std::forward<Args>(args)...);
   }
 
diff --git a/tst/unit/test_metadata.cpp b/tst/unit/test_metadata.cpp
index 417fed077a1c..23a3a3fea6fe 100644
--- a/tst/unit/test_metadata.cpp
+++ b/tst/unit/test_metadata.cpp
@@ -82,8 +82,7 @@ TEST_CASE("A Metadata flag is allocated", "[Metadata]") {
     // rightfully break later - this test needn't be maintained if so.
     //
     // Checks that an allocated flag was ended to the end of existing flags
-    REQUIRE(f.InternalFlagValue() ==
-            static_cast<int>(Metadata::num_flags)-1);
+    REQUIRE(f.InternalFlagValue() == static_cast<int>(Metadata::num_flags) - 1);
     REQUIRE(name == f.Name());
 
     // Metadata should be able to report that this flag exists and
diff --git a/tst/unit/test_state_descriptor.cpp b/tst/unit/test_state_descriptor.cpp
index c383e6d20c15..3028f706b298 100644
--- a/tst/unit/test_state_descriptor.cpp
+++ b/tst/unit/test_state_descriptor.cpp
@@ -541,17 +541,16 @@ TEST_CASE("Test getting a vector of variable names given criteria", "[StateDescr
   state.AddField("indc", m_indc);
   state.AddField("indf", m_indf);
   state.AddField("derc", m_derc);
-  state.AddSparsePool("sp", m_sparse, std::vector<int>{3,7,12});
+  state.AddSparsePool("sp", m_sparse, std::vector<int>{3, 7, 12});
   GIVEN("A state descriptor with some fields with differenet metadata") {
     WHEN("We ask for fields by name") {
       std::vector<std::string> req_names({"indc", "derc"});
       auto names = state.GetVariableNames(req_names);
-      THEN("The vector contains the requested fields") {
-        REQUIRE(names == req_names);
-      }
+      THEN("The vector contains the requested fields") { REQUIRE(names == req_names); }
     }
     WHEN("We ask for fields by metadata") {
-      FC_t fc = FC_t({Metadata::Independent, Metadata::FillGhost}) - FC_t({Metadata::Face});
+      FC_t fc =
+          FC_t({Metadata::Independent, Metadata::FillGhost}) - FC_t({Metadata::Face});
       auto names = state.GetVariableNames(fc);
       THEN("The vector contains the request fields") {
         REQUIRE(names == std::vector<std::string>({"indc"}));
@@ -566,7 +565,7 @@ TEST_CASE("Test getting a vector of variable names given criteria", "[StateDescr
     }
     WHEN("We try to filter based on sparse ids") {
       std::vector<std::string> bname({"sp"});
-      std::vector<int> sids({3,12});
+      std::vector<int> sids({3, 12});
       auto names = state.GetVariableNames(bname, sids);
       THEN("We should get only the ids we asked for") {
         REQUIRE(names == std::vector<std::string>({"sp_3", "sp_12"}));
@@ -574,7 +573,7 @@ TEST_CASE("Test getting a vector of variable names given criteria", "[StateDescr
     }
     WHEN("We use metadata and filter on sparse ids") {
       FC_t fc({Metadata::OneCopy});
-      std::vector<int> sids({3,12});
+      std::vector<int> sids({3, 12});
       auto names = state.GetVariableNames(fc, sids);
       THEN("We should get fields that satisfy the metadata criteria including only "
            "specific sparse ids") {
@@ -584,7 +583,7 @@ TEST_CASE("Test getting a vector of variable names given criteria", "[StateDescr
     WHEN("We use all possible filters") {
       std::vector<std::string> req_names({"indc", "indf"});
       FC_t fc = FC_t({Metadata::OneCopy}) - FC_t({Metadata::Cell});
-      std::vector<int> sids({3,7});
+      std::vector<int> sids({3, 7});
       auto names = state.GetVariableNames(req_names, fc, sids);
       THEN("We should get just what we asked for") {
         REQUIRE(names == std::vector<std::string>({"indc", "indf", "sp_3", "sp_7"}));
@@ -595,11 +594,9 @@ TEST_CASE("Test getting a vector of variable names given criteria", "[StateDescr
       REQUIRE_THROWS(state.GetVariableNames(req_names));
     }
     WHEN("We ask for metadata that are not satisfied for any variable") {
-      FC_t fc = FC_t({Metadata::Independent,Metadata::Sparse});
+      FC_t fc = FC_t({Metadata::Independent, Metadata::Sparse});
       auto names = state.GetVariableNames(fc);
-      THEN("We should get nothing back") {
-        REQUIRE(names == std::vector<std::string>());
-      }
+      THEN("We should get nothing back") { REQUIRE(names == std::vector<std::string>()); }
     }
   }
 }
\ No newline at end of file

From ade63bf6f766e83a8329b3b686ca75270dc2dd3c Mon Sep 17 00:00:00 2001
From: jdolence <jcdolence@gmail.com>
Date: Fri, 11 Aug 2023 15:13:39 -0600
Subject: [PATCH 07/36] newline for the linter

---
 tst/unit/test_state_descriptor.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tst/unit/test_state_descriptor.cpp b/tst/unit/test_state_descriptor.cpp
index 3028f706b298..32f22820de0b 100644
--- a/tst/unit/test_state_descriptor.cpp
+++ b/tst/unit/test_state_descriptor.cpp
@@ -599,4 +599,4 @@ TEST_CASE("Test getting a vector of variable names given criteria", "[StateDescr
       THEN("We should get nothing back") { REQUIRE(names == std::vector<std::string>()); }
     }
   }
-}
\ No newline at end of file
+}

From eeaf23d985327c1b754fdf27a585063d630befc2 Mon Sep 17 00:00:00 2001
From: jdolence <jcdolence@gmail.com>
Date: Fri, 11 Aug 2023 15:16:30 -0600
Subject: [PATCH 08/36] remove dead code

---
 src/interface/meshblock_data.cpp | 29 +----------------------------
 1 file changed, 1 insertion(+), 28 deletions(-)

diff --git a/src/interface/meshblock_data.cpp b/src/interface/meshblock_data.cpp
index 874699f00316..2448e19e0302 100644
--- a/src/interface/meshblock_data.cpp
+++ b/src/interface/meshblock_data.cpp
@@ -73,7 +73,7 @@ void MeshBlockData<T>::AddField(const std::string &base_name, const Metadata &me
   }
 }
 
-// TODO(JMM): Add CopyFrom that takes unique IDs
+// TODO(JMM): Move to unique IDs at some point
 template <typename T>
 void MeshBlockData<T>::Copy(const MeshBlockData<T> *src,
                             const std::vector<std::string> &names,
@@ -110,33 +110,6 @@ void MeshBlockData<T>::Copy(const MeshBlockData<T> *src,
   }
 }
 
-// Constructor for getting sub-containers
-// the variables returned are all shallow copies of the src container.
-// Optionally extract only some of the sparse ids of src variable.
-// template <typename T>
-// MeshBlockData<T>::MeshBlockData(const MeshBlockData<T> &src,
-// const std::vector<std::string> &names,
-// const std::vector<int> &sparse_ids) {
-// CopyFrom(src, true, names, {}, sparse_ids);
-//}
-
-// TODO(JMM): Add constructor that takes unique IDs
-// template <typename T>
-// MeshBlockData<T>::MeshBlockData(const MeshBlockData<T> &src,
-// const std::vector<MetadataFlag> &flags,
-// const std::vector<int> &sparse_ids) {
-// CopyFrom(src, true, {}, flags, sparse_ids);
-//}
-
-// provides a container that has a single sparse slice
-// template <typename T>
-// std::shared_ptr<MeshBlockData<T>>
-// MeshBlockData<T>::SparseSlice(const std::vector<int> &sparse_ids) const {
-// auto c = std::make_shared<MeshBlockData<T>>();
-// c->CopyFrom(*this, true, {}, {}, sparse_ids);
-// return c;
-//}
-
 /// Queries related to variable packs
 /// This is a helper function that queries the cache for the given pack.
 /// The strings are the keys and the lists are the values.

From 3c72945560c95d876510445af83180f46ed73c8a Mon Sep 17 00:00:00 2001
From: jdolence <jcdolence@gmail.com>
Date: Fri, 11 Aug 2023 16:07:22 -0600
Subject: [PATCH 09/36] sneak in templated versions of the
 AddField/SparsePool/Swarm/SwarmValue functions

---
 src/interface/state_descriptor.hpp | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/src/interface/state_descriptor.hpp b/src/interface/state_descriptor.hpp
index 6d295c573ded..c24d60eacc31 100644
--- a/src/interface/state_descriptor.hpp
+++ b/src/interface/state_descriptor.hpp
@@ -169,9 +169,17 @@ class StateDescriptor {
 
     return true;
   }
+  template <typename T>
+  bool AddSwarm(const Metadata &m) {
+    return AddSwarm(T::name(), m);
+  }
 
   bool AddSwarmValue(const std::string &value_name, const std::string &swarm_name,
                      const Metadata &m);
+  template <typename T, typename V>
+  bool AddSwarmValue(const Metadata &m) {
+    return AddSwarmValue(T::name(), V::name(), m);
+  }
 
   // field addition / retrieval routines
  private:
@@ -195,6 +203,10 @@ class StateDescriptor {
     if (controlling_field == "") controller = VarID(field_name);
     return AddFieldImpl(VarID(field_name), m, controller);
   }
+  template <typename T>
+  bool AddField(const Metadata &m, const std::string &controlling_field = "") {
+    return AddField(T::name(), m, controlling_field);
+  }
 
   // add sparse pool, all arguments will be forwarded to the SparsePool constructor, so
   // one can pass in a reference to a SparsePool or arguments that match one of the
@@ -209,6 +221,10 @@ class StateDescriptor {
     if (!m.IsSet(GetMetadataFlag())) m.Set(GetMetadataFlag());
     return AddSparsePoolImpl(SparsePool(base_name, m, std::forward<Args>(args)...));
   }
+  template <typename T, typename... Args>
+  bool AddSparsePool(const Metadata &m_in, Args &&...args) {
+    return AddSparsePool(T::name(), m_in, std::forward<Args>(args)...);
+  }
 
   // retrieve number of fields
   int size() const noexcept { return metadataMap_.size(); }

From fa1ce4f8f1dc2da79de44460b37035d8e793db68 Mon Sep 17 00:00:00 2001
From: jdolence <jcdolence@gmail.com>
Date: Sat, 12 Aug 2023 15:35:02 -0600
Subject: [PATCH 10/36] template SparsePool constructor on variable type

---
 src/interface/sparse_pool.hpp | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/src/interface/sparse_pool.hpp b/src/interface/sparse_pool.hpp
index ec21b1e390a0..ead32927d5f8 100644
--- a/src/interface/sparse_pool.hpp
+++ b/src/interface/sparse_pool.hpp
@@ -71,6 +71,12 @@ class SparsePool {
              const std::vector<std::vector<std::string>> &component_labels)
       : SparsePool(base_name, metadata, sparse_ids, {}, {}, component_labels, "") {}
 
+  // template on variable type
+  template <typename T, typename... Args>
+  SparsePool(Args &&... args) {
+    SparsePool(T::name(), std::forward<Args>(args)...);
+  }
+
   const std::string &base_name() const { return base_name_; }
   const std::string &controller_base_name() const { return controller_base_name_; }
   const Metadata &shared_metadata() const { return shared_metadata_; }

From c127627e2e49f25c82c7db6202f6cbd04ef37f78 Mon Sep 17 00:00:00 2001
From: jdolence <jcdolence@gmail.com>
Date: Sat, 12 Aug 2023 16:11:13 -0600
Subject: [PATCH 11/36] that didnt work.  try this.

---
 src/interface/sparse_pool.hpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/interface/sparse_pool.hpp b/src/interface/sparse_pool.hpp
index ead32927d5f8..6f9915dac095 100644
--- a/src/interface/sparse_pool.hpp
+++ b/src/interface/sparse_pool.hpp
@@ -73,8 +73,8 @@ class SparsePool {
 
   // template on variable type
   template <typename T, typename... Args>
-  SparsePool(Args &&... args) {
-    SparsePool(T::name(), std::forward<Args>(args)...);
+  static SparsePool MakeSparsePool(Args &&... args) {
+    return SparsePool(T::name(), std::forward<Args>(args)...);
   }
 
   const std::string &base_name() const { return base_name_; }

From c58d82e485ce2f8965aef184c864240ab0110e50 Mon Sep 17 00:00:00 2001
From: jdolence <jcdolence@gmail.com>
Date: Sat, 12 Aug 2023 16:22:12 -0600
Subject: [PATCH 12/36] change name

---
 src/interface/sparse_pool.hpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/interface/sparse_pool.hpp b/src/interface/sparse_pool.hpp
index 6f9915dac095..d6628275051e 100644
--- a/src/interface/sparse_pool.hpp
+++ b/src/interface/sparse_pool.hpp
@@ -73,7 +73,7 @@ class SparsePool {
 
   // template on variable type
   template <typename T, typename... Args>
-  static SparsePool MakeSparsePool(Args &&... args) {
+  static SparsePool Make(Args &&... args) {
     return SparsePool(T::name(), std::forward<Args>(args)...);
   }
 

From 305089b7d7136f203c8bdfe361ecfd1273f6f59d Mon Sep 17 00:00:00 2001
From: jdolence <jcdolence@gmail.com>
Date: Sat, 12 Aug 2023 17:02:14 -0600
Subject: [PATCH 13/36] DataCollection Add functions should returns references

---
 src/interface/data_collection.cpp | 10 +++++-----
 src/interface/data_collection.hpp | 12 ++++++------
 2 files changed, 11 insertions(+), 11 deletions(-)

diff --git a/src/interface/data_collection.cpp b/src/interface/data_collection.cpp
index ad8c28efbd0e..05e2f696f397 100644
--- a/src/interface/data_collection.cpp
+++ b/src/interface/data_collection.cpp
@@ -22,7 +22,7 @@
 namespace parthenon {
 
 template <typename T>
-std::shared_ptr<T>
+std::shared_ptr<T>&
 DataCollection<T>::Add(const std::string &name, const std::shared_ptr<T> &src,
                        const std::vector<std::string> &field_names, const bool shallow) {
   auto it = containers_.find(name);
@@ -41,24 +41,24 @@ DataCollection<T>::Add(const std::string &name, const std::shared_ptr<T> &src,
   return containers_[name];
 }
 template <typename T>
-std::shared_ptr<T> DataCollection<T>::Add(const std::string &name,
+std::shared_ptr<T>& DataCollection<T>::Add(const std::string &name,
                                           const std::shared_ptr<T> &src,
                                           const std::vector<std::string> &field_names) {
   return Add(name, src, field_names, false);
 }
 template <typename T>
-std::shared_ptr<T>
+std::shared_ptr<T>&
 DataCollection<T>::AddShallow(const std::string &name, const std::shared_ptr<T> &src,
                               const std::vector<std::string> &field_names) {
   return Add(name, src, field_names, true);
 }
 template <typename T>
-std::shared_ptr<T> DataCollection<T>::Add(const std::string &name,
+std::shared_ptr<T>& DataCollection<T>::Add(const std::string &name,
                                           const std::shared_ptr<T> &src) {
   return Add(name, src, {}, false);
 }
 template <typename T>
-std::shared_ptr<T> DataCollection<T>::AddShallow(const std::string &name,
+std::shared_ptr<T>& DataCollection<T>::AddShallow(const std::string &name,
                                                  const std::shared_ptr<T> &src) {
   return Add(name, src, {}, true);
 }
diff --git a/src/interface/data_collection.hpp b/src/interface/data_collection.hpp
index 039146afceac..0494f8213e5b 100644
--- a/src/interface/data_collection.hpp
+++ b/src/interface/data_collection.hpp
@@ -38,15 +38,15 @@ class DataCollection {
 
   void SetMeshPointer(Mesh *pmesh) { pmy_mesh_ = pmesh; }
 
-  std::shared_ptr<T> Add(const std::string &label, const std::shared_ptr<T> &src,
+  std::shared_ptr<T>& Add(const std::string &label, const std::shared_ptr<T> &src,
                          const std::vector<std::string> &flags, const bool shallow);
-  std::shared_ptr<T> Add(const std::string &label, const std::shared_ptr<T> &src,
+  std::shared_ptr<T>& Add(const std::string &label, const std::shared_ptr<T> &src,
                          const std::vector<std::string> &flags);
-  std::shared_ptr<T> AddShallow(const std::string &label, const std::shared_ptr<T> &src,
+  std::shared_ptr<T>& AddShallow(const std::string &label, const std::shared_ptr<T> &src,
                                 const std::vector<std::string> &flags);
-  std::shared_ptr<T> Add(const std::string &label, const std::shared_ptr<T> &src);
-  std::shared_ptr<T> AddShallow(const std::string &label, const std::shared_ptr<T> &src);
-  std::shared_ptr<T> Add(const std::string &label) {
+  std::shared_ptr<T>& Add(const std::string &label, const std::shared_ptr<T> &src);
+  std::shared_ptr<T>& AddShallow(const std::string &label, const std::shared_ptr<T> &src);
+  std::shared_ptr<T>& Add(const std::string &label) {
     // error check for duplicate names
     auto it = containers_.find(label);
     if (it != containers_.end()) {

From 56abe2a19a7ec072d73e5adbc3b80feb6a7d724d Mon Sep 17 00:00:00 2001
From: jdolence <jcdolence@gmail.com>
Date: Sun, 13 Aug 2023 09:22:27 -0600
Subject: [PATCH 14/36] formatting

---
 src/interface/data_collection.cpp | 18 +++++++++---------
 src/interface/data_collection.hpp | 18 +++++++++---------
 src/interface/sparse_pool.hpp     |  2 +-
 3 files changed, 19 insertions(+), 19 deletions(-)

diff --git a/src/interface/data_collection.cpp b/src/interface/data_collection.cpp
index 05e2f696f397..fc1b78122cb1 100644
--- a/src/interface/data_collection.cpp
+++ b/src/interface/data_collection.cpp
@@ -22,7 +22,7 @@
 namespace parthenon {
 
 template <typename T>
-std::shared_ptr<T>&
+std::shared_ptr<T> &
 DataCollection<T>::Add(const std::string &name, const std::shared_ptr<T> &src,
                        const std::vector<std::string> &field_names, const bool shallow) {
   auto it = containers_.find(name);
@@ -41,25 +41,25 @@ DataCollection<T>::Add(const std::string &name, const std::shared_ptr<T> &src,
   return containers_[name];
 }
 template <typename T>
-std::shared_ptr<T>& DataCollection<T>::Add(const std::string &name,
-                                          const std::shared_ptr<T> &src,
-                                          const std::vector<std::string> &field_names) {
+std::shared_ptr<T> &DataCollection<T>::Add(const std::string &name,
+                                           const std::shared_ptr<T> &src,
+                                           const std::vector<std::string> &field_names) {
   return Add(name, src, field_names, false);
 }
 template <typename T>
-std::shared_ptr<T>&
+std::shared_ptr<T> &
 DataCollection<T>::AddShallow(const std::string &name, const std::shared_ptr<T> &src,
                               const std::vector<std::string> &field_names) {
   return Add(name, src, field_names, true);
 }
 template <typename T>
-std::shared_ptr<T>& DataCollection<T>::Add(const std::string &name,
-                                          const std::shared_ptr<T> &src) {
+std::shared_ptr<T> &DataCollection<T>::Add(const std::string &name,
+                                           const std::shared_ptr<T> &src) {
   return Add(name, src, {}, false);
 }
 template <typename T>
-std::shared_ptr<T>& DataCollection<T>::AddShallow(const std::string &name,
-                                                 const std::shared_ptr<T> &src) {
+std::shared_ptr<T> &DataCollection<T>::AddShallow(const std::string &name,
+                                                  const std::shared_ptr<T> &src) {
   return Add(name, src, {}, true);
 }
 
diff --git a/src/interface/data_collection.hpp b/src/interface/data_collection.hpp
index 0494f8213e5b..a07cfc01dd63 100644
--- a/src/interface/data_collection.hpp
+++ b/src/interface/data_collection.hpp
@@ -38,15 +38,15 @@ class DataCollection {
 
   void SetMeshPointer(Mesh *pmesh) { pmy_mesh_ = pmesh; }
 
-  std::shared_ptr<T>& Add(const std::string &label, const std::shared_ptr<T> &src,
-                         const std::vector<std::string> &flags, const bool shallow);
-  std::shared_ptr<T>& Add(const std::string &label, const std::shared_ptr<T> &src,
-                         const std::vector<std::string> &flags);
-  std::shared_ptr<T>& AddShallow(const std::string &label, const std::shared_ptr<T> &src,
-                                const std::vector<std::string> &flags);
-  std::shared_ptr<T>& Add(const std::string &label, const std::shared_ptr<T> &src);
-  std::shared_ptr<T>& AddShallow(const std::string &label, const std::shared_ptr<T> &src);
-  std::shared_ptr<T>& Add(const std::string &label) {
+  std::shared_ptr<T> &Add(const std::string &label, const std::shared_ptr<T> &src,
+                          const std::vector<std::string> &flags, const bool shallow);
+  std::shared_ptr<T> &Add(const std::string &label, const std::shared_ptr<T> &src,
+                          const std::vector<std::string> &flags);
+  std::shared_ptr<T> &AddShallow(const std::string &label, const std::shared_ptr<T> &src,
+                                 const std::vector<std::string> &flags);
+  std::shared_ptr<T> &Add(const std::string &label, const std::shared_ptr<T> &src);
+  std::shared_ptr<T> &AddShallow(const std::string &label, const std::shared_ptr<T> &src);
+  std::shared_ptr<T> &Add(const std::string &label) {
     // error check for duplicate names
     auto it = containers_.find(label);
     if (it != containers_.end()) {
diff --git a/src/interface/sparse_pool.hpp b/src/interface/sparse_pool.hpp
index d6628275051e..f8a698d7d09a 100644
--- a/src/interface/sparse_pool.hpp
+++ b/src/interface/sparse_pool.hpp
@@ -73,7 +73,7 @@ class SparsePool {
 
   // template on variable type
   template <typename T, typename... Args>
-  static SparsePool Make(Args &&... args) {
+  static SparsePool Make(Args &&...args) {
     return SparsePool(T::name(), std::forward<Args>(args)...);
   }
 

From 3800f489aa4646d0c0606c8890e63f613ab9b7cd Mon Sep 17 00:00:00 2001
From: jdolence <jcdolence@gmail.com>
Date: Mon, 14 Aug 2023 10:21:47 -0600
Subject: [PATCH 15/36] fix some things to make adding to the mesh_data
 collection directly work

---
 src/interface/data_collection.cpp | 10 +++++++++-
 src/interface/data_collection.hpp |  2 ++
 src/interface/mesh_data.hpp       |  1 +
 src/interface/meshblock_data.cpp  |  1 +
 src/interface/meshblock_data.hpp  | 12 ++++++++----
 src/mesh/mesh.cpp                 |  1 +
 6 files changed, 22 insertions(+), 5 deletions(-)

diff --git a/src/interface/data_collection.cpp b/src/interface/data_collection.cpp
index fc1b78122cb1..2f0da74b5997 100644
--- a/src/interface/data_collection.cpp
+++ b/src/interface/data_collection.cpp
@@ -37,7 +37,15 @@ DataCollection<T>::Add(const std::string &name, const std::shared_ptr<T> &src,
   auto c = std::make_shared<T>();
   c->Copy(src.get(), field_names, shallow);
 
-  containers_[name] = c;
+  Set(name, c);
+
+  if constexpr (std::is_same<T, MeshData<Real>>::value) {
+    for (int b = 0; b < pmy_mesh_->block_list.size(); b++) {
+      auto &mbd = pmy_mesh_->block_list[b]->meshblock_data;
+      mbd.Set(name, c->GetBlockData(b));
+    }
+  }
+
   return containers_[name];
 }
 template <typename T>
diff --git a/src/interface/data_collection.hpp b/src/interface/data_collection.hpp
index a07cfc01dd63..2fe45f1c5d0b 100644
--- a/src/interface/data_collection.hpp
+++ b/src/interface/data_collection.hpp
@@ -69,6 +69,8 @@ class DataCollection {
     return it->second;
   }
 
+  void Set(const std::string &name, std::shared_ptr<T> &d) { containers_[name] = d; }
+
   std::shared_ptr<T> &GetOrAdd(const std::string &mbd_label, const int &partition_id);
 
   void PurgeNonBase() {
diff --git a/src/interface/mesh_data.hpp b/src/interface/mesh_data.hpp
index 7aee69801fe6..cb238061f2c7 100644
--- a/src/interface/mesh_data.hpp
+++ b/src/interface/mesh_data.hpp
@@ -243,6 +243,7 @@ class MeshData {
     const int nblocks = src->NumBlocks();
     block_data_.resize(nblocks);
     for (int i = 0; i < nblocks; i++) {
+      block_data_[i] = std::make_shared<MeshBlockData<T>>();
       block_data_[i]->Copy(src->GetBlockData(i).get(), std::forward<Args>(args)...);
     }
   }
diff --git a/src/interface/meshblock_data.cpp b/src/interface/meshblock_data.cpp
index 2448e19e0302..5aed2a939f7a 100644
--- a/src/interface/meshblock_data.cpp
+++ b/src/interface/meshblock_data.cpp
@@ -78,6 +78,7 @@ template <typename T>
 void MeshBlockData<T>::Copy(const MeshBlockData<T> *src,
                             const std::vector<std::string> &names,
                             const bool shallow_copy) {
+  assert(src != nullptr);
   SetBlockPointer(src);
   resolved_packages_ = src->resolved_packages_;
 
diff --git a/src/interface/meshblock_data.hpp b/src/interface/meshblock_data.hpp
index 429aea08a003..6f76c071e661 100644
--- a/src/interface/meshblock_data.hpp
+++ b/src/interface/meshblock_data.hpp
@@ -93,12 +93,16 @@ class MeshBlockData {
             const bool shallow);
 
   /// Set the pointer to the mesh block for this container
-  void SetBlockPointer(std::weak_ptr<MeshBlock> pmb) { pmy_block = pmb; }
+  void SetBlockPointer(std::weak_ptr<MeshBlock> pmb) { pmy_block = pmb.lock(); }
   void SetBlockPointer(const std::shared_ptr<MeshBlockData<T>> &other) {
-    SetBlockPointer(*other);
+    SetBlockPointer(other.get());
+  }
+  void SetBlockPointer(const MeshBlockData<T> &other) {
+    pmy_block = other.GetBlockPointer();
+  }
+  void SetBlockPointer(const MeshBlockData<T> *other) {
+    pmy_block = other->GetBlockPointer();
   }
-  void SetBlockPointer(const MeshBlockData<T> &other) { pmy_block = other.pmy_block; }
-  void SetBlockPointer(const MeshBlockData<T> *other) { pmy_block = other->pmy_block; }
 
   void Initialize(const std::shared_ptr<StateDescriptor> resolved_packages,
                   const std::shared_ptr<MeshBlock> pmb);
diff --git a/src/mesh/mesh.cpp b/src/mesh/mesh.cpp
index 1a86309da05e..833ef1b1a953 100644
--- a/src/mesh/mesh.cpp
+++ b/src/mesh/mesh.cpp
@@ -492,6 +492,7 @@ Mesh::Mesh(ParameterInput *pin, ApplicationInput *app_in, Packages_t &packages,
                         packages, resolved_packages, gflag);
     block_list[i - nbs]->SearchAndSetNeighbors(tree, ranklist.data(), nslist.data());
   }
+  mesh_data.Get()->Set(block_list, "base");
 
   ResetLoadBalanceVariables();
 

From 2c828fdcb85c3f076cda3680cdf95ac0e802b7da Mon Sep 17 00:00:00 2001
From: Luke Roberts <lfroberts@lanl.gov>
Date: Mon, 14 Aug 2023 15:26:13 -0600
Subject: [PATCH 16/36] Add MakePackDescriptor for uids

---
 src/interface/make_pack_descriptor.hpp |  18 ++++
 src/interface/sparse_pack_base.cpp     |  44 +++-------
 src/interface/sparse_pack_base.hpp     | 110 ++++++++++++++++++-------
 3 files changed, 108 insertions(+), 64 deletions(-)

diff --git a/src/interface/make_pack_descriptor.hpp b/src/interface/make_pack_descriptor.hpp
index dc51b088b78a..bd838e37363f 100644
--- a/src/interface/make_pack_descriptor.hpp
+++ b/src/interface/make_pack_descriptor.hpp
@@ -31,6 +31,7 @@
 #include "interface/metadata.hpp"
 #include "interface/sparse_pack.hpp"
 #include "interface/state_descriptor.hpp"
+#include "interface/variable.hpp"
 #include "mesh/mesh.hpp"
 
 namespace parthenon {
@@ -116,6 +117,23 @@ inline auto MakePackDescriptor(
   return MakePackDescriptor(psd, vars, use_regex, flags, options);
 }
 
+inline auto MakePackDescriptor(StateDescriptor *psd, const std::vector<Uid_t> &var_ids,
+                               const std::vector<MetadataFlag> &flags = {},
+                               const std::set<PDOpt> &options = {}) {
+  auto selector = [&](int vidx, const VarID &id, const Metadata &md) {
+    if (flags.size() > 0) {
+      for (const auto &flag : flags) {
+        if (!md.IsSet(flag)) return false;
+      }
+    }
+    if (Variable<Real>::GetUniqueID(id.label()) == var_ids[vidx]) return true;
+    return false;
+  };
+
+  impl::PackDescriptor base_desc(psd, var_ids, selector, options);
+  return typename SparsePack<>::Descriptor(base_desc);
+}
+
 } // namespace parthenon
 
 #endif // INTERFACE_MAKE_PACK_DESCRIPTOR_HPP_
diff --git a/src/interface/sparse_pack_base.cpp b/src/interface/sparse_pack_base.cpp
index aa8384e7635c..e138f77e29aa 100644
--- a/src/interface/sparse_pack_base.cpp
+++ b/src/interface/sparse_pack_base.cpp
@@ -31,40 +31,6 @@
 #include "utils/utils.hpp"
 namespace parthenon {
 namespace impl {
-PackDescriptor::PackDescriptor(StateDescriptor *psd,
-                               const std::vector<std::string> &var_group_names,
-                               const SelectorFunction_t &selector,
-                               const std::set<PDOpt> &options)
-    : nvar_groups(var_group_names.size()), var_group_names(var_group_names),
-      var_groups(BuildUids(var_group_names.size(), psd, selector)),
-      with_fluxes(options.count(PDOpt::WithFluxes)), coarse(options.count(PDOpt::Coarse)),
-      flat(options.count(PDOpt::Flatten)) {
-  PARTHENON_REQUIRE(!(with_fluxes && coarse),
-                    "Probably shouldn't be making a coarse pack with fine fluxes.");
-}
-
-std::vector<PackDescriptor::VariableGroup_t>
-PackDescriptor::BuildUids(int nvgs, const StateDescriptor *const psd,
-                          const SelectorFunction_t &selector) {
-  auto fields = psd->AllFields();
-  std::vector<VariableGroup_t> vgs(nvgs);
-  for (auto [id, md] : fields) {
-    for (int i = 0; i < nvgs; ++i) {
-      if (selector(i, id, md)) {
-        vgs[i].push_back({id, Variable<Real>::GetUniqueID(id.label())});
-      }
-    }
-  }
-  // Ensure ordering in terms of value of sparse indices
-  for (auto &vg : vgs) {
-    std::sort(vg.begin(), vg.end(), [](const auto &a, const auto &b) {
-      if (a.first.base_name == b.first.base_name)
-        return a.first.sparse_id < b.first.sparse_id;
-      return a.first.base_name < b.first.base_name;
-    });
-  }
-  return vgs;
-}
 
 void PackDescriptor::Print() const {
   printf("--------------------\n");
@@ -100,6 +66,16 @@ namespace parthenon {
 
 using namespace impl;
 
+SparsePackIdxMap SparsePackBase::GetIdxMap(const impl::PackDescriptor &desc) {
+  SparsePackIdxMap map;
+  std::size_t idx = 0;
+  for (const auto &var : desc.var_group_names) {
+    map[var] = idx;
+    ++idx;
+  }
+  return map;
+}
+
 template <class T>
 SparsePackBase::alloc_t SparsePackBase::GetAllocStatus(T *pmd,
                                                        const PackDescriptor &desc) {
diff --git a/src/interface/sparse_pack_base.hpp b/src/interface/sparse_pack_base.hpp
index c17882bd7ba9..e7b17d9cc622 100644
--- a/src/interface/sparse_pack_base.hpp
+++ b/src/interface/sparse_pack_base.hpp
@@ -33,6 +33,9 @@
 
 namespace parthenon {
 class SparsePackCache;
+namespace impl {
+class PackDescriptor; 
+}
 
 // Map for going from variable names to sparse pack variable indices
 using SparsePackIdxMap = std::unordered_map<std::string, std::size_t>;
@@ -41,28 +44,7 @@ class StateDescriptor;
 
 enum class PDOpt { WithFluxes, Coarse, Flatten };
 
-namespace impl {
-struct PackDescriptor {
-  using VariableGroup_t = std::vector<std::pair<VarID, Uid_t>>;
-  using SelectorFunction_t = std::function<bool(int, const VarID &, const Metadata &)>;
-
-  PackDescriptor(StateDescriptor *psd, const std::vector<std::string> &var_group_names,
-                 const SelectorFunction_t &selector, const std::set<PDOpt> &options);
-
-  void Print() const;
-
-  const int nvar_groups;
-  const std::vector<std::string> var_group_names;
-  const std::vector<VariableGroup_t> var_groups;
-  const bool with_fluxes;
-  const bool coarse;
-  const bool flat;
 
- private:
-  std::vector<VariableGroup_t> BuildUids(int nvgs, const StateDescriptor *const psd,
-                                         const SelectorFunction_t &selector);
-};
-} // namespace impl
 
 class SparsePackBase {
  public:
@@ -87,15 +69,7 @@ class SparsePackBase {
   }
 
   // Return a map from variable names to pack variable indices
-  static SparsePackIdxMap GetIdxMap(const impl::PackDescriptor &desc) {
-    SparsePackIdxMap map;
-    std::size_t idx = 0;
-    for (const auto &var : desc.var_group_names) {
-      map[var] = idx;
-      ++idx;
-    }
-    return map;
-  }
+  static SparsePackIdxMap GetIdxMap(const impl::PackDescriptor &desc);
 
   // Get a list of booleans of the allocation status of every variable in pmd matching the
   // PackDescriptor desc
@@ -146,6 +120,82 @@ class SparsePackCache {
   friend class SparsePackBase;
 };
 
+namespace impl {
+struct PackDescriptor {
+  using VariableGroup_t = std::vector<std::pair<VarID, Uid_t>>;
+  using SelectorFunction_t = std::function<bool(int, const VarID &, const Metadata &)>;
+  using SelectorFunctionUid_t = std::function<bool(int, const Uid_t &, const Metadata &)>;
+  
+  void Print() const;
+   
+  template <class GROUP_t, class SELECTOR_t> 
+  PackDescriptor(StateDescriptor *psd,
+                                 const std::vector<GROUP_t> &var_groups_in,
+                                 const SELECTOR_t &selector,
+                                 const std::set<PDOpt> &options)
+      : nvar_groups(var_groups_in.size()), var_group_names(MakeGroupNames(var_groups_in)),
+        var_groups(BuildUids(var_groups_in.size(), psd, selector)),
+        with_fluxes(options.count(PDOpt::WithFluxes)), coarse(options.count(PDOpt::Coarse)),
+        flat(options.count(PDOpt::Flatten)) {
+    PARTHENON_REQUIRE(!(with_fluxes && coarse),
+                      "Probably shouldn't be making a coarse pack with fine fluxes.");
+  }
+
+  const int nvar_groups;
+  const std::vector<std::string> var_group_names;
+  const std::vector<VariableGroup_t> var_groups;
+  const bool with_fluxes;
+  const bool coarse;
+  const bool flat;
+
+ private:
+  
+  template <class FUNC_t>
+  std::vector<PackDescriptor::VariableGroup_t>
+  BuildUids(int nvgs, const StateDescriptor *const psd,
+                            const FUNC_t &selector) {
+    auto fields = psd->AllFields();
+    std::vector<VariableGroup_t> vgs(nvgs);
+    for (auto [id, md] : fields) {
+      for (int i = 0; i < nvgs; ++i) {
+        auto uid = Variable<Real>::GetUniqueID(id.label());
+        if constexpr (std::is_invocable<FUNC_t, int, VarID, Metadata>::value) {  
+          if (selector(i, id, md)) {
+            vgs[i].push_back({id, uid});
+          }
+        } else if constexpr (std::is_invocable<FUNC_t, int, Uid_t, Metadata>::value) {  
+          if (selector(i, uid, md)) {
+            vgs[i].push_back({id, uid});
+          }
+        } else { 
+          PARTHENON_FAIL("Passing the wrong sort of selector.");
+        }
+      }
+    }
+    // Ensure ordering in terms of value of sparse indices
+    for (auto &vg : vgs) {
+      std::sort(vg.begin(), vg.end(), [](const auto &a, const auto &b) {
+        if (a.first.base_name == b.first.base_name)
+          return a.first.sparse_id < b.first.sparse_id;
+        return a.first.base_name < b.first.base_name;
+      });
+    }
+    return vgs;
+  }
+
+  template <class base_t> 
+  std::vector<std::string> MakeGroupNames(const std::vector<base_t> &var_groups) {
+    if constexpr (std::is_same<base_t, std::string>::value) { 
+      return var_groups; 
+    } else if constexpr (std::is_same<base_t, Uid_t>::value) { 
+      std::vector<std::string> var_group_names; 
+      for (auto &vg : var_groups) var_group_names.push_back(std::to_string(vg)); 
+      return var_group_names;
+    }
+  }
+};
+} // namespace impl
+
 } // namespace parthenon
 
 #endif // INTERFACE_SPARSE_PACK_BASE_HPP_

From 7dff9e443ad22e67e459c4e9a0485a4447ae423f Mon Sep 17 00:00:00 2001
From: jdolence <jcdolence@gmail.com>
Date: Mon, 14 Aug 2023 10:36:15 -0600
Subject: [PATCH 17/36] fix style

---
 src/interface/sparse_pool.hpp | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/interface/sparse_pool.hpp b/src/interface/sparse_pool.hpp
index f8a698d7d09a..0b029e8c2eb7 100644
--- a/src/interface/sparse_pool.hpp
+++ b/src/interface/sparse_pool.hpp
@@ -16,6 +16,7 @@
 #include <map>
 #include <string>
 #include <type_traits>
+#include <utility>
 #include <vector>
 
 #include "metadata.hpp"

From 097d0bb8aec321004aa88c0f654db7d1fa24b2a3 Mon Sep 17 00:00:00 2001
From: jdolence <jcdolence@gmail.com>
Date: Mon, 14 Aug 2023 13:25:04 -0600
Subject: [PATCH 18/36] compute uid intersections of variables in different
 MeshBlockData classes

---
 src/interface/meshblock_data.hpp | 21 +++++++++++++++++++++
 src/utils/unique_id.cpp          | 15 ++++++++++++++-
 src/utils/unique_id.hpp          |  3 +++
 3 files changed, 38 insertions(+), 1 deletion(-)

diff --git a/src/interface/meshblock_data.hpp b/src/interface/meshblock_data.hpp
index 6f76c071e661..56123cc2dba2 100644
--- a/src/interface/meshblock_data.hpp
+++ b/src/interface/meshblock_data.hpp
@@ -26,6 +26,7 @@
 #include "interface/variable_pack.hpp"
 #include "mesh/domain.hpp"
 #include "utils/error_checking.hpp"
+#include "utils/unique_id.hpp"
 
 namespace parthenon {
 
@@ -187,6 +188,18 @@ class MeshBlockData {
     return GetVariablesByFlag(Metadata::FlagCollection(), sparse_ids);
   }
 
+  std::vector<Uid_t> GetVariableUIDs(const std::vector<std::string> &names,
+                                    const std::vector<int> &sparse_ids = {}) {
+    return GetVariablesByName(names, sparse_ids).unique_ids();
+  }
+  std::vector<Uid_t> GetVariableUIDs(const Metadata::FlagCollection &flags,
+                                    const std::vector<int> &sparse_ids = {}) {
+    return GetVariablesByFlag(flags, sparse_ids).unique_ids();
+  }
+  std::vector<Uid_t> GetVariableUIDs(const std::vector<int> &sparse_ids = {}) {
+    return GetAllVariables(sparse_ids).unique_ids();
+  }
+
   /// Queries related to variable packs
   /// For all of these functions, vmap and key are optional output parameters, they will
   /// be set if not null.
@@ -643,6 +656,14 @@ class MeshBlockData {
                                            vpack_types::VPackKey_t *key);
 };
 
+template <typename T, typename... Args>
+std::vector<Uid_t> UidIntersection(MeshBlockData<T> *mbd1, MeshBlockData<T> *mbd2,
+                                   Args &&... args) {
+  auto u1 = mbd1->GetVariableUIDs(std::forward<Args>(args)...);
+  auto u2 = mbd2->GetVariableUIDs(std::forward<Args>(args)...);
+  return UidIntersection(u1, u2);
+}
+
 } // namespace parthenon
 
 #endif // INTERFACE_MESHBLOCK_DATA_HPP_
diff --git a/src/utils/unique_id.cpp b/src/utils/unique_id.cpp
index 7cec7afa6fec..4d493c07a4ad 100644
--- a/src/utils/unique_id.cpp
+++ b/src/utils/unique_id.cpp
@@ -11,9 +11,22 @@
 // the public, perform publicly and display publicly, and to permit others to do so.
 //========================================================================================
 
-#include "unique_id.hpp"
+#include <algorithm>
 #include <string>
+#include <vector>
+
+#include "unique_id.hpp"
 
 namespace parthenon {
 template class UniqueIDGenerator<std::string>;
+
+std::vector<Uid_t> UidIntersection(std::vector<Uid_t> v1, std::vector<Uid_t> v2) {
+  std::vector<Uid_t> vout;
+  std::sort(v1.begin(), v1.end());
+  std::sort(v2.begin(), v2.end());
+  std::set_intersection(v1.begin(), v1.end(), v2.begin(), v2.end(),
+                        std::back_inserter(vout));
+  return vout;
+}
+
 } // namespace parthenon
diff --git a/src/utils/unique_id.hpp b/src/utils/unique_id.hpp
index c92f5256289e..a0168297b479 100644
--- a/src/utils/unique_id.hpp
+++ b/src/utils/unique_id.hpp
@@ -38,6 +38,9 @@ class UniqueIDGenerator {
  private:
   std::unordered_map<T, Uid_t> uids_;
 };
+
+std::vector<Uid_t> UidIntersection(std::vector<Uid_t> v1, std::vector<Uid_t> v2);
+
 } // namespace parthenon
 
 #endif // UTILS_UNIQUE_ID_HPP_

From 19bc17b30b944657f608280170ddfb8b58f7af33 Mon Sep 17 00:00:00 2001
From: jdolence <jcdolence@gmail.com>
Date: Mon, 14 Aug 2023 15:41:38 -0600
Subject: [PATCH 19/36] add a MeshData intersection function

---
 src/interface/mesh_data.hpp | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/src/interface/mesh_data.hpp b/src/interface/mesh_data.hpp
index cb238061f2c7..5c17ce80eb71 100644
--- a/src/interface/mesh_data.hpp
+++ b/src/interface/mesh_data.hpp
@@ -31,6 +31,7 @@
 #include "utils/communication_buffer.hpp"
 #include "utils/error_checking.hpp"
 #include "utils/object_pool.hpp"
+#include "utils/unique_id.hpp"
 #include "utils/utils.hpp"
 
 namespace parthenon {
@@ -436,6 +437,13 @@ class MeshData {
   BvarsCache_t bvars_cache_;
 };
 
+template <typename T, typename... Args>
+std::vector<Uid_t> UidIntersection(MeshData<T> *md1, MeshData<T> *md2, Args &&... args) {
+  auto u1 = md1->GetBlockData(0)->GetVariableUIDs(std::forward<Args>(args)...);
+  auto u2 = md2->GetBlockData(0)->GetVariableUIDs(std::forward<Args>(args)...);
+  return UidIntersection(u1, u2);
+}
+
 } // namespace parthenon
 
 #endif // INTERFACE_MESH_DATA_HPP_

From 61e345ac87d3f312db280870639a05b745ee6239 Mon Sep 17 00:00:00 2001
From: jdolence <jcdolence@gmail.com>
Date: Mon, 14 Aug 2023 16:19:51 -0600
Subject: [PATCH 20/36] a little hack to avoid calling a member of
 StateDescriptor from the header

---
 src/interface/sparse_pack_base.cpp | 6 ++++++
 src/interface/sparse_pack_base.hpp | 5 ++++-
 2 files changed, 10 insertions(+), 1 deletion(-)

diff --git a/src/interface/sparse_pack_base.cpp b/src/interface/sparse_pack_base.cpp
index e138f77e29aa..315b7db8282f 100644
--- a/src/interface/sparse_pack_base.cpp
+++ b/src/interface/sparse_pack_base.cpp
@@ -313,4 +313,10 @@ std::string SparsePackCache::GetIdentifier(const PackDescriptor &desc) const {
   return identifier;
 }
 
+namespace impl {
+const SD_MetadataMap_t &PackDescriptor::GetAllFields(const StateDescriptor *const psd) const {
+  return psd->AllFields();
+}
+}
+
 } // namespace parthenon
diff --git a/src/interface/sparse_pack_base.hpp b/src/interface/sparse_pack_base.hpp
index e7b17d9cc622..d88246d54b85 100644
--- a/src/interface/sparse_pack_base.hpp
+++ b/src/interface/sparse_pack_base.hpp
@@ -39,6 +39,7 @@ class PackDescriptor;
 
 // Map for going from variable names to sparse pack variable indices
 using SparsePackIdxMap = std::unordered_map<std::string, std::size_t>;
+using SD_MetadataMap_t = std::unordered_map<VarID, Metadata, VarIDHasher>;
 
 class StateDescriptor;
 
@@ -154,7 +155,7 @@ struct PackDescriptor {
   std::vector<PackDescriptor::VariableGroup_t>
   BuildUids(int nvgs, const StateDescriptor *const psd,
                             const FUNC_t &selector) {
-    auto fields = psd->AllFields();
+    const auto fields = GetAllFields(psd);
     std::vector<VariableGroup_t> vgs(nvgs);
     for (auto [id, md] : fields) {
       for (int i = 0; i < nvgs; ++i) {
@@ -183,6 +184,8 @@ struct PackDescriptor {
     return vgs;
   }
 
+  const SD_MetadataMap_t &GetAllFields(const StateDescriptor *const psd) const;
+
   template <class base_t> 
   std::vector<std::string> MakeGroupNames(const std::vector<base_t> &var_groups) {
     if constexpr (std::is_same<base_t, std::string>::value) { 

From 346e86486001fbdd738452451b9fa24c8558a1ab Mon Sep 17 00:00:00 2001
From: jdolence <jcdolence@gmail.com>
Date: Mon, 14 Aug 2023 16:34:56 -0600
Subject: [PATCH 21/36] get rid of the hack

---
 src/amr_criteria/amr_criteria.hpp  | 3 ++-
 src/interface/sparse_pack_base.cpp | 6 ------
 src/interface/sparse_pack_base.hpp | 6 ++----
 3 files changed, 4 insertions(+), 11 deletions(-)

diff --git a/src/amr_criteria/amr_criteria.hpp b/src/amr_criteria/amr_criteria.hpp
index 411ef0e90e18..42fcb6d65502 100644
--- a/src/amr_criteria/amr_criteria.hpp
+++ b/src/amr_criteria/amr_criteria.hpp
@@ -17,12 +17,13 @@
 #include <string>
 
 #include "defs.hpp"
-#include "interface/meshblock_data.hpp"
+//#include "interface/meshblock_data.hpp"
 #include "mesh/domain.hpp"
 
 namespace parthenon {
 
 class ParameterInput;
+template <class> class MeshBlockData;
 
 struct AMRBounds {
   AMRBounds(const IndexRange &ib, const IndexRange &jb, const IndexRange &kb)
diff --git a/src/interface/sparse_pack_base.cpp b/src/interface/sparse_pack_base.cpp
index 315b7db8282f..e138f77e29aa 100644
--- a/src/interface/sparse_pack_base.cpp
+++ b/src/interface/sparse_pack_base.cpp
@@ -313,10 +313,4 @@ std::string SparsePackCache::GetIdentifier(const PackDescriptor &desc) const {
   return identifier;
 }
 
-namespace impl {
-const SD_MetadataMap_t &PackDescriptor::GetAllFields(const StateDescriptor *const psd) const {
-  return psd->AllFields();
-}
-}
-
 } // namespace parthenon
diff --git a/src/interface/sparse_pack_base.hpp b/src/interface/sparse_pack_base.hpp
index d88246d54b85..1249b890714f 100644
--- a/src/interface/sparse_pack_base.hpp
+++ b/src/interface/sparse_pack_base.hpp
@@ -27,6 +27,7 @@
 #include <vector>
 
 #include "coordinates/coordinates.hpp"
+#include "interface/state_descriptor.hpp"
 #include "interface/variable.hpp"
 #include "interface/variable_state.hpp"
 #include "utils/utils.hpp"
@@ -39,7 +40,6 @@ class PackDescriptor;
 
 // Map for going from variable names to sparse pack variable indices
 using SparsePackIdxMap = std::unordered_map<std::string, std::size_t>;
-using SD_MetadataMap_t = std::unordered_map<VarID, Metadata, VarIDHasher>;
 
 class StateDescriptor;
 
@@ -155,7 +155,7 @@ struct PackDescriptor {
   std::vector<PackDescriptor::VariableGroup_t>
   BuildUids(int nvgs, const StateDescriptor *const psd,
                             const FUNC_t &selector) {
-    const auto fields = GetAllFields(psd);
+    auto fields = psd->AllFields();
     std::vector<VariableGroup_t> vgs(nvgs);
     for (auto [id, md] : fields) {
       for (int i = 0; i < nvgs; ++i) {
@@ -184,8 +184,6 @@ struct PackDescriptor {
     return vgs;
   }
 
-  const SD_MetadataMap_t &GetAllFields(const StateDescriptor *const psd) const;
-
   template <class base_t> 
   std::vector<std::string> MakeGroupNames(const std::vector<base_t> &var_groups) {
     if constexpr (std::is_same<base_t, std::string>::value) { 

From 53e9f06bd448fba88ed9e2dd7b947c9771d97941 Mon Sep 17 00:00:00 2001
From: jdolence <jcdolence@gmail.com>
Date: Mon, 14 Aug 2023 16:38:17 -0600
Subject: [PATCH 22/36] expose Uid_t in the driver/prelude namespace

---
 src/parthenon/driver.hpp | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/parthenon/driver.hpp b/src/parthenon/driver.hpp
index 38a93f53a023..9fd8c3e8ea3e 100644
--- a/src/parthenon/driver.hpp
+++ b/src/parthenon/driver.hpp
@@ -31,6 +31,7 @@
 #include <tasks/task_types.hpp>
 #include <utils/partition_stl_containers.hpp>
 #include <utils/reductions.hpp>
+#include <utils/unique_id.hpp>
 
 // Local Includes
 #include "prelude.hpp"
@@ -70,6 +71,7 @@ using ::parthenon::TaskStatus;
 using ::parthenon::TaskType;
 using ::parthenon::DriverUtils::ConstructAndExecuteBlockTasks;
 using ::parthenon::DriverUtils::ConstructAndExecuteTaskLists;
+using ::parthenon::Uid_t;
 
 namespace partition {
 using ::parthenon::partition::Partition_t;

From 85e0d672059712f7836ef728a2c21c6c6836c8af Mon Sep 17 00:00:00 2001
From: jdolence <jcdolence@gmail.com>
Date: Tue, 15 Aug 2023 08:27:16 -0600
Subject: [PATCH 23/36] treat shallow copies differently for sparse allocation

---
 src/interface/meshblock_data.cpp | 1 +
 src/interface/meshblock_data.hpp | 3 +++
 src/mesh/meshblock.cpp           | 2 +-
 3 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/src/interface/meshblock_data.cpp b/src/interface/meshblock_data.cpp
index 5aed2a939f7a..af23a1e8a8d6 100644
--- a/src/interface/meshblock_data.cpp
+++ b/src/interface/meshblock_data.cpp
@@ -81,6 +81,7 @@ void MeshBlockData<T>::Copy(const MeshBlockData<T> *src,
   assert(src != nullptr);
   SetBlockPointer(src);
   resolved_packages_ = src->resolved_packages_;
+  is_shallow_ = shallow_copy;
 
   auto add_var = [=](auto var) {
     if (shallow_copy || var->IsSet(Metadata::OneCopy)) {
diff --git a/src/interface/meshblock_data.hpp b/src/interface/meshblock_data.hpp
index 56123cc2dba2..0221e0d6eaab 100644
--- a/src/interface/meshblock_data.hpp
+++ b/src/interface/meshblock_data.hpp
@@ -403,6 +403,8 @@ class MeshBlockData {
     return all_initialized;
   }
 
+  bool IsShallow() const { return is_shallow_; }
+
  private:
   void AddField(const std::string &base_name, const Metadata &metadata,
                 int sparse_id = InvalidSparseID);
@@ -455,6 +457,7 @@ class MeshBlockData {
 
   std::weak_ptr<MeshBlock> pmy_block;
   std::shared_ptr<StateDescriptor> resolved_packages_;
+  bool is_shallow_ = false;
 
   VariableVector<T> varVector_; ///< the saved variable array
   std::map<Uid_t, std::shared_ptr<Variable<T>>> varUidMap_;
diff --git a/src/mesh/meshblock.cpp b/src/mesh/meshblock.cpp
index 9dbaaef4e6f0..fe91988d1e3c 100644
--- a/src/mesh/meshblock.cpp
+++ b/src/mesh/meshblock.cpp
@@ -291,7 +291,7 @@ void MeshBlock::AllocateSparse(std::string const &label, bool only_control,
 
       auto v = stage.second->GetVarPtr(l);
 
-      if (v->IsSet(Metadata::OneCopy)) {
+      if (v->IsSet(Metadata::OneCopy) || stage.second->IsShallow()) {
         // nothing to do, we already allocated variable on base stage, and all other
         // stages share that variable
         continue;

From 0d286a3a0691c4a8c7d0008252d290ec66708690 Mon Sep 17 00:00:00 2001
From: jdolence <jcdolence@gmail.com>
Date: Tue, 15 Aug 2023 09:58:00 -0600
Subject: [PATCH 24/36] only allocate/deallocate sparse if it exists in the
 MeshBlockData object

---
 src/mesh/meshblock.cpp | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/src/mesh/meshblock.cpp b/src/mesh/meshblock.cpp
index fe91988d1e3c..c96282e3d9a3 100644
--- a/src/mesh/meshblock.cpp
+++ b/src/mesh/meshblock.cpp
@@ -289,6 +289,8 @@ void MeshBlock::AllocateSparse(std::string const &label, bool only_control,
         continue;
       }
 
+      if (!stage.second->HasVariable(l)) continue;
+
       auto v = stage.second->GetVarPtr(l);
 
       if (v->IsSet(Metadata::OneCopy) || stage.second->IsShallow()) {
@@ -327,7 +329,9 @@ void MeshBlock::DeallocateSparse(std::string const &label) {
   auto &mbd = meshblock_data;
   auto DeallocateVar = [&mbd](const std::string &l) {
     for (auto stage : mbd.Stages()) {
-      stage.second->DeallocateSparse(l);
+      if (!stage.second->IsShallow() && stage.second->HasVariable(l)) {
+        stage.second->DeallocateSparse(l);
+      }
     }
   };
 

From c153a904da586858db891b891198bbd263b45a5a Mon Sep 17 00:00:00 2001
From: jdolence <jcdolence@gmail.com>
Date: Tue, 15 Aug 2023 10:22:06 -0600
Subject: [PATCH 25/36] formatting and style

---
 src/amr_criteria/amr_criteria.hpp  |  3 ++-
 src/interface/mesh_data.hpp        |  2 +-
 src/interface/meshblock_data.hpp   |  6 ++---
 src/interface/sparse_pack_base.hpp | 43 +++++++++++++-----------------
 src/parthenon/driver.hpp           |  2 +-
 src/utils/unique_id.hpp            |  1 +
 6 files changed, 27 insertions(+), 30 deletions(-)

diff --git a/src/amr_criteria/amr_criteria.hpp b/src/amr_criteria/amr_criteria.hpp
index 42fcb6d65502..86b09c9728a1 100644
--- a/src/amr_criteria/amr_criteria.hpp
+++ b/src/amr_criteria/amr_criteria.hpp
@@ -23,7 +23,8 @@
 namespace parthenon {
 
 class ParameterInput;
-template <class> class MeshBlockData;
+template <class>
+class MeshBlockData;
 
 struct AMRBounds {
   AMRBounds(const IndexRange &ib, const IndexRange &jb, const IndexRange &kb)
diff --git a/src/interface/mesh_data.hpp b/src/interface/mesh_data.hpp
index 5c17ce80eb71..614c72656a27 100644
--- a/src/interface/mesh_data.hpp
+++ b/src/interface/mesh_data.hpp
@@ -438,7 +438,7 @@ class MeshData {
 };
 
 template <typename T, typename... Args>
-std::vector<Uid_t> UidIntersection(MeshData<T> *md1, MeshData<T> *md2, Args &&... args) {
+std::vector<Uid_t> UidIntersection(MeshData<T> *md1, MeshData<T> *md2, Args &&...args) {
   auto u1 = md1->GetBlockData(0)->GetVariableUIDs(std::forward<Args>(args)...);
   auto u2 = md2->GetBlockData(0)->GetVariableUIDs(std::forward<Args>(args)...);
   return UidIntersection(u1, u2);
diff --git a/src/interface/meshblock_data.hpp b/src/interface/meshblock_data.hpp
index 0221e0d6eaab..4ca32223efae 100644
--- a/src/interface/meshblock_data.hpp
+++ b/src/interface/meshblock_data.hpp
@@ -189,11 +189,11 @@ class MeshBlockData {
   }
 
   std::vector<Uid_t> GetVariableUIDs(const std::vector<std::string> &names,
-                                    const std::vector<int> &sparse_ids = {}) {
+                                     const std::vector<int> &sparse_ids = {}) {
     return GetVariablesByName(names, sparse_ids).unique_ids();
   }
   std::vector<Uid_t> GetVariableUIDs(const Metadata::FlagCollection &flags,
-                                    const std::vector<int> &sparse_ids = {}) {
+                                     const std::vector<int> &sparse_ids = {}) {
     return GetVariablesByFlag(flags, sparse_ids).unique_ids();
   }
   std::vector<Uid_t> GetVariableUIDs(const std::vector<int> &sparse_ids = {}) {
@@ -661,7 +661,7 @@ class MeshBlockData {
 
 template <typename T, typename... Args>
 std::vector<Uid_t> UidIntersection(MeshBlockData<T> *mbd1, MeshBlockData<T> *mbd2,
-                                   Args &&... args) {
+                                   Args &&...args) {
   auto u1 = mbd1->GetVariableUIDs(std::forward<Args>(args)...);
   auto u2 = mbd2->GetVariableUIDs(std::forward<Args>(args)...);
   return UidIntersection(u1, u2);
diff --git a/src/interface/sparse_pack_base.hpp b/src/interface/sparse_pack_base.hpp
index 1249b890714f..732862e307d9 100644
--- a/src/interface/sparse_pack_base.hpp
+++ b/src/interface/sparse_pack_base.hpp
@@ -35,7 +35,7 @@
 namespace parthenon {
 class SparsePackCache;
 namespace impl {
-class PackDescriptor; 
+class PackDescriptor;
 }
 
 // Map for going from variable names to sparse pack variable indices
@@ -45,8 +45,6 @@ class StateDescriptor;
 
 enum class PDOpt { WithFluxes, Coarse, Flatten };
 
-
-
 class SparsePackBase {
  public:
   SparsePackBase() = default;
@@ -126,18 +124,16 @@ struct PackDescriptor {
   using VariableGroup_t = std::vector<std::pair<VarID, Uid_t>>;
   using SelectorFunction_t = std::function<bool(int, const VarID &, const Metadata &)>;
   using SelectorFunctionUid_t = std::function<bool(int, const Uid_t &, const Metadata &)>;
-  
+
   void Print() const;
-   
-  template <class GROUP_t, class SELECTOR_t> 
-  PackDescriptor(StateDescriptor *psd,
-                                 const std::vector<GROUP_t> &var_groups_in,
-                                 const SELECTOR_t &selector,
-                                 const std::set<PDOpt> &options)
+
+  template <class GROUP_t, class SELECTOR_t>
+  PackDescriptor(StateDescriptor *psd, const std::vector<GROUP_t> &var_groups_in,
+                 const SELECTOR_t &selector, const std::set<PDOpt> &options)
       : nvar_groups(var_groups_in.size()), var_group_names(MakeGroupNames(var_groups_in)),
         var_groups(BuildUids(var_groups_in.size(), psd, selector)),
-        with_fluxes(options.count(PDOpt::WithFluxes)), coarse(options.count(PDOpt::Coarse)),
-        flat(options.count(PDOpt::Flatten)) {
+        with_fluxes(options.count(PDOpt::WithFluxes)),
+        coarse(options.count(PDOpt::Coarse)), flat(options.count(PDOpt::Flatten)) {
     PARTHENON_REQUIRE(!(with_fluxes && coarse),
                       "Probably shouldn't be making a coarse pack with fine fluxes.");
   }
@@ -150,25 +146,23 @@ struct PackDescriptor {
   const bool flat;
 
  private:
-  
   template <class FUNC_t>
   std::vector<PackDescriptor::VariableGroup_t>
-  BuildUids(int nvgs, const StateDescriptor *const psd,
-                            const FUNC_t &selector) {
+  BuildUids(int nvgs, const StateDescriptor *const psd, const FUNC_t &selector) {
     auto fields = psd->AllFields();
     std::vector<VariableGroup_t> vgs(nvgs);
     for (auto [id, md] : fields) {
       for (int i = 0; i < nvgs; ++i) {
         auto uid = Variable<Real>::GetUniqueID(id.label());
-        if constexpr (std::is_invocable<FUNC_t, int, VarID, Metadata>::value) {  
+        if constexpr (std::is_invocable<FUNC_t, int, VarID, Metadata>::value) {
           if (selector(i, id, md)) {
             vgs[i].push_back({id, uid});
           }
-        } else if constexpr (std::is_invocable<FUNC_t, int, Uid_t, Metadata>::value) {  
+        } else if constexpr (std::is_invocable<FUNC_t, int, Uid_t, Metadata>::value) {
           if (selector(i, uid, md)) {
             vgs[i].push_back({id, uid});
           }
-        } else { 
+        } else {
           PARTHENON_FAIL("Passing the wrong sort of selector.");
         }
       }
@@ -184,13 +178,14 @@ struct PackDescriptor {
     return vgs;
   }
 
-  template <class base_t> 
+  template <class base_t>
   std::vector<std::string> MakeGroupNames(const std::vector<base_t> &var_groups) {
-    if constexpr (std::is_same<base_t, std::string>::value) { 
-      return var_groups; 
-    } else if constexpr (std::is_same<base_t, Uid_t>::value) { 
-      std::vector<std::string> var_group_names; 
-      for (auto &vg : var_groups) var_group_names.push_back(std::to_string(vg)); 
+    if constexpr (std::is_same<base_t, std::string>::value) {
+      return var_groups;
+    } else if constexpr (std::is_same<base_t, Uid_t>::value) {
+      std::vector<std::string> var_group_names;
+      for (auto &vg : var_groups)
+        var_group_names.push_back(std::to_string(vg));
       return var_group_names;
     }
   }
diff --git a/src/parthenon/driver.hpp b/src/parthenon/driver.hpp
index 9fd8c3e8ea3e..a62773cbcc87 100644
--- a/src/parthenon/driver.hpp
+++ b/src/parthenon/driver.hpp
@@ -69,9 +69,9 @@ using ::parthenon::TaskListStatus;
 using ::parthenon::TaskRegion;
 using ::parthenon::TaskStatus;
 using ::parthenon::TaskType;
+using ::parthenon::Uid_t;
 using ::parthenon::DriverUtils::ConstructAndExecuteBlockTasks;
 using ::parthenon::DriverUtils::ConstructAndExecuteTaskLists;
-using ::parthenon::Uid_t;
 
 namespace partition {
 using ::parthenon::partition::Partition_t;
diff --git a/src/utils/unique_id.hpp b/src/utils/unique_id.hpp
index a0168297b479..eca507f886a1 100644
--- a/src/utils/unique_id.hpp
+++ b/src/utils/unique_id.hpp
@@ -15,6 +15,7 @@
 #define UTILS_UNIQUE_ID_HPP_
 
 #include <cstddef>
+#include <vector>
 
 #include <unordered_map>
 

From 811e08e3e94d8646d9c7731e311b5f80f18416cf Mon Sep 17 00:00:00 2001
From: jdolence <jcdolence@gmail.com>
Date: Tue, 15 Aug 2023 13:21:16 -0600
Subject: [PATCH 26/36] add default descriptors/sparse packs for weird usage
 downstream

---
 src/amr_criteria/amr_criteria.hpp      | 1 -
 src/interface/make_pack_descriptor.hpp | 2 ++
 src/interface/sparse_pack.hpp          | 1 +
 src/interface/sparse_pack_base.hpp     | 5 +++++
 4 files changed, 8 insertions(+), 1 deletion(-)

diff --git a/src/amr_criteria/amr_criteria.hpp b/src/amr_criteria/amr_criteria.hpp
index 86b09c9728a1..e31bd8b381d8 100644
--- a/src/amr_criteria/amr_criteria.hpp
+++ b/src/amr_criteria/amr_criteria.hpp
@@ -17,7 +17,6 @@
 #include <string>
 
 #include "defs.hpp"
-//#include "interface/meshblock_data.hpp"
 #include "mesh/domain.hpp"
 
 namespace parthenon {
diff --git a/src/interface/make_pack_descriptor.hpp b/src/interface/make_pack_descriptor.hpp
index bd838e37363f..f3cfe17ec4f3 100644
--- a/src/interface/make_pack_descriptor.hpp
+++ b/src/interface/make_pack_descriptor.hpp
@@ -36,6 +36,8 @@
 
 namespace parthenon {
 
+inline auto MakeDefaultPackDescriptor() { return typename SparsePack<>::Descriptor(); }
+
 inline auto MakePackDescriptor(StateDescriptor *psd, const std::vector<std::string> &vars,
                                const std::vector<bool> &use_regex,
                                const std::vector<MetadataFlag> &flags = {},
diff --git a/src/interface/sparse_pack.hpp b/src/interface/sparse_pack.hpp
index 4b646abc0793..ef21ee16fa98 100644
--- a/src/interface/sparse_pack.hpp
+++ b/src/interface/sparse_pack.hpp
@@ -122,6 +122,7 @@ class SparsePack : public SparsePackBase {
 
   class Descriptor : public impl::PackDescriptor {
    public:
+    Descriptor() = default;
     explicit Descriptor(const impl::PackDescriptor &desc_in)
         : impl::PackDescriptor(desc_in) {}
 
diff --git a/src/interface/sparse_pack_base.hpp b/src/interface/sparse_pack_base.hpp
index 732862e307d9..97c95995c8ac 100644
--- a/src/interface/sparse_pack_base.hpp
+++ b/src/interface/sparse_pack_base.hpp
@@ -127,6 +127,11 @@ struct PackDescriptor {
 
   void Print() const;
 
+  // default constructor needed for certain use cases
+  PackDescriptor()
+      : nvar_groups(0), var_group_names({}), var_groups({}), with_fluxes(false),
+        coarse(false), flat(false) {}
+
   template <class GROUP_t, class SELECTOR_t>
   PackDescriptor(StateDescriptor *psd, const std::vector<GROUP_t> &var_groups_in,
                  const SELECTOR_t &selector, const std::set<PDOpt> &options)

From 1da52bb760099a0e77a2ff28e489ad9f36d8e342 Mon Sep 17 00:00:00 2001
From: Josh Dolence <jdolence@lanl.gov>
Date: Tue, 15 Aug 2023 13:22:28 -0600
Subject: [PATCH 27/36] Update src/interface/state_descriptor.cpp

Co-authored-by: Luke Roberts <lfroberts@lanl.gov>
---
 src/interface/state_descriptor.cpp | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/interface/state_descriptor.cpp b/src/interface/state_descriptor.cpp
index 2e0dcc64ee32..5d846e4e4767 100644
--- a/src/interface/state_descriptor.cpp
+++ b/src/interface/state_descriptor.cpp
@@ -489,7 +489,6 @@ StateDescriptor::GetVariableNames(const std::vector<std::string> &req_names,
       auto &name = p.first;
       auto &pool = p.second;
       auto &meta = pool.shared_metadata();
-      printf("checking %s\n", name.c_str());
       if (MetadataUtils::MatchFlags(flags, meta)) {
         printf("matched flag\n");
         for (const auto &s : pool.pool()) {

From 4e51cc12675d297f5c960c59bfd1c14681c88b58 Mon Sep 17 00:00:00 2001
From: Josh Dolence <jdolence@lanl.gov>
Date: Tue, 15 Aug 2023 13:23:17 -0600
Subject: [PATCH 28/36] Update src/interface/state_descriptor.cpp

Co-authored-by: Luke Roberts <lfroberts@lanl.gov>
---
 src/interface/state_descriptor.cpp | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/interface/state_descriptor.cpp b/src/interface/state_descriptor.cpp
index 5d846e4e4767..950686ceb520 100644
--- a/src/interface/state_descriptor.cpp
+++ b/src/interface/state_descriptor.cpp
@@ -490,7 +490,6 @@ StateDescriptor::GetVariableNames(const std::vector<std::string> &req_names,
       auto &pool = p.second;
       auto &meta = pool.shared_metadata();
       if (MetadataUtils::MatchFlags(flags, meta)) {
-        printf("matched flag\n");
         for (const auto &s : pool.pool()) {
           if (sparse_ids_set.empty() || sparse_ids_set.count(s.first) > 0) {
             if (names.count(MakeVarLabel(name, s.first)) == 0)

From 134f1d19f97958b2ca85514fa8f9fdaeec327f40 Mon Sep 17 00:00:00 2001
From: jdolence <jcdolence@gmail.com>
Date: Tue, 15 Aug 2023 20:41:02 -0600
Subject: [PATCH 29/36] fix bug and address comments

---
 src/interface/data_collection.cpp  | 24 +++++++-----------------
 src/interface/data_collection.hpp  |  6 ++----
 src/interface/mesh_data.hpp        |  5 ++---
 src/interface/metadata.hpp         | 18 ++++++++++--------
 src/interface/state_descriptor.cpp | 20 ++++++++++++++------
 src/mesh/mesh.cpp                  |  4 +++-
 6 files changed, 38 insertions(+), 39 deletions(-)

diff --git a/src/interface/data_collection.cpp b/src/interface/data_collection.cpp
index 2f0da74b5997..37c2a66fe4c1 100644
--- a/src/interface/data_collection.cpp
+++ b/src/interface/data_collection.cpp
@@ -49,26 +49,16 @@ DataCollection<T>::Add(const std::string &name, const std::shared_ptr<T> &src,
   return containers_[name];
 }
 template <typename T>
-std::shared_ptr<T> &DataCollection<T>::Add(const std::string &name,
+std::shared_ptr<T> &DataCollection<T>::Add(const std::string &label,
                                            const std::shared_ptr<T> &src,
-                                           const std::vector<std::string> &field_names) {
-  return Add(name, src, field_names, false);
+                                           const std::vector<std::string> &flags) {
+  return Add(label, src, flags, false);
 }
 template <typename T>
-std::shared_ptr<T> &
-DataCollection<T>::AddShallow(const std::string &name, const std::shared_ptr<T> &src,
-                              const std::vector<std::string> &field_names) {
-  return Add(name, src, field_names, true);
-}
-template <typename T>
-std::shared_ptr<T> &DataCollection<T>::Add(const std::string &name,
-                                           const std::shared_ptr<T> &src) {
-  return Add(name, src, {}, false);
-}
-template <typename T>
-std::shared_ptr<T> &DataCollection<T>::AddShallow(const std::string &name,
-                                                  const std::shared_ptr<T> &src) {
-  return Add(name, src, {}, true);
+std::shared_ptr<T> &DataCollection<T>::AddShallow(const std::string &label,
+                                                  const std::shared_ptr<T> &src,
+                                                  const std::vector<std::string> &flags) {
+  return Add(label, src, flags, true);
 }
 
 template <>
diff --git a/src/interface/data_collection.hpp b/src/interface/data_collection.hpp
index 2fe45f1c5d0b..527df2055fac 100644
--- a/src/interface/data_collection.hpp
+++ b/src/interface/data_collection.hpp
@@ -41,11 +41,9 @@ class DataCollection {
   std::shared_ptr<T> &Add(const std::string &label, const std::shared_ptr<T> &src,
                           const std::vector<std::string> &flags, const bool shallow);
   std::shared_ptr<T> &Add(const std::string &label, const std::shared_ptr<T> &src,
-                          const std::vector<std::string> &flags);
+                          const std::vector<std::string> &flags = {});
   std::shared_ptr<T> &AddShallow(const std::string &label, const std::shared_ptr<T> &src,
-                                 const std::vector<std::string> &flags);
-  std::shared_ptr<T> &Add(const std::string &label, const std::shared_ptr<T> &src);
-  std::shared_ptr<T> &AddShallow(const std::string &label, const std::shared_ptr<T> &src);
+                                 const std::vector<std::string> &flags = {});
   std::shared_ptr<T> &Add(const std::string &label) {
     // error check for duplicate names
     auto it = containers_.find(label);
diff --git a/src/interface/mesh_data.hpp b/src/interface/mesh_data.hpp
index 614c72656a27..88eb45d6e54b 100644
--- a/src/interface/mesh_data.hpp
+++ b/src/interface/mesh_data.hpp
@@ -439,9 +439,8 @@ class MeshData {
 
 template <typename T, typename... Args>
 std::vector<Uid_t> UidIntersection(MeshData<T> *md1, MeshData<T> *md2, Args &&...args) {
-  auto u1 = md1->GetBlockData(0)->GetVariableUIDs(std::forward<Args>(args)...);
-  auto u2 = md2->GetBlockData(0)->GetVariableUIDs(std::forward<Args>(args)...);
-  return UidIntersection(u1, u2);
+  return UidIntersection(md1->GetBlockData(0).get(), md2->GetBlockData(0).get(),
+                         std::forward<Args>(args)...);
 }
 
 } // namespace parthenon
diff --git a/src/interface/metadata.hpp b/src/interface/metadata.hpp
index f5ba4807e6cc..6d6f47aa1ace 100644
--- a/src/interface/metadata.hpp
+++ b/src/interface/metadata.hpp
@@ -491,8 +491,9 @@ class Metadata {
   /**
    * @brief Returns true if any flag is set
    */
-  template <template <class...> class Container_t, class... extra>
-  bool AnyFlagsSet(const Container_t<MetadataFlag, extra...> &flags) const {
+  template <class Container_t,
+            REQUIRES(std::is_same<typename Container_t::value_type, MetadataFlag>::value)>
+  bool AnyFlagsSet(const Container_t &flags) const {
     return std::any_of(flags.begin(), flags.end(),
                        [this](MetadataFlag const &f) { return IsSet(f); });
   }
@@ -501,8 +502,9 @@ class Metadata {
     return AnyFlagsSet(FlagVec{flag, std::forward<Args>(args)...});
   }
 
-  template <template <class...> class Container_t, class... extra>
-  bool AllFlagsSet(const Container_t<MetadataFlag, extra...> &flags) const {
+  template <class Container_t,
+            REQUIRES(std::is_same<typename Container_t::value_type, MetadataFlag>::value)>
+  bool AllFlagsSet(const Container_t &flags) const {
     return std::all_of(flags.begin(), flags.end(),
                        [this](MetadataFlag const &f) { return IsSet(f); });
   }
@@ -510,15 +512,15 @@ class Metadata {
   bool AllFlagsSet(const MetadataFlag &flag, Args... args) const {
     return AllFlagsSet(FlagVec{flag, std::forward<Args>(args)...});
   }
-
-  template <template <class...> class Container_t, class... extra>
-  bool NoFlagsSet(const Container_t<MetadataFlag, extra...> &flags) const {
+  template <class Container_t,
+            REQUIRES(std::is_same<typename Container_t::value_type, MetadataFlag>::value)>
+  bool NoFlagsSet(const Container_t &flags) const {
     return std::none_of(flags.begin(), flags.end(),
                         [this](MetadataFlag const &f) { return IsSet(f); });
   }
   template <typename... Args>
   bool NoFlagsSet(const MetadataFlag &flag, Args... args) const {
-    return AllFlagsSet(FlagVec{flag, std::forward<Args>(args)...});
+    return NoFlagsSet(FlagVec{flag, std::forward<Args>(args)...});
   }
 
   template <template <class...> class Container_t, class... extra>
diff --git a/src/interface/state_descriptor.cpp b/src/interface/state_descriptor.cpp
index 950686ceb520..3ab618a722d1 100644
--- a/src/interface/state_descriptor.cpp
+++ b/src/interface/state_descriptor.cpp
@@ -444,15 +444,23 @@ StateDescriptor::CreateResolvedStateDescriptor(Packages_t &packages) {
   return state;
 }
 
+// Build a list of variables in the following order
+// 1. fields requested by name (if present), in the order they're requested
+// 2. non-sparse fields picked up by the provided Metadata::FlagCollection
+// 3. sparse fields picked up by metadata
+// For sparse fields, only those ids in sparse_ids are included, unless
+// sparse_ids is empty, in which case all ids are included.
+// Variable names are only added to the list once the first time they
+// are encountered.
 std::vector<std::string>
-StateDescriptor::GetVariableNames(const std::vector<std::string> &req_names,
+StateDescriptor::GetVariableNames(const std::vector<std::string> &requested_names,
                                   const Metadata::FlagCollection &flags,
                                   const std::vector<int> &sparse_ids) {
   std::unordered_set<int> sparse_ids_set(sparse_ids.begin(), sparse_ids.end());
   std::unordered_set<std::string> names;
   std::vector<std::string> names_vec;
   // first add names that are present
-  for (const auto &name : req_names) {
+  for (const auto &name : requested_names) {
     if (FieldPresent(name)) {
       if (metadataMap_[VarID(name)].IsSet(Metadata::Sparse)) {
         PARTHENON_THROW("Cannot ask for variables by name with specific sparse ids");
@@ -504,9 +512,9 @@ StateDescriptor::GetVariableNames(const std::vector<std::string> &req_names,
   return names_vec;
 }
 std::vector<std::string>
-StateDescriptor::GetVariableNames(const std::vector<std::string> &req_names,
+StateDescriptor::GetVariableNames(const std::vector<std::string> &requested_names,
                                   const std::vector<int> &sparse_ids) {
-  return GetVariableNames(req_names, Metadata::FlagCollection(), sparse_ids);
+  return GetVariableNames(requested_names, Metadata::FlagCollection(), sparse_ids);
 }
 std::vector<std::string>
 StateDescriptor::GetVariableNames(const Metadata::FlagCollection &flags,
@@ -514,8 +522,8 @@ StateDescriptor::GetVariableNames(const Metadata::FlagCollection &flags,
   return GetVariableNames({}, flags, sparse_ids);
 }
 std::vector<std::string>
-StateDescriptor::GetVariableNames(const std::vector<std::string> &req_names) {
-  return GetVariableNames(req_names, Metadata::FlagCollection(), {});
+StateDescriptor::GetVariableNames(const std::vector<std::string> &requested_names) {
+  return GetVariableNames(requested_names, Metadata::FlagCollection(), {});
 }
 std::vector<std::string>
 StateDescriptor::GetVariableNames(const Metadata::FlagCollection &flags) {
diff --git a/src/mesh/mesh.cpp b/src/mesh/mesh.cpp
index 833ef1b1a953..937e64b46436 100644
--- a/src/mesh/mesh.cpp
+++ b/src/mesh/mesh.cpp
@@ -492,7 +492,6 @@ Mesh::Mesh(ParameterInput *pin, ApplicationInput *app_in, Packages_t &packages,
                         packages, resolved_packages, gflag);
     block_list[i - nbs]->SearchAndSetNeighbors(tree, ranklist.data(), nslist.data());
   }
-  mesh_data.Get()->Set(block_list, "base");
 
   ResetLoadBalanceVariables();
 
@@ -1201,6 +1200,9 @@ void Mesh::Initialize(bool init_problem, ParameterInput *pin, ApplicationInput *
     }
   } while (!init_done);
 
+  // Initialize the "base" MeshData object
+  mesh_data.Get()->Set(block_list, "base");
+
   Kokkos::Profiling::popRegion(); // Mesh::Initialize
 }
 

From 4cdd3799a2b71f70d9c04d794ad669eb526f3f99 Mon Sep 17 00:00:00 2001
From: jdolence <jcdolence@gmail.com>
Date: Tue, 15 Aug 2023 21:38:02 -0600
Subject: [PATCH 30/36] move from Copy to Initialize

---
 src/interface/data_collection.cpp |  2 +-
 src/interface/mesh_data.hpp       |  5 +++--
 src/interface/meshblock_data.cpp  |  6 +++---
 src/interface/meshblock_data.hpp  | 12 ++++++------
 4 files changed, 13 insertions(+), 12 deletions(-)

diff --git a/src/interface/data_collection.cpp b/src/interface/data_collection.cpp
index 37c2a66fe4c1..fc7c3c0c6c3c 100644
--- a/src/interface/data_collection.cpp
+++ b/src/interface/data_collection.cpp
@@ -35,7 +35,7 @@ DataCollection<T>::Add(const std::string &name, const std::shared_ptr<T> &src,
   }
 
   auto c = std::make_shared<T>();
-  c->Copy(src.get(), field_names, shallow);
+  c->Initialize(src.get(), field_names, shallow);
 
   Set(name, c);
 
diff --git a/src/interface/mesh_data.hpp b/src/interface/mesh_data.hpp
index 88eb45d6e54b..7e37c7288eb3 100644
--- a/src/interface/mesh_data.hpp
+++ b/src/interface/mesh_data.hpp
@@ -237,15 +237,16 @@ class MeshData {
   }
 
   template <typename... Args>
-  void Copy(const MeshData<T> *src, Args &&...args) {
+  void Initialize(const MeshData<T> *src, Args &&...args) {
     if (src == nullptr) {
       PARTHENON_THROW("src points at null");
     }
+    pmy_mesh_ = src->GetParentPointer();
     const int nblocks = src->NumBlocks();
     block_data_.resize(nblocks);
     for (int i = 0; i < nblocks; i++) {
       block_data_[i] = std::make_shared<MeshBlockData<T>>();
-      block_data_[i]->Copy(src->GetBlockData(i).get(), std::forward<Args>(args)...);
+      block_data_[i]->Initialize(src->GetBlockData(i).get(), std::forward<Args>(args)...);
     }
   }
 
diff --git a/src/interface/meshblock_data.cpp b/src/interface/meshblock_data.cpp
index af23a1e8a8d6..a17760b4a36e 100644
--- a/src/interface/meshblock_data.cpp
+++ b/src/interface/meshblock_data.cpp
@@ -75,9 +75,9 @@ void MeshBlockData<T>::AddField(const std::string &base_name, const Metadata &me
 
 // TODO(JMM): Move to unique IDs at some point
 template <typename T>
-void MeshBlockData<T>::Copy(const MeshBlockData<T> *src,
-                            const std::vector<std::string> &names,
-                            const bool shallow_copy) {
+void MeshBlockData<T>::Initialize(const MeshBlockData<T> *src,
+                                  const std::vector<std::string> &names,
+                                  const bool shallow_copy) {
   assert(src != nullptr);
   SetBlockPointer(src);
   resolved_packages_ = src->resolved_packages_;
diff --git a/src/interface/meshblock_data.hpp b/src/interface/meshblock_data.hpp
index 4ca32223efae..261a72802fc2 100644
--- a/src/interface/meshblock_data.hpp
+++ b/src/interface/meshblock_data.hpp
@@ -87,12 +87,6 @@ class MeshBlockData {
     return GetBlockPointer()->cellbounds.GetBoundsK(domain);
   }
 
-  /// Create copy of MeshBlockData, possibly with a subset of named fields,
-  /// and possibly shallow.  Note when shallow=false, new storage is allocated
-  /// for non-OneCopy vars, but the data from src is not actually deep copied
-  void Copy(const MeshBlockData<T> *src, const std::vector<std::string> &names,
-            const bool shallow);
-
   /// Set the pointer to the mesh block for this container
   void SetBlockPointer(std::weak_ptr<MeshBlock> pmb) { pmy_block = pmb.lock(); }
   void SetBlockPointer(const std::shared_ptr<MeshBlockData<T>> &other) {
@@ -108,6 +102,12 @@ class MeshBlockData {
   void Initialize(const std::shared_ptr<StateDescriptor> resolved_packages,
                   const std::shared_ptr<MeshBlock> pmb);
 
+  /// Create copy of MeshBlockData, possibly with a subset of named fields,
+  /// and possibly shallow.  Note when shallow=false, new storage is allocated
+  /// for non-OneCopy vars, but the data from src is not actually deep copied
+  void Initialize(const MeshBlockData<T> *src, const std::vector<std::string> &names,
+                  const bool shallow);
+
   //
   // Queries related to Variable objects
   //

From 7ea5de257a8ff9236e9fcc669d8c000335f5ed0f Mon Sep 17 00:00:00 2001
From: jdolence <jcdolence@gmail.com>
Date: Wed, 16 Aug 2023 08:34:59 -0600
Subject: [PATCH 31/36] change name of package metadata flag

---
 src/interface/state_descriptor.hpp | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/interface/state_descriptor.hpp b/src/interface/state_descriptor.hpp
index c24d60eacc31..d5d172520b13 100644
--- a/src/interface/state_descriptor.hpp
+++ b/src/interface/state_descriptor.hpp
@@ -102,16 +102,16 @@ class StateDescriptor {
   // Preferred constructor
   explicit StateDescriptor(std::string const &label) : label_(label) {
     if (Metadata::FlagNameExists(label)) {
-      AddParam("MetadataFlag", Metadata::GetUserFlag(label));
+      AddParam("PackageMetadataFlag_", Metadata::GetUserFlag(label));
     } else {
-      AddParam("MetadataFlag", Metadata::AddUserFlag(label));
+      AddParam("PackageMetadataFlag_", Metadata::AddUserFlag(label));
     }
   }
 
   static std::shared_ptr<StateDescriptor>
   CreateResolvedStateDescriptor(Packages_t &packages);
 
-  MetadataFlag GetMetadataFlag() { return params_.Get<MetadataFlag>("MetadataFlag"); }
+  MetadataFlag GetMetadataFlag() { return params_.Get<MetadataFlag>("PackageMetadataFlag_"); }
 
   template <typename T>
   void AddParam(const std::string &key, T value, Params::Mutability mutability) {

From 048051924e4a259bea1cddd07fb04abb0d4b9672 Mon Sep 17 00:00:00 2001
From: jdolence <jcdolence@gmail.com>
Date: Wed, 16 Aug 2023 08:37:07 -0600
Subject: [PATCH 32/36] update changelog

---
 CHANGELOG.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 29feeb213f33..0f5c6a795483 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -3,6 +3,7 @@
 ## Current develop
 
 ### Added (new features/APIs/variables/...)
+- [[PR 921]](https://github.com/parthenon-hpc-lab/parthenon/pull/921) Add more flexible ways of adding and using MeshData/MeshBlockData objects to DataCollections
 - [[PR 900]](https://github.com/parthenon-hpc-lab/parthenon/pull/900) Add Morton numbers and expand functionality of LogicalLocation
 - [[PR 902]](https://github.com/parthenon-hpc-lab/parthenon/pull/902) Add ability to output NaNs for de-allocated sparse fields
 - [[PR 887]](https://github.com/parthenon-hpc-lab/parthenon/pull/887) Add ability to dump more types of params and read them from restarts

From 0657d9308095da4d7ba3e738879621c054939379 Mon Sep 17 00:00:00 2001
From: jdolence <jcdolence@gmail.com>
Date: Wed, 16 Aug 2023 09:17:45 -0600
Subject: [PATCH 33/36] update docs

---
 doc/sphinx/src/interface/state.rst | 28 ++++++++++++++++++++++++++++
 1 file changed, 28 insertions(+)

diff --git a/doc/sphinx/src/interface/state.rst b/doc/sphinx/src/interface/state.rst
index 24e162d1a029..b2a91fa8e4cc 100644
--- a/doc/sphinx/src/interface/state.rst
+++ b/doc/sphinx/src/interface/state.rst
@@ -66,6 +66,22 @@ several useful features and functions.
 - ``T *MutableParam(const std::string &key)`` returns a pointer to a
   parameter that has been marked mutable when it was added. Note this
   pointer is *not* marked ``const``.
+- ``MetadataFlag GetMetadataFlag()`` returns a ``MetadataFlag`` that is
+  automatically added to all fields, sparse pools, and swarms that are
+  added to the ``StateDescriptor``.
+- ``std::vector<std::string> GetVariableNames(...)`` provides a means of
+  getting a list of variables that satisfy specified requirements and that
+  are associated with the ``StateDescriptor``.  Optional arguments, in order,
+  include a vector of variable names, a ``Metadata::FlagCollection`` to select,
+  variables by flags, and a vector of sparse ids to allow selecting subsets of
+  sparse fields.  Selection of sparse fields by name requires providing the
+  base name of the sparse field.  Names of a sparse field tied to a specific
+  sparse index are not supported (and will throw).  The returned list is
+  appropriate for use in adding ``MeshData`` and/or ``MeshBlockData`` objects
+  with specified fields to the ``DataCollection`` objects in ``Mesh`` and
+  ``MeshBlock``.  For convenience, the ``Mesh`` class also provides this
+  function, which provides a list of variables gathered from all the package
+  ``StateDescriptor``s.
 - ``void FillDerivedBlock(MeshBlockData<Real>* rc)`` delgates to the
   ``std::function`` member ``FillDerivedBlock`` if set (defaults to
   ``nullptr`` and therefore a no-op) that allows an application to provide
@@ -293,6 +309,18 @@ downstream applications to allocate storage in a more targeted fashion,
 as might be desirable to hold source terms for particular equations, for
 example.
 
+Analogously, ``DataCollection`` provides ``AddShallow`` functions that
+differ from ``Add`` only in that ***all*** included variables, even
+non-``Metadata::OncCopy`` variables, are simply shallow copies.  For
+these functions, no new storage for variables is ever allocated.
+
+Finally, all of the functionality just described for ``MeshBlockData``
+objects is also provided for ``MeshData`` objects.  Adding a new
+``MeshData`` object to the ``Mesh``-level ``DataCollection`` automatically
+adds the corresponding ``MeshBlockData`` objects to each of the
+``MeshBlock``-level ``DataCollection``s.  Using this ``Mesh`` level
+functionality can be more convenient.
+
 Two simple examples of usage of these new containers are 1) to provide
 storage for multistage integration schemes and 2) to provide a mechanism
 to allocate storage for right hand sides, deltas, etc. Both of these

From fe1a6afaff39dfcc48b7c71fc241697ea6d8a05e Mon Sep 17 00:00:00 2001
From: jdolence <jcdolence@gmail.com>
Date: Wed, 16 Aug 2023 09:26:16 -0600
Subject: [PATCH 34/36] update all the copyrights

---
 src/interface/data_collection.cpp         | 2 +-
 src/interface/data_collection.hpp         | 2 +-
 src/interface/mesh_data.hpp               | 2 +-
 src/interface/metadata.cpp                | 2 +-
 src/mesh/mesh.cpp                         | 2 +-
 src/mesh/meshblock.cpp                    | 2 +-
 src/parthenon/driver.hpp                  | 2 +-
 tst/unit/test_meshblock_data_iterator.cpp | 2 +-
 tst/unit/test_metadata.cpp                | 2 +-
 9 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/src/interface/data_collection.cpp b/src/interface/data_collection.cpp
index fc7c3c0c6c3c..53dd6d8cc968 100644
--- a/src/interface/data_collection.cpp
+++ b/src/interface/data_collection.cpp
@@ -1,5 +1,5 @@
 //========================================================================================
-// (C) (or copyright) 2020. Triad National Security, LLC. All rights reserved.
+// (C) (or copyright) 2020-2023. Triad National Security, LLC. All rights reserved.
 //
 // This program was produced under U.S. Government contract 89233218CNA000001 for Los
 // Alamos National Laboratory (LANL), which is operated by Triad National Security, LLC
diff --git a/src/interface/data_collection.hpp b/src/interface/data_collection.hpp
index 527df2055fac..264a272ecd1b 100644
--- a/src/interface/data_collection.hpp
+++ b/src/interface/data_collection.hpp
@@ -1,5 +1,5 @@
 //========================================================================================
-// (C) (or copyright) 2020. Triad National Security, LLC. All rights reserved.
+// (C) (or copyright) 2020-2023. Triad National Security, LLC. All rights reserved.
 //
 // This program was produced under U.S. Government contract 89233218CNA000001 for Los
 // Alamos National Laboratory (LANL), which is operated by Triad National Security, LLC
diff --git a/src/interface/mesh_data.hpp b/src/interface/mesh_data.hpp
index 7e37c7288eb3..c0295118af92 100644
--- a/src/interface/mesh_data.hpp
+++ b/src/interface/mesh_data.hpp
@@ -1,5 +1,5 @@
 //========================================================================================
-// (C) (or copyright) 2020-2022. Triad National Security, LLC. All rights reserved.
+// (C) (or copyright) 2020-2023. Triad National Security, LLC. All rights reserved.
 //
 // This program was produced under U.S. Government contract 89233218CNA000001 for Los
 // Alamos National Laboratory (LANL), which is operated by Triad National Security, LLC
diff --git a/src/interface/metadata.cpp b/src/interface/metadata.cpp
index 89a70d7468a8..d91147af2491 100644
--- a/src/interface/metadata.cpp
+++ b/src/interface/metadata.cpp
@@ -1,5 +1,5 @@
 //========================================================================================
-// (C) (or copyright) 2020-2022. Triad National Security, LLC. All rights reserved.
+// (C) (or copyright) 2020-2023. Triad National Security, LLC. All rights reserved.
 //
 // This program was produced under U.S. Government contract 89233218CNA000001 for Los
 // Alamos National Laboratory (LANL), which is operated by Triad National Security, LLC
diff --git a/src/mesh/mesh.cpp b/src/mesh/mesh.cpp
index 937e64b46436..8eb59faf8a60 100644
--- a/src/mesh/mesh.cpp
+++ b/src/mesh/mesh.cpp
@@ -3,7 +3,7 @@
 // Copyright(C) 2014 James M. Stone <jmstone@princeton.edu> and other code contributors
 // Licensed under the 3-clause BSD License, see LICENSE file for details
 //========================================================================================
-// (C) (or copyright) 2020-2022. Triad National Security, LLC. All rights reserved.
+// (C) (or copyright) 2020-2023. Triad National Security, LLC. All rights reserved.
 //
 // This program was produced under U.S. Government contract 89233218CNA000001 for Los
 // Alamos National Laboratory (LANL), which is operated by Triad National Security, LLC
diff --git a/src/mesh/meshblock.cpp b/src/mesh/meshblock.cpp
index c96282e3d9a3..82f21316b80c 100644
--- a/src/mesh/meshblock.cpp
+++ b/src/mesh/meshblock.cpp
@@ -3,7 +3,7 @@
 // Copyright(C) 2014 James M. Stone <jmstone@princeton.edu> and other code contributors
 // Licensed under the 3-clause BSD License, see LICENSE file for details
 //========================================================================================
-// (C) (or copyright) 2020-2022. Triad National Security, LLC. All rights reserved.
+// (C) (or copyright) 2020-2023. Triad National Security, LLC. All rights reserved.
 //
 // This program was produced under U.S. Government contract 89233218CNA000001 for Los
 // Alamos National Laboratory (LANL), which is operated by Triad National Security, LLC
diff --git a/src/parthenon/driver.hpp b/src/parthenon/driver.hpp
index a62773cbcc87..60dd5f3bd7b5 100644
--- a/src/parthenon/driver.hpp
+++ b/src/parthenon/driver.hpp
@@ -1,5 +1,5 @@
 //========================================================================================
-// (C) (or copyright) 2020-2021. Triad National Security, LLC. All rights reserved.
+// (C) (or copyright) 2020-2023. Triad National Security, LLC. All rights reserved.
 //
 // This program was produced under U.S. Government contract 89233218CNA000001 for Los
 // Alamos National Laboratory (LANL), which is operated by Triad National Security, LLC
diff --git a/tst/unit/test_meshblock_data_iterator.cpp b/tst/unit/test_meshblock_data_iterator.cpp
index 1ed919f042b7..00baaa30d4e2 100644
--- a/tst/unit/test_meshblock_data_iterator.cpp
+++ b/tst/unit/test_meshblock_data_iterator.cpp
@@ -3,7 +3,7 @@
 // Copyright(C) 2014 James M. Stone <jmstone@princeton.edu> and other code contributors
 // Licensed under the 3-clause BSD License, see LICENSE file for details
 //========================================================================================
-// (C) (or copyright) 2020-2022. Triad National Security, LLC. All rights reserved.
+// (C) (or copyright) 2020-2023. Triad National Security, LLC. All rights reserved.
 //
 // This program was produced under U.S. Government contract 89233218CNA000001 for Los
 // Alamos National Laboratory (LANL), which is operated by Triad National Security, LLC
diff --git a/tst/unit/test_metadata.cpp b/tst/unit/test_metadata.cpp
index 23a3a3fea6fe..fbe7ef870d9a 100644
--- a/tst/unit/test_metadata.cpp
+++ b/tst/unit/test_metadata.cpp
@@ -1,5 +1,5 @@
 //========================================================================================
-// (C) (or copyright) 2020. Triad National Security, LLC. All rights reserved.
+// (C) (or copyright) 2020-2023. Triad National Security, LLC. All rights reserved.
 //
 // This program was produced under U.S. Government contract 89233218CNA000001 for Los
 // Alamos National Laboratory (LANL), which is operated by Triad National Security, LLC

From 42e678eb808f03ef3aa60d9f73baa0a08de9584d Mon Sep 17 00:00:00 2001
From: jdolence <jcdolence@gmail.com>
Date: Wed, 16 Aug 2023 09:29:30 -0600
Subject: [PATCH 35/36] gah! formatting!

---
 src/interface/state_descriptor.hpp | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/interface/state_descriptor.hpp b/src/interface/state_descriptor.hpp
index d5d172520b13..ff21e628147c 100644
--- a/src/interface/state_descriptor.hpp
+++ b/src/interface/state_descriptor.hpp
@@ -111,7 +111,9 @@ class StateDescriptor {
   static std::shared_ptr<StateDescriptor>
   CreateResolvedStateDescriptor(Packages_t &packages);
 
-  MetadataFlag GetMetadataFlag() { return params_.Get<MetadataFlag>("PackageMetadataFlag_"); }
+  MetadataFlag GetMetadataFlag() {
+    return params_.Get<MetadataFlag>("PackageMetadataFlag_");
+  }
 
   template <typename T>
   void AddParam(const std::string &key, T value, Params::Mutability mutability) {

From 6807b613bd991548bfd36febeecc6b65cee7ae0d Mon Sep 17 00:00:00 2001
From: jdolence <jcdolence@gmail.com>
Date: Wed, 16 Aug 2023 10:00:18 -0600
Subject: [PATCH 36/36] add a comment as requested

---
 src/interface/data_collection.hpp | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/src/interface/data_collection.hpp b/src/interface/data_collection.hpp
index 264a272ecd1b..4c07b0cd252a 100644
--- a/src/interface/data_collection.hpp
+++ b/src/interface/data_collection.hpp
@@ -28,6 +28,11 @@ class Mesh;
 /// Current usage includes (but is not limited to) storing MeshBlockData for different
 /// stages in multi-stage drivers or the corresponding MeshBlockPacks in a
 /// DataCollection of MeshData.
+///
+/// T must implement:
+///   bool Contains(std::vector<std::string>)
+///   Initialize(T*, std::vector<std::string>, bool)
+/// TODO: implement a concept
 template <typename T>
 class DataCollection {
  public: