Merge branch 'develop' into lroberts36/add-combined-buffer-communication

CHANGELOG.md

@@ -12,6 +12,7 @@
 - [[PR 1161]](https://github.com/parthenon-hpc-lab/parthenon/pull/1161) Make flux field Metadata accessible, add Metadata::CellMemAligned flag, small perfomance upgrades
 
 ### Changed (changing behavior/API/variables/...)
+- [[PR 1209]](https://github.com/parthenon-hpc-lab/parthenon/pull/1209) Ordered history output
 - [[PR 1206]](https://github.com/parthenon-hpc-lab/parthenon/pull/1206) Leapfrog fix
 - [[PR1203]](https://github.com/parthenon-hpc-lab/parthenon/pull/1203) Pin Ubuntu CI image
 - [[PR1177]](https://github.com/parthenon-hpc-lab/parthenon/pull/1177) Make mesh-level boundary conditions usable without the "user" flag
@@ -21,6 +22,7 @@
 - [[PR 1172]](https://github.com/parthenon-hpc-lab/parthenon/pull/1172) Make parthenon manager robust against external MPI init and finalize calls
 
 ### Fixed (not changing behavior/API/variables/...)
+- [[PR 1188]](https://github.com/parthenon-hpc-lab/parthenon/pull/1188) Fix hdf5 output issue for metadata none variables, update test.
 - [[PR 1170]](https://github.com/parthenon-hpc-lab/parthenon/pull/1170) Fixed incorrect initialization of array by a const not constexpr
 - [[PR 1189]](https://github.com/parthenon-hpc-lab/parthenon/pull/1189) Address CUDA MPI/ICP issue with Kokkos <=4.4.1
 - [[PR 1178]](https://github.com/parthenon-hpc-lab/parthenon/pull/1178) Fix issue with mesh pointer when using relative residual tolerance in BiCGSTAB solver.

CMakeLists.txt

@@ -75,9 +75,9 @@ include(cmake/Format.cmake)
 include(cmake/Lint.cmake)
 
 # regression test reference data
-set(REGRESSION_GOLD_STANDARD_VER 24 CACHE STRING "Version of gold standard to download and use")
+set(REGRESSION_GOLD_STANDARD_VER 25 CACHE STRING "Version of gold standard to download and use")
 set(REGRESSION_GOLD_STANDARD_HASH
-  "SHA512=e220df92a335131131e42ddb52dc221a6dbd6bb56361483b4af0292620eeb82ffb21ef3b95fd9a7c5cc158fb754da0bf1a1015bec98b5bbad05f4bceb1ee99bc"
+  "SHA512=314dc8312366d81ba33d1fde25812e9a7697b2f529de29e22662df0d458f1c4bc5b5bb4e649888170f66ffec0df1be20a9cf401944531c1c1ad835e26eaad28f"
   CACHE STRING "Hash of default gold standard file to download")
 option(REGRESSION_GOLD_STANDARD_SYNC "Automatically sync gold standard files." ON)
 

doc/sphinx/src/outputs.rst

@@ -194,7 +194,9 @@ block might look like
 
 This will produce a text file (``.hst``) output file every 1 units of
 simulation time. The content of the file is determined by the functions
-enrolled by specific packages, see :ref:`state history output`.
+enrolled by specific packages, see :ref:`state history output`. Per-package history
+outputs will always be in alphabetical order by package name, which may not match
+the order in which packages were added to a simulation.
 
 Histograms
 ----------

example/advection/advection_package.cpp

@@ -196,6 +196,20 @@ std::shared_ptr<StateDescriptor> Initialize(ParameterInput *pin) {
   m = Metadata({Metadata::Cell, Metadata::OneCopy}, std::vector<int>({1}));
   pkg->AddField("my_derived_var", m);
 
+  // Create a Metadata::None variable for IO testing purposes.
+  // Only load if test_metadata_none is specified in the Advection block
+  auto test_metadata_none =
+      pin->GetOrAddBoolean("Advection", "test_metadata_none", false);
+  pkg->AddParam<bool>("test_metadata_none", test_metadata_none);
+  if (test_metadata_none) {
+    const int nx1 = pin->GetOrAddInteger("parthenon/meshblock", "nx1", 1);
+    const int nx2 = pin->GetOrAddInteger("parthenon/meshblock", "nx2", 1);
+    const int nx3 = pin->GetOrAddInteger("parthenon/meshblock", "nx3", 1);
+    std::vector<int> test_shape = {nx1 + 1, nx2 + 1, nx3 + 1, 3};
+    m = Metadata({Metadata::OneCopy, Metadata::None}, test_shape);
+    pkg->AddField("metadata_none_var", m);
+  }
+
   // List (vector) of HistoryOutputVar that will all be enrolled as output variables
   parthenon::HstVar_list hst_vars = {};
   // Now we add a couple of callback functions
@@ -281,6 +295,7 @@ AmrTag CheckRefinement(MeshBlockData<Real> *rc) {
 void PreFill(MeshBlockData<Real> *rc) {
   auto pmb = rc->GetBlockPointer();
   auto pkg = pmb->packages.Get("advection_package");
+  const bool test_metadata_none = pkg->Param<bool>("test_metadata_none");
   bool fill_derived = pkg->Param<bool>("fill_derived");
 
   if (fill_derived) {
@@ -302,6 +317,24 @@ void PreFill(MeshBlockData<Real> *rc) {
           v(out + n, k, j, i) = 1.0 - v(in + n, k, j, i);
         });
   }
+
+  // Fill the metadata::None var with index gymnastics.
+  if (test_metadata_none) {
+    const int nx1 = pmb->cellbounds.ncellsi(IndexDomain::interior);
+    const int nx2 = pmb->cellbounds.ncellsj(IndexDomain::interior);
+    const int nx3 = pmb->cellbounds.ncellsk(IndexDomain::interior);
+
+    // packing in principle unnecessary/convoluted here and just done for demonstration
+    std::vector<std::string> vars({"metadata_none_var"});
+    PackIndexMap imap;
+    const auto &v = rc->PackVariables(vars, imap);
+
+    pmb->par_for(
+        PARTHENON_AUTO_LABEL, 0, 2, 0, nx3, 0, nx2, 0, nx1,
+        KOKKOS_LAMBDA(const int n, const int k, const int j, const int i) {
+          v(n, k, j, i) = n + k + j + i;
+        });
+  }
 }
 
 // this is the package registered function to fill derived

src/outputs/history.cpp

@@ -18,6 +18,7 @@
 //  \brief writes history output data, volume-averaged quantities that are output
 //         frequently in time to trace their history.
 
+#include <algorithm>
 #include <cstdio>
 #include <cstdlib>
 #include <iomanip>
@@ -93,9 +94,16 @@ void HistoryOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm,
     md_base->Initialize(pm->block_list, pm);
   }
 
-  // Loop over all packages of the application
-  for (const auto &pkg : packages) {
-    const auto &params = pkg.second->AllParams();
+  // Loop over all packages of the application in alphabetical order to ensure consistency
+  // of ordering of data in columns.
+  std::vector<std::string> keys;
+  for (const auto &pair : packages) {
+    keys.push_back(pair.first);
+  }
+  std::sort(keys.begin(), keys.end());
+  for (const auto &key : keys) {
+    const auto &pkg = packages[key];
+    const auto &params = pkg->AllParams();
 
     // Check if the package has enrolled scalar history functions which are stored in the
     // Params under the `hist_param_key` name.

src/outputs/output_utils.hpp

@@ -312,11 +312,11 @@ void PackOrUnpackVar(const VarInfo &info, bool do_ghosts, idx_t &idx, Function_t
   auto [kb, jb, ib] = info.GetPaddedBoundsKJI(domain);
   if (info.where == MetadataFlag({Metadata::None})) {
     kb.s = 0;
-    kb.e = shape[4];
+    kb.e = std::max(0, shape[4] - 1);
     jb.s = 0;
-    jb.e = shape[5];
+    jb.e = std::max(0, shape[5] - 1);
     ib.s = 0;
-    ib.e = shape[6];
+    ib.e = std::max(0, shape[6] - 1);
   }
   for (int topo = 0; topo < shape[0]; ++topo) {
     for (int t = 0; t < shape[1]; ++t) {

tst/regression/test_suites/output_hdf5/output_hdf5.py

@@ -190,6 +190,18 @@ def Analyse(self, parameters):
                 )
                 analyze_status = False
 
+        # check contents of matadata_none_var
+        for dim in [2, 3]:
+            data = phdf.phdf(f"advection_{dim}d.out0.final.phdf")
+            profile = data.Get("metadata_none_var", flatten=False)
+            for index in np.ndindex(profile.shape):
+                ib, j, k, l, m = index
+                expected_value = l + k + j + m
+                actual_value = profile[ib, j, k, l, m]
+                if expected_value != actual_value:
+                    print("metadata_none_var is incorrect")
+                    analyze_status = False
+
         # Checking Parthenon histograms versus numpy ones
         for dim in [2, 3]:
             # 1D histogram with binning of a variable with bins defined by a var

tst/regression/test_suites/output_hdf5/parthinput.advection

@@ -54,6 +54,7 @@ vx = 1.0
 vy = 1.0
 vz = 1.0
 profile = hard_sphere
+test_metadata_none = true
 
 refine_tol = 0.3    # control the package specific refinement tagging function
 derefine_tol = 0.03
@@ -64,7 +65,8 @@ file_type = hdf5
 dt = 1.0
 variables = advected, one_minus_advected, & # comments are ok
             one_minus_advected_sq, & # on every (& characters are ok in comments)
-            one_minus_sqrt_one_minus_advected_sq # line
+            one_minus_sqrt_one_minus_advected_sq, & # line
+            metadata_none_var
 
 <parthenon/output1>
 file_type = hst