diff --git a/CHANGELOG.md b/CHANGELOG.md index 7373e6fae8d0..cc734b960614 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ ### Added (new features/APIs/variables/...) - [[PR 1174]](https://github.com/parthenon-hpc-lab/parthenon/pull/1174) Add CG solver and custom solver prolongation operator options +- [[PR 1179]](https://github.com/parthenon-hpc-lab/parthenon/pull/1179) Make a global variable for whether simulation is a restart - [[PR 1171]](https://github.com/parthenon-hpc-lab/parthenon/pull/1171) Add PARTHENON_USE_SYSTEM_PACKAGES build option - [[PR 1161]](https://github.com/parthenon-hpc-lab/parthenon/pull/1161) Make flux field Metadata accessible, add Metadata::CellMemAligned flag, small perfomance upgrades diff --git a/src/globals.cpp b/src/globals.cpp index 139a01fbddf6..b00db585a3fe 100644 --- a/src/globals.cpp +++ b/src/globals.cpp @@ -3,7 +3,7 @@ // Copyright(C) 2014 James M. Stone and other code contributors // Licensed under the 3-clause BSD License, see LICENSE file for details //======================================================================================== -// (C) (or copyright) 2020-2021. Triad National Security, LLC. All rights reserved. +// (C) (or copyright) 2020-2024. Triad National Security, LLC. All rights reserved. // // This program was produced under U.S. Government contract 89233218CNA000001 for Los // Alamos National Laboratory (LANL), which is operated by Triad National Security, LLC @@ -30,8 +30,9 @@ namespace Globals { int nghost; // all of these global variables are set at the start of main(): -int my_rank; // MPI rank of this process -int nranks; // total number of MPI ranks +int my_rank; // MPI rank of this process +int nranks; // total number of MPI ranks +bool is_restart; // Whether this simulation is restarted from a checkpoint file // sparse configuration values that are needed in various places SparseConfig sparse_config; diff --git a/src/globals.hpp b/src/globals.hpp index 870a0803d8ce..539c158d6570 100644 --- a/src/globals.hpp +++ b/src/globals.hpp @@ -3,7 +3,7 @@ // Copyright(C) 2014 James M. Stone and other code contributors // Licensed under the 3-clause BSD License, see LICENSE file for details //======================================================================================== -// (C) (or copyright) 2020-2021. Triad National Security, LLC. All rights reserved. +// (C) (or copyright) 2020-2024. Triad National Security, LLC. All rights reserved. // // This program was produced under U.S. Government contract 89233218CNA000001 for Los // Alamos National Laboratory (LANL), which is operated by Triad National Security, LLC @@ -36,6 +36,7 @@ struct SparseConfig { }; extern int my_rank, nranks, nghost; +extern bool is_restart; extern SparseConfig sparse_config; diff --git a/src/parthenon_manager.cpp b/src/parthenon_manager.cpp index b0ec6ac971ae..26b5909e8775 100644 --- a/src/parthenon_manager.cpp +++ b/src/parthenon_manager.cpp @@ -70,6 +70,8 @@ ParthenonStatus ParthenonManager::ParthenonInitEnv(int argc, char *argv[]) { Globals::nranks = 1; #endif // MPI_PARALLEL + Globals::is_restart = IsRestart(); + Kokkos::initialize(argc, argv); // pgrete: This is a hack to disable allocation tracking until the Kokkos diff --git a/src/utils/robust.hpp b/src/utils/robust.hpp index 1714d9359b17..ca205db2430c 100644 --- a/src/utils/robust.hpp +++ b/src/utils/robust.hpp @@ -60,6 +60,18 @@ KOKKOS_INLINE_FUNCTION auto ratio(const A &a, const B &b) { const B sgn = b >= 0 ? 1 : -1; return a / (b + sgn * SMALL()); } + +// Return true equivalence if value and reference differ by less than precision +// Optionally return true if reference value is close to zero +KOKKOS_FORCEINLINE_FUNCTION +bool SoftEquiv(const Real &val, const Real &ref, + const Real eps = 10. * std::numeric_limits::epsilon(), + const bool pass_on_small = true) { + const bool is_close = std::abs(val - ref) < eps * std::abs(ref); + const bool is_small = std::abs(ref) < std::numeric_limits::min(); + return (is_close || (is_small && pass_on_small)); +} + } // namespace robust } // namespace parthenon #endif // UTILS_ROBUST_HPP_ diff --git a/tst/unit/CMakeLists.txt b/tst/unit/CMakeLists.txt index c892572bdde9..7d36ebb2be49 100644 --- a/tst/unit/CMakeLists.txt +++ b/tst/unit/CMakeLists.txt @@ -25,7 +25,7 @@ list(APPEND unit_tests_SOURCES test_unit_constants.cpp test_unit_domain.cpp test_unit_sort.cpp - kokkos_abstraction.cpp + test_kokkos_abstraction.cpp test_index_split.cpp test_logical_location.cpp test_forest.cpp diff --git a/tst/unit/kokkos_abstraction.cpp b/tst/unit/test_kokkos_abstraction.cpp similarity index 98% rename from tst/unit/kokkos_abstraction.cpp rename to tst/unit/test_kokkos_abstraction.cpp index 525877375121..767e40f38315 100644 --- a/tst/unit/kokkos_abstraction.cpp +++ b/tst/unit/test_kokkos_abstraction.cpp @@ -26,12 +26,14 @@ #include "basic_types.hpp" #include "kokkos_abstraction.hpp" +#include "utils/robust.hpp" using parthenon::DevExecSpace; using parthenon::ParArray1D; using parthenon::ParArray2D; using parthenon::ParArray3D; using parthenon::ParArray4D; +using parthenon::robust::SoftEquiv; using Real = double; template @@ -316,7 +318,6 @@ bool test_wrapper_nested_3d(OuterLoopPattern outer_loop_pattern, // Copy array back from device to host Kokkos::deep_copy(host_du, dev_du); - Real max_rel_err = -1; const Real rel_tol = std::numeric_limits::epsilon(); // compare data on the host @@ -324,14 +325,15 @@ bool test_wrapper_nested_3d(OuterLoopPattern outer_loop_pattern, for (int j = 0; j < N; j++) { for (int i = 1; i < N - 1; i++) { const Real analytic = 2.0 * (i + 1) * pow((j + 2) * (k + 3), 2.0); - const Real err = host_du(k, j, i - 1) - analytic; - max_rel_err = fmax(fabs(err / analytic), max_rel_err); + if (!SoftEquiv(host_du(k, j, i - 1), analytic, rel_tol)) { + return false; + } } } } - return max_rel_err < rel_tol; + return true; } template @@ -385,7 +387,6 @@ bool test_wrapper_nested_4d(OuterLoopPattern outer_loop_pattern, // Copy array back from device to host Kokkos::deep_copy(host_du, dev_du); - Real max_rel_err = -1; const Real rel_tol = std::numeric_limits::epsilon(); // compare data on the host @@ -394,15 +395,16 @@ bool test_wrapper_nested_4d(OuterLoopPattern outer_loop_pattern, for (int j = 0; j < N; j++) { for (int i = 1; i < N - 1; i++) { const Real analytic = 2.0 * (i + 1) * pow((j + 2) * (k + 3) * (n + 4), 2.0); - const Real err = host_du(n, k, j, i - 1) - analytic; - max_rel_err = fmax(fabs(err / analytic), max_rel_err); + if (!SoftEquiv(host_du(n, k, j, i - 1), analytic, rel_tol)) { + return false; + } } } } } - return max_rel_err < rel_tol; + return true; } TEST_CASE("nested par_for loops", "[wrapper]") {