Skip to content

Commit

Permalink
feat: added metadata info on neighbours
Browse files Browse the repository at this point in the history
add metadata info for periodic and non-periodic neighbours i.e., halo
sizes, starting indices etc.

add an associated unit test to check values are read correctly from
file.
  • Loading branch information
TomMelt committed Nov 25, 2024
1 parent 502e126 commit 88f9e68
Show file tree
Hide file tree
Showing 6 changed files with 524 additions and 7 deletions.
86 changes: 81 additions & 5 deletions core/src/ModelMetadata.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,9 @@
#include "include/IStructure.hpp"
#include "include/NextsimModule.hpp"
#include "include/gridNames.hpp"
#include "mpi.h"
#include <cstddef>
#include <vector>

#ifdef USE_MPI
#include <ncDim.h>
Expand Down Expand Up @@ -39,6 +42,75 @@ void ModelMetadata::setMpiMetadata(MPI_Comm comm)
MPI_Comm_rank(mpiComm, &mpiMyRank);
}

void ModelMetadata::readNeighbourData(netCDF::NcFile& ncFile)
{
netCDF::NcGroup neighbourGroup(ncFile.getGroup(neighbourName));
std::string varName {};
for (auto edge : edges) {
size_t nStart {}; // start point in metadata arrays
size_t count {}; // number of elements to read from metadata arrays
std::vector<int> numNeighbours = std::vector<int>(mpiSize, 0);
std::vector<int> offsets = std::vector<int>(mpiSize, 0);

// non-periodic neighbours
varName = edgeNames[edge] + "_neighbours";
neighbourGroup.getVar(varName).getVar(
{ 0 }, { static_cast<size_t>(mpiSize) }, &numNeighbours[0]);

// compute start index for each process
MPI_Exscan(&numNeighbours[mpiMyRank], &nStart, 1, MPI_INT, MPI_SUM, mpiComm);
// how many elements to read for each process
count = numNeighbours[mpiMyRank];

if (count) {
// initialize neighbour info to zero and correct size
neighbourRanks[edge].resize(count, 0);
neighbourExtents[edge].resize(count, 0);
neighbourHaloStarts[edge].resize(count, 0);

varName = edgeNames[edge] + "_neighbour_ids";
neighbourGroup.getVar(varName).getVar({ nStart }, { count }, &neighbourRanks[edge][0]);

varName = edgeNames[edge] + "_neighbour_halos";
neighbourGroup.getVar(varName).getVar(
{ nStart }, { count }, &neighbourExtents[edge][0]);

varName = edgeNames[edge] + "_neighbour_halo_starts";
neighbourGroup.getVar(varName).getVar(
{ nStart }, { count }, &neighbourHaloStarts[edge][0]);
}

// periodic neighbours
varName = edgeNames[edge] + "_neighbours_periodic";
neighbourGroup.getVar(varName).getVar(
{ 0 }, { static_cast<size_t>(mpiSize) }, &numNeighbours[0]);

// compute start index for each process
MPI_Exscan(&numNeighbours[mpiMyRank], &nStart, 1, MPI_INT, MPI_SUM, mpiComm);
// how many elements to read for each process
count = numNeighbours[mpiMyRank];

if (count) {
// initialize neighbour info to zero and correct size
neighbourRanksPeriodic[edge].resize(count, 0);
neighbourExtentsPeriodic[edge].resize(count, 0);
neighbourHaloStartsPeriodic[edge].resize(count, 0);

varName = edgeNames[edge] + "_neighbour_ids_periodic";
neighbourGroup.getVar(varName).getVar(
{ nStart }, { count }, &neighbourRanksPeriodic[edge][0]);

varName = edgeNames[edge] + "_neighbour_halos_periodic";
neighbourGroup.getVar(varName).getVar(
{ nStart }, { count }, &neighbourExtentsPeriodic[edge][0]);

varName = edgeNames[edge] + "_neighbour_halo_starts_periodic";
neighbourGroup.getVar(varName).getVar(
{ nStart }, { count }, &neighbourHaloStartsPeriodic[edge][0]);
}
}
}

void ModelMetadata::getPartitionMetadata(std::string partitionFile)
{
// TODO: Move the reading of the partition file to its own class
Expand All @@ -53,11 +125,15 @@ void ModelMetadata::getPartitionMetadata(std::string partitionFile)
globalExtentX = ncFile.getDim("NX").getSize();
globalExtentY = ncFile.getDim("NY").getSize();
netCDF::NcGroup bboxGroup(ncFile.getGroup(bboxName));
std::vector<size_t> index(1, mpiMyRank);
bboxGroup.getVar("domain_x").getVar(index, &localCornerX);
bboxGroup.getVar("domain_y").getVar(index, &localCornerY);
bboxGroup.getVar("domain_extent_x").getVar(index, &localExtentX);
bboxGroup.getVar("domain_extent_y").getVar(index, &localExtentY);

std::vector<size_t> rank(1, mpiMyRank);
bboxGroup.getVar("domain_x").getVar(rank, &localCornerX);
bboxGroup.getVar("domain_y").getVar(rank, &localCornerY);
bboxGroup.getVar("domain_extent_x").getVar(rank, &localExtentX);
bboxGroup.getVar("domain_extent_y").getVar(rank, &localExtentY);

readNeighbourData(ncFile);

ncFile.close();
}

Expand Down
26 changes: 25 additions & 1 deletion core/src/include/ModelMetadata.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,9 @@
#include "include/ModelState.hpp"
#include "include/Time.hpp"

#include <ncFile.h>
#include <string>
#include <vector>

#ifdef USE_MPI
#include <mpi.h>
Expand Down Expand Up @@ -93,13 +95,34 @@ class ModelMetadata {
*/
void getPartitionMetadata(std::string partitionFile);

enum Edge { BOTTOM, RIGHT, TOP, LEFT, N_EDGE };
// An array to allow the edges to be accessed in the correct order.
static constexpr std::array<Edge, N_EDGE> edges = { BOTTOM, RIGHT, TOP, LEFT };
std::array<std::string, N_EDGE> edgeNames = { "bottom", "right", "top", "left" };

MPI_Comm mpiComm;
int mpiSize = 0;
int mpiMyRank = -1;
int localCornerX, localCornerY, localExtentX, localExtentY, globalExtentX, globalExtentY;
int localCornerX, localCornerY;
int localExtentX, localExtentY;
int globalExtentX, globalExtentY;
// mpi rank ID and extent for each edge direction
std::array<std::vector<int>, N_EDGE> neighbourRanks;
std::array<std::vector<int>, N_EDGE> neighbourExtents;
std::array<std::vector<int>, N_EDGE> neighbourHaloStarts;
std::array<std::vector<int>, N_EDGE> neighbourRanksPeriodic;
std::array<std::vector<int>, N_EDGE> neighbourExtentsPeriodic;
std::array<std::vector<int>, N_EDGE> neighbourHaloStartsPeriodic;
#endif

private:
/*!
* @brief Read neighbour metadata from netCDF file
*
* @param netCDF file with partition metadata
*/
void readNeighbourData(netCDF::NcFile& ncFile);

TimePoint m_time;
ConfigMap m_config;

Expand All @@ -116,6 +139,7 @@ class ModelMetadata {
bool hasParameters;
#ifdef USE_MPI
const std::string bboxName = "bounding_boxes";
const std::string neighbourName = "connectivity";
#endif
};

Expand Down
30 changes: 29 additions & 1 deletion core/test/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -32,10 +32,24 @@ if(ENABLE_MPI)
ncgen -b -o partition_metadata_2.nc ${CMAKE_CURRENT_SOURCE_DIR}/partition_metadata_2.cdl
DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/partition_metadata_2.cdl
)
add_custom_command(
OUTPUT partition_metadata_3_cb.nc partition_metadata_3_pb.nc
COMMAND
ncgen -b -o partition_metadata_3_cb.nc ${CMAKE_CURRENT_SOURCE_DIR}/partition_metadata_3_cb.cdl
COMMAND
ncgen -b -o partition_metadata_3_pb.nc ${CMAKE_CURRENT_SOURCE_DIR}/partition_metadata_3_pb.cdl
DEPENDS
${CMAKE_CURRENT_SOURCE_DIR}/partition_metadata_3_cb.cdl
${CMAKE_CURRENT_SOURCE_DIR}/partition_metadata_3_pb.cdl
)
add_custom_target(
generate_partition_files
ALL
DEPENDS partition_metadata_3.nc partition_metadata_2.nc
DEPENDS
partition_metadata_3.nc
partition_metadata_2.nc
partition_metadata_3_cb.nc
partition_metadata_3_pb.nc
)

add_executable(testRectGrid_MPI3 "RectGrid_test.cpp" "MainMPI.cpp")
Expand All @@ -49,6 +63,20 @@ if(ENABLE_MPI)
)
target_link_libraries(testRectGrid_MPI3 PRIVATE nextsimlib doctest::doctest)

add_executable(testModelMetadata_MPI3 "ModelMetadata_test.cpp" "MainMPI.cpp")
target_compile_definitions(
testModelMetadata_MPI3
PRIVATE
USE_MPI
TEST_FILES_DIR=\"${CMAKE_CURRENT_BINARY_DIR}\"
TEST_FILE_SOURCE=\"${CMAKE_CURRENT_SOURCE_DIR}\"
)
target_include_directories(
testModelMetadata_MPI3
PRIVATE ${MODEL_INCLUDE_DIR} "${ModulesRoot}/StructureModule"
)
target_link_libraries(testModelMetadata_MPI3 PRIVATE nextsimlib doctest::doctest)

add_executable(testParaGrid_MPI2 "ParaGrid_test.cpp" "MainMPI.cpp")
target_compile_definitions(
testParaGrid_MPI2
Expand Down
149 changes: 149 additions & 0 deletions core/test/ModelMetadata_test.cpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,149 @@
/*!
* @file ModelMetadata_test.cpp
*
* @date 18 Nov 2024
* @author Tom Meltzer <tdm39@cam.ac.uk>
*/

#include <doctest/extensions/doctest_mpi.h>
#include <iostream>

#include "ModelMetadata.hpp"

const std::string testFilesDir = TEST_FILES_DIR;
const std::string filename = testFilesDir + "/paraGrid_test.nc";
const std::string diagFile = "paraGrid_diag.nc";
const std::string dateString = "2000-01-01T00:00:00Z";
const std::string partitionFilenameCB = testFilesDir + "/partition_metadata_3_cb.nc";
const std::string partitionFilenamePB = testFilesDir + "/partition_metadata_3_pb.nc";

namespace Nextsim {

TEST_SUITE_BEGIN("ModelMetadata");
MPI_TEST_CASE("Test getPartitionMetadata closed boundary", 3)
{
ModelMetadata metadata(partitionFilenameCB, test_comm);
REQUIRE(metadata.mpiComm == test_comm);
// this metadata is specific to the non-periodic boundary conditions
if (test_rank == 0) {
REQUIRE(metadata.neighbourRanks[ModelMetadata::LEFT].size() == 0);
REQUIRE(metadata.neighbourRanks[ModelMetadata::RIGHT] == std::vector<int> { 2 });
REQUIRE(metadata.neighbourExtents[ModelMetadata::RIGHT] == std::vector<int> { 4 });
REQUIRE(metadata.neighbourHaloStarts[ModelMetadata::RIGHT] == std::vector<int> { 0 });
REQUIRE(metadata.neighbourRanks[ModelMetadata::BOTTOM].size() == 0);
REQUIRE(metadata.neighbourRanks[ModelMetadata::TOP] == std::vector<int> { 1 });
REQUIRE(metadata.neighbourExtents[ModelMetadata::TOP] == std::vector<int> { 7 });
REQUIRE(metadata.neighbourHaloStarts[ModelMetadata::TOP] == std::vector<int> { 0 });
} else if (test_rank == 1) {
REQUIRE(metadata.neighbourRanks[ModelMetadata::LEFT].size() == 0);
REQUIRE(metadata.neighbourRanks[ModelMetadata::RIGHT] == std::vector<int> { 2 });
REQUIRE(metadata.neighbourExtents[ModelMetadata::RIGHT] == std::vector<int> { 5 });
REQUIRE(metadata.neighbourHaloStarts[ModelMetadata::RIGHT] == std::vector<int> { 12 });
REQUIRE(metadata.neighbourRanks[ModelMetadata::BOTTOM] == std::vector<int> { 0 });
REQUIRE(metadata.neighbourExtents[ModelMetadata::BOTTOM] == std::vector<int> { 7 });
REQUIRE(metadata.neighbourHaloStarts[ModelMetadata::BOTTOM] == std::vector<int> { 21 });
REQUIRE(metadata.neighbourRanks[ModelMetadata::TOP].size() == 0);
} else if (test_rank == 2) {
REQUIRE(metadata.neighbourRanks[ModelMetadata::LEFT] == std::vector<int> { 0, 1 });
REQUIRE(metadata.neighbourExtents[ModelMetadata::LEFT] == std::vector<int> { 4, 5 });
REQUIRE(metadata.neighbourHaloStarts[ModelMetadata::LEFT] == std::vector<int> { 6, 6 });
REQUIRE(metadata.neighbourRanks[ModelMetadata::RIGHT].size() == 0);
REQUIRE(metadata.neighbourRanks[ModelMetadata::BOTTOM].size() == 0);
REQUIRE(metadata.neighbourRanks[ModelMetadata::TOP].size() == 0);
} else {
std::cerr << "only valid for 3 ranks" << std::endl;
exit(1);
}

// This metadata is specific to the periodic boundary conditions.
// They are all zero because the input metadata file `partitionFilenameCB` does not use periodic
// boundary conditions.
REQUIRE(metadata.neighbourRanksPeriodic[ModelMetadata::LEFT].size() == 0);
REQUIRE(metadata.neighbourRanksPeriodic[ModelMetadata::RIGHT].size() == 0);
REQUIRE(metadata.neighbourRanksPeriodic[ModelMetadata::BOTTOM].size() == 0);
REQUIRE(metadata.neighbourRanksPeriodic[ModelMetadata::TOP].size() == 0);
}

MPI_TEST_CASE("Test getPartitionMetadata periodic boundary", 3)
{
ModelMetadata metadata(partitionFilenamePB, test_comm);
REQUIRE(metadata.mpiComm == test_comm);
// this metadata should be identical to the Closed Boundary version so we check it again
if (test_rank == 0) {
REQUIRE(metadata.neighbourRanks[ModelMetadata::LEFT].size() == 0);
REQUIRE(metadata.neighbourRanks[ModelMetadata::RIGHT] == std::vector<int> { 2 });
REQUIRE(metadata.neighbourExtents[ModelMetadata::RIGHT] == std::vector<int> { 4 });
REQUIRE(metadata.neighbourHaloStarts[ModelMetadata::RIGHT] == std::vector<int> { 0 });
REQUIRE(metadata.neighbourRanks[ModelMetadata::BOTTOM].size() == 0);
REQUIRE(metadata.neighbourRanks[ModelMetadata::TOP] == std::vector<int> { 1 });
REQUIRE(metadata.neighbourExtents[ModelMetadata::TOP] == std::vector<int> { 7 });
REQUIRE(metadata.neighbourHaloStarts[ModelMetadata::TOP] == std::vector<int> { 0 });
} else if (test_rank == 1) {
REQUIRE(metadata.neighbourRanks[ModelMetadata::LEFT].size() == 0);
REQUIRE(metadata.neighbourRanks[ModelMetadata::RIGHT] == std::vector<int> { 2 });
REQUIRE(metadata.neighbourExtents[ModelMetadata::RIGHT] == std::vector<int> { 5 });
REQUIRE(metadata.neighbourHaloStarts[ModelMetadata::RIGHT] == std::vector<int> { 12 });
REQUIRE(metadata.neighbourRanks[ModelMetadata::BOTTOM] == std::vector<int> { 0 });
REQUIRE(metadata.neighbourExtents[ModelMetadata::BOTTOM] == std::vector<int> { 7 });
REQUIRE(metadata.neighbourHaloStarts[ModelMetadata::BOTTOM] == std::vector<int> { 21 });
REQUIRE(metadata.neighbourRanks[ModelMetadata::TOP].size() == 0);
} else if (test_rank == 2) {
REQUIRE(metadata.neighbourRanks[ModelMetadata::LEFT] == std::vector<int> { 0, 1 });
REQUIRE(metadata.neighbourExtents[ModelMetadata::LEFT] == std::vector<int> { 4, 5 });
REQUIRE(metadata.neighbourHaloStarts[ModelMetadata::LEFT] == std::vector<int> { 6, 6 });
REQUIRE(metadata.neighbourRanks[ModelMetadata::RIGHT].size() == 0);
REQUIRE(metadata.neighbourRanks[ModelMetadata::BOTTOM].size() == 0);
REQUIRE(metadata.neighbourRanks[ModelMetadata::TOP].size() == 0);
} else {
std::cerr << "only valid for 3 ranks" << std::endl;
exit(1);
}

// this metadata is specific to the periodic boundary conditions
if (test_rank == 0) {
// clang-format off
REQUIRE(metadata.neighbourRanksPeriodic[ModelMetadata::LEFT] == std::vector<int> { 2 });
REQUIRE(metadata.neighbourExtentsPeriodic[ModelMetadata::LEFT] == std::vector<int> { 4 });
REQUIRE(metadata.neighbourHaloStartsPeriodic[ModelMetadata::LEFT] == std::vector<int> { 2 });

REQUIRE(metadata.neighbourRanksPeriodic[ModelMetadata::RIGHT].size() == 0);

REQUIRE(metadata.neighbourRanksPeriodic[ModelMetadata::BOTTOM] == std::vector<int> { 1 });
REQUIRE(metadata.neighbourExtentsPeriodic[ModelMetadata::BOTTOM] == std::vector<int> { 7 });
REQUIRE(metadata.neighbourHaloStartsPeriodic[ModelMetadata::BOTTOM] == std::vector<int> { 28 });

REQUIRE(metadata.neighbourRanksPeriodic[ModelMetadata::TOP].size() == 0);
} else if (test_rank == 1) {
REQUIRE(metadata.neighbourRanksPeriodic[ModelMetadata::LEFT] == std::vector<int> { 2 });
REQUIRE(metadata.neighbourExtentsPeriodic[ModelMetadata::LEFT] == std::vector<int> { 5 });
REQUIRE(metadata.neighbourHaloStartsPeriodic[ModelMetadata::LEFT] == std::vector<int> { 14 });

REQUIRE(metadata.neighbourRanksPeriodic[ModelMetadata::RIGHT].size() == 0);

REQUIRE(metadata.neighbourRanksPeriodic[ModelMetadata::BOTTOM].size() == 0);

REQUIRE(metadata.neighbourRanksPeriodic[ModelMetadata::TOP] == std::vector<int> { 0 });
REQUIRE(metadata.neighbourExtentsPeriodic[ModelMetadata::TOP] == std::vector<int> { 7 });
REQUIRE(metadata.neighbourHaloStartsPeriodic[ModelMetadata::TOP] == std::vector<int> { 0 });
} else if (test_rank == 2) {
REQUIRE(metadata.neighbourRanksPeriodic[ModelMetadata::LEFT].size() == 0);

REQUIRE(metadata.neighbourRanksPeriodic[ModelMetadata::RIGHT] == std::vector<int> { 0, 1 });
REQUIRE(metadata.neighbourExtentsPeriodic[ModelMetadata::RIGHT] == std::vector<int> { 4, 5 });
REQUIRE(metadata.neighbourHaloStartsPeriodic[ModelMetadata::RIGHT] == std::vector<int> { 0, 0 });

REQUIRE(metadata.neighbourRanksPeriodic[ModelMetadata::BOTTOM] == std::vector<int> { 2 });
REQUIRE(metadata.neighbourExtentsPeriodic[ModelMetadata::BOTTOM] == std::vector<int> { 3 });
REQUIRE(metadata.neighbourHaloStartsPeriodic[ModelMetadata::BOTTOM] == std::vector<int> { 24 });

REQUIRE(metadata.neighbourRanksPeriodic[ModelMetadata::TOP] == std::vector<int> { 2 });
REQUIRE(metadata.neighbourExtentsPeriodic[ModelMetadata::TOP] == std::vector<int> { 3 });
REQUIRE(metadata.neighbourHaloStartsPeriodic[ModelMetadata::TOP] == std::vector<int> { 0 });
// clang-format on
} else {
std::cerr << "only valid for 3 ranks" << std::endl;
exit(1);
}
}

}
Loading

0 comments on commit 88f9e68

Please sign in to comment.