Skip to content

Commit

Permalink
Use mpi_comm in CubedSphere(Dual)MeshGenerator
Browse files Browse the repository at this point in the history
  • Loading branch information
@wdeconinck
wdeconinck committed Sep 6, 2023
1 parent 28939e9 commit 503011f
Show file tree
Hide file tree
Showing 3 changed files with 70 additions and 8 deletions.
21 changes: 18 additions & 3 deletions src/atlas/meshgenerator/detail/CubedSphereDualMeshGenerator.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,11 @@ namespace meshgenerator {
// -----------------------------------------------------------------------------

CubedSphereDualMeshGenerator::CubedSphereDualMeshGenerator(const eckit::Parametrisation& p) {
// Get mpi_comm
std::string mpi_comm = mpi::comm().name();
p.get("mpi_comm", mpi_comm);
options.set("mpi_comm", mpi_comm);

configure_defaults();

// Get number of partitions.
Expand All @@ -58,7 +63,9 @@ CubedSphereDualMeshGenerator::CubedSphereDualMeshGenerator(const eckit::Parametr

// Get partitioner.
std::string partitioner;
if (p.get("partitioner", partitioner)) {
try { p.get("partitioner.type", partitioner); } catch( std::exception& ) {}
p.get("partitioner", partitioner);
if( partitioner.size() ) {
options.set("partitioner", partitioner);
}
}
Expand All @@ -67,11 +74,13 @@ CubedSphereDualMeshGenerator::CubedSphereDualMeshGenerator(const eckit::Parametr


void CubedSphereDualMeshGenerator::configure_defaults() {
auto& comm = mpi::comm(options.getString("mpi_comm"));

// This option sets number of partitions.
options.set("nb_parts", mpi::size());
options.set("nb_parts", comm.size());

// This option sets the part that will be generated.
options.set("part", mpi::rank());
options.set("part", comm.rank());

// This options sets the number of halo elements around each partition.
options.set("halo", 0);
Expand All @@ -90,13 +99,15 @@ void CubedSphereDualMeshGenerator::generate(const Grid& grid, Mesh& mesh) const
auto partConfig = util::Config{};
partConfig.set("type", partType);
partConfig.set("partitions", nParts);
partConfig.set("mpi_comm",options.getString("mpi_comm"));

// Use lonlat instead of xy for non cubedsphere partitioner.
if (partType != "cubedsphere") {
partConfig.set("coordinates", "lonlat");
}

// Set distribution.
mpi::Scope mpi_scope(options.getString("mpi_comm"));
const auto partitioner = grid::Partitioner(partConfig);
const auto distribution = grid::Distribution(grid, partitioner);

Expand Down Expand Up @@ -126,6 +137,7 @@ void CubedSphereDualMeshGenerator::generate(const Grid& grid, const grid::Distri
// Clone some grid properties.
setGrid(mesh, csGrid, distribution);

mpi::Scope mpi_scope(options.getString("mpi_comm"));
generate_mesh(csGrid, distribution, mesh);
}

Expand Down Expand Up @@ -462,6 +474,9 @@ void CubedSphereDualMeshGenerator::set_metadata(Mesh& mesh) const {
mesh.nodes().metadata().set("parallel", true);
mesh.cells().metadata().set("parallel", true);

mesh.metadata().set("mpi_comm",options.getString("mpi_comm"));


// Loop over nodes and count number of halo elements.
auto nNodes = std::vector<idx_t>(nHalo + 2, 0);
const auto nodeHalo = array::make_view<int, 1>(mesh.nodes().halo());
Expand Down
17 changes: 14 additions & 3 deletions src/atlas/meshgenerator/detail/CubedSphereMeshGenerator.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,11 @@ namespace meshgenerator {
// -----------------------------------------------------------------------------

CubedSphereMeshGenerator::CubedSphereMeshGenerator(const eckit::Parametrisation& p) {
// Get mpi_comm
std::string mpi_comm = mpi::comm().name();
p.get("mpi_comm", mpi_comm);
options.set("mpi_comm", mpi_comm);

configure_defaults();

// Get number of partitions.
Expand All @@ -68,7 +73,9 @@ CubedSphereMeshGenerator::CubedSphereMeshGenerator(const eckit::Parametrisation&

// Get partitioner.
std::string partitioner;
if (p.get("partitioner", partitioner)) {
try { p.get("partitioner.type", partitioner); } catch( std::exception& ) {}
p.get("partitioner", partitioner);
if( partitioner.size() ) {
options.set("partitioner", partitioner);
}
}
Expand All @@ -77,11 +84,13 @@ CubedSphereMeshGenerator::CubedSphereMeshGenerator(const eckit::Parametrisation&


void CubedSphereMeshGenerator::configure_defaults() {
auto& comm = mpi::comm(options.getString("mpi_comm"));

// This option sets number of partitions.
options.set("nb_parts", mpi::size());
options.set("nb_parts", comm.size());

// This option sets the part that will be generated.
options.set("part", mpi::rank());
options.set("part", comm.rank());

// This options sets the number of halo elements around each partition.
options.set("halo", 0);
Expand All @@ -107,6 +116,7 @@ void CubedSphereMeshGenerator::generate(const Grid& grid, Mesh& mesh) const {
}

// Set distribution.
mpi::Scope mpi_scope(options.getString("mpi_comm"));
const auto partitioner = grid::Partitioner(partConfig);
const auto distribution = grid::Distribution(grid, partitioner);

Expand Down Expand Up @@ -149,6 +159,7 @@ void CubedSphereMeshGenerator::generate(const Grid& grid, const grid::Distributi

// Clone some grid properties.
setGrid(mesh, csGrid, distribution);
mesh.metadata().set("mpi_comm",options.getString("mpi_comm"));

generate_mesh(csGrid, distribution, mesh);
}
Expand Down
40 changes: 38 additions & 2 deletions src/tests/mesh/test_meshgen_splitcomm.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,11 +39,20 @@ Grid grid() {
return g;
}

Grid grid_A() {
return grid();
}

Grid grid_B() {
static Grid g (color() == 0 ? "F64" : "O64" );
return g;
}

Grid grid_CS() {
static Grid g (color() == 0 ? "CS-LFR-C-8" : "CS-LFR-C-16" );
return g;
}

struct Fixture {
Fixture() {
mpi::comm().split(color(),"split");
Expand Down Expand Up @@ -74,7 +83,7 @@ CASE("Partitioners") {
CASE("StructuredMeshGenerator") {
Fixture fixture;

StructuredMeshGenerator meshgen{option::mpi_comm("split")};
MeshGenerator meshgen{"structured", option::mpi_comm("split")};
Mesh mesh = meshgen.generate(grid());
EXPECT_EQUAL(mesh.nb_parts(),mpi::comm("split").size());
EXPECT_EQUAL(mesh.part(),mpi::comm("split").rank());
Expand All @@ -89,6 +98,33 @@ CASE("StructuredMeshGenerator") {
mesh.polygon().outputPythonScript(grid().name()+"_polygons_1.py");
}

CASE("CubedSphereDualMeshGenerator") {
Fixture fixture;

MeshGenerator meshgen{"cubedsphere_dual", option::mpi_comm("split")};
Mesh mesh = meshgen.generate(grid_CS());
EXPECT_EQUAL(mesh.nb_parts(),mpi::comm("split").size());
EXPECT_EQUAL(mesh.part(),mpi::comm("split").rank());
EXPECT_EQUAL(mesh.mpi_comm(),"split");
EXPECT_EQUAL(mpi::comm().name(),"world");
output::Gmsh gmsh(grid_CS().name()+"_1.msh");
gmsh.write(mesh);
}

CASE("CubedSphereMeshGenerator") {
Fixture fixture;

MeshGenerator meshgen{"cubedsphere", option::mpi_comm("split")};
Mesh mesh = meshgen.generate(grid_CS());
EXPECT_EQUAL(mesh.nb_parts(),mpi::comm("split").size());
EXPECT_EQUAL(mesh.part(),mpi::comm("split").rank());
EXPECT_EQUAL(mesh.mpi_comm(),"split");
EXPECT_EQUAL(mpi::comm().name(),"world");
output::Gmsh gmsh(grid_CS().name()+"_2.msh");
gmsh.write(mesh);
}


CASE("Mesh constructor") {
Fixture fixture;

Expand Down Expand Up @@ -128,7 +164,7 @@ CASE("Mesh constructor with partitioner") {
CASE("MatchingPartitioner") {
Fixture fixture;

auto mesh_A = Mesh(grid(), option::mpi_comm("split"));
auto mesh_A = Mesh(grid_A(), option::mpi_comm("split"));
auto mesh_B = Mesh(grid_B(), grid::MatchingPartitioner(mesh_A), option::mpi_comm("split"));

output::Gmsh gmsh_B(grid_B().name()+"_3.msh");
Expand Down

0 comments on commit 503011f

Please sign in to comment.