Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Register and use non-default MPI communicators in Atlas objects #159

Merged
merged 43 commits into from
Sep 21, 2023
Merged

Register and use non-default MPI communicators in Atlas objects #159

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
43 commits
Select commit Hold shift + click to select a range
b24cdeb
Capability to access mpi comm by name
wdeconinck Aug 24, 2023
4e2b898
Add "mpi_comm" configuration option to StructuredMeshGenerator
wdeconinck Aug 25, 2023
d2f2aa4
Add mpi::Scope, mpi::push, mpi::pop to easily switch default MPI comm…
wdeconinck Aug 25, 2023
d26f02a
Implement Mesh(grid,...,config) supporting "mpi_comm" configuration
wdeconinck Aug 25, 2023
3074c7c
Rename nb_partitions and partition to nb_parts and part
wdeconinck Aug 25, 2023
9f54998
Clean atlas_test_meshgen_splitcomm
wdeconinck Aug 30, 2023
0b07ccd
Add MPI scope to mesh::actions for BuildParallelFields, BuildPeriodic…
wdeconinck Aug 25, 2023
850244a
Add FunctionSpace::mpi_comm()
wdeconinck Aug 29, 2023
c2326f6
Use mpi_comm in GmshIO
wdeconinck Aug 29, 2023
5973015
Test gmsh mesh writing with mpi_comm
wdeconinck Aug 30, 2023
68e6541
Add custom mpi_comm to HaloExchange::setup
wdeconinck Aug 29, 2023
351adf5
Add custom mpi_comm to GatherScatter::setup
wdeconinck Aug 29, 2023
cbeb0bc
Add custom mpi_comm to Checksum::setup
wdeconinck Aug 29, 2023
9ed2026
Use mesh mpi_comm in functionspace NodeColumns
wdeconinck Aug 25, 2023
a2d7022
Use mesh mpi_comm in functionspace CellColumns
wdeconinck Aug 29, 2023
965c23b
Use mesh mpi_comm in functionspace EdgeColumns
wdeconinck Aug 29, 2023
6f76e8d
Use mpi_comm in StructuredColumns
wdeconinck Aug 29, 2023
66041f9
Add atlas_test_functionspace_splitcomm
wdeconinck Aug 30, 2023
8d9e1b3
Register mpi_comm in Mesh with MeshBuilder for further use
wdeconinck Aug 30, 2023
01ffa4e
Use mesh mpi_comm in mesh::PartitionPolygon and mesh::PartitionGraph
wdeconinck Aug 30, 2023
9ebc7e9
Use functionspace mpi_comm in grid::StructuredPartitionPolygon
wdeconinck Aug 30, 2023
ce6c123
Test mpi_comm with mesh partitionGraph and polygons
wdeconinck Aug 30, 2023
1a90dd7
Use mpi_comm of partitioned in MatchingPartitioners
wdeconinck Aug 30, 2023
7de2507
Test MatchingPartitioner in atlas_test_meshgen_splitcomm
wdeconinck Aug 30, 2023
53583ba
Test MatchingPartitioner in atlas_test_functionspace_splitcomm
wdeconinck Aug 30, 2023
980f0fe
Configurable mpi_comm in atlas::grid::Partitioner
wdeconinck Sep 4, 2023
2e685e7
SerialDistribution with part argument
wdeconinck Sep 4, 2023
bd2fd9b
Rework Partitioners
wdeconinck Sep 4, 2023
e15cf75
fixup structuredcolumns
wdeconinck Sep 4, 2023
369e0dc
StructuredColumns inspects Partitioner for mpi_comm
wdeconinck Sep 4, 2023
75bb8b1
Mesh(grid,partitioner) constructor inspects partitioner for mpi_comm
wdeconinck Sep 4, 2023
8d25481
Test mpi_comm with Mesh(grid,partitioner)
wdeconinck Sep 4, 2023
aaa6436
Use mpi_comm in EqualRegionsPartitioner
wdeconinck Sep 5, 2023
d5d826a
Add test of mpi_comm with partitioners
wdeconinck Sep 5, 2023
4e3e199
Use mpi_comm in CubedSphere(Dual)MeshGenerator
wdeconinck Sep 5, 2023
074a547
Use mpi_comm in Redistribute
wdeconinck Sep 5, 2023
bf65e5a
Make Partitioner(std::string) explicit
wdeconinck Sep 6, 2023
21d707a
Use mpi_comm in DelaunayMeshGenerator
wdeconinck Sep 12, 2023
9495fa8
Add Configuration to PointCloud
wdeconinck Sep 13, 2023
8d5b072
Use mpi_comm in functionspace::PointCloud
wdeconinck Sep 13, 2023
403aac7
Add mpi_comm to RegularMeshgenerator
wdeconinck Sep 20, 2023
87764de
Add mpi_comm to HealpixMeshgenerator
wdeconinck Sep 20, 2023
1d98b4b
Add test for mpi_comm with functionspace::BlockStructuredColumns
wdeconinck Sep 21, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 6 additions & 4 deletions src/atlas/functionspace/CellColumns.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -95,7 +95,8 @@ class CellColumnsHaloExchangeCache : public util::Cache<std::string, parallel::H

static value_type* create(const Mesh& mesh) {
value_type* value = new value_type();
value->setup(array::make_view<int, 1>(mesh.cells().partition()).data(),
value->setup(mesh.mpi_comm(),
array::make_view<int, 1>(mesh.cells().partition()).data(),
array::make_view<idx_t, 1>(mesh.cells().remote_index()).data(), REMOTE_IDX_BASE,
mesh.cells().size());
return value;
Expand Down Expand Up @@ -129,7 +130,8 @@ class CellColumnsGatherScatterCache : public util::Cache<std::string, parallel::

static value_type* create(const Mesh& mesh) {
value_type* value = new value_type();
value->setup(array::make_view<int, 1>(mesh.cells().partition()).data(),
value->setup(mesh.mpi_comm(),
array::make_view<int, 1>(mesh.cells().partition()).data(),
array::make_view<idx_t, 1>(mesh.cells().remote_index()).data(), REMOTE_IDX_BASE,
array::make_view<gidx_t, 1>(mesh.cells().global_index()).data(), mesh.cells().size());
return value;
Expand Down Expand Up @@ -193,14 +195,14 @@ void CellColumns::set_field_metadata(const eckit::Configuration& config, Field&
}

idx_t CellColumns::config_size(const eckit::Configuration& config) const {
const idx_t rank = mpi::rank();
idx_t size = nb_cells();
bool global(false);
if (config.get("global", global)) {
if (global) {
idx_t owner(0);
config.get("owner", owner);
idx_t _nb_cells_global(nb_cells_global());
const idx_t rank = mpi::comm(mpi_comm()).rank();
size = (rank == owner ? _nb_cells_global : 0);
}
}
Expand Down Expand Up @@ -269,7 +271,7 @@ CellColumns::CellColumns(const Mesh& mesh, const eckit::Configuration& config):
return nb_cells;
};

mesh::actions::build_nodes_parallel_fields(mesh_.nodes());
mesh::actions::build_nodes_parallel_fields(mesh_);
mesh::actions::build_cells_parallel_fields(mesh_);
mesh::actions::build_periodic_boundaries(mesh_);

Expand Down
2 changes: 2 additions & 0 deletions src/atlas/functionspace/CellColumns.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -99,6 +99,8 @@ class CellColumns : public functionspace::FunctionSpaceImpl {

Field partition() const override;

std::string mpi_comm() const override { return mesh_.mpi_comm(); }

private: // methods
idx_t config_size(const eckit::Configuration& config) const;
array::DataType config_datatype(const eckit::Configuration&) const;
Expand Down
10 changes: 6 additions & 4 deletions src/atlas/functionspace/EdgeColumns.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -96,7 +96,8 @@ class EdgeColumnsHaloExchangeCache : public util::Cache<std::string, parallel::H

static value_type* create(const Mesh& mesh) {
value_type* value = new value_type();
value->setup(array::make_view<int, 1>(mesh.edges().partition()).data(),
value->setup(mesh.mpi_comm(),
array::make_view<int, 1>(mesh.edges().partition()).data(),
array::make_view<idx_t, 1>(mesh.edges().remote_index()).data(), REMOTE_IDX_BASE,
mesh.edges().size());
return value;
Expand Down Expand Up @@ -130,7 +131,8 @@ class EdgeColumnsGatherScatterCache : public util::Cache<std::string, parallel::

static value_type* create(const Mesh& mesh) {
value_type* value = new value_type();
value->setup(array::make_view<int, 1>(mesh.edges().partition()).data(),
value->setup(mesh.mpi_comm(),
array::make_view<int, 1>(mesh.edges().partition()).data(),
array::make_view<idx_t, 1>(mesh.edges().remote_index()).data(), REMOTE_IDX_BASE,
array::make_view<gidx_t, 1>(mesh.edges().global_index()).data(), mesh.edges().size());
return value;
Expand Down Expand Up @@ -194,14 +196,14 @@ void EdgeColumns::set_field_metadata(const eckit::Configuration& config, Field&
}

idx_t EdgeColumns::config_size(const eckit::Configuration& config) const {
const idx_t rank = mpi::rank();
idx_t size = nb_edges();
bool global(false);
if (config.get("global", global)) {
if (global) {
idx_t owner(0);
config.get("owner", owner);
idx_t _nb_edges_global(nb_edges_global());
const idx_t rank = mpi::comm(mpi_comm()).rank();
size = (rank == owner ? _nb_edges_global : 0);
}
}
Expand Down Expand Up @@ -266,7 +268,7 @@ EdgeColumns::EdgeColumns(const Mesh& mesh, const eckit::Configuration& config):
return _nb_edges;
};

mesh::actions::build_nodes_parallel_fields(mesh_.nodes());
mesh::actions::build_nodes_parallel_fields(mesh_);
mesh::actions::build_periodic_boundaries(mesh_);

if (halo_.size() > 0) {
Expand Down
2 changes: 2 additions & 0 deletions src/atlas/functionspace/EdgeColumns.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -93,6 +93,8 @@ class EdgeColumns : public functionspace::FunctionSpaceImpl {

Field partition() const override;

std::string mpi_comm() const override { return mesh_.mpi_comm(); }

private: // methods
idx_t config_size(const eckit::Configuration& config) const;
array::DataType config_datatype(const eckit::Configuration&) const;
Expand Down
12 changes: 10 additions & 2 deletions src/atlas/functionspace/FunctionSpace.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -58,8 +58,12 @@ idx_t FunctionSpace::size() const {
return get()->size();
}

idx_t FunctionSpace::nb_partitions() const {
return get()->nb_partitions();
idx_t FunctionSpace::part() const {
return get()->part();
}

idx_t FunctionSpace::nb_parts() const {
return get()->nb_parts();
}

Field FunctionSpace::lonlat() const {
Expand Down Expand Up @@ -114,6 +118,10 @@ const parallel::GatherScatter& FunctionSpace::scatter() const {
return get()->scatter();
}

std::string FunctionSpace::mpi_comm() const {
return get()->mpi_comm();
}

const util::PartitionPolygon& FunctionSpace::polygon(idx_t halo) const {
return get()->polygon(halo);
}
Expand Down
6 changes: 5 additions & 1 deletion src/atlas/functionspace/FunctionSpace.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -79,7 +79,9 @@ class FunctionSpace : DOXYGEN_HIDE(public util::ObjectHandle<functionspace::Func

const Projection& projection() const;

idx_t nb_partitions() const;
idx_t part() const;

idx_t nb_parts() const;

idx_t size() const;

Expand All @@ -95,6 +97,8 @@ class FunctionSpace : DOXYGEN_HIDE(public util::ObjectHandle<functionspace::Func

const parallel::GatherScatter& gather() const;
const parallel::GatherScatter& scatter() const;

std::string mpi_comm() const;
};

//------------------------------------------------------------------------------------------------------
Expand Down
10 changes: 6 additions & 4 deletions src/atlas/functionspace/NodeColumns.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -105,7 +105,7 @@ class NodeColumnsHaloExchangeCache : public util::Cache<std::string, parallel::H
idx_t nb_nodes(mesh.nodes().size());
mesh.metadata().get(ss.str(), nb_nodes);

value->setup(array::make_view<int, 1>(mesh.nodes().partition()).data(),
value->setup(mesh.mpi_comm(), array::make_view<int, 1>(mesh.nodes().partition()).data(),
array::make_view<idx_t, 1>(mesh.nodes().remote_index()).data(), REMOTE_IDX_BASE, nb_nodes);

return value;
Expand Down Expand Up @@ -154,7 +154,8 @@ class NodeColumnsGatherScatterCache : public util::Cache<std::string, parallel::
//}
}

value->setup(array::make_view<int, 1>(mesh.nodes().partition()).data(),
value->setup(mesh.mpi_comm(),
array::make_view<int, 1>(mesh.nodes().partition()).data(),
array::make_view<idx_t, 1>(mesh.nodes().remote_index()).data(), REMOTE_IDX_BASE,
array::make_view<gidx_t, 1>(mesh.nodes().global_index()).data(), mask.data(), mesh.nodes().size());
return value;
Expand Down Expand Up @@ -206,7 +207,7 @@ NodeColumns::NodeColumns(Mesh mesh, const eckit::Configuration& config):
else {
halo_ = mesh::Halo(mesh);
}
mesh::actions::build_nodes_parallel_fields(mesh_.nodes());
mesh::actions::build_nodes_parallel_fields(mesh_);
mesh::actions::build_periodic_boundaries(mesh_);

if (halo_.size() > 0) {
Expand Down Expand Up @@ -255,7 +256,8 @@ idx_t NodeColumns::config_nb_nodes(const eckit::Configuration& config) const {
idx_t owner(0);
config.get("owner", owner);
idx_t _nb_nodes_global = nb_nodes_global();
size = (mpi::rank() == owner ? _nb_nodes_global : 0);
idx_t rank = mpi::comm(mpi_comm()).rank();
size = (rank == owner ? _nb_nodes_global : 0);
}
}
return size;
Expand Down
6 changes: 5 additions & 1 deletion src/atlas/functionspace/NodeColumns.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -258,7 +258,9 @@ class NodeColumns : public functionspace::FunctionSpaceImpl {

virtual idx_t size() const override { return nb_nodes_; }

idx_t nb_partitions() const override { return mesh_.nb_partitions(); }
idx_t part() const override { return mesh_.part(); }

idx_t nb_parts() const override { return mesh_.nb_parts(); }

Field lonlat() const override { return nodes_.lonlat(); }

Expand All @@ -276,6 +278,8 @@ class NodeColumns : public functionspace::FunctionSpaceImpl {

const Projection& projection() const override { return mesh_.projection(); }

std::string mpi_comm() const override { return mesh_.mpi_comm(); }

private: // methods
void constructor();

Expand Down
Loading
Loading