56 : _src(map.src().begin(), map.src().end()),
57 _dest(map.dest().begin(), map.dest().end()),
58 _sizes_remote(_src.size(), 0), _displs_remote(_src.size() + 1),
59 _sizes_local(_dest.size()), _displs_local(_dest.size() + 1)
68 assert(std::ranges::is_sorted(_src));
69 assert(std::ranges::is_sorted(_dest));
75 ierr = MPI_Dist_graph_create_adjacent(
76 map.
comm(), _src.size(), _src.data(), MPI_UNWEIGHTED, _dest.size(),
77 _dest.data(), MPI_UNWEIGHTED, MPI_INFO_NULL,
false, &comm0);
82 ierr = MPI_Dist_graph_create_adjacent(
83 map.
comm(), _dest.size(), _dest.data(), MPI_UNWEIGHTED, _src.size(),
84 _src.data(), MPI_UNWEIGHTED, MPI_INFO_NULL,
false, &comm1);
89 std::span owners = map.
owners();
90 std::vector<std::int32_t> perm(owners.size());
91 std::iota(perm.begin(), perm.end(), 0);
96 std::span ghosts = map.
ghosts();
97 std::vector<int> owners_sorted(owners.size());
98 std::vector<std::int64_t> ghosts_sorted(owners.size());
99 std::ranges::transform(perm, owners_sorted.begin(),
100 [&owners](
auto idx) { return owners[idx]; });
101 std::ranges::transform(perm, ghosts_sorted.begin(),
102 [&ghosts](
auto idx) { return ghosts[idx]; });
110 assert(_sizes_remote.size() == _src.size());
111 assert(_displs_remote.size() == _src.size() + 1);
112 auto begin = owners_sorted.begin();
113 for (std::size_t i = 0; i < _src.size(); i++)
115 auto upper = std::upper_bound(begin, owners_sorted.end(), _src[i]);
116 std::size_t num_ind = std::distance(begin, upper);
117 _displs_remote[i + 1] = _displs_remote[i] + num_ind;
118 _sizes_remote[i] = num_ind;
129 assert(_sizes_local.size() == _dest.size());
130 assert(_displs_local.size() == _dest.size() + 1);
131 _sizes_remote.reserve(1);
132 _sizes_local.reserve(1);
133 ierr = MPI_Neighbor_alltoall(_sizes_remote.data(), 1, MPI_INT32_T,
134 _sizes_local.data(), 1, MPI_INT32_T,
138 std::partial_sum(_sizes_local.begin(), _sizes_local.end(),
139 std::next(_displs_local.begin()));
141 assert((std::int32_t)ghosts_sorted.size() == _displs_remote.back());
142 assert((std::int32_t)ghosts_sorted.size() == _displs_remote.back());
146 std::vector<std::int64_t> recv_buffer(_displs_local.back(), 0);
147 ierr = MPI_Neighbor_alltoallv(
148 ghosts_sorted.data(), _sizes_remote.data(), _displs_remote.data(),
149 MPI_INT64_T, recv_buffer.data(), _sizes_local.data(),
150 _displs_local.data(), MPI_INT64_T, _comm1.comm());
153 const std::array<std::int64_t, 2> range = map.
local_range();
156 std::ranges::for_each(recv_buffer, [range](
auto idx)
157 { assert(idx >= range[0] and idx < range[1]); });
162 for (
auto& x : {std::ref(_sizes_local), std::ref(_displs_local),
163 std::ref(_sizes_remote), std::ref(_displs_remote)})
165 std::ranges::transform(x.get(), x.get().begin(),
166 [bs](
auto e) { return e *= bs; });
173 std::vector<typename container_type::value_type> idx(recv_buffer.size()
175 std::int64_t offset = range[0] * bs;
176 for (std::size_t i = 0; i < recv_buffer.size(); i++)
177 for (
int j = 0; j < bs; j++)
178 idx[i * bs + j] = (recv_buffer[i] * bs + j) - offset;
179 _local_inds = std::move(idx);
184 std::vector<typename container_type::value_type> idx(perm.size() * bs);
185 for (std::size_t i = 0; i < perm.size(); i++)
186 for (
int j = 0; j < bs; j++)
187 idx[i * bs + j] = perm[i] * bs + j;
188 _remote_inds = std::move(idx);
247 std::span<MPI_Request> requests)
const
249 if (requests.size() != _dest.size() + _src.size())
251 throw std::runtime_error(
252 "Point-to-point scatterer has wrong number of MPI_Requests.");
256 if (_sizes_local.empty() and _sizes_remote.empty())
259 for (std::size_t i = 0; i < _src.size(); ++i)
261 int ierr = MPI_Irecv(recv_buffer + _displs_remote[i], _sizes_remote[i],
263 _comm0.comm(), &requests[i]);
267 for (std::size_t i = 0; i < _dest.size(); ++i)
269 int ierr = MPI_Isend(send_buffer + _displs_local[i], _sizes_local[i],
271 &requests[i + _src.size()]);
332 std::span<MPI_Request> requests)
const
334 if (requests.size() != _dest.size() + _src.size())
336 throw std::runtime_error(
337 "Point-to-point scatterer has wrong number of MPI_Requests.");
341 if (_sizes_local.empty() and _sizes_remote.empty())
345 for (std::size_t i = 0; i < _dest.size(); i++)
347 int ierr = MPI_Irecv(recv_buffer + _displs_local[i], _sizes_local[i],
349 _comm0.comm(), &requests[i]);
355 for (std::size_t i = 0; i < _src.size(); i++)
357 int ierr = MPI_Isend(send_buffer + _displs_remote[i], _sizes_remote[i],
359 &requests[i + _dest.size()]);