16 #ifndef dealii_mpi_consensus_algorithm_h
17 #define dealii_mpi_consensus_algorithm_h
22 #include <deal.II/base/mpi.templates.h>
132 namespace ConsensusAlgorithms
158 template <
typename RequestType,
typename AnswerType>
174 virtual std::vector<unsigned int>
203 const RequestType &buffer_recv,
204 AnswerType & request_buffer);
215 const AnswerType & recv_buffer);
233 template <
typename RequestType,
typename AnswerType>
256 const MPI_Comm &
comm);
274 std::vector<unsigned int>
285 std::vector<unsigned int>
311 virtual std::vector<unsigned int>
313 const std::vector<unsigned int> & targets,
314 const std::function<RequestType(
const unsigned int)> &create_request,
315 const std::function<AnswerType(
const unsigned int,
316 const RequestType &)> &answer_request,
317 const std::function<
void(
const unsigned int,
const AnswerType &)>
319 const MPI_Comm &
comm) = 0;
356 template <
typename RequestType,
typename AnswerType>
391 virtual std::vector<unsigned int>
393 const std::vector<unsigned int> & targets,
394 const std::function<RequestType(
const unsigned int)> &create_request,
395 const std::function<AnswerType(
const unsigned int,
396 const RequestType &)> &answer_request,
397 const std::function<
void(
const unsigned int,
const AnswerType &)>
399 const MPI_Comm &
comm)
override;
402 #ifdef DEAL_II_WITH_MPI
448 const std::function<
void(
const unsigned int,
const AnswerType &)>
450 const MPI_Comm &
comm);
474 const std::function<AnswerType(
const unsigned int,
475 const RequestType &)> &answer_request,
476 const MPI_Comm &
comm);
484 const std::vector<unsigned int> & targets,
485 const std::function<RequestType(
const unsigned int)> &create_request,
486 const MPI_Comm &
comm);
530 template <
typename RequestType,
typename AnswerType>
531 std::vector<unsigned int>
532 nbx(
const std::vector<unsigned int> & targets,
533 const std::function<RequestType(
const unsigned int)> &create_request,
534 const std::function<AnswerType(
const unsigned int,
535 const RequestType &)> &answer_request,
536 const std::function<
void(
const unsigned int,
const AnswerType &)>
538 const MPI_Comm &
comm);
566 template <
typename RequestType>
567 std::vector<unsigned int>
568 nbx(
const std::vector<unsigned int> & targets,
569 const std::function<RequestType(
const unsigned int)> &create_request,
570 const std::function<
void(
const unsigned int,
const RequestType &)>
572 const MPI_Comm &
comm);
599 template <
typename RequestType,
typename AnswerType>
635 virtual std::vector<unsigned int>
637 const std::vector<unsigned int> & targets,
638 const std::function<RequestType(
const unsigned int)> &create_request,
639 const std::function<AnswerType(
const unsigned int,
640 const RequestType &)> &answer_request,
641 const std::function<
void(
const unsigned int,
const AnswerType &)>
643 const MPI_Comm &
comm)
override;
646 #ifdef DEAL_II_WITH_MPI
683 const std::vector<unsigned int> & targets,
684 const std::function<RequestType(
const unsigned int)> &create_request,
685 const MPI_Comm &
comm);
693 const unsigned int index,
694 const std::function<AnswerType(
const unsigned int,
695 const RequestType &)> &answer_request,
696 const MPI_Comm &
comm);
704 const unsigned int n_targets,
705 const std::function<
void(
const unsigned int,
const AnswerType &)>
707 const MPI_Comm &
comm);
764 template <
typename RequestType,
typename AnswerType>
765 std::vector<unsigned int>
766 pex(
const std::vector<unsigned int> & targets,
767 const std::function<RequestType(
const unsigned int)> &create_request,
768 const std::function<AnswerType(
const unsigned int,
769 const RequestType &)> &answer_request,
770 const std::function<
void(
const unsigned int,
const AnswerType &)>
772 const MPI_Comm &
comm);
800 template <
typename RequestType>
801 std::vector<unsigned int>
802 pex(
const std::vector<unsigned int> & targets,
803 const std::function<RequestType(
const unsigned int)> &create_request,
804 const std::function<
void(
const unsigned int,
const RequestType &)>
806 const MPI_Comm &
comm);
813 template <
typename RequestType,
typename AnswerType>
843 virtual std::vector<unsigned int>
845 const std::vector<unsigned int> & targets,
846 const std::function<RequestType(
const unsigned int)> &create_request,
847 const std::function<AnswerType(
const unsigned int,
848 const RequestType &)> &answer_request,
849 const std::function<
void(
const unsigned int,
const AnswerType &)>
851 const MPI_Comm &
comm)
override;
885 template <
typename RequestType,
typename AnswerType>
886 std::vector<unsigned int>
888 const std::vector<unsigned int> & targets,
889 const std::function<RequestType(
const unsigned int)> &create_request,
890 const std::function<AnswerType(
const unsigned int,
const RequestType &)>
892 const std::function<
void(
const unsigned int,
const AnswerType &)>
894 const MPI_Comm &
comm);
922 template <
typename RequestType>
923 std::vector<unsigned int>
925 const std::vector<unsigned int> & targets,
926 const std::function<RequestType(
const unsigned int)> &create_request,
927 const std::function<
void(
const unsigned int,
const RequestType &)>
929 const MPI_Comm &
comm);
945 template <
typename RequestType,
typename AnswerType>
968 const MPI_Comm &
comm);
983 virtual std::vector<unsigned int>
985 const std::vector<unsigned int> & targets,
986 const std::function<RequestType(
const unsigned int)> &create_request,
987 const std::function<AnswerType(
const unsigned int,
988 const RequestType &)> &answer_request,
989 const std::function<
void(
const unsigned int,
const AnswerType &)>
991 const MPI_Comm &
comm)
override;
1033 template <
typename RequestType,
typename AnswerType>
1034 std::vector<unsigned int>
1036 const std::vector<unsigned int> & targets,
1037 const std::function<RequestType(
const unsigned int)> &create_request,
1038 const std::function<AnswerType(
const unsigned int,
const RequestType &)>
1040 const std::function<
void(
const unsigned int,
const AnswerType &)>
1042 const MPI_Comm &
comm);
1070 template <
typename RequestType>
1071 std::vector<unsigned int>
1073 const std::vector<unsigned int> & targets,
1074 const std::function<RequestType(
const unsigned int)> &create_request,
1075 const std::function<
void(
const unsigned int,
const RequestType &)>
1077 const MPI_Comm &
comm);
1086 template <
typename RequestType,
typename AnswerType>
1088 :
public Process<RequestType, AnswerType>
1101 const std::function<std::vector<unsigned int>()>
1102 &function_compute_targets,
1103 const std::function<
void(
const unsigned int, RequestType &)>
1104 & function_create_request = {},
1105 const std::function<void(
const unsigned int,
1106 const RequestType &,
1107 AnswerType &)> &function_answer_request = {},
1108 const std::function<void(
const unsigned int,
const AnswerType &)>
1109 &function_read_answer = {});
1114 std::vector<unsigned int>
1122 RequestType & send_buffer)
override;
1129 const RequestType &buffer_recv,
1130 AnswerType & request_buffer)
override;
1137 const AnswerType & recv_buffer)
override;
1140 const std::function<std::vector<unsigned int>()>
1142 const std::function<void(
const int, RequestType &)>
1144 const std::function<
1145 void(
const unsigned int,
const RequestType &, AnswerType &)>
1147 const std::function<void(
const int,
const AnswerType &)>
1155 template <
typename RequestType,
typename AnswerType>
1156 std::vector<unsigned int>
1157 nbx(
const std::vector<unsigned int> & targets,
1158 const std::function<RequestType(
const unsigned int)> &create_request,
1159 const std::function<AnswerType(
const unsigned int,
1160 const RequestType &)> &answer_request,
1161 const std::function<
void(
const unsigned int,
const AnswerType &)>
1163 const MPI_Comm &
comm)
1166 targets, create_request, answer_request, process_answer,
comm);
1171 template <
typename RequestType>
1172 std::vector<unsigned int>
1173 nbx(
const std::vector<unsigned int> & targets,
1174 const std::function<RequestType(
const unsigned int)> &create_request,
1175 const std::function<
void(
const unsigned int,
const RequestType &)>
1177 const MPI_Comm &
comm)
1184 using EmptyType = std::tuple<>;
1186 return nbx<RequestType, EmptyType>(
1190 [&process_request](
const unsigned int source_rank,
1191 const RequestType &request) -> EmptyType {
1192 process_request(source_rank, request);
1200 [](
const unsigned int ,
1201 const EmptyType & ) {},
1207 template <
typename RequestType,
typename AnswerType>
1208 std::vector<unsigned int>
1209 pex(
const std::vector<unsigned int> & targets,
1210 const std::function<RequestType(
const unsigned int)> &create_request,
1211 const std::function<AnswerType(
const unsigned int,
1212 const RequestType &)> &answer_request,
1213 const std::function<
void(
const unsigned int,
const AnswerType &)>
1215 const MPI_Comm &
comm)
1217 return PEX<RequestType, AnswerType>().run(
1218 targets, create_request, answer_request, process_answer,
comm);
1223 template <
typename RequestType>
1224 std::vector<unsigned int>
1225 pex(
const std::vector<unsigned int> & targets,
1226 const std::function<RequestType(
const unsigned int)> &create_request,
1227 const std::function<
void(
const unsigned int,
const RequestType &)>
1229 const MPI_Comm &
comm)
1236 using EmptyType = std::tuple<>;
1238 return pex<RequestType, EmptyType>(
1242 [&process_request](
const unsigned int source_rank,
1243 const RequestType &request) -> EmptyType {
1244 process_request(source_rank, request);
1252 [](
const unsigned int ,
1253 const EmptyType & ) {},
1259 template <
typename RequestType,
typename AnswerType>
1260 std::vector<unsigned int>
1262 const std::vector<unsigned int> & targets,
1263 const std::function<RequestType(
const unsigned int)> &create_request,
1264 const std::function<AnswerType(
const unsigned int,
const RequestType &)>
1266 const std::function<
void(
const unsigned int,
const AnswerType &)>
1268 const MPI_Comm &
comm)
1270 return Serial<RequestType, AnswerType>().run(
1271 targets, create_request, answer_request, process_answer,
comm);
1276 template <
typename RequestType>
1277 std::vector<unsigned int>
1279 const std::vector<unsigned int> & targets,
1280 const std::function<RequestType(
const unsigned int)> &create_request,
1281 const std::function<
void(
const unsigned int,
const RequestType &)>
1283 const MPI_Comm &
comm)
1290 using EmptyType = std::tuple<>;
1292 return serial<RequestType, EmptyType>(
1296 [&process_request](
const unsigned int source_rank,
1297 const RequestType &request) -> EmptyType {
1298 process_request(source_rank, request);
1306 [](
const unsigned int ,
1307 const EmptyType & ) {},
1313 template <
typename RequestType,
typename AnswerType>
1314 std::vector<unsigned int>
1316 const std::vector<unsigned int> & targets,
1317 const std::function<RequestType(
const unsigned int)> &create_request,
1318 const std::function<AnswerType(
const unsigned int,
const RequestType &)>
1320 const std::function<
void(
const unsigned int,
const AnswerType &)>
1322 const MPI_Comm &
comm)
1324 return Selector<RequestType, AnswerType>().run(
1325 targets, create_request, answer_request, process_answer,
comm);
1330 template <
typename RequestType>
1331 std::vector<unsigned int>
1333 const std::vector<unsigned int> & targets,
1334 const std::function<RequestType(
const unsigned int)> &create_request,
1335 const std::function<
void(
const unsigned int,
const RequestType &)>
1337 const MPI_Comm &
comm)
1344 using EmptyType = std::tuple<>;
1346 return selector<RequestType, EmptyType>(
1350 [&process_request](
const unsigned int source_rank,
1351 const RequestType &request) -> EmptyType {
1352 process_request(source_rank, request);
1360 [](
const unsigned int ,
1361 const EmptyType & ) {},
1367 template <
typename RequestType,
typename AnswerType>
1369 const std::function<std::vector<unsigned int>()>
1370 &function_compute_targets,
1371 const std::function<
void(
const unsigned int, RequestType &)>
1372 & function_create_request,
1373 const std::function<
void(
const unsigned int,
1374 const RequestType &,
1375 AnswerType &)> &function_answer_request,
1376 const std::function<
void(
const unsigned int,
const AnswerType &)>
1377 &function_read_answer)
1378 : function_compute_targets(function_compute_targets)
1379 , function_create_request(function_create_request)
1380 , function_answer_request(function_answer_request)
1381 , function_read_answer(function_read_answer)
1386 template <
typename RequestType,
typename AnswerType>
1387 std::vector<unsigned int>
1390 return function_compute_targets();
1395 template <
typename RequestType,
typename AnswerType>
1398 const unsigned int other_rank,
1399 RequestType & send_buffer)
1401 if (function_create_request)
1402 function_create_request(other_rank, send_buffer);
1407 template <
typename RequestType,
typename AnswerType>
1410 const unsigned int other_rank,
1411 const RequestType &buffer_recv,
1412 AnswerType & request_buffer)
1414 if (function_answer_request)
1415 function_answer_request(other_rank, buffer_recv, request_buffer);
1420 template <
typename RequestType,
typename AnswerType>
1423 const unsigned int other_rank,
1424 const AnswerType & recv_buffer)
1426 if (function_read_answer)
1427 function_read_answer(other_rank, recv_buffer);
1447 namespace ConsensusAlgorithms
1466 # ifndef DEAL_II_MSVC
1470 has_unique_elements(
const std::vector<unsigned int> &targets)
1472 std::vector<unsigned int> my_destinations = targets;
1473 std::sort(my_destinations.begin(), my_destinations.end());
1474 return (std::adjacent_find(my_destinations.begin(),
1475 my_destinations.end()) ==
1476 my_destinations.end());
1482 template <
typename RequestType,
typename AnswerType>
1485 const RequestType &,
1493 template <
typename RequestType,
typename AnswerType>
1503 template <
typename RequestType,
typename AnswerType>
1513 template <
typename RequestType,
typename AnswerType>
1515 Process<RequestType, AnswerType> &process,
1516 const MPI_Comm &
comm)
1523 template <
typename RequestType,
typename AnswerType>
1526 ,
comm(MPI_COMM_NULL)
1531 template <
typename RequestType,
typename AnswerType>
1532 std::vector<unsigned int>
1535 Assert(process !=
nullptr,
1536 ExcMessage(
"This function can only be called if the "
1537 "deprecated non-default constructor of this class "
1538 "has previously been called to set the Process "
1539 "object and a communicator."));
1545 template <
typename RequestType,
typename AnswerType>
1546 std::vector<unsigned int>
1548 Process<RequestType, AnswerType> &process,
1549 const MPI_Comm &
comm)
1554 process.compute_targets(),
1556 [&process](
const unsigned int target) {
1557 RequestType request;
1558 process.create_request(target, request);
1562 [&process](
const unsigned int source,
const RequestType &request) {
1564 process.answer_request(source, request, answer);
1568 [&process](
const unsigned int target,
const AnswerType &answer) {
1569 process.read_answer(target, answer);
1576 template <
typename RequestType,
typename AnswerType>
1578 Process<RequestType, AnswerType> &process,
1579 const MPI_Comm &
comm)
1580 : Interface<RequestType, AnswerType>(process,
comm)
1585 template <
typename RequestType,
typename AnswerType>
1586 std::vector<unsigned int>
1588 const std::vector<unsigned int> & targets,
1589 const std::function<RequestType(
const unsigned int)> &create_request,
1590 const std::function<AnswerType(
const unsigned int,
const RequestType &)>
1592 const std::function<
void(
const unsigned int,
const AnswerType &)>
1594 const MPI_Comm &
comm)
1596 Assert(has_unique_elements(targets),
1597 ExcMessage(
"The consensus algorithms expect that each process "
1598 "only sends a single message to another process, "
1599 "but the targets provided include duplicates."));
1601 static CollectiveMutex mutex;
1602 CollectiveMutex::ScopedLock lock(mutex,
comm);
1606 start_communication(targets, create_request,
comm);
1622 while (all_locally_originated_receives_are_completed(process_answer,
1624 maybe_answer_one_request(answer_request,
comm);
1628 signal_finish(
comm);
1633 while (all_remotely_originated_receives_are_completed() ==
false)
1634 maybe_answer_one_request(answer_request,
comm);
1637 clean_up_and_end_communication(
comm);
1639 return std::vector<unsigned int>(requesting_processes.begin(),
1640 requesting_processes.end());
1645 template <
typename RequestType,
typename AnswerType>
1648 const std::vector<unsigned int> & targets,
1649 const std::function<RequestType(
const unsigned int)> &create_request,
1650 const MPI_Comm &
comm)
1652 # ifdef DEAL_II_WITH_MPI
1654 const auto n_targets = targets.size();
1660 send_requests.resize(n_targets);
1661 send_buffers.resize(n_targets);
1665 for (
unsigned int index = 0;
index < n_targets; ++
index)
1667 const unsigned int rank = targets[
index];
1670 auto &send_buffer = send_buffers[
index];
1673 std::vector<char>());
1676 auto ierr = MPI_Isend(send_buffer.data(),
1682 &send_requests[index]);
1688 n_outstanding_answers = n_targets;
1692 (void)create_request;
1699 template <
typename RequestType,
typename AnswerType>
1703 const std::function<
void(
const unsigned int,
const AnswerType &)>
1705 const MPI_Comm &
comm)
1707 # ifdef DEAL_II_WITH_MPI
1715 if (n_outstanding_answers == 0)
1722 int request_is_pending;
1724 const auto ierr = MPI_Iprobe(
1725 MPI_ANY_SOURCE, tag_deliver,
comm, &request_is_pending, &status);
1731 if (request_is_pending == 0)
1738 const auto target = status.MPI_SOURCE;
1745 MPI_Get_count(&status, MPI_CHAR, &message_size);
1748 std::vector<char> recv_buffer(message_size);
1754 const int ierr = MPI_Recv(recv_buffer.data(),
1765 process_answer(target,
1766 Utilities::unpack<AnswerType>(recv_buffer,
1771 --n_outstanding_answers;
1781 return (n_outstanding_answers == 0);
1786 (void)process_answer;
1795 template <
typename RequestType,
typename AnswerType>
1798 const std::function<AnswerType(
const unsigned int,
const RequestType &)>
1800 const MPI_Comm &
comm)
1802 # ifdef DEAL_II_WITH_MPI
1817 int request_is_pending;
1818 const auto ierr = MPI_Iprobe(
1819 MPI_ANY_SOURCE, tag_request,
comm, &request_is_pending, &status);
1822 if (request_is_pending != 0)
1826 const auto other_rank = status.MPI_SOURCE;
1828 Assert(requesting_processes.find(other_rank) ==
1829 requesting_processes.end(),
1830 ExcMessage(
"Process is requesting a second time!"));
1831 requesting_processes.insert(other_rank);
1835 auto ierr = MPI_Get_count(&status, MPI_CHAR, &number_amount);
1839 std::vector<char> buffer_recv(number_amount);
1840 ierr = MPI_Recv(buffer_recv.data(),
1851 request_buffers.emplace_back(std::make_unique<std::vector<char>>());
1852 auto &request_buffer = *request_buffers.back();
1856 Utilities::unpack<RequestType>(
1857 buffer_recv,
false)),
1861 request_requests.emplace_back(std::make_unique<MPI_Request>());
1862 ierr = MPI_Isend(request_buffer.data(),
1863 request_buffer.size(),
1868 request_requests.back().get());
1872 (void)answer_request;
1879 template <
typename RequestType,
typename AnswerType>
1883 # ifdef DEAL_II_WITH_MPI
1884 const auto ierr = MPI_Ibarrier(
comm, &barrier_request);
1893 template <
typename RequestType,
typename AnswerType>
1896 AnswerType>::all_remotely_originated_receives_are_completed()
1898 # ifdef DEAL_II_WITH_MPI
1899 int all_ranks_reached_barrier;
1900 const auto ierr = MPI_Test(&barrier_request,
1901 &all_ranks_reached_barrier,
1902 MPI_STATUSES_IGNORE);
1904 return all_ranks_reached_barrier != 0;
1912 template <
typename RequestType,
typename AnswerType>
1915 const MPI_Comm &
comm)
1918 # ifdef DEAL_II_WITH_MPI
1921 if (send_requests.size() > 0)
1923 const int ierr = MPI_Waitall(send_requests.size(),
1924 send_requests.data(),
1925 MPI_STATUSES_IGNORE);
1929 int ierr = MPI_Wait(&barrier_request, MPI_STATUS_IGNORE);
1932 for (
auto &i : request_requests)
1934 ierr = MPI_Wait(i.get(), MPI_STATUS_IGNORE);
1941 ierr = MPI_Barrier(
comm);
1950 template <
typename RequestType,
typename AnswerType>
1952 Process<RequestType, AnswerType> &process,
1953 const MPI_Comm &
comm)
1954 : Interface<RequestType, AnswerType>(process,
comm)
1959 template <
typename RequestType,
typename AnswerType>
1960 std::vector<unsigned int>
1962 const std::vector<unsigned int> & targets,
1963 const std::function<RequestType(
const unsigned int)> &create_request,
1964 const std::function<AnswerType(
const unsigned int,
const RequestType &)>
1966 const std::function<
void(
const unsigned int,
const AnswerType &)>
1968 const MPI_Comm &
comm)
1970 Assert(has_unique_elements(targets),
1971 ExcMessage(
"The consensus algorithms expect that each process "
1972 "only sends a single message to another process, "
1973 "but the targets provided include duplicates."));
1975 static CollectiveMutex mutex;
1976 CollectiveMutex::ScopedLock lock(mutex,
comm);
1981 const unsigned int n_requests =
1982 start_communication(targets, create_request,
comm);
1985 for (
unsigned int request = 0; request < n_requests; ++request)
1986 answer_one_request(request, answer_request,
comm);
1989 process_incoming_answers(targets.size(), process_answer,
comm);
1992 clean_up_and_end_communication();
1994 return std::vector<unsigned int>(requesting_processes.begin(),
1995 requesting_processes.end());
2000 template <
typename RequestType,
typename AnswerType>
2003 const std::vector<unsigned int> & targets,
2004 const std::function<RequestType(
const unsigned int)> &create_request,
2005 const MPI_Comm &
comm)
2007 # ifdef DEAL_II_WITH_MPI
2013 const unsigned int n_targets = targets.size();
2016 const unsigned int n_sources =
2020 recv_buffers.resize(n_targets);
2021 send_buffers.resize(n_targets);
2022 send_request_requests.resize(n_targets);
2024 send_answer_requests.resize(n_sources);
2025 requests_buffers.resize(n_sources);
2028 for (
unsigned int i = 0; i < n_targets; ++i)
2030 const unsigned int rank = targets[i];
2034 auto &send_buffer = send_buffers[i];
2039 auto ierr = MPI_Isend(send_buffer.data(),
2045 &send_request_requests[i]);
2052 (void)create_request;
2060 template <
typename RequestType,
typename AnswerType>
2063 const unsigned int index,
2064 const std::function<AnswerType(
const unsigned int,
const RequestType &)>
2066 const MPI_Comm &
comm)
2068 # ifdef DEAL_II_WITH_MPI
2077 int ierr = MPI_Probe(MPI_ANY_SOURCE, tag_request,
comm, &status);
2081 const unsigned int other_rank = status.MPI_SOURCE;
2083 Assert(requesting_processes.find(other_rank) ==
2084 requesting_processes.end(),
2086 "A process is sending a request after a request from "
2087 "the same process has previously already been "
2088 "received. This algorithm does not expect this to happen."));
2089 requesting_processes.insert(other_rank);
2093 ierr = MPI_Get_count(&status, MPI_CHAR, &number_amount);
2096 std::vector<char> buffer_recv(number_amount);
2097 ierr = MPI_Recv(buffer_recv.data(),
2108 auto &request_buffer = requests_buffers[
index];
2112 Utilities::unpack<RequestType>(
2113 buffer_recv,
false)),
2115 std::vector<char>());
2117 ierr = MPI_Isend(request_buffer.data(),
2118 request_buffer.size(),
2123 &send_answer_requests[index]);
2126 (void)answer_request;
2134 template <
typename RequestType,
typename AnswerType>
2137 const unsigned int n_targets,
2138 const std::function<
void(
const unsigned int,
const AnswerType &)>
2140 const MPI_Comm &
comm)
2142 # ifdef DEAL_II_WITH_MPI
2151 for (
unsigned int i = 0; i < n_targets; ++i)
2156 MPI_Probe(MPI_ANY_SOURCE, tag_deliver,
comm, &status);
2160 const auto other_rank = status.MPI_SOURCE;
2163 const int ierr = MPI_Get_count(&status, MPI_CHAR, &message_size);
2166 std::vector<char> recv_buffer(message_size);
2172 const int ierr = MPI_Recv(recv_buffer.data(),
2183 process_answer(other_rank,
2184 Utilities::unpack<AnswerType>(recv_buffer,
false));
2188 (void)process_answer;
2195 template <
typename RequestType,
typename AnswerType>
2199 # ifdef DEAL_II_WITH_MPI
2202 if (send_request_requests.size() > 0)
2204 const int ierr = MPI_Waitall(send_request_requests.size(),
2205 send_request_requests.data(),
2206 MPI_STATUSES_IGNORE);
2211 if (send_answer_requests.size() > 0)
2213 const int ierr = MPI_Waitall(send_answer_requests.size(),
2214 send_answer_requests.data(),
2215 MPI_STATUSES_IGNORE);
2223 template <
typename RequestType,
typename AnswerType>
2225 Process<RequestType, AnswerType> &process,
2226 const MPI_Comm &
comm)
2227 : Interface<RequestType, AnswerType>(process,
comm)
2232 template <
typename RequestType,
typename AnswerType>
2233 std::vector<unsigned int>
2235 const std::vector<unsigned int> & targets,
2236 const std::function<RequestType(
const unsigned int)> &create_request,
2237 const std::function<AnswerType(
const unsigned int,
const RequestType &)>
2239 const std::function<
void(
const unsigned int,
const AnswerType &)>
2241 const MPI_Comm &
comm)
2246 ExcMessage(
"You shouldn't use the 'Serial' class on "
2247 "communicators that have more than one process "
2248 "associated with it."));
2251 if (targets.size() != 0)
2253 Assert(targets.size() == 1,
2255 "On a single process, the only valid target "
2256 "is process zero (the process itself), which can only be "
2263 const RequestType request =
2264 (create_request ? create_request(0) : RequestType());
2265 const AnswerType answer =
2266 (answer_request ? answer_request(0, request) : AnswerType());
2269 process_answer(0, answer);
2277 template <
typename RequestType,
typename AnswerType>
2279 Process<RequestType, AnswerType> &process,
2280 const MPI_Comm &
comm)
2281 : Interface<RequestType, AnswerType>(process,
comm)
2286 template <
typename RequestType,
typename AnswerType>
2287 std::vector<unsigned int>
2289 const std::vector<unsigned int> & targets,
2290 const std::function<RequestType(
const unsigned int)> &create_request,
2291 const std::function<AnswerType(
const unsigned int,
const RequestType &)>
2293 const std::function<
void(
const unsigned int,
const AnswerType &)>
2295 const MPI_Comm &
comm)
2306 # ifdef DEAL_II_WITH_MPI
2312 consensus_algo.reset(
new NBX<RequestType, AnswerType>());
2316 consensus_algo.reset(
new PEX<RequestType, AnswerType>());
2318 consensus_algo.reset(
new Serial<RequestType, AnswerType>());
2320 return consensus_algo->run(
2321 targets, create_request, answer_request, process_answer,
comm);
const std::function< void(const unsigned int, const RequestType &, AnswerType &)> function_answer_request
void create_request(const unsigned int other_rank, RequestType &send_buffer) override
const std::function< void(const int, RequestType &)> function_create_request
void read_answer(const unsigned int other_rank, const AnswerType &recv_buffer) override
std::vector< unsigned int > compute_targets() override
void answer_request(const unsigned int other_rank, const RequestType &buffer_recv, AnswerType &request_buffer) override
const std::function< void(const int, const AnswerType &)> function_read_answer
AnonymousProcess(const std::function< std::vector< unsigned int >()> &function_compute_targets, const std::function< void(const unsigned int, RequestType &)> &function_create_request={}, const std::function< void(const unsigned int, const RequestType &, AnswerType &)> &function_answer_request={}, const std::function< void(const unsigned int, const AnswerType &)> &function_read_answer={})
const std::function< std::vector< unsigned int >)> function_compute_targets
virtual std::vector< unsigned int > run(const std::vector< unsigned int > &targets, const std::function< RequestType(const unsigned int)> &create_request, const std::function< AnswerType(const unsigned int, const RequestType &)> &answer_request, const std::function< void(const unsigned int, const AnswerType &)> &process_answer, const MPI_Comm &comm)=0
virtual ~Interface()=default
std::vector< unsigned int > run(Process< RequestType, AnswerType > &process, const MPI_Comm &comm)
Interface(Process< RequestType, AnswerType > &process, const MPI_Comm &comm)
std::vector< unsigned int > run()
Process< RequestType, AnswerType > * process
bool all_locally_originated_receives_are_completed(const std::function< void(const unsigned int, const AnswerType &)> &process_answer, const MPI_Comm &comm)
void maybe_answer_one_request(const std::function< AnswerType(const unsigned int, const RequestType &)> &answer_request, const MPI_Comm &comm)
virtual std::vector< unsigned int > run(const std::vector< unsigned int > &targets, const std::function< RequestType(const unsigned int)> &create_request, const std::function< AnswerType(const unsigned int, const RequestType &)> &answer_request, const std::function< void(const unsigned int, const AnswerType &)> &process_answer, const MPI_Comm &comm) override
void signal_finish(const MPI_Comm &comm)
std::vector< std::unique_ptr< std::vector< char > > > request_buffers
std::set< unsigned int > requesting_processes
MPI_Request barrier_request
NBX(Process< RequestType, AnswerType > &process, const MPI_Comm &comm)
std::vector< std::unique_ptr< MPI_Request > > request_requests
std::vector< std::vector< char > > send_buffers
bool all_remotely_originated_receives_are_completed()
unsigned int n_outstanding_answers
void clean_up_and_end_communication(const MPI_Comm &comm)
void start_communication(const std::vector< unsigned int > &targets, const std::function< RequestType(const unsigned int)> &create_request, const MPI_Comm &comm)
std::vector< MPI_Request > send_requests
void clean_up_and_end_communication()
std::vector< MPI_Request > send_answer_requests
std::vector< MPI_Request > send_request_requests
void answer_one_request(const unsigned int index, const std::function< AnswerType(const unsigned int, const RequestType &)> &answer_request, const MPI_Comm &comm)
std::vector< std::vector< char > > requests_buffers
unsigned int start_communication(const std::vector< unsigned int > &targets, const std::function< RequestType(const unsigned int)> &create_request, const MPI_Comm &comm)
std::vector< std::vector< char > > send_buffers
PEX(Process< RequestType, AnswerType > &process, const MPI_Comm &comm)
virtual std::vector< unsigned int > run(const std::vector< unsigned int > &targets, const std::function< RequestType(const unsigned int)> &create_request, const std::function< AnswerType(const unsigned int, const RequestType &)> &answer_request, const std::function< void(const unsigned int, const AnswerType &)> &process_answer, const MPI_Comm &comm) override
void process_incoming_answers(const unsigned int n_targets, const std::function< void(const unsigned int, const AnswerType &)> &process_answer, const MPI_Comm &comm)
std::set< unsigned int > requesting_processes
std::vector< std::vector< char > > recv_buffers
virtual ~Process()=default
virtual void answer_request(const unsigned int other_rank, const RequestType &buffer_recv, AnswerType &request_buffer)
virtual void read_answer(const unsigned int other_rank, const AnswerType &recv_buffer)
virtual std::vector< unsigned int > compute_targets()=0
virtual void create_request(const unsigned int other_rank, RequestType &send_buffer)
virtual ~Selector()=default
Selector(Process< RequestType, AnswerType > &process, const MPI_Comm &comm)
virtual std::vector< unsigned int > run(const std::vector< unsigned int > &targets, const std::function< RequestType(const unsigned int)> &create_request, const std::function< AnswerType(const unsigned int, const RequestType &)> &answer_request, const std::function< void(const unsigned int, const AnswerType &)> &process_answer, const MPI_Comm &comm) override
std::shared_ptr< Interface< RequestType, AnswerType > > consensus_algo
virtual std::vector< unsigned int > run(const std::vector< unsigned int > &targets, const std::function< RequestType(const unsigned int)> &create_request, const std::function< AnswerType(const unsigned int, const RequestType &)> &answer_request, const std::function< void(const unsigned int, const AnswerType &)> &process_answer, const MPI_Comm &comm) override
Serial(Process< RequestType, AnswerType > &process, const MPI_Comm &comm)
#define DEAL_II_DEPRECATED
#define DEAL_II_NAMESPACE_OPEN
#define DEAL_II_NAMESPACE_CLOSE
#define Assert(cond, exc)
#define AssertDimension(dim1, dim2)
#define AssertThrowMPI(error_code)
#define AssertIndexRange(index, range)
static ::ExceptionBase & ExcMessage(std::string arg1)
std::vector< unsigned int > serial(const std::vector< unsigned int > &targets, const std::function< RequestType(const unsigned int)> &create_request, const std::function< AnswerType(const unsigned int, const RequestType &)> &answer_request, const std::function< void(const unsigned int, const AnswerType &)> &process_answer, const MPI_Comm &comm)
std::vector< unsigned int > nbx(const std::vector< unsigned int > &targets, const std::function< RequestType(const unsigned int)> &create_request, const std::function< AnswerType(const unsigned int, const RequestType &)> &answer_request, const std::function< void(const unsigned int, const AnswerType &)> &process_answer, const MPI_Comm &comm)
std::vector< unsigned int > pex(const std::vector< unsigned int > &targets, const std::function< RequestType(const unsigned int)> &create_request, const std::function< AnswerType(const unsigned int, const RequestType &)> &answer_request, const std::function< void(const unsigned int, const AnswerType &)> &process_answer, const MPI_Comm &comm)
std::vector< unsigned int > selector(const std::vector< unsigned int > &targets, const std::function< RequestType(const unsigned int)> &create_request, const std::function< AnswerType(const unsigned int, const RequestType &)> &answer_request, const std::function< void(const unsigned int, const AnswerType &)> &process_answer, const MPI_Comm &comm)
unsigned int compute_n_point_to_point_communications(const MPI_Comm &mpi_comm, const std::vector< unsigned int > &destinations)
unsigned int n_mpi_processes(const MPI_Comm &mpi_communicator)
size_t pack(const T &object, std::vector< char > &dest_buffer, const bool allow_compression=true)
void run(const Iterator &begin, const typename identity< Iterator >::type &end, Worker worker, Copier copier, const ScratchData &sample_scratch_data, const CopyData &sample_copy_data, const unsigned int queue_length, const unsigned int chunk_size)