Reference documentation for deal.II version GIT d9d8a449a2 2022-08-17 08:45:02+00:00
\(\newcommand{\dealvcentcolon}{\mathrel{\mathop{:}}}\) \(\newcommand{\dealcoloneq}{\dealvcentcolon\mathrel{\mkern-1.2mu}=}\) \(\newcommand{\jump}[1]{\left[\!\left[ #1 \right]\!\right]}\) \(\newcommand{\average}[1]{\left\{\!\left\{ #1 \right\}\!\right\}}\)
mpi_consensus_algorithms.h
Go to the documentation of this file.
1 // ---------------------------------------------------------------------
2 //
3 // Copyright (C) 2020 - 2022 by the deal.II authors
4 //
5 // This file is part of the deal.II library.
6 //
7 // The deal.II library is free software; you can use it, redistribute
8 // it, and/or modify it under the terms of the GNU Lesser General
9 // Public License as published by the Free Software Foundation; either
10 // version 2.1 of the License, or (at your option) any later version.
11 // The full text of the license can be found in the file LICENSE.md at
12 // the top level directory of deal.II.
13 //
14 // ---------------------------------------------------------------------
15 
16 #ifndef dealii_mpi_consensus_algorithm_h
17 #define dealii_mpi_consensus_algorithm_h
18 
19 #include <deal.II/base/config.h>
20 
21 #include <deal.II/base/mpi.h>
22 #include <deal.II/base/mpi.templates.h>
23 #include <deal.II/base/mpi_tags.h>
24 
26 
27 
28 namespace Utilities
29 {
30  namespace MPI
31  {
132  namespace ConsensusAlgorithms
133  {
158  template <typename RequestType, typename AnswerType>
159  class Process
160  {
161  public:
166  virtual ~Process() = default;
167 
174  virtual std::vector<unsigned int>
176 
186  virtual void
187  create_request(const unsigned int other_rank, RequestType &send_buffer);
188 
201  virtual void
202  answer_request(const unsigned int other_rank,
203  const RequestType &buffer_recv,
204  AnswerType & request_buffer);
205 
213  virtual void
214  read_answer(const unsigned int other_rank,
215  const AnswerType & recv_buffer);
216  };
217 
218 
219 
233  template <typename RequestType, typename AnswerType>
234  class Interface
235  {
236  public:
241 
256  const MPI_Comm & comm);
257 
262  virtual ~Interface() = default;
263 
274  std::vector<unsigned int>
275  run();
276 
285  std::vector<unsigned int>
287 
311  virtual std::vector<unsigned int>
313  const std::vector<unsigned int> & targets,
314  const std::function<RequestType(const unsigned int)> &create_request,
315  const std::function<AnswerType(const unsigned int,
316  const RequestType &)> &answer_request,
317  const std::function<void(const unsigned int, const AnswerType &)>
318  & process_answer,
319  const MPI_Comm &comm) = 0;
320 
321  private:
331 
339  MPI_Comm comm;
340  };
341 
342 
356  template <typename RequestType, typename AnswerType>
357  class NBX : public Interface<RequestType, AnswerType>
358  {
359  public:
363  NBX() = default;
364 
379 
383  virtual ~NBX() = default;
384 
385  // Import the declarations from the base class.
387 
391  virtual std::vector<unsigned int>
393  const std::vector<unsigned int> & targets,
394  const std::function<RequestType(const unsigned int)> &create_request,
395  const std::function<AnswerType(const unsigned int,
396  const RequestType &)> &answer_request,
397  const std::function<void(const unsigned int, const AnswerType &)>
398  & process_answer,
399  const MPI_Comm &comm) override;
400 
401  private:
402 #ifdef DEAL_II_WITH_MPI
406  std::vector<std::vector<char>> send_buffers;
407 
411  std::vector<MPI_Request> send_requests;
412 
420  std::vector<std::unique_ptr<std::vector<char>>> request_buffers;
421 
425  std::vector<std::unique_ptr<MPI_Request>> request_requests;
426 
430  unsigned int n_outstanding_answers;
431 
432  // request for barrier
433  MPI_Request barrier_request;
434 #endif
435 
439  std::set<unsigned int> requesting_processes;
440 
446  bool
448  const std::function<void(const unsigned int, const AnswerType &)>
449  & process_answer,
450  const MPI_Comm &comm);
451 
456  void
457  signal_finish(const MPI_Comm &comm);
458 
464  bool
466 
472  void
474  const std::function<AnswerType(const unsigned int,
475  const RequestType &)> &answer_request,
476  const MPI_Comm & comm);
477 
482  void
484  const std::vector<unsigned int> & targets,
485  const std::function<RequestType(const unsigned int)> &create_request,
486  const MPI_Comm & comm);
487 
492  void
494  };
495 
496 
530  template <typename RequestType, typename AnswerType>
531  std::vector<unsigned int>
532  nbx(const std::vector<unsigned int> & targets,
533  const std::function<RequestType(const unsigned int)> &create_request,
534  const std::function<AnswerType(const unsigned int,
535  const RequestType &)> &answer_request,
536  const std::function<void(const unsigned int, const AnswerType &)>
537  & process_answer,
538  const MPI_Comm &comm);
539 
566  template <typename RequestType>
567  std::vector<unsigned int>
568  nbx(const std::vector<unsigned int> & targets,
569  const std::function<RequestType(const unsigned int)> &create_request,
570  const std::function<void(const unsigned int, const RequestType &)>
571  & process_request,
572  const MPI_Comm &comm);
573 
599  template <typename RequestType, typename AnswerType>
600  class PEX : public Interface<RequestType, AnswerType>
601  {
602  public:
606  PEX() = default;
607 
608 
623 
627  virtual ~PEX() = default;
628 
629  // Import the declarations from the base class.
631 
635  virtual std::vector<unsigned int>
637  const std::vector<unsigned int> & targets,
638  const std::function<RequestType(const unsigned int)> &create_request,
639  const std::function<AnswerType(const unsigned int,
640  const RequestType &)> &answer_request,
641  const std::function<void(const unsigned int, const AnswerType &)>
642  & process_answer,
643  const MPI_Comm &comm) override;
644 
645  private:
646 #ifdef DEAL_II_WITH_MPI
650  std::vector<std::vector<char>> send_buffers;
651 
655  std::vector<std::vector<char>> recv_buffers;
656 
660  std::vector<MPI_Request> send_request_requests;
661 
665  std::vector<std::vector<char>> requests_buffers;
666 
670  std::vector<MPI_Request> send_answer_requests;
671 #endif
675  std::set<unsigned int> requesting_processes;
676 
681  unsigned int
683  const std::vector<unsigned int> & targets,
684  const std::function<RequestType(const unsigned int)> &create_request,
685  const MPI_Comm & comm);
686 
691  void
693  const unsigned int index,
694  const std::function<AnswerType(const unsigned int,
695  const RequestType &)> &answer_request,
696  const MPI_Comm & comm);
697 
702  void
704  const unsigned int n_targets,
705  const std::function<void(const unsigned int, const AnswerType &)>
706  & process_answer,
707  const MPI_Comm &comm);
708 
713  void
715  };
716 
717 
718 
764  template <typename RequestType, typename AnswerType>
765  std::vector<unsigned int>
766  pex(const std::vector<unsigned int> & targets,
767  const std::function<RequestType(const unsigned int)> &create_request,
768  const std::function<AnswerType(const unsigned int,
769  const RequestType &)> &answer_request,
770  const std::function<void(const unsigned int, const AnswerType &)>
771  & process_answer,
772  const MPI_Comm &comm);
773 
800  template <typename RequestType>
801  std::vector<unsigned int>
802  pex(const std::vector<unsigned int> & targets,
803  const std::function<RequestType(const unsigned int)> &create_request,
804  const std::function<void(const unsigned int, const RequestType &)>
805  & process_request,
806  const MPI_Comm &comm);
807 
808 
813  template <typename RequestType, typename AnswerType>
814  class Serial : public Interface<RequestType, AnswerType>
815  {
816  public:
820  Serial() = default;
821 
836 
837  // Import the declarations from the base class.
839 
843  virtual std::vector<unsigned int>
845  const std::vector<unsigned int> & targets,
846  const std::function<RequestType(const unsigned int)> &create_request,
847  const std::function<AnswerType(const unsigned int,
848  const RequestType &)> &answer_request,
849  const std::function<void(const unsigned int, const AnswerType &)>
850  & process_answer,
851  const MPI_Comm &comm) override;
852  };
853 
854 
855 
885  template <typename RequestType, typename AnswerType>
886  std::vector<unsigned int>
888  const std::vector<unsigned int> & targets,
889  const std::function<RequestType(const unsigned int)> &create_request,
890  const std::function<AnswerType(const unsigned int, const RequestType &)>
891  &answer_request,
892  const std::function<void(const unsigned int, const AnswerType &)>
893  & process_answer,
894  const MPI_Comm &comm);
895 
922  template <typename RequestType>
923  std::vector<unsigned int>
925  const std::vector<unsigned int> & targets,
926  const std::function<RequestType(const unsigned int)> &create_request,
927  const std::function<void(const unsigned int, const RequestType &)>
928  & process_request,
929  const MPI_Comm &comm);
930 
931 
932 
945  template <typename RequestType, typename AnswerType>
946  class Selector : public Interface<RequestType, AnswerType>
947  {
948  public:
952  Selector() = default;
953 
968  const MPI_Comm & comm);
969 
973  virtual ~Selector() = default;
974 
975  // Import the declarations from the base class.
977 
983  virtual std::vector<unsigned int>
985  const std::vector<unsigned int> & targets,
986  const std::function<RequestType(const unsigned int)> &create_request,
987  const std::function<AnswerType(const unsigned int,
988  const RequestType &)> &answer_request,
989  const std::function<void(const unsigned int, const AnswerType &)>
990  & process_answer,
991  const MPI_Comm &comm) override;
992 
993  private:
994  // Pointer to the actual ConsensusAlgorithms::Interface implementation.
995  std::shared_ptr<Interface<RequestType, AnswerType>> consensus_algo;
996  };
997 
998 
999 
1033  template <typename RequestType, typename AnswerType>
1034  std::vector<unsigned int>
1036  const std::vector<unsigned int> & targets,
1037  const std::function<RequestType(const unsigned int)> &create_request,
1038  const std::function<AnswerType(const unsigned int, const RequestType &)>
1039  &answer_request,
1040  const std::function<void(const unsigned int, const AnswerType &)>
1041  & process_answer,
1042  const MPI_Comm &comm);
1043 
1070  template <typename RequestType>
1071  std::vector<unsigned int>
1073  const std::vector<unsigned int> & targets,
1074  const std::function<RequestType(const unsigned int)> &create_request,
1075  const std::function<void(const unsigned int, const RequestType &)>
1076  & process_request,
1077  const MPI_Comm &comm);
1078 
1079 
1086  template <typename RequestType, typename AnswerType>
1088  : public Process<RequestType, AnswerType>
1089  {
1090  public:
1101  const std::function<std::vector<unsigned int>()>
1102  &function_compute_targets,
1103  const std::function<void(const unsigned int, RequestType &)>
1104  & function_create_request = {},
1105  const std::function<void(const unsigned int,
1106  const RequestType &,
1107  AnswerType &)> &function_answer_request = {},
1108  const std::function<void(const unsigned int, const AnswerType &)>
1109  &function_read_answer = {});
1110 
1114  std::vector<unsigned int>
1115  compute_targets() override;
1116 
1120  void
1121  create_request(const unsigned int other_rank,
1122  RequestType & send_buffer) override;
1123 
1127  void
1128  answer_request(const unsigned int other_rank,
1129  const RequestType &buffer_recv,
1130  AnswerType & request_buffer) override;
1131 
1135  void
1136  read_answer(const unsigned int other_rank,
1137  const AnswerType & recv_buffer) override;
1138 
1139  private:
1140  const std::function<std::vector<unsigned int>()>
1142  const std::function<void(const int, RequestType &)>
1144  const std::function<
1145  void(const unsigned int, const RequestType &, AnswerType &)>
1147  const std::function<void(const int, const AnswerType &)>
1149  };
1150 
1151 
1152 #ifndef DOXYGEN
1153  // Implementation of the functions in this namespace.
1154 
1155  template <typename RequestType, typename AnswerType>
1156  std::vector<unsigned int>
1157  nbx(const std::vector<unsigned int> & targets,
1158  const std::function<RequestType(const unsigned int)> &create_request,
1159  const std::function<AnswerType(const unsigned int,
1160  const RequestType &)> &answer_request,
1161  const std::function<void(const unsigned int, const AnswerType &)>
1162  & process_answer,
1163  const MPI_Comm &comm)
1164  {
1166  targets, create_request, answer_request, process_answer, comm);
1167  }
1168 
1169 
1170 
1171  template <typename RequestType>
1172  std::vector<unsigned int>
1173  nbx(const std::vector<unsigned int> & targets,
1174  const std::function<RequestType(const unsigned int)> &create_request,
1175  const std::function<void(const unsigned int, const RequestType &)>
1176  & process_request,
1177  const MPI_Comm &comm)
1178  {
1179  // TODO: For the moment, simply implement this special case by
1180  // forwarding to the other function with rewritten function
1181  // objects and using an empty type as answer type. This way,
1182  // we have the interface in place and can provide a more
1183  // efficient implementation later on.
1184  using EmptyType = std::tuple<>;
1185 
1186  return nbx<RequestType, EmptyType>(
1187  targets,
1188  create_request,
1189  // answer_request:
1190  [&process_request](const unsigned int source_rank,
1191  const RequestType &request) -> EmptyType {
1192  process_request(source_rank, request);
1193  // Return something. What it is is arbitrary here, except that
1194  // we want it to be as small an object as possible. Using
1195  // std::tuple<> is interpreted as an empty object that is packed
1196  // down to a zero-length char array.
1197  return {};
1198  },
1199  // process_answer:
1200  [](const unsigned int /*target_rank */,
1201  const EmptyType & /*answer*/) {},
1202  comm);
1203  }
1204 
1205 
1206 
1207  template <typename RequestType, typename AnswerType>
1208  std::vector<unsigned int>
1209  pex(const std::vector<unsigned int> & targets,
1210  const std::function<RequestType(const unsigned int)> &create_request,
1211  const std::function<AnswerType(const unsigned int,
1212  const RequestType &)> &answer_request,
1213  const std::function<void(const unsigned int, const AnswerType &)>
1214  & process_answer,
1215  const MPI_Comm &comm)
1216  {
1217  return PEX<RequestType, AnswerType>().run(
1218  targets, create_request, answer_request, process_answer, comm);
1219  }
1220 
1221 
1222 
1223  template <typename RequestType>
1224  std::vector<unsigned int>
1225  pex(const std::vector<unsigned int> & targets,
1226  const std::function<RequestType(const unsigned int)> &create_request,
1227  const std::function<void(const unsigned int, const RequestType &)>
1228  & process_request,
1229  const MPI_Comm &comm)
1230  {
1231  // TODO: For the moment, simply implement this special case by
1232  // forwarding to the other function with rewritten function
1233  // objects and using an empty type as answer type. This way,
1234  // we have the interface in place and can provide a more
1235  // efficient implementation later on.
1236  using EmptyType = std::tuple<>;
1237 
1238  return pex<RequestType, EmptyType>(
1239  targets,
1240  create_request,
1241  // answer_request:
1242  [&process_request](const unsigned int source_rank,
1243  const RequestType &request) -> EmptyType {
1244  process_request(source_rank, request);
1245  // Return something. What it is is arbitrary here, except that
1246  // we want it to be as small an object as possible. Using
1247  // std::tuple<> is interpreted as an empty object that is packed
1248  // down to a zero-length char array.
1249  return {};
1250  },
1251  // process_answer:
1252  [](const unsigned int /*target_rank */,
1253  const EmptyType & /*answer*/) {},
1254  comm);
1255  }
1256 
1257 
1258 
1259  template <typename RequestType, typename AnswerType>
1260  std::vector<unsigned int>
1261  serial(
1262  const std::vector<unsigned int> & targets,
1263  const std::function<RequestType(const unsigned int)> &create_request,
1264  const std::function<AnswerType(const unsigned int, const RequestType &)>
1265  &answer_request,
1266  const std::function<void(const unsigned int, const AnswerType &)>
1267  & process_answer,
1268  const MPI_Comm &comm)
1269  {
1270  return Serial<RequestType, AnswerType>().run(
1271  targets, create_request, answer_request, process_answer, comm);
1272  }
1273 
1274 
1275 
1276  template <typename RequestType>
1277  std::vector<unsigned int>
1278  serial(
1279  const std::vector<unsigned int> & targets,
1280  const std::function<RequestType(const unsigned int)> &create_request,
1281  const std::function<void(const unsigned int, const RequestType &)>
1282  & process_request,
1283  const MPI_Comm &comm)
1284  {
1285  // TODO: For the moment, simply implement this special case by
1286  // forwarding to the other function with rewritten function
1287  // objects and using an empty type as answer type. This way,
1288  // we have the interface in place and can provide a more
1289  // efficient implementation later on.
1290  using EmptyType = std::tuple<>;
1291 
1292  return serial<RequestType, EmptyType>(
1293  targets,
1294  create_request,
1295  // answer_request:
1296  [&process_request](const unsigned int source_rank,
1297  const RequestType &request) -> EmptyType {
1298  process_request(source_rank, request);
1299  // Return something. What it is is arbitrary here, except that
1300  // we want it to be as small an object as possible. Using
1301  // std::tuple<> is interpreted as an empty object that is packed
1302  // down to a zero-length char array.
1303  return {};
1304  },
1305  // process_answer:
1306  [](const unsigned int /*target_rank */,
1307  const EmptyType & /*answer*/) {},
1308  comm);
1309  }
1310 
1311 
1312 
1313  template <typename RequestType, typename AnswerType>
1314  std::vector<unsigned int>
1315  selector(
1316  const std::vector<unsigned int> & targets,
1317  const std::function<RequestType(const unsigned int)> &create_request,
1318  const std::function<AnswerType(const unsigned int, const RequestType &)>
1319  &answer_request,
1320  const std::function<void(const unsigned int, const AnswerType &)>
1321  & process_answer,
1322  const MPI_Comm &comm)
1323  {
1324  return Selector<RequestType, AnswerType>().run(
1325  targets, create_request, answer_request, process_answer, comm);
1326  }
1327 
1328 
1329 
1330  template <typename RequestType>
1331  std::vector<unsigned int>
1332  selector(
1333  const std::vector<unsigned int> & targets,
1334  const std::function<RequestType(const unsigned int)> &create_request,
1335  const std::function<void(const unsigned int, const RequestType &)>
1336  & process_request,
1337  const MPI_Comm &comm)
1338  {
1339  // TODO: For the moment, simply implement this special case by
1340  // forwarding to the other function with rewritten function
1341  // objects and using an empty type as answer type. This way,
1342  // we have the interface in place and can provide a more
1343  // efficient implementation later on.
1344  using EmptyType = std::tuple<>;
1345 
1346  return selector<RequestType, EmptyType>(
1347  targets,
1348  create_request,
1349  // answer_request:
1350  [&process_request](const unsigned int source_rank,
1351  const RequestType &request) -> EmptyType {
1352  process_request(source_rank, request);
1353  // Return something. What it is is arbitrary here, except that
1354  // we want it to be as small an object as possible. Using
1355  // std::tuple<> is interpreted as an empty object that is packed
1356  // down to a zero-length char array.
1357  return {};
1358  },
1359  // process_answer:
1360  [](const unsigned int /*target_rank */,
1361  const EmptyType & /*answer*/) {},
1362  comm);
1363  }
1364 
1365 
1366 
1367  template <typename RequestType, typename AnswerType>
1369  const std::function<std::vector<unsigned int>()>
1370  &function_compute_targets,
1371  const std::function<void(const unsigned int, RequestType &)>
1372  & function_create_request,
1373  const std::function<void(const unsigned int,
1374  const RequestType &,
1375  AnswerType &)> &function_answer_request,
1376  const std::function<void(const unsigned int, const AnswerType &)>
1377  &function_read_answer)
1378  : function_compute_targets(function_compute_targets)
1379  , function_create_request(function_create_request)
1380  , function_answer_request(function_answer_request)
1381  , function_read_answer(function_read_answer)
1382  {}
1383 
1384 
1385 
1386  template <typename RequestType, typename AnswerType>
1387  std::vector<unsigned int>
1389  {
1390  return function_compute_targets();
1391  }
1392 
1393 
1394 
1395  template <typename RequestType, typename AnswerType>
1396  void
1398  const unsigned int other_rank,
1399  RequestType & send_buffer)
1400  {
1401  if (function_create_request)
1402  function_create_request(other_rank, send_buffer);
1403  }
1404 
1405 
1406 
1407  template <typename RequestType, typename AnswerType>
1408  void
1410  const unsigned int other_rank,
1411  const RequestType &buffer_recv,
1412  AnswerType & request_buffer)
1413  {
1414  if (function_answer_request)
1415  function_answer_request(other_rank, buffer_recv, request_buffer);
1416  }
1417 
1418 
1419 
1420  template <typename RequestType, typename AnswerType>
1421  void
1423  const unsigned int other_rank,
1424  const AnswerType & recv_buffer)
1425  {
1426  if (function_read_answer)
1427  function_read_answer(other_rank, recv_buffer);
1428  }
1429 
1430 #endif
1431 
1432 
1433  } // namespace ConsensusAlgorithms
1434  } // end of namespace MPI
1435 } // end of namespace Utilities
1436 
1437 
1438 
1439 #ifndef DOXYGEN
1440 
1441 // ----------------- Implementation of template functions
1442 
1443 namespace Utilities
1444 {
1445  namespace MPI
1446  {
1447  namespace ConsensusAlgorithms
1448  {
1449  namespace
1450  {
1466 # ifndef DEAL_II_MSVC
1467  [[gnu::unused]]
1468 # endif
1469  inline bool
1470  has_unique_elements(const std::vector<unsigned int> &targets)
1471  {
1472  std::vector<unsigned int> my_destinations = targets;
1473  std::sort(my_destinations.begin(), my_destinations.end());
1474  return (std::adjacent_find(my_destinations.begin(),
1475  my_destinations.end()) ==
1476  my_destinations.end());
1477  }
1478  } // namespace
1479 
1480 
1481 
1482  template <typename RequestType, typename AnswerType>
1483  void
1485  const RequestType &,
1486  AnswerType &)
1487  {
1488  // nothing to do
1489  }
1490 
1491 
1492 
1493  template <typename RequestType, typename AnswerType>
1494  void
1496  RequestType &)
1497  {
1498  // nothing to do
1499  }
1500 
1501 
1502 
1503  template <typename RequestType, typename AnswerType>
1504  void
1506  const AnswerType &)
1507  {
1508  // nothing to do
1509  }
1510 
1511 
1512 
1513  template <typename RequestType, typename AnswerType>
1515  Process<RequestType, AnswerType> &process,
1516  const MPI_Comm & comm)
1517  : process(&process)
1518  , comm(comm)
1519  {}
1520 
1521 
1522 
1523  template <typename RequestType, typename AnswerType>
1525  : process(nullptr)
1526  , comm(MPI_COMM_NULL)
1527  {}
1528 
1529 
1530 
1531  template <typename RequestType, typename AnswerType>
1532  std::vector<unsigned int>
1534  {
1535  Assert(process != nullptr,
1536  ExcMessage("This function can only be called if the "
1537  "deprecated non-default constructor of this class "
1538  "has previously been called to set the Process "
1539  "object and a communicator."));
1540  return run(*process, comm);
1541  }
1542 
1543 
1544 
1545  template <typename RequestType, typename AnswerType>
1546  std::vector<unsigned int>
1548  Process<RequestType, AnswerType> &process,
1549  const MPI_Comm & comm)
1550  {
1551  // Unpack the 'process' object and call the function that takes
1552  // function objects for all operations.
1553  return run(
1554  process.compute_targets(),
1555  /* create_request: */
1556  [&process](const unsigned int target) {
1557  RequestType request;
1558  process.create_request(target, request);
1559  return request;
1560  },
1561  /* answer_request: */
1562  [&process](const unsigned int source, const RequestType &request) {
1563  AnswerType answer;
1564  process.answer_request(source, request, answer);
1565  return answer;
1566  },
1567  /* process_answer: */
1568  [&process](const unsigned int target, const AnswerType &answer) {
1569  process.read_answer(target, answer);
1570  },
1571  comm);
1572  }
1573 
1574 
1575 
1576  template <typename RequestType, typename AnswerType>
1578  Process<RequestType, AnswerType> &process,
1579  const MPI_Comm & comm)
1580  : Interface<RequestType, AnswerType>(process, comm)
1581  {}
1582 
1583 
1584 
1585  template <typename RequestType, typename AnswerType>
1586  std::vector<unsigned int>
1588  const std::vector<unsigned int> & targets,
1589  const std::function<RequestType(const unsigned int)> &create_request,
1590  const std::function<AnswerType(const unsigned int, const RequestType &)>
1591  &answer_request,
1592  const std::function<void(const unsigned int, const AnswerType &)>
1593  & process_answer,
1594  const MPI_Comm &comm)
1595  {
1596  Assert(has_unique_elements(targets),
1597  ExcMessage("The consensus algorithms expect that each process "
1598  "only sends a single message to another process, "
1599  "but the targets provided include duplicates."));
1600 
1601  static CollectiveMutex mutex;
1602  CollectiveMutex::ScopedLock lock(mutex, comm);
1603 
1604  // 1) Send data to identified targets and start receiving
1605  // the answers from these very same processes.
1606  start_communication(targets, create_request, comm);
1607 
1608  // 2) Until all posted receive operations are known to have completed,
1609  // answer requests and keep checking whether all requests of
1610  // this process have been answered.
1611  //
1612  // The requests that we catch in the answer_requests() function
1613  // originate elsewhere, that is, they are not in response
1614  // to our own messages
1615  //
1616  // Note also that we may not catch all incoming requests in
1617  // the following two lines: our own requests may have been
1618  // satisfied before we've dealt with all incoming requests.
1619  // That's ok: We will get around to dealing with all remaining
1620  // message later. We just want to move on to the next step
1621  // as early as possible.
1622  while (all_locally_originated_receives_are_completed(process_answer,
1623  comm) == false)
1624  maybe_answer_one_request(answer_request, comm);
1625 
1626  // 3) Signal to all other processes that all requests of this process
1627  // have been answered
1628  signal_finish(comm);
1629 
1630  // 4) Nevertheless, this process has to keep on answering (potential)
1631  // incoming requests until all processes have received the
1632  // answer to all requests
1633  while (all_remotely_originated_receives_are_completed() == false)
1634  maybe_answer_one_request(answer_request, comm);
1635 
1636  // 5) process the answer to all requests
1637  clean_up_and_end_communication(comm);
1638 
1639  return std::vector<unsigned int>(requesting_processes.begin(),
1640  requesting_processes.end());
1641  }
1642 
1643 
1644 
1645  template <typename RequestType, typename AnswerType>
1646  void
1648  const std::vector<unsigned int> & targets,
1649  const std::function<RequestType(const unsigned int)> &create_request,
1650  const MPI_Comm & comm)
1651  {
1652 # ifdef DEAL_II_WITH_MPI
1653  // 1)
1654  const auto n_targets = targets.size();
1655 
1656  const int tag_request = Utilities::MPI::internal::Tags::
1658 
1659  // 2) allocate memory
1660  send_requests.resize(n_targets);
1661  send_buffers.resize(n_targets);
1662 
1663  {
1664  // 4) send and receive
1665  for (unsigned int index = 0; index < n_targets; ++index)
1666  {
1667  const unsigned int rank = targets[index];
1669 
1670  auto &send_buffer = send_buffers[index];
1671  send_buffer =
1672  (create_request ? Utilities::pack(create_request(rank), false) :
1673  std::vector<char>());
1674 
1675  // Post a request to send data
1676  auto ierr = MPI_Isend(send_buffer.data(),
1677  send_buffer.size(),
1678  MPI_CHAR,
1679  rank,
1680  tag_request,
1681  comm,
1682  &send_requests[index]);
1683  AssertThrowMPI(ierr);
1684  }
1685 
1686  // Also record that we expect an answer from each target we sent
1687  // a request to:
1688  n_outstanding_answers = n_targets;
1689  }
1690 # else
1691  (void)targets;
1692  (void)create_request;
1693  (void)comm;
1694 # endif
1695  }
1696 
1697 
1698 
1699  template <typename RequestType, typename AnswerType>
1700  bool
1703  const std::function<void(const unsigned int, const AnswerType &)>
1704  & process_answer,
1705  const MPI_Comm &comm)
1706  {
1707 # ifdef DEAL_II_WITH_MPI
1708  // We know that all requests have come in when we have pending
1709  // messages from all targets with the right tag (some of which we may
1710  // have already taken care of below, after discovering their existence).
1711  // We can check for pending messages with MPI_IProbe, which returns
1712  // immediately with a return code that indicates whether
1713  // it has found a message from any process with a given
1714  // tag.
1715  if (n_outstanding_answers == 0)
1716  return true;
1717  else
1718  {
1719  const int tag_deliver = Utilities::MPI::internal::Tags::
1721 
1722  int request_is_pending;
1723  MPI_Status status;
1724  const auto ierr = MPI_Iprobe(
1725  MPI_ANY_SOURCE, tag_deliver, comm, &request_is_pending, &status);
1726  AssertThrowMPI(ierr);
1727 
1728  // If there is no pending message with this tag,
1729  // then we are clearly not done receiving everything
1730  // yet -- so return false.
1731  if (request_is_pending == 0)
1732  return false;
1733  else
1734  {
1735  // OK, so we have gotten a reply to our answer from
1736  // one rank. Let us process it, after double checking
1737  // that it is indeed one we were still expecting:
1738  const auto target = status.MPI_SOURCE;
1739 
1740  // Then query the size of the message, allocate enough memory,
1741  // receive the data, and process it.
1742  int message_size;
1743  {
1744  const int ierr =
1745  MPI_Get_count(&status, MPI_CHAR, &message_size);
1746  AssertThrowMPI(ierr);
1747  }
1748  std::vector<char> recv_buffer(message_size);
1749 
1750  {
1751  const int tag_deliver = Utilities::MPI::internal::Tags::
1753 
1754  const int ierr = MPI_Recv(recv_buffer.data(),
1755  recv_buffer.size(),
1756  MPI_CHAR,
1757  target,
1758  tag_deliver,
1759  comm,
1760  MPI_STATUS_IGNORE);
1761  AssertThrowMPI(ierr);
1762  }
1763 
1764  if (process_answer)
1765  process_answer(target,
1766  Utilities::unpack<AnswerType>(recv_buffer,
1767  false));
1768 
1769  // Finally, remove this rank from the list of outstanding
1770  // targets:
1771  --n_outstanding_answers;
1772 
1773  // We could do another go-around from the top of this
1774  // else-branch to see whether there are actually other messages
1775  // that are currently pending. But that would mean spending
1776  // substantial time in receiving answers while we should also be
1777  // sending answers to requests we have received from other
1778  // places. So let it be enough for now. If there are outstanding
1779  // answers, we will get back to this function before long and
1780  // can take care of them then.
1781  return (n_outstanding_answers == 0);
1782  }
1783  }
1784 
1785 # else
1786  (void)process_answer;
1787  (void)comm;
1788 
1789  return true;
1790 # endif
1791  }
1792 
1793 
1794 
1795  template <typename RequestType, typename AnswerType>
1796  void
1798  const std::function<AnswerType(const unsigned int, const RequestType &)>
1799  & answer_request,
1800  const MPI_Comm &comm)
1801  {
1802 # ifdef DEAL_II_WITH_MPI
1803 
1804  const int tag_request = Utilities::MPI::internal::Tags::
1806  const int tag_deliver = Utilities::MPI::internal::Tags::
1808 
1809  // Check if there is a request pending. By selecting the
1810  // tag_request tag, these are other processes asking for
1811  // our own replies, not these other processes' replies
1812  // to our own requests.
1813  //
1814  // There may be multiple such pending messages. We
1815  // only answer one.
1816  MPI_Status status;
1817  int request_is_pending;
1818  const auto ierr = MPI_Iprobe(
1819  MPI_ANY_SOURCE, tag_request, comm, &request_is_pending, &status);
1820  AssertThrowMPI(ierr);
1821 
1822  if (request_is_pending != 0)
1823  {
1824  // Get the rank of the requesting process and add it to the
1825  // list of requesting processes (which may contain duplicates).
1826  const auto other_rank = status.MPI_SOURCE;
1827 
1828  Assert(requesting_processes.find(other_rank) ==
1829  requesting_processes.end(),
1830  ExcMessage("Process is requesting a second time!"));
1831  requesting_processes.insert(other_rank);
1832 
1833  // get size of incoming message
1834  int number_amount;
1835  auto ierr = MPI_Get_count(&status, MPI_CHAR, &number_amount);
1836  AssertThrowMPI(ierr);
1837 
1838  // allocate memory for incoming message
1839  std::vector<char> buffer_recv(number_amount);
1840  ierr = MPI_Recv(buffer_recv.data(),
1841  number_amount,
1842  MPI_CHAR,
1843  other_rank,
1844  tag_request,
1845  comm,
1846  MPI_STATUS_IGNORE);
1847  AssertThrowMPI(ierr);
1848 
1849  // Allocate memory for an answer message to the current request,
1850  // and ask the 'process' object to produce an answer:
1851  request_buffers.emplace_back(std::make_unique<std::vector<char>>());
1852  auto &request_buffer = *request_buffers.back();
1853  if (answer_request)
1854  request_buffer =
1855  Utilities::pack(answer_request(other_rank,
1856  Utilities::unpack<RequestType>(
1857  buffer_recv, false)),
1858  false);
1859 
1860  // Then initiate sending the answer back to the requester.
1861  request_requests.emplace_back(std::make_unique<MPI_Request>());
1862  ierr = MPI_Isend(request_buffer.data(),
1863  request_buffer.size(),
1864  MPI_CHAR,
1865  other_rank,
1866  tag_deliver,
1867  comm,
1868  request_requests.back().get());
1869  AssertThrowMPI(ierr);
1870  }
1871 # else
1872  (void)answer_request;
1873  (void)comm;
1874 # endif
1875  }
1876 
1877 
1878 
1879  template <typename RequestType, typename AnswerType>
1880  void
1882  {
1883 # ifdef DEAL_II_WITH_MPI
1884  const auto ierr = MPI_Ibarrier(comm, &barrier_request);
1885  AssertThrowMPI(ierr);
1886 # else
1887  (void)comm;
1888 # endif
1889  }
1890 
1891 
1892 
1893  template <typename RequestType, typename AnswerType>
1894  bool
1895  NBX<RequestType,
1896  AnswerType>::all_remotely_originated_receives_are_completed()
1897  {
1898 # ifdef DEAL_II_WITH_MPI
1899  int all_ranks_reached_barrier;
1900  const auto ierr = MPI_Test(&barrier_request,
1901  &all_ranks_reached_barrier,
1902  MPI_STATUSES_IGNORE);
1903  AssertThrowMPI(ierr);
1904  return all_ranks_reached_barrier != 0;
1905 # else
1906  return true;
1907 # endif
1908  }
1909 
1910 
1911 
1912  template <typename RequestType, typename AnswerType>
1913  void
1915  const MPI_Comm &comm)
1916  {
1917  (void)comm;
1918 # ifdef DEAL_II_WITH_MPI
1919  // clean up
1920  {
1921  if (send_requests.size() > 0)
1922  {
1923  const int ierr = MPI_Waitall(send_requests.size(),
1924  send_requests.data(),
1925  MPI_STATUSES_IGNORE);
1926  AssertThrowMPI(ierr);
1927  }
1928 
1929  int ierr = MPI_Wait(&barrier_request, MPI_STATUS_IGNORE);
1930  AssertThrowMPI(ierr);
1931 
1932  for (auto &i : request_requests)
1933  {
1934  ierr = MPI_Wait(i.get(), MPI_STATUS_IGNORE);
1935  AssertThrowMPI(ierr);
1936  }
1937 
1938 # ifdef DEBUG
1939  // note: IBarrier seems to make problem during testing, this
1940  // additional Barrier seems to help
1941  ierr = MPI_Barrier(comm);
1942  AssertThrowMPI(ierr);
1943 # endif
1944  }
1945 # endif
1946  }
1947 
1948 
1949 
1950  template <typename RequestType, typename AnswerType>
1952  Process<RequestType, AnswerType> &process,
1953  const MPI_Comm & comm)
1954  : Interface<RequestType, AnswerType>(process, comm)
1955  {}
1956 
1957 
1958 
1959  template <typename RequestType, typename AnswerType>
1960  std::vector<unsigned int>
1962  const std::vector<unsigned int> & targets,
1963  const std::function<RequestType(const unsigned int)> &create_request,
1964  const std::function<AnswerType(const unsigned int, const RequestType &)>
1965  &answer_request,
1966  const std::function<void(const unsigned int, const AnswerType &)>
1967  & process_answer,
1968  const MPI_Comm &comm)
1969  {
1970  Assert(has_unique_elements(targets),
1971  ExcMessage("The consensus algorithms expect that each process "
1972  "only sends a single message to another process, "
1973  "but the targets provided include duplicates."));
1974 
1975  static CollectiveMutex mutex;
1976  CollectiveMutex::ScopedLock lock(mutex, comm);
1977 
1978  // 1) Send requests and start receiving the answers.
1979  // In particular, determine how many requests we should expect
1980  // on the current process.
1981  const unsigned int n_requests =
1982  start_communication(targets, create_request, comm);
1983 
1984  // 2) Answer requests:
1985  for (unsigned int request = 0; request < n_requests; ++request)
1986  answer_one_request(request, answer_request, comm);
1987 
1988  // 3) Process answers:
1989  process_incoming_answers(targets.size(), process_answer, comm);
1990 
1991  // 4) Make sure all sends have successfully terminated:
1992  clean_up_and_end_communication();
1993 
1994  return std::vector<unsigned int>(requesting_processes.begin(),
1995  requesting_processes.end());
1996  }
1997 
1998 
1999 
2000  template <typename RequestType, typename AnswerType>
2001  unsigned int
2003  const std::vector<unsigned int> & targets,
2004  const std::function<RequestType(const unsigned int)> &create_request,
2005  const MPI_Comm & comm)
2006  {
2007 # ifdef DEAL_II_WITH_MPI
2008  const int tag_request = Utilities::MPI::internal::Tags::
2010 
2011  // 1) determine with which processes this process wants to communicate
2012  // with
2013  const unsigned int n_targets = targets.size();
2014 
2015  // 2) determine who wants to communicate with this process
2016  const unsigned int n_sources =
2018 
2019  // 2) allocate memory
2020  recv_buffers.resize(n_targets);
2021  send_buffers.resize(n_targets);
2022  send_request_requests.resize(n_targets);
2023 
2024  send_answer_requests.resize(n_sources);
2025  requests_buffers.resize(n_sources);
2026 
2027  // 4) send and receive
2028  for (unsigned int i = 0; i < n_targets; ++i)
2029  {
2030  const unsigned int rank = targets[i];
2032 
2033  // pack data which should be sent
2034  auto &send_buffer = send_buffers[i];
2035  if (create_request)
2036  send_buffer = Utilities::pack(create_request(rank), false);
2037 
2038  // start to send data
2039  auto ierr = MPI_Isend(send_buffer.data(),
2040  send_buffer.size(),
2041  MPI_CHAR,
2042  rank,
2043  tag_request,
2044  comm,
2045  &send_request_requests[i]);
2046  AssertThrowMPI(ierr);
2047  }
2048 
2049  return n_sources;
2050 # else
2051  (void)targets;
2052  (void)create_request;
2053  (void)comm;
2054  return 0;
2055 # endif
2056  }
2057 
2058 
2059 
2060  template <typename RequestType, typename AnswerType>
2061  void
2063  const unsigned int index,
2064  const std::function<AnswerType(const unsigned int, const RequestType &)>
2065  & answer_request,
2066  const MPI_Comm &comm)
2067  {
2068 # ifdef DEAL_II_WITH_MPI
2069  const int tag_request = Utilities::MPI::internal::Tags::
2071  const int tag_deliver = Utilities::MPI::internal::Tags::
2073 
2074  // Wait until we have a message ready for retrieval, though we don't
2075  // care which process it is from.
2076  MPI_Status status;
2077  int ierr = MPI_Probe(MPI_ANY_SOURCE, tag_request, comm, &status);
2078  AssertThrowMPI(ierr);
2079 
2080  // Get rank of incoming message and verify that it makes sense
2081  const unsigned int other_rank = status.MPI_SOURCE;
2082 
2083  Assert(requesting_processes.find(other_rank) ==
2084  requesting_processes.end(),
2085  ExcMessage(
2086  "A process is sending a request after a request from "
2087  "the same process has previously already been "
2088  "received. This algorithm does not expect this to happen."));
2089  requesting_processes.insert(other_rank);
2090 
2091  // Actually get the incoming message:
2092  int number_amount;
2093  ierr = MPI_Get_count(&status, MPI_CHAR, &number_amount);
2094  AssertThrowMPI(ierr);
2095 
2096  std::vector<char> buffer_recv(number_amount);
2097  ierr = MPI_Recv(buffer_recv.data(),
2098  number_amount,
2099  MPI_CHAR,
2100  other_rank,
2101  tag_request,
2102  comm,
2103  &status);
2104  AssertThrowMPI(ierr);
2105 
2106  // Process request by asking the user-provided function for
2107  // the answer and post a send for it.
2108  auto &request_buffer = requests_buffers[index];
2109  request_buffer =
2110  (answer_request ?
2111  Utilities::pack(answer_request(other_rank,
2112  Utilities::unpack<RequestType>(
2113  buffer_recv, false)),
2114  false) :
2115  std::vector<char>());
2116 
2117  ierr = MPI_Isend(request_buffer.data(),
2118  request_buffer.size(),
2119  MPI_CHAR,
2120  other_rank,
2121  tag_deliver,
2122  comm,
2123  &send_answer_requests[index]);
2124  AssertThrowMPI(ierr);
2125 # else
2126  (void)answer_request;
2127  (void)comm;
2128  (void)index;
2129 # endif
2130  }
2131 
2132 
2133 
2134  template <typename RequestType, typename AnswerType>
2135  void
2137  const unsigned int n_targets,
2138  const std::function<void(const unsigned int, const AnswerType &)>
2139  & process_answer,
2140  const MPI_Comm &comm)
2141  {
2142 # ifdef DEAL_II_WITH_MPI
2143  const int tag_deliver = Utilities::MPI::internal::Tags::
2145 
2146  // We know how many targets we have sent requests to. These
2147  // targets will all eventually send us their responses, but
2148  // we need not process them in order -- rather, just see what
2149  // comes in and then look at message originators' ranks and
2150  // message sizes
2151  for (unsigned int i = 0; i < n_targets; ++i)
2152  {
2153  MPI_Status status;
2154  {
2155  const int ierr =
2156  MPI_Probe(MPI_ANY_SOURCE, tag_deliver, comm, &status);
2157  AssertThrowMPI(ierr);
2158  }
2159 
2160  const auto other_rank = status.MPI_SOURCE;
2161  int message_size;
2162  {
2163  const int ierr = MPI_Get_count(&status, MPI_CHAR, &message_size);
2164  AssertThrowMPI(ierr);
2165  }
2166  std::vector<char> recv_buffer(message_size);
2167 
2168  // Now actually receive the answer. Because the MPI_Probe
2169  // above blocks until we have a message, we know that the
2170  // following MPI_Recv call will immediately succeed.
2171  {
2172  const int ierr = MPI_Recv(recv_buffer.data(),
2173  recv_buffer.size(),
2174  MPI_CHAR,
2175  other_rank,
2176  tag_deliver,
2177  comm,
2178  MPI_STATUS_IGNORE);
2179  AssertThrowMPI(ierr);
2180  }
2181 
2182  if (process_answer)
2183  process_answer(other_rank,
2184  Utilities::unpack<AnswerType>(recv_buffer, false));
2185  }
2186 # else
2187  (void)n_targets;
2188  (void)process_answer;
2189  (void)comm;
2190 # endif
2191  }
2192 
2193 
2194 
2195  template <typename RequestType, typename AnswerType>
2196  void
2198  {
2199 # ifdef DEAL_II_WITH_MPI
2200  // Finalize all MPI_Request objects for both the
2201  // send-request and receive-answer operations.
2202  if (send_request_requests.size() > 0)
2203  {
2204  const int ierr = MPI_Waitall(send_request_requests.size(),
2205  send_request_requests.data(),
2206  MPI_STATUSES_IGNORE);
2207  AssertThrowMPI(ierr);
2208  }
2209 
2210  // Then also check the send-answer requests.
2211  if (send_answer_requests.size() > 0)
2212  {
2213  const int ierr = MPI_Waitall(send_answer_requests.size(),
2214  send_answer_requests.data(),
2215  MPI_STATUSES_IGNORE);
2216  AssertThrowMPI(ierr);
2217  }
2218 # endif
2219  }
2220 
2221 
2222 
2223  template <typename RequestType, typename AnswerType>
2225  Process<RequestType, AnswerType> &process,
2226  const MPI_Comm & comm)
2227  : Interface<RequestType, AnswerType>(process, comm)
2228  {}
2229 
2230 
2231 
2232  template <typename RequestType, typename AnswerType>
2233  std::vector<unsigned int>
2235  const std::vector<unsigned int> & targets,
2236  const std::function<RequestType(const unsigned int)> &create_request,
2237  const std::function<AnswerType(const unsigned int, const RequestType &)>
2238  &answer_request,
2239  const std::function<void(const unsigned int, const AnswerType &)>
2240  & process_answer,
2241  const MPI_Comm &comm)
2242  {
2243  (void)comm;
2246  ExcMessage("You shouldn't use the 'Serial' class on "
2247  "communicators that have more than one process "
2248  "associated with it."));
2249 
2250  // The only valid target for a serial program is itself.
2251  if (targets.size() != 0)
2252  {
2253  Assert(targets.size() == 1,
2254  ExcMessage(
2255  "On a single process, the only valid target "
2256  "is process zero (the process itself), which can only be "
2257  "listed once."));
2258  AssertDimension(targets[0], 0);
2259 
2260  // Since the caller indicates that there is a target, and since we
2261  // know that it is the current process, let the process send
2262  // something to itself.
2263  const RequestType request =
2264  (create_request ? create_request(0) : RequestType());
2265  const AnswerType answer =
2266  (answer_request ? answer_request(0, request) : AnswerType());
2267 
2268  if (process_answer)
2269  process_answer(0, answer);
2270  }
2271 
2272  return targets; // nothing to do
2273  }
2274 
2275 
2276 
2277  template <typename RequestType, typename AnswerType>
2279  Process<RequestType, AnswerType> &process,
2280  const MPI_Comm & comm)
2281  : Interface<RequestType, AnswerType>(process, comm)
2282  {}
2283 
2284 
2285 
2286  template <typename RequestType, typename AnswerType>
2287  std::vector<unsigned int>
2289  const std::vector<unsigned int> & targets,
2290  const std::function<RequestType(const unsigned int)> &create_request,
2291  const std::function<AnswerType(const unsigned int, const RequestType &)>
2292  &answer_request,
2293  const std::function<void(const unsigned int, const AnswerType &)>
2294  & process_answer,
2295  const MPI_Comm &comm)
2296  {
2297  // Depending on the number of processes we switch between
2298  // implementations. We reduce the threshold for debug mode to be
2299  // able to test also the non-blocking implementation. This feature
2300  // is tested by:
2301  // tests/multigrid/transfer_matrix_free_06.with_mpi=true.with_p4est=true.with_trilinos=true.mpirun=10.output
2302 
2303  const unsigned int n_procs = (Utilities::MPI::job_supports_mpi() ?
2305  1);
2306 # ifdef DEAL_II_WITH_MPI
2307 # ifdef DEBUG
2308  if (n_procs > 10)
2309 # else
2310  if (n_procs > 99)
2311 # endif
2312  consensus_algo.reset(new NBX<RequestType, AnswerType>());
2313  else
2314 # endif
2315  if (n_procs > 1)
2316  consensus_algo.reset(new PEX<RequestType, AnswerType>());
2317  else
2318  consensus_algo.reset(new Serial<RequestType, AnswerType>());
2319 
2320  return consensus_algo->run(
2321  targets, create_request, answer_request, process_answer, comm);
2322  }
2323 
2324 
2325  } // namespace ConsensusAlgorithms
2326  } // end of namespace MPI
2327 } // end of namespace Utilities
2328 
2329 #endif // DOXYGEN
2330 
2331 
2333 
2334 #endif
const std::function< void(const unsigned int, const RequestType &, AnswerType &)> function_answer_request
void create_request(const unsigned int other_rank, RequestType &send_buffer) override
const std::function< void(const int, RequestType &)> function_create_request
void read_answer(const unsigned int other_rank, const AnswerType &recv_buffer) override
std::vector< unsigned int > compute_targets() override
void answer_request(const unsigned int other_rank, const RequestType &buffer_recv, AnswerType &request_buffer) override
const std::function< void(const int, const AnswerType &)> function_read_answer
AnonymousProcess(const std::function< std::vector< unsigned int >()> &function_compute_targets, const std::function< void(const unsigned int, RequestType &)> &function_create_request={}, const std::function< void(const unsigned int, const RequestType &, AnswerType &)> &function_answer_request={}, const std::function< void(const unsigned int, const AnswerType &)> &function_read_answer={})
const std::function< std::vector< unsigned int >)> function_compute_targets
virtual std::vector< unsigned int > run(const std::vector< unsigned int > &targets, const std::function< RequestType(const unsigned int)> &create_request, const std::function< AnswerType(const unsigned int, const RequestType &)> &answer_request, const std::function< void(const unsigned int, const AnswerType &)> &process_answer, const MPI_Comm &comm)=0
std::vector< unsigned int > run(Process< RequestType, AnswerType > &process, const MPI_Comm &comm)
Interface(Process< RequestType, AnswerType > &process, const MPI_Comm &comm)
bool all_locally_originated_receives_are_completed(const std::function< void(const unsigned int, const AnswerType &)> &process_answer, const MPI_Comm &comm)
void maybe_answer_one_request(const std::function< AnswerType(const unsigned int, const RequestType &)> &answer_request, const MPI_Comm &comm)
virtual std::vector< unsigned int > run(const std::vector< unsigned int > &targets, const std::function< RequestType(const unsigned int)> &create_request, const std::function< AnswerType(const unsigned int, const RequestType &)> &answer_request, const std::function< void(const unsigned int, const AnswerType &)> &process_answer, const MPI_Comm &comm) override
void signal_finish(const MPI_Comm &comm)
std::vector< std::unique_ptr< std::vector< char > > > request_buffers
NBX(Process< RequestType, AnswerType > &process, const MPI_Comm &comm)
std::vector< std::unique_ptr< MPI_Request > > request_requests
std::vector< std::vector< char > > send_buffers
void clean_up_and_end_communication(const MPI_Comm &comm)
void start_communication(const std::vector< unsigned int > &targets, const std::function< RequestType(const unsigned int)> &create_request, const MPI_Comm &comm)
void answer_one_request(const unsigned int index, const std::function< AnswerType(const unsigned int, const RequestType &)> &answer_request, const MPI_Comm &comm)
std::vector< std::vector< char > > requests_buffers
unsigned int start_communication(const std::vector< unsigned int > &targets, const std::function< RequestType(const unsigned int)> &create_request, const MPI_Comm &comm)
std::vector< std::vector< char > > send_buffers
PEX(Process< RequestType, AnswerType > &process, const MPI_Comm &comm)
virtual std::vector< unsigned int > run(const std::vector< unsigned int > &targets, const std::function< RequestType(const unsigned int)> &create_request, const std::function< AnswerType(const unsigned int, const RequestType &)> &answer_request, const std::function< void(const unsigned int, const AnswerType &)> &process_answer, const MPI_Comm &comm) override
void process_incoming_answers(const unsigned int n_targets, const std::function< void(const unsigned int, const AnswerType &)> &process_answer, const MPI_Comm &comm)
std::vector< std::vector< char > > recv_buffers
virtual void answer_request(const unsigned int other_rank, const RequestType &buffer_recv, AnswerType &request_buffer)
virtual void read_answer(const unsigned int other_rank, const AnswerType &recv_buffer)
virtual std::vector< unsigned int > compute_targets()=0
virtual void create_request(const unsigned int other_rank, RequestType &send_buffer)
Selector(Process< RequestType, AnswerType > &process, const MPI_Comm &comm)
virtual std::vector< unsigned int > run(const std::vector< unsigned int > &targets, const std::function< RequestType(const unsigned int)> &create_request, const std::function< AnswerType(const unsigned int, const RequestType &)> &answer_request, const std::function< void(const unsigned int, const AnswerType &)> &process_answer, const MPI_Comm &comm) override
std::shared_ptr< Interface< RequestType, AnswerType > > consensus_algo
virtual std::vector< unsigned int > run(const std::vector< unsigned int > &targets, const std::function< RequestType(const unsigned int)> &create_request, const std::function< AnswerType(const unsigned int, const RequestType &)> &answer_request, const std::function< void(const unsigned int, const AnswerType &)> &process_answer, const MPI_Comm &comm) override
Serial(Process< RequestType, AnswerType > &process, const MPI_Comm &comm)
#define DEAL_II_DEPRECATED
Definition: config.h:164
#define DEAL_II_NAMESPACE_OPEN
Definition: config.h:442
#define DEAL_II_NAMESPACE_CLOSE
Definition: config.h:443
#define Assert(cond, exc)
Definition: exceptions.h:1473
#define AssertDimension(dim1, dim2)
Definition: exceptions.h:1667
#define AssertThrowMPI(error_code)
Definition: exceptions.h:1790
#define AssertIndexRange(index, range)
Definition: exceptions.h:1732
static ::ExceptionBase & ExcMessage(std::string arg1)
std::vector< unsigned int > serial(const std::vector< unsigned int > &targets, const std::function< RequestType(const unsigned int)> &create_request, const std::function< AnswerType(const unsigned int, const RequestType &)> &answer_request, const std::function< void(const unsigned int, const AnswerType &)> &process_answer, const MPI_Comm &comm)
std::vector< unsigned int > nbx(const std::vector< unsigned int > &targets, const std::function< RequestType(const unsigned int)> &create_request, const std::function< AnswerType(const unsigned int, const RequestType &)> &answer_request, const std::function< void(const unsigned int, const AnswerType &)> &process_answer, const MPI_Comm &comm)
std::vector< unsigned int > pex(const std::vector< unsigned int > &targets, const std::function< RequestType(const unsigned int)> &create_request, const std::function< AnswerType(const unsigned int, const RequestType &)> &answer_request, const std::function< void(const unsigned int, const AnswerType &)> &process_answer, const MPI_Comm &comm)
std::vector< unsigned int > selector(const std::vector< unsigned int > &targets, const std::function< RequestType(const unsigned int)> &create_request, const std::function< AnswerType(const unsigned int, const RequestType &)> &answer_request, const std::function< void(const unsigned int, const AnswerType &)> &process_answer, const MPI_Comm &comm)
@ consensus_algorithm_nbx_process_deliver
ConsensusAlgorithms::NBX::process.
Definition: mpi_tags.h:91
@ consensus_algorithm_pex_process_deliver
ConsensusAlgorithms::PEX::process.
Definition: mpi_tags.h:96
@ consensus_algorithm_nbx_answer_request
ConsensusAlgorithms::NBX::process.
Definition: mpi_tags.h:89
@ consensus_algorithm_pex_answer_request
ConsensusAlgorithms::PEX::process.
Definition: mpi_tags.h:94
unsigned int compute_n_point_to_point_communications(const MPI_Comm &mpi_comm, const std::vector< unsigned int > &destinations)
Definition: mpi.cc:424
bool job_supports_mpi()
Definition: mpi.cc:1027
unsigned int n_mpi_processes(const MPI_Comm &mpi_communicator)
Definition: mpi.cc:145
size_t pack(const T &object, std::vector< char > &dest_buffer, const bool allow_compression=true)
Definition: utilities.h:1483
void run(const Iterator &begin, const typename identity< Iterator >::type &end, Worker worker, Copier copier, const ScratchData &sample_scratch_data, const CopyData &sample_copy_data, const unsigned int queue_length, const unsigned int chunk_size)
Definition: work_stream.h:474
const MPI_Comm & comm