Reference documentation for deal.II version GIT 9042b9283b 2023-12-02 14:50:02+00:00
\(\newcommand{\dealvcentcolon}{\mathrel{\mathop{:}}}\) \(\newcommand{\dealcoloneq}{\dealvcentcolon\mathrel{\mkern-1.2mu}=}\) \(\newcommand{\jump}[1]{\left[\!\left[ #1 \right]\!\right]}\) \(\newcommand{\average}[1]{\left\{\!\left\{ #1 \right\}\!\right\}}\)
mpi_consensus_algorithms.h
Go to the documentation of this file.
1 // ---------------------------------------------------------------------
2 //
3 // Copyright (C) 2020 - 2023 by the deal.II authors
4 //
5 // This file is part of the deal.II library.
6 //
7 // The deal.II library is free software; you can use it, redistribute
8 // it, and/or modify it under the terms of the GNU Lesser General
9 // Public License as published by the Free Software Foundation; either
10 // version 2.1 of the License, or (at your option) any later version.
11 // The full text of the license can be found in the file LICENSE.md at
12 // the top level directory of deal.II.
13 //
14 // ---------------------------------------------------------------------
15 
16 #ifndef dealii_mpi_consensus_algorithm_h
17 #define dealii_mpi_consensus_algorithm_h
18 
19 #include <deal.II/base/config.h>
20 
21 #include <deal.II/base/mpi.h>
22 #include <deal.II/base/mpi.templates.h>
23 #include <deal.II/base/mpi_tags.h>
24 
26 
27 
28 namespace Utilities
29 {
30  namespace MPI
31  {
132  namespace ConsensusAlgorithms
133  {
158  template <typename RequestType, typename AnswerType>
159  class Process
160  {
161  public:
166  virtual ~Process() = default;
167 
174  virtual std::vector<unsigned int>
176 
186  virtual void
187  create_request(const unsigned int other_rank, RequestType &send_buffer);
188 
201  virtual void
202  answer_request(const unsigned int other_rank,
203  const RequestType &buffer_recv,
204  AnswerType &request_buffer);
205 
213  virtual void
214  read_answer(const unsigned int other_rank,
215  const AnswerType &recv_buffer);
216  };
217 
218 
219 
233  template <typename RequestType, typename AnswerType>
234  class Interface
235  {
236  public:
240  Interface() = default;
241 
246  virtual ~Interface() = default;
247 
256  std::vector<unsigned int>
257  run(Process<RequestType, AnswerType> &process, const MPI_Comm comm);
258 
282  virtual std::vector<unsigned int>
284  const std::vector<unsigned int> &targets,
285  const std::function<RequestType(const unsigned int)> &create_request,
286  const std::function<AnswerType(const unsigned int,
287  const RequestType &)> &answer_request,
288  const std::function<void(const unsigned int, const AnswerType &)>
289  &process_answer,
290  const MPI_Comm comm) = 0;
291  };
292 
293 
307  template <typename RequestType, typename AnswerType>
308  class NBX : public Interface<RequestType, AnswerType>
309  {
310  public:
314  NBX() = default;
315 
319  virtual ~NBX() = default;
320 
321  // Import the declarations from the base class.
323 
327  virtual std::vector<unsigned int>
329  const std::vector<unsigned int> &targets,
330  const std::function<RequestType(const unsigned int)> &create_request,
331  const std::function<AnswerType(const unsigned int,
332  const RequestType &)> &answer_request,
333  const std::function<void(const unsigned int, const AnswerType &)>
334  &process_answer,
335  const MPI_Comm comm) override;
336 
337  private:
338 #ifdef DEAL_II_WITH_MPI
342  std::vector<std::vector<char>> send_buffers;
343 
347  std::vector<MPI_Request> send_requests;
348 
356  std::vector<std::unique_ptr<std::vector<char>>> request_buffers;
357 
361  std::vector<std::unique_ptr<MPI_Request>> request_requests;
362 
366  unsigned int n_outstanding_answers;
367 
368  // request for barrier
369  MPI_Request barrier_request;
370 #endif
371 
375  std::set<unsigned int> requesting_processes;
376 
382  bool
384  const std::function<void(const unsigned int, const AnswerType &)>
385  &process_answer,
386  const MPI_Comm comm);
387 
392  void
393  signal_finish(const MPI_Comm comm);
394 
400  bool
402 
408  void
410  const std::function<AnswerType(const unsigned int,
411  const RequestType &)> &answer_request,
412  const MPI_Comm comm);
413 
418  void
420  const std::vector<unsigned int> &targets,
421  const std::function<RequestType(const unsigned int)> &create_request,
422  const MPI_Comm comm);
423 
428  void
430  };
431 
432 
477  template <typename RequestType, typename AnswerType>
478  std::vector<unsigned int>
479  nbx(const std::vector<unsigned int> &targets,
480  const std::function<RequestType(const unsigned int)> &create_request,
481  const std::function<AnswerType(const unsigned int,
482  const RequestType &)> &answer_request,
483  const std::function<void(const unsigned int, const AnswerType &)>
484  &process_answer,
485  const MPI_Comm comm);
486 
524  template <typename RequestType>
525  std::vector<unsigned int>
526  nbx(const std::vector<unsigned int> &targets,
527  const std::function<RequestType(const unsigned int)> &create_request,
528  const std::function<void(const unsigned int, const RequestType &)>
529  &process_request,
530  const MPI_Comm comm);
531 
557  template <typename RequestType, typename AnswerType>
558  class PEX : public Interface<RequestType, AnswerType>
559  {
560  public:
564  PEX() = default;
565 
569  virtual ~PEX() = default;
570 
571  // Import the declarations from the base class.
573 
577  virtual std::vector<unsigned int>
579  const std::vector<unsigned int> &targets,
580  const std::function<RequestType(const unsigned int)> &create_request,
581  const std::function<AnswerType(const unsigned int,
582  const RequestType &)> &answer_request,
583  const std::function<void(const unsigned int, const AnswerType &)>
584  &process_answer,
585  const MPI_Comm comm) override;
586 
587  private:
588 #ifdef DEAL_II_WITH_MPI
592  std::vector<std::vector<char>> send_buffers;
593 
597  std::vector<std::vector<char>> recv_buffers;
598 
602  std::vector<MPI_Request> send_request_requests;
603 
607  std::vector<std::vector<char>> requests_buffers;
608 
612  std::vector<MPI_Request> send_answer_requests;
613 #endif
617  std::set<unsigned int> requesting_processes;
618 
623  unsigned int
625  const std::vector<unsigned int> &targets,
626  const std::function<RequestType(const unsigned int)> &create_request,
627  const MPI_Comm comm);
628 
633  void
635  const unsigned int index,
636  const std::function<AnswerType(const unsigned int,
637  const RequestType &)> &answer_request,
638  const MPI_Comm comm);
639 
644  void
646  const unsigned int n_targets,
647  const std::function<void(const unsigned int, const AnswerType &)>
648  &process_answer,
649  const MPI_Comm comm);
650 
655  void
657  };
658 
659 
660 
717  template <typename RequestType, typename AnswerType>
718  std::vector<unsigned int>
719  pex(const std::vector<unsigned int> &targets,
720  const std::function<RequestType(const unsigned int)> &create_request,
721  const std::function<AnswerType(const unsigned int,
722  const RequestType &)> &answer_request,
723  const std::function<void(const unsigned int, const AnswerType &)>
724  &process_answer,
725  const MPI_Comm comm);
726 
764  template <typename RequestType>
765  std::vector<unsigned int>
766  pex(const std::vector<unsigned int> &targets,
767  const std::function<RequestType(const unsigned int)> &create_request,
768  const std::function<void(const unsigned int, const RequestType &)>
769  &process_request,
770  const MPI_Comm comm);
771 
772 
777  template <typename RequestType, typename AnswerType>
778  class Serial : public Interface<RequestType, AnswerType>
779  {
780  public:
784  Serial() = default;
785 
786  // Import the declarations from the base class.
788 
792  virtual std::vector<unsigned int>
794  const std::vector<unsigned int> &targets,
795  const std::function<RequestType(const unsigned int)> &create_request,
796  const std::function<AnswerType(const unsigned int,
797  const RequestType &)> &answer_request,
798  const std::function<void(const unsigned int, const AnswerType &)>
799  &process_answer,
800  const MPI_Comm comm) override;
801  };
802 
803 
804 
837  template <typename RequestType, typename AnswerType>
838  std::vector<unsigned int>
840  const std::vector<unsigned int> &targets,
841  const std::function<RequestType(const unsigned int)> &create_request,
842  const std::function<AnswerType(const unsigned int, const RequestType &)>
843  &answer_request,
844  const std::function<void(const unsigned int, const AnswerType &)>
845  &process_answer,
846  const MPI_Comm comm);
847 
877  template <typename RequestType>
878  std::vector<unsigned int>
880  const std::vector<unsigned int> &targets,
881  const std::function<RequestType(const unsigned int)> &create_request,
882  const std::function<void(const unsigned int, const RequestType &)>
883  &process_request,
884  const MPI_Comm comm);
885 
886 
887 
900  template <typename RequestType, typename AnswerType>
901  class Selector : public Interface<RequestType, AnswerType>
902  {
903  public:
907  Selector() = default;
908 
912  virtual ~Selector() = default;
913 
914  // Import the declarations from the base class.
916 
922  virtual std::vector<unsigned int>
924  const std::vector<unsigned int> &targets,
925  const std::function<RequestType(const unsigned int)> &create_request,
926  const std::function<AnswerType(const unsigned int,
927  const RequestType &)> &answer_request,
928  const std::function<void(const unsigned int, const AnswerType &)>
929  &process_answer,
930  const MPI_Comm comm) override;
931 
932  private:
933  // Pointer to the actual ConsensusAlgorithms::Interface implementation.
934  std::shared_ptr<Interface<RequestType, AnswerType>> consensus_algo;
935  };
936 
937 
938 
983  template <typename RequestType, typename AnswerType>
984  std::vector<unsigned int>
986  const std::vector<unsigned int> &targets,
987  const std::function<RequestType(const unsigned int)> &create_request,
988  const std::function<AnswerType(const unsigned int, const RequestType &)>
989  &answer_request,
990  const std::function<void(const unsigned int, const AnswerType &)>
991  &process_answer,
992  const MPI_Comm comm);
993 
1031  template <typename RequestType>
1032  std::vector<unsigned int>
1034  const std::vector<unsigned int> &targets,
1035  const std::function<RequestType(const unsigned int)> &create_request,
1036  const std::function<void(const unsigned int, const RequestType &)>
1037  &process_request,
1038  const MPI_Comm comm);
1039 
1040 
1041 
1042 #ifndef DOXYGEN
1043  // Implementation of the functions in this namespace.
1044 
1045  template <typename RequestType, typename AnswerType>
1046  std::vector<unsigned int>
1047  nbx(const std::vector<unsigned int> &targets,
1048  const std::function<RequestType(const unsigned int)> &create_request,
1049  const std::function<AnswerType(const unsigned int,
1050  const RequestType &)> &answer_request,
1051  const std::function<void(const unsigned int, const AnswerType &)>
1052  &process_answer,
1053  const MPI_Comm comm)
1054  {
1056  targets, create_request, answer_request, process_answer, comm);
1057  }
1058 
1059 
1060 
1061  template <typename RequestType>
1062  std::vector<unsigned int>
1063  nbx(const std::vector<unsigned int> &targets,
1064  const std::function<RequestType(const unsigned int)> &create_request,
1065  const std::function<void(const unsigned int, const RequestType &)>
1066  &process_request,
1067  const MPI_Comm comm)
1068  {
1069  // TODO: For the moment, simply implement this special case by
1070  // forwarding to the other function with rewritten function
1071  // objects and using an empty type as answer type. This way,
1072  // we have the interface in place and can provide a more
1073  // efficient implementation later on.
1074  using EmptyType = std::tuple<>;
1075 
1076  return nbx<RequestType, EmptyType>(
1077  targets,
1078  create_request,
1079  // answer_request:
1080  [&process_request](const unsigned int source_rank,
1081  const RequestType &request) -> EmptyType {
1082  process_request(source_rank, request);
1083  // Return something. What it is is arbitrary here, except that
1084  // we want it to be as small an object as possible. Using
1085  // std::tuple<> is interpreted as an empty object that is packed
1086  // down to a zero-length char array.
1087  return {};
1088  },
1089  // process_answer:
1090  [](const unsigned int /*target_rank */,
1091  const EmptyType & /*answer*/) {},
1092  comm);
1093  }
1094 
1095 
1096 
1097  template <typename RequestType, typename AnswerType>
1098  std::vector<unsigned int>
1099  pex(const std::vector<unsigned int> &targets,
1100  const std::function<RequestType(const unsigned int)> &create_request,
1101  const std::function<AnswerType(const unsigned int,
1102  const RequestType &)> &answer_request,
1103  const std::function<void(const unsigned int, const AnswerType &)>
1104  &process_answer,
1105  const MPI_Comm comm)
1106  {
1107  return PEX<RequestType, AnswerType>().run(
1108  targets, create_request, answer_request, process_answer, comm);
1109  }
1110 
1111 
1112 
1113  template <typename RequestType>
1114  std::vector<unsigned int>
1115  pex(const std::vector<unsigned int> &targets,
1116  const std::function<RequestType(const unsigned int)> &create_request,
1117  const std::function<void(const unsigned int, const RequestType &)>
1118  &process_request,
1119  const MPI_Comm comm)
1120  {
1121  // TODO: For the moment, simply implement this special case by
1122  // forwarding to the other function with rewritten function
1123  // objects and using an empty type as answer type. This way,
1124  // we have the interface in place and can provide a more
1125  // efficient implementation later on.
1126  using EmptyType = std::tuple<>;
1127 
1128  return pex<RequestType, EmptyType>(
1129  targets,
1130  create_request,
1131  // answer_request:
1132  [&process_request](const unsigned int source_rank,
1133  const RequestType &request) -> EmptyType {
1134  process_request(source_rank, request);
1135  // Return something. What it is is arbitrary here, except that
1136  // we want it to be as small an object as possible. Using
1137  // std::tuple<> is interpreted as an empty object that is packed
1138  // down to a zero-length char array.
1139  return {};
1140  },
1141  // process_answer:
1142  [](const unsigned int /*target_rank */,
1143  const EmptyType & /*answer*/) {},
1144  comm);
1145  }
1146 
1147 
1148 
1149  template <typename RequestType, typename AnswerType>
1150  std::vector<unsigned int>
1151  serial(
1152  const std::vector<unsigned int> &targets,
1153  const std::function<RequestType(const unsigned int)> &create_request,
1154  const std::function<AnswerType(const unsigned int, const RequestType &)>
1155  &answer_request,
1156  const std::function<void(const unsigned int, const AnswerType &)>
1157  &process_answer,
1158  const MPI_Comm comm)
1159  {
1160  return Serial<RequestType, AnswerType>().run(
1161  targets, create_request, answer_request, process_answer, comm);
1162  }
1163 
1164 
1165 
1166  template <typename RequestType>
1167  std::vector<unsigned int>
1168  serial(
1169  const std::vector<unsigned int> &targets,
1170  const std::function<RequestType(const unsigned int)> &create_request,
1171  const std::function<void(const unsigned int, const RequestType &)>
1172  &process_request,
1173  const MPI_Comm comm)
1174  {
1175  // TODO: For the moment, simply implement this special case by
1176  // forwarding to the other function with rewritten function
1177  // objects and using an empty type as answer type. This way,
1178  // we have the interface in place and can provide a more
1179  // efficient implementation later on.
1180  using EmptyType = std::tuple<>;
1181 
1182  return serial<RequestType, EmptyType>(
1183  targets,
1184  create_request,
1185  // answer_request:
1186  [&process_request](const unsigned int source_rank,
1187  const RequestType &request) -> EmptyType {
1188  process_request(source_rank, request);
1189  // Return something. What it is is arbitrary here, except that
1190  // we want it to be as small an object as possible. Using
1191  // std::tuple<> is interpreted as an empty object that is packed
1192  // down to a zero-length char array.
1193  return {};
1194  },
1195  // process_answer:
1196  [](const unsigned int /*target_rank */,
1197  const EmptyType & /*answer*/) {},
1198  comm);
1199  }
1200 
1201 
1202 
1203  template <typename RequestType, typename AnswerType>
1204  std::vector<unsigned int>
1205  selector(
1206  const std::vector<unsigned int> &targets,
1207  const std::function<RequestType(const unsigned int)> &create_request,
1208  const std::function<AnswerType(const unsigned int, const RequestType &)>
1209  &answer_request,
1210  const std::function<void(const unsigned int, const AnswerType &)>
1211  &process_answer,
1212  const MPI_Comm comm)
1213  {
1214  return Selector<RequestType, AnswerType>().run(
1215  targets, create_request, answer_request, process_answer, comm);
1216  }
1217 
1218 
1219 
1220  template <typename RequestType>
1221  std::vector<unsigned int>
1222  selector(
1223  const std::vector<unsigned int> &targets,
1224  const std::function<RequestType(const unsigned int)> &create_request,
1225  const std::function<void(const unsigned int, const RequestType &)>
1226  &process_request,
1227  const MPI_Comm comm)
1228  {
1229  // TODO: For the moment, simply implement this special case by
1230  // forwarding to the other function with rewritten function
1231  // objects and using an empty type as answer type. This way,
1232  // we have the interface in place and can provide a more
1233  // efficient implementation later on.
1234  using EmptyType = std::tuple<>;
1235 
1236  return selector<RequestType, EmptyType>(
1237  targets,
1238  create_request,
1239  // answer_request:
1240  [&process_request](const unsigned int source_rank,
1241  const RequestType &request) -> EmptyType {
1242  process_request(source_rank, request);
1243  // Return something. What it is is arbitrary here, except that
1244  // we want it to be as small an object as possible. Using
1245  // std::tuple<> is interpreted as an empty object that is packed
1246  // down to a zero-length char array.
1247  return {};
1248  },
1249  // process_answer:
1250  [](const unsigned int /*target_rank */,
1251  const EmptyType & /*answer*/) {},
1252  comm);
1253  }
1254 
1255 #endif
1256 
1257 
1258  } // namespace ConsensusAlgorithms
1259  } // end of namespace MPI
1260 } // end of namespace Utilities
1261 
1262 
1263 
1264 #ifndef DOXYGEN
1265 
1266 // ----------------- Implementation of template functions
1267 
1268 namespace Utilities
1269 {
1270  namespace MPI
1271  {
1272  namespace ConsensusAlgorithms
1273  {
1274  namespace
1275  {
1291 # ifndef DEAL_II_MSVC
1292  [[gnu::unused]]
1293 # endif
1294  inline bool
1295  has_unique_elements(const std::vector<unsigned int> &targets)
1296  {
1297  std::vector<unsigned int> my_destinations = targets;
1298  std::sort(my_destinations.begin(), my_destinations.end());
1299  return (std::adjacent_find(my_destinations.begin(),
1300  my_destinations.end()) ==
1301  my_destinations.end());
1302  }
1303 
1304 
1305 
1309  inline void
1310  handle_exception(std::exception_ptr &&exception, const MPI_Comm comm)
1311  {
1312 # ifdef DEAL_II_WITH_MPI
1313  // an exception within a ConsensusAlgorithm likely causes an
1314  // MPI deadlock. Abort with a reasonable error message instead.
1315  try
1316  {
1317  std::rethrow_exception(exception);
1318  }
1319  catch (ExceptionBase &exc)
1320  {
1321  // report name of the deal.II exception:
1322  std::cerr
1323  << std::endl
1324  << std::endl
1325  << "----------------------------------------------------"
1326  << std::endl;
1327  std::cerr
1328  << "Exception '" << exc.get_exc_name() << "'"
1329  << " on rank " << Utilities::MPI::this_mpi_process(comm)
1330  << " on processing: " << std::endl
1331  << exc.what() << std::endl
1332  << "Aborting!" << std::endl
1333  << "----------------------------------------------------"
1334  << std::endl;
1335 
1336  // Then bring down the whole MPI world
1337  MPI_Abort(comm, 255);
1338  }
1339  catch (std::exception &exc)
1340  {
1341  std::cerr
1342  << std::endl
1343  << std::endl
1344  << "----------------------------------------------------"
1345  << std::endl;
1346  std::cerr
1347  << "Exception within ConsensusAlgorithm"
1348  << " on rank " << Utilities::MPI::this_mpi_process(comm)
1349  << " on processing: " << std::endl
1350  << exc.what() << std::endl
1351  << "Aborting!" << std::endl
1352  << "----------------------------------------------------"
1353  << std::endl;
1354 
1355  // Then bring down the whole MPI world
1356  MPI_Abort(comm, 255);
1357  }
1358  catch (...)
1359  {
1360  std::cerr
1361  << std::endl
1362  << std::endl
1363  << "----------------------------------------------------"
1364  << std::endl;
1365  std::cerr
1366  << "Unknown exception within ConsensusAlgorithm!" << std::endl
1367  << "Aborting!" << std::endl
1368  << "----------------------------------------------------"
1369  << std::endl;
1370 
1371  // Then bring down the whole MPI world
1372  MPI_Abort(comm, 255);
1373  }
1374 # else
1375  (void)comm;
1376 
1377  // No need to be concerned about deadlocks without MPI.
1378  // Defer to exception handling further up the callstack.
1379  std::rethrow_exception(exception);
1380 # endif
1381  }
1382  } // namespace
1383 
1384 
1385 
1386  template <typename RequestType, typename AnswerType>
1387  void
1389  const RequestType &,
1390  AnswerType &)
1391  {
1392  // nothing to do
1393  }
1394 
1395 
1396 
1397  template <typename RequestType, typename AnswerType>
1398  void
1400  RequestType &)
1401  {
1402  // nothing to do
1403  }
1404 
1405 
1406 
1407  template <typename RequestType, typename AnswerType>
1408  void
1410  const AnswerType &)
1411  {
1412  // nothing to do
1413  }
1414 
1415 
1416 
1417  template <typename RequestType, typename AnswerType>
1418  std::vector<unsigned int>
1420  Process<RequestType, AnswerType> &process,
1421  const MPI_Comm comm)
1422  {
1423  // Unpack the 'process' object and call the function that takes
1424  // function objects for all operations.
1425  return run(
1426  process.compute_targets(),
1427  /* create_request: */
1428  [&process](const unsigned int target) {
1429  RequestType request;
1430  process.create_request(target, request);
1431  return request;
1432  },
1433  /* answer_request: */
1434  [&process](const unsigned int source, const RequestType &request) {
1435  AnswerType answer;
1436  process.answer_request(source, request, answer);
1437  return answer;
1438  },
1439  /* process_answer: */
1440  [&process](const unsigned int target, const AnswerType &answer) {
1441  process.read_answer(target, answer);
1442  },
1443  comm);
1444  }
1445 
1446 
1447 
1448  template <typename RequestType, typename AnswerType>
1449  std::vector<unsigned int>
1451  const std::vector<unsigned int> &targets,
1452  const std::function<RequestType(const unsigned int)> &create_request,
1453  const std::function<AnswerType(const unsigned int, const RequestType &)>
1454  &answer_request,
1455  const std::function<void(const unsigned int, const AnswerType &)>
1456  &process_answer,
1457  const MPI_Comm comm)
1458  {
1459  Assert(has_unique_elements(targets),
1460  ExcMessage("The consensus algorithms expect that each process "
1461  "only sends a single message to another process, "
1462  "but the targets provided include duplicates."));
1463 
1464  static CollectiveMutex mutex;
1465  CollectiveMutex::ScopedLock lock(mutex, comm);
1466 
1467  try
1468  {
1469  // 1) Send data to identified targets and start receiving
1470  // the answers from these very same processes.
1471  start_communication(targets, create_request, comm);
1472 
1473  // 2) Until all posted receive operations are known to have
1474  // completed, answer requests and keep checking whether all
1475  // requests of this process have been answered.
1476  //
1477  // The requests that we catch in the answer_requests()
1478  // function originate elsewhere, that is, they are not in
1479  // response to our own messages
1480  //
1481  // Note also that we may not catch all incoming requests in
1482  // the following two lines: our own requests may have been
1483  // satisfied before we've dealt with all incoming requests.
1484  // That's ok: We will get around to dealing with all
1485  // remaining message later. We just want to move on to the
1486  // next step as early as possible.
1487  while (all_locally_originated_receives_are_completed(process_answer,
1488  comm) == false)
1489  maybe_answer_one_request(answer_request, comm);
1490 
1491  // 3) Signal to all other processes that all requests of this
1492  // process have been answered
1493  signal_finish(comm);
1494 
1495  // 4) Nevertheless, this process has to keep on answering
1496  // (potential) incoming requests until all processes have
1497  // received the answer to all requests
1498  while (all_remotely_originated_receives_are_completed() == false)
1499  maybe_answer_one_request(answer_request, comm);
1500 
1501  // 5) process the answer to all requests
1502  clean_up_and_end_communication(comm);
1503  }
1504  catch (...)
1505  {
1506  handle_exception(std::current_exception(), comm);
1507  }
1508 
1509  return std::vector<unsigned int>(requesting_processes.begin(),
1510  requesting_processes.end());
1511  }
1512 
1513 
1514 
1515  template <typename RequestType, typename AnswerType>
1516  void
1518  const std::vector<unsigned int> &targets,
1519  const std::function<RequestType(const unsigned int)> &create_request,
1520  const MPI_Comm comm)
1521  {
1522 # ifdef DEAL_II_WITH_MPI
1523  // 1)
1524  const auto n_targets = targets.size();
1525 
1526  const int tag_request = Utilities::MPI::internal::Tags::
1528 
1529  // 2) allocate memory
1530  send_requests.resize(n_targets);
1531  send_buffers.resize(n_targets);
1532 
1533  {
1534  // 4) send and receive
1535  for (unsigned int index = 0; index < n_targets; ++index)
1536  {
1537  const unsigned int rank = targets[index];
1539 
1540  auto &send_buffer = send_buffers[index];
1541  send_buffer =
1542  (create_request ? Utilities::pack(create_request(rank), false) :
1543  std::vector<char>());
1544 
1545  // Post a request to send data
1546  auto ierr = MPI_Isend(send_buffer.data(),
1547  send_buffer.size(),
1548  MPI_CHAR,
1549  rank,
1550  tag_request,
1551  comm,
1552  &send_requests[index]);
1553  AssertThrowMPI(ierr);
1554  }
1555 
1556  // Also record that we expect an answer from each target we sent
1557  // a request to:
1558  n_outstanding_answers = n_targets;
1559  }
1560 # else
1561  (void)targets;
1562  (void)create_request;
1563  (void)comm;
1564 # endif
1565  }
1566 
1567 
1568 
1569  template <typename RequestType, typename AnswerType>
1570  bool
1573  const std::function<void(const unsigned int, const AnswerType &)>
1574  &process_answer,
1575  const MPI_Comm comm)
1576  {
1577 # ifdef DEAL_II_WITH_MPI
1578  // We know that all requests have come in when we have pending
1579  // messages from all targets with the right tag (some of which we may
1580  // have already taken care of below, after discovering their existence).
1581  // We can check for pending messages with MPI_IProbe, which returns
1582  // immediately with a return code that indicates whether
1583  // it has found a message from any process with a given
1584  // tag.
1585  if (n_outstanding_answers == 0)
1586  return true;
1587  else
1588  {
1589  const int tag_deliver = Utilities::MPI::internal::Tags::
1591 
1592  int request_is_pending;
1593  MPI_Status status;
1594  const auto ierr = MPI_Iprobe(
1595  MPI_ANY_SOURCE, tag_deliver, comm, &request_is_pending, &status);
1596  AssertThrowMPI(ierr);
1597 
1598  // If there is no pending message with this tag,
1599  // then we are clearly not done receiving everything
1600  // yet -- so return false.
1601  if (request_is_pending == 0)
1602  return false;
1603  else
1604  {
1605  // OK, so we have gotten a reply to our request from
1606  // one rank. Let us process it.
1607  const auto target = status.MPI_SOURCE;
1608 
1609  // Then query the size of the message, allocate enough memory,
1610  // receive the data, and process it.
1611  int message_size;
1612  {
1613  const int ierr =
1614  MPI_Get_count(&status, MPI_CHAR, &message_size);
1615  AssertThrowMPI(ierr);
1616  }
1617  std::vector<char> recv_buffer(message_size);
1618 
1619  {
1620  const int tag_deliver = Utilities::MPI::internal::Tags::
1622 
1623  const int ierr = MPI_Recv(recv_buffer.data(),
1624  recv_buffer.size(),
1625  MPI_CHAR,
1626  target,
1627  tag_deliver,
1628  comm,
1629  MPI_STATUS_IGNORE);
1630  AssertThrowMPI(ierr);
1631  }
1632 
1633  if (process_answer)
1634  process_answer(target,
1635  Utilities::unpack<AnswerType>(recv_buffer,
1636  false));
1637 
1638  // Finally, remove this rank from the list of outstanding
1639  // targets:
1640  --n_outstanding_answers;
1641 
1642  // We could do another go-around from the top of this
1643  // else-branch to see whether there are actually other messages
1644  // that are currently pending. But that would mean spending
1645  // substantial time in receiving answers while we should also be
1646  // sending answers to requests we have received from other
1647  // places. So let it be enough for now. If there are outstanding
1648  // answers, we will get back to this function before long and
1649  // can take care of them then.
1650  return (n_outstanding_answers == 0);
1651  }
1652  }
1653 
1654 # else
1655  (void)process_answer;
1656  (void)comm;
1657 
1658  return true;
1659 # endif
1660  }
1661 
1662 
1663 
1664  template <typename RequestType, typename AnswerType>
1665  void
1667  const std::function<AnswerType(const unsigned int, const RequestType &)>
1668  &answer_request,
1669  const MPI_Comm comm)
1670  {
1671 # ifdef DEAL_II_WITH_MPI
1672 
1673  const int tag_request = Utilities::MPI::internal::Tags::
1675  const int tag_deliver = Utilities::MPI::internal::Tags::
1677 
1678  // Check if there is a request pending. By selecting the
1679  // tag_request tag, these are other processes asking for
1680  // our own replies, not these other processes' replies
1681  // to our own requests.
1682  //
1683  // There may be multiple such pending messages. We
1684  // only answer one.
1685  MPI_Status status;
1686  int request_is_pending;
1687  const auto ierr = MPI_Iprobe(
1688  MPI_ANY_SOURCE, tag_request, comm, &request_is_pending, &status);
1689  AssertThrowMPI(ierr);
1690 
1691  if (request_is_pending != 0)
1692  {
1693  // Get the rank of the requesting process and add it to the
1694  // list of requesting processes (which may contain duplicates).
1695  const auto other_rank = status.MPI_SOURCE;
1696 
1697  Assert(requesting_processes.find(other_rank) ==
1698  requesting_processes.end(),
1699  ExcMessage("Process is requesting a second time!"));
1700  requesting_processes.insert(other_rank);
1701 
1702  // get size of incoming message
1703  int number_amount;
1704  auto ierr = MPI_Get_count(&status, MPI_CHAR, &number_amount);
1705  AssertThrowMPI(ierr);
1706 
1707  // allocate memory for incoming message
1708  std::vector<char> buffer_recv(number_amount);
1709  ierr = MPI_Recv(buffer_recv.data(),
1710  number_amount,
1711  MPI_CHAR,
1712  other_rank,
1713  tag_request,
1714  comm,
1715  MPI_STATUS_IGNORE);
1716  AssertThrowMPI(ierr);
1717 
1718  // Allocate memory for an answer message to the current request,
1719  // and ask the 'process' object to produce an answer:
1720  request_buffers.emplace_back(std::make_unique<std::vector<char>>());
1721  auto &request_buffer = *request_buffers.back();
1722  if (answer_request)
1723  request_buffer =
1724  Utilities::pack(answer_request(other_rank,
1725  Utilities::unpack<RequestType>(
1726  buffer_recv, false)),
1727  false);
1728 
1729  // Then initiate sending the answer back to the requester.
1730  request_requests.emplace_back(std::make_unique<MPI_Request>());
1731  ierr = MPI_Isend(request_buffer.data(),
1732  request_buffer.size(),
1733  MPI_CHAR,
1734  other_rank,
1735  tag_deliver,
1736  comm,
1737  request_requests.back().get());
1738  AssertThrowMPI(ierr);
1739  }
1740 # else
1741  (void)answer_request;
1742  (void)comm;
1743 # endif
1744  }
1745 
1746 
1747 
1748  template <typename RequestType, typename AnswerType>
1749  void
1751  {
1752 # ifdef DEAL_II_WITH_MPI
1753  const auto ierr = MPI_Ibarrier(comm, &barrier_request);
1754  AssertThrowMPI(ierr);
1755 # else
1756  (void)comm;
1757 # endif
1758  }
1759 
1760 
1761 
1762  template <typename RequestType, typename AnswerType>
1763  bool
1764  NBX<RequestType,
1765  AnswerType>::all_remotely_originated_receives_are_completed()
1766  {
1767 # ifdef DEAL_II_WITH_MPI
1768  int all_ranks_reached_barrier;
1769  const auto ierr = MPI_Test(&barrier_request,
1770  &all_ranks_reached_barrier,
1771  MPI_STATUS_IGNORE);
1772  AssertThrowMPI(ierr);
1773  return all_ranks_reached_barrier != 0;
1774 # else
1775  return true;
1776 # endif
1777  }
1778 
1779 
1780 
1781  template <typename RequestType, typename AnswerType>
1782  void
1784  const MPI_Comm comm)
1785  {
1786  (void)comm;
1787 # ifdef DEAL_II_WITH_MPI
1788  // clean up
1789  {
1790  if (send_requests.size() > 0)
1791  {
1792  const int ierr = MPI_Waitall(send_requests.size(),
1793  send_requests.data(),
1794  MPI_STATUSES_IGNORE);
1795  AssertThrowMPI(ierr);
1796  }
1797 
1798  int ierr = MPI_Wait(&barrier_request, MPI_STATUS_IGNORE);
1799  AssertThrowMPI(ierr);
1800 
1801  for (auto &i : request_requests)
1802  {
1803  ierr = MPI_Wait(i.get(), MPI_STATUS_IGNORE);
1804  AssertThrowMPI(ierr);
1805  }
1806 
1807 # ifdef DEBUG
1808  // note: IBarrier seems to make problem during testing, this
1809  // additional Barrier seems to help
1810  ierr = MPI_Barrier(comm);
1811  AssertThrowMPI(ierr);
1812 # endif
1813  }
1814 # endif
1815  }
1816 
1817 
1818 
1819  template <typename RequestType, typename AnswerType>
1820  std::vector<unsigned int>
1822  const std::vector<unsigned int> &targets,
1823  const std::function<RequestType(const unsigned int)> &create_request,
1824  const std::function<AnswerType(const unsigned int, const RequestType &)>
1825  &answer_request,
1826  const std::function<void(const unsigned int, const AnswerType &)>
1827  &process_answer,
1828  const MPI_Comm comm)
1829  {
1830  Assert(has_unique_elements(targets),
1831  ExcMessage("The consensus algorithms expect that each process "
1832  "only sends a single message to another process, "
1833  "but the targets provided include duplicates."));
1834 
1835  static CollectiveMutex mutex;
1836  CollectiveMutex::ScopedLock lock(mutex, comm);
1837 
1838  try
1839  {
1840  // 1) Send requests and start receiving the answers.
1841  // In particular, determine how many requests we should expect
1842  // on the current process.
1843  const unsigned int n_requests =
1844  start_communication(targets, create_request, comm);
1845 
1846  // 2) Answer requests:
1847  for (unsigned int request = 0; request < n_requests; ++request)
1848  answer_one_request(request, answer_request, comm);
1849 
1850  // 3) Process answers:
1851  process_incoming_answers(targets.size(), process_answer, comm);
1852 
1853  // 4) Make sure all sends have successfully terminated:
1854  clean_up_and_end_communication();
1855  }
1856  catch (...)
1857  {
1858  handle_exception(std::current_exception(), comm);
1859  }
1860 
1861  return std::vector<unsigned int>(requesting_processes.begin(),
1862  requesting_processes.end());
1863  }
1864 
1865 
1866 
1867  template <typename RequestType, typename AnswerType>
1868  unsigned int
1870  const std::vector<unsigned int> &targets,
1871  const std::function<RequestType(const unsigned int)> &create_request,
1872  const MPI_Comm comm)
1873  {
1874 # ifdef DEAL_II_WITH_MPI
1875  const int tag_request = Utilities::MPI::internal::Tags::
1877 
1878  // 1) determine with which processes this process wants to communicate
1879  // with
1880  const unsigned int n_targets = targets.size();
1881 
1882  // 2) determine who wants to communicate with this process
1883  const unsigned int n_sources =
1885 
1886  // 2) allocate memory
1887  recv_buffers.resize(n_targets);
1888  send_buffers.resize(n_targets);
1889  send_request_requests.resize(n_targets);
1890 
1891  send_answer_requests.resize(n_sources);
1892  requests_buffers.resize(n_sources);
1893 
1894  // 4) send and receive
1895  for (unsigned int i = 0; i < n_targets; ++i)
1896  {
1897  const unsigned int rank = targets[i];
1899 
1900  // pack data which should be sent
1901  auto &send_buffer = send_buffers[i];
1902  if (create_request)
1903  send_buffer = Utilities::pack(create_request(rank), false);
1904 
1905  // start to send data
1906  auto ierr = MPI_Isend(send_buffer.data(),
1907  send_buffer.size(),
1908  MPI_CHAR,
1909  rank,
1910  tag_request,
1911  comm,
1912  &send_request_requests[i]);
1913  AssertThrowMPI(ierr);
1914  }
1915 
1916  return n_sources;
1917 # else
1918  (void)targets;
1919  (void)create_request;
1920  (void)comm;
1921  return 0;
1922 # endif
1923  }
1924 
1925 
1926 
1927  template <typename RequestType, typename AnswerType>
1928  void
1930  const unsigned int index,
1931  const std::function<AnswerType(const unsigned int, const RequestType &)>
1932  &answer_request,
1933  const MPI_Comm comm)
1934  {
1935 # ifdef DEAL_II_WITH_MPI
1936  const int tag_request = Utilities::MPI::internal::Tags::
1938  const int tag_deliver = Utilities::MPI::internal::Tags::
1940 
1941  // Wait until we have a message ready for retrieval, though we don't
1942  // care which process it is from.
1943  MPI_Status status;
1944  int ierr = MPI_Probe(MPI_ANY_SOURCE, tag_request, comm, &status);
1945  AssertThrowMPI(ierr);
1946 
1947  // Get rank of incoming message and verify that it makes sense
1948  const unsigned int other_rank = status.MPI_SOURCE;
1949 
1950  Assert(requesting_processes.find(other_rank) ==
1951  requesting_processes.end(),
1952  ExcMessage(
1953  "A process is sending a request after a request from "
1954  "the same process has previously already been "
1955  "received. This algorithm does not expect this to happen."));
1956  requesting_processes.insert(other_rank);
1957 
1958  // Actually get the incoming message:
1959  int number_amount;
1960  ierr = MPI_Get_count(&status, MPI_CHAR, &number_amount);
1961  AssertThrowMPI(ierr);
1962 
1963  std::vector<char> buffer_recv(number_amount);
1964  ierr = MPI_Recv(buffer_recv.data(),
1965  number_amount,
1966  MPI_CHAR,
1967  other_rank,
1968  tag_request,
1969  comm,
1970  &status);
1971  AssertThrowMPI(ierr);
1972 
1973  // Process request by asking the user-provided function for
1974  // the answer and post a send for it.
1975  auto &request_buffer = requests_buffers[index];
1976  request_buffer =
1977  (answer_request ?
1978  Utilities::pack(answer_request(other_rank,
1979  Utilities::unpack<RequestType>(
1980  buffer_recv, false)),
1981  false) :
1982  std::vector<char>());
1983 
1984  ierr = MPI_Isend(request_buffer.data(),
1985  request_buffer.size(),
1986  MPI_CHAR,
1987  other_rank,
1988  tag_deliver,
1989  comm,
1990  &send_answer_requests[index]);
1991  AssertThrowMPI(ierr);
1992 # else
1993  (void)answer_request;
1994  (void)comm;
1995  (void)index;
1996 # endif
1997  }
1998 
1999 
2000 
2001  template <typename RequestType, typename AnswerType>
2002  void
2004  const unsigned int n_targets,
2005  const std::function<void(const unsigned int, const AnswerType &)>
2006  &process_answer,
2007  const MPI_Comm comm)
2008  {
2009 # ifdef DEAL_II_WITH_MPI
2010  const int tag_deliver = Utilities::MPI::internal::Tags::
2012 
2013  // We know how many targets we have sent requests to. These
2014  // targets will all eventually send us their responses, but
2015  // we need not process them in order -- rather, just see what
2016  // comes in and then look at message originators' ranks and
2017  // message sizes
2018  for (unsigned int i = 0; i < n_targets; ++i)
2019  {
2020  MPI_Status status;
2021  {
2022  const int ierr =
2023  MPI_Probe(MPI_ANY_SOURCE, tag_deliver, comm, &status);
2024  AssertThrowMPI(ierr);
2025  }
2026 
2027  const auto other_rank = status.MPI_SOURCE;
2028  int message_size;
2029  {
2030  const int ierr = MPI_Get_count(&status, MPI_CHAR, &message_size);
2031  AssertThrowMPI(ierr);
2032  }
2033  std::vector<char> recv_buffer(message_size);
2034 
2035  // Now actually receive the answer. Because the MPI_Probe
2036  // above blocks until we have a message, we know that the
2037  // following MPI_Recv call will immediately succeed.
2038  {
2039  const int ierr = MPI_Recv(recv_buffer.data(),
2040  recv_buffer.size(),
2041  MPI_CHAR,
2042  other_rank,
2043  tag_deliver,
2044  comm,
2045  MPI_STATUS_IGNORE);
2046  AssertThrowMPI(ierr);
2047  }
2048 
2049  if (process_answer)
2050  process_answer(other_rank,
2051  Utilities::unpack<AnswerType>(recv_buffer, false));
2052  }
2053 # else
2054  (void)n_targets;
2055  (void)process_answer;
2056  (void)comm;
2057 # endif
2058  }
2059 
2060 
2061 
2062  template <typename RequestType, typename AnswerType>
2063  void
2065  {
2066 # ifdef DEAL_II_WITH_MPI
2067  // Finalize all MPI_Request objects for both the
2068  // send-request and receive-answer operations.
2069  if (send_request_requests.size() > 0)
2070  {
2071  const int ierr = MPI_Waitall(send_request_requests.size(),
2072  send_request_requests.data(),
2073  MPI_STATUSES_IGNORE);
2074  AssertThrowMPI(ierr);
2075  }
2076 
2077  // Then also check the send-answer requests.
2078  if (send_answer_requests.size() > 0)
2079  {
2080  const int ierr = MPI_Waitall(send_answer_requests.size(),
2081  send_answer_requests.data(),
2082  MPI_STATUSES_IGNORE);
2083  AssertThrowMPI(ierr);
2084  }
2085 # endif
2086  }
2087 
2088 
2089 
2090  template <typename RequestType, typename AnswerType>
2091  std::vector<unsigned int>
2093  const std::vector<unsigned int> &targets,
2094  const std::function<RequestType(const unsigned int)> &create_request,
2095  const std::function<AnswerType(const unsigned int, const RequestType &)>
2096  &answer_request,
2097  const std::function<void(const unsigned int, const AnswerType &)>
2098  &process_answer,
2099  const MPI_Comm comm)
2100  {
2101  (void)comm;
2103  ExcMessage("You shouldn't use the 'Serial' class on "
2104  "communicators that have more than one process "
2105  "associated with it."));
2106 
2107  // The only valid target for a serial program is itself.
2108  if (targets.size() != 0)
2109  {
2110  Assert(targets.size() == 1,
2111  ExcMessage(
2112  "On a single process, the only valid target "
2113  "is process zero (the process itself), which can only be "
2114  "listed once."));
2115  AssertDimension(targets[0], 0);
2116 
2117  // Since the caller indicates that there is a target, and since we
2118  // know that it is the current process, let the process send
2119  // something to itself.
2120  const RequestType request =
2121  (create_request ? create_request(0) : RequestType());
2122  const AnswerType answer =
2123  (answer_request ? answer_request(0, request) : AnswerType());
2124 
2125  if (process_answer)
2126  process_answer(0, answer);
2127  }
2128 
2129  return targets; // nothing to do
2130  }
2131 
2132 
2133 
2134  template <typename RequestType, typename AnswerType>
2135  std::vector<unsigned int>
2137  const std::vector<unsigned int> &targets,
2138  const std::function<RequestType(const unsigned int)> &create_request,
2139  const std::function<AnswerType(const unsigned int, const RequestType &)>
2140  &answer_request,
2141  const std::function<void(const unsigned int, const AnswerType &)>
2142  &process_answer,
2143  const MPI_Comm comm)
2144  {
2145  // Depending on the number of processes we switch between
2146  // implementations. We reduce the threshold for debug mode to be
2147  // able to test also the non-blocking implementation. This feature
2148  // is tested by:
2149  // tests/multigrid/transfer_matrix_free_06.with_mpi=true.with_p4est=true.with_trilinos=true.mpirun=10.output
2150 
2151  const unsigned int n_procs = (Utilities::MPI::job_supports_mpi() ?
2153  1);
2154 # ifdef DEAL_II_WITH_MPI
2155 # ifdef DEBUG
2156  if (n_procs > 10)
2157 # else
2158  if (n_procs > 99)
2159 # endif
2160  consensus_algo.reset(new NBX<RequestType, AnswerType>());
2161  else
2162 # endif
2163  if (n_procs > 1)
2164  consensus_algo.reset(new PEX<RequestType, AnswerType>());
2165  else
2166  consensus_algo.reset(new Serial<RequestType, AnswerType>());
2167 
2168  return consensus_algo->run(
2169  targets, create_request, answer_request, process_answer, comm);
2170  }
2171 
2172 
2173  } // namespace ConsensusAlgorithms
2174  } // end of namespace MPI
2175 } // end of namespace Utilities
2176 
2177 #endif // DOXYGEN
2178 
2179 
2181 
2182 #endif
const char * get_exc_name() const
Definition: exceptions.cc:173
virtual const char * what() const noexcept override
Definition: exceptions.cc:161
std::vector< unsigned int > run(Process< RequestType, AnswerType > &process, const MPI_Comm comm)
virtual std::vector< unsigned int > run(const std::vector< unsigned int > &targets, const std::function< RequestType(const unsigned int)> &create_request, const std::function< AnswerType(const unsigned int, const RequestType &)> &answer_request, const std::function< void(const unsigned int, const AnswerType &)> &process_answer, const MPI_Comm comm)=0
void start_communication(const std::vector< unsigned int > &targets, const std::function< RequestType(const unsigned int)> &create_request, const MPI_Comm comm)
void maybe_answer_one_request(const std::function< AnswerType(const unsigned int, const RequestType &)> &answer_request, const MPI_Comm comm)
std::vector< std::unique_ptr< std::vector< char > > > request_buffers
virtual std::vector< unsigned int > run(const std::vector< unsigned int > &targets, const std::function< RequestType(const unsigned int)> &create_request, const std::function< AnswerType(const unsigned int, const RequestType &)> &answer_request, const std::function< void(const unsigned int, const AnswerType &)> &process_answer, const MPI_Comm comm) override
void clean_up_and_end_communication(const MPI_Comm comm)
bool all_locally_originated_receives_are_completed(const std::function< void(const unsigned int, const AnswerType &)> &process_answer, const MPI_Comm comm)
std::vector< std::unique_ptr< MPI_Request > > request_requests
std::vector< std::vector< char > > send_buffers
void signal_finish(const MPI_Comm comm)
unsigned int start_communication(const std::vector< unsigned int > &targets, const std::function< RequestType(const unsigned int)> &create_request, const MPI_Comm comm)
std::vector< std::vector< char > > requests_buffers
virtual std::vector< unsigned int > run(const std::vector< unsigned int > &targets, const std::function< RequestType(const unsigned int)> &create_request, const std::function< AnswerType(const unsigned int, const RequestType &)> &answer_request, const std::function< void(const unsigned int, const AnswerType &)> &process_answer, const MPI_Comm comm) override
std::vector< std::vector< char > > send_buffers
void answer_one_request(const unsigned int index, const std::function< AnswerType(const unsigned int, const RequestType &)> &answer_request, const MPI_Comm comm)
std::vector< std::vector< char > > recv_buffers
void process_incoming_answers(const unsigned int n_targets, const std::function< void(const unsigned int, const AnswerType &)> &process_answer, const MPI_Comm comm)
virtual void answer_request(const unsigned int other_rank, const RequestType &buffer_recv, AnswerType &request_buffer)
virtual void read_answer(const unsigned int other_rank, const AnswerType &recv_buffer)
virtual std::vector< unsigned int > compute_targets()=0
virtual void create_request(const unsigned int other_rank, RequestType &send_buffer)
virtual std::vector< unsigned int > run(const std::vector< unsigned int > &targets, const std::function< RequestType(const unsigned int)> &create_request, const std::function< AnswerType(const unsigned int, const RequestType &)> &answer_request, const std::function< void(const unsigned int, const AnswerType &)> &process_answer, const MPI_Comm comm) override
std::shared_ptr< Interface< RequestType, AnswerType > > consensus_algo
virtual std::vector< unsigned int > run(const std::vector< unsigned int > &targets, const std::function< RequestType(const unsigned int)> &create_request, const std::function< AnswerType(const unsigned int, const RequestType &)> &answer_request, const std::function< void(const unsigned int, const AnswerType &)> &process_answer, const MPI_Comm comm) override
#define DEAL_II_NAMESPACE_OPEN
Definition: config.h:477
#define DEAL_II_NAMESPACE_CLOSE
Definition: config.h:478
#define Assert(cond, exc)
Definition: exceptions.h:1631
#define AssertDimension(dim1, dim2)
Definition: exceptions.h:1820
#define AssertThrowMPI(error_code)
Definition: exceptions.h:1947
#define AssertIndexRange(index, range)
Definition: exceptions.h:1888
static ::ExceptionBase & ExcMessage(std::string arg1)
std::vector< unsigned int > serial(const std::vector< unsigned int > &targets, const std::function< RequestType(const unsigned int)> &create_request, const std::function< AnswerType(const unsigned int, const RequestType &)> &answer_request, const std::function< void(const unsigned int, const AnswerType &)> &process_answer, const MPI_Comm comm)
std::vector< unsigned int > nbx(const std::vector< unsigned int > &targets, const std::function< RequestType(const unsigned int)> &create_request, const std::function< AnswerType(const unsigned int, const RequestType &)> &answer_request, const std::function< void(const unsigned int, const AnswerType &)> &process_answer, const MPI_Comm comm)
std::vector< unsigned int > selector(const std::vector< unsigned int > &targets, const std::function< RequestType(const unsigned int)> &create_request, const std::function< AnswerType(const unsigned int, const RequestType &)> &answer_request, const std::function< void(const unsigned int, const AnswerType &)> &process_answer, const MPI_Comm comm)
std::vector< unsigned int > pex(const std::vector< unsigned int > &targets, const std::function< RequestType(const unsigned int)> &create_request, const std::function< AnswerType(const unsigned int, const RequestType &)> &answer_request, const std::function< void(const unsigned int, const AnswerType &)> &process_answer, const MPI_Comm comm)
@ consensus_algorithm_nbx_process_deliver
ConsensusAlgorithms::NBX::process.
Definition: mpi_tags.h:91
@ consensus_algorithm_pex_process_deliver
ConsensusAlgorithms::PEX::process.
Definition: mpi_tags.h:96
@ consensus_algorithm_nbx_answer_request
ConsensusAlgorithms::NBX::process.
Definition: mpi_tags.h:89
@ consensus_algorithm_pex_answer_request
ConsensusAlgorithms::PEX::process.
Definition: mpi_tags.h:94
unsigned int n_mpi_processes(const MPI_Comm mpi_communicator)
Definition: mpi.cc:149
unsigned int this_mpi_process(const MPI_Comm mpi_communicator)
Definition: mpi.cc:164
bool job_supports_mpi()
Definition: mpi.cc:1073
unsigned int compute_n_point_to_point_communications(const MPI_Comm mpi_comm, const std::vector< unsigned int > &destinations)
Definition: mpi.cc:436
size_t pack(const T &object, std::vector< char > &dest_buffer, const bool allow_compression=true)
Definition: utilities.h:1343
void run(const Iterator &begin, const std_cxx20::type_identity_t< Iterator > &end, Worker worker, Copier copier, const ScratchData &sample_scratch_data, const CopyData &sample_copy_data, const unsigned int queue_length, const unsigned int chunk_size)
Definition: work_stream.h:472
const MPI_Comm comm