Reference documentation for deal.II version Git f39d5fa 2018-06-22 10:03:07 -0500
mpi.h
1 // ---------------------------------------------------------------------
2 //
3 // Copyright (C) 2011 - 2018 by the deal.II authors
4 //
5 // This file is part of the deal.II library.
6 //
7 // The deal.II library is free software; you can use it, redistribute
8 // it, and/or modify it under the terms of the GNU Lesser General
9 // Public License as published by the Free Software Foundation; either
10 // version 2.1 of the License, or (at your option) any later version.
11 // The full text of the license can be found in the file LICENSE.md at
12 // the top level directory of deal.II.
13 //
14 // ---------------------------------------------------------------------
15 
16 #ifndef dealii_mpi_h
17 #define dealii_mpi_h
18 
19 #include <deal.II/base/config.h>
20 
21 #include <deal.II/base/array_view.h>
22 
23 #include <map>
24 #include <vector>
25 
26 #if !defined(DEAL_II_WITH_MPI) && !defined(DEAL_II_WITH_PETSC)
27 // without MPI, we would still like to use
28 // some constructs with MPI data
29 // types. Therefore, create some dummies
30 typedef int MPI_Comm;
31 typedef int MPI_Datatype;
32 typedef int MPI_Op;
33 # ifndef MPI_COMM_WORLD
34 # define MPI_COMM_WORLD 0
35 # endif
36 # ifndef MPI_COMM_SELF
37 # define MPI_COMM_SELF 0
38 # endif
39 # ifndef MPI_MIN
40 # define MPI_MIN 0
41 # endif
42 # ifndef MPI_MAX
43 # define MPI_MAX 0
44 # endif
45 # ifndef MPI_SUM
46 # define MPI_SUM 0
47 # endif
48 #endif
49 
50 DEAL_II_NAMESPACE_OPEN
51 
52 
53 // Forward type declarations to allow MPI sums over tensorial types
54 template <int rank, int dim, typename Number>
55 class Tensor;
56 template <int rank, int dim, typename Number>
58 template <typename Number>
60 
61 namespace Utilities
62 {
70  namespace MPI
71  {
80  unsigned int
81  n_mpi_processes(const MPI_Comm &mpi_communicator);
82 
91  unsigned int
92  this_mpi_process(const MPI_Comm &mpi_communicator);
93 
115  std::vector<unsigned int>
117  const MPI_Comm & mpi_comm,
118  const std::vector<unsigned int> &destinations);
119 
133  MPI_Comm
134  duplicate_communicator(const MPI_Comm &mpi_communicator);
135 
163 #ifdef DEAL_II_WITH_MPI
164  int
165  create_group(const MPI_Comm & comm,
166  const MPI_Group &group,
167  const int tag,
168  MPI_Comm * new_comm);
169 #endif
170 
190  template <typename T>
191  T
192  sum(const T &t, const MPI_Comm &mpi_communicator);
193 
203  template <typename T, typename U>
204  void
205  sum(const T &values, const MPI_Comm &mpi_communicator, U &sums);
206 
216  template <typename T>
217  void
218  sum(const ArrayView<const T> &values,
219  const MPI_Comm & mpi_communicator,
220  const ArrayView<T> & sums);
221 
227  template <int rank, int dim, typename Number>
230  const MPI_Comm & mpi_communicator);
231 
237  template <int rank, int dim, typename Number>
239  sum(const Tensor<rank, dim, Number> &local,
240  const MPI_Comm & mpi_communicator);
241 
250  template <typename Number>
251  void
252  sum(const SparseMatrix<Number> &local,
253  const MPI_Comm & mpi_communicator,
254  SparseMatrix<Number> & global);
255 
275  template <typename T>
276  T
277  max(const T &t, const MPI_Comm &mpi_communicator);
278 
288  template <typename T, typename U>
289  void
290  max(const T &values, const MPI_Comm &mpi_communicator, U &maxima);
291 
301  template <typename T>
302  void
303  max(const ArrayView<const T> &values,
304  const MPI_Comm & mpi_communicator,
305  const ArrayView<T> & maxima);
306 
326  template <typename T>
327  T
328  min(const T &t, const MPI_Comm &mpi_communicator);
329 
339  template <typename T, typename U>
340  void
341  min(const T &values, const MPI_Comm &mpi_communicator, U &minima);
342 
352  template <typename T>
353  void
354  min(const ArrayView<const T> &values,
355  const MPI_Comm & mpi_communicator,
356  const ArrayView<T> & minima);
357 
372  struct MinMaxAvg
373  {
378  double sum;
379 
384  double min;
385 
390  double max;
391 
400  unsigned int min_index;
401 
410  unsigned int max_index;
411 
416  double avg;
417  };
418 
433  MinMaxAvg
434  min_max_avg(const double my_value, const MPI_Comm &mpi_communicator);
435 
480  {
481  public:
528  int & argc,
529  char **& argv,
530  const unsigned int max_num_threads = numbers::invalid_unsigned_int);
531 
537  };
538 
550  bool
552 
569  template <typename T>
570  std::map<unsigned int, T>
571  some_to_some(const MPI_Comm & comm,
572  const std::map<unsigned int, T> &objects_to_send);
573 
589  template <typename T>
590  std::vector<T>
591  all_gather(const MPI_Comm &comm, const T &object_to_send);
592 
610  template <typename T>
611  std::vector<T>
612  gather(const MPI_Comm & comm,
613  const T & object_to_send,
614  const unsigned int root_process = 0);
615 
616 #ifndef DOXYGEN
617  // declaration for an internal function that lives in mpi.templates.h
618  namespace internal
619  {
620  template <typename T>
621  void
622  all_reduce(const MPI_Op & mpi_op,
623  const ArrayView<const T> &values,
624  const MPI_Comm & mpi_communicator,
625  const ArrayView<T> & output);
626  }
627 
628  // Since these depend on N they must live in the header file
629  template <typename T, unsigned int N>
630  void
631  sum(const T (&values)[N], const MPI_Comm &mpi_communicator, T (&sums)[N])
632  {
633  internal::all_reduce(MPI_SUM,
634  ArrayView<const T>(values, N),
635  mpi_communicator,
636  ArrayView<T>(sums, N));
637  }
638 
639  template <typename T, unsigned int N>
640  void
641  max(const T (&values)[N], const MPI_Comm &mpi_communicator, T (&maxima)[N])
642  {
643  internal::all_reduce(MPI_MAX,
644  ArrayView<const T>(values, N),
645  mpi_communicator,
646  ArrayView<T>(maxima, N));
647  }
648 
649  template <typename T, unsigned int N>
650  void
651  min(const T (&values)[N], const MPI_Comm &mpi_communicator, T (&minima)[N])
652  {
653  internal::all_reduce(MPI_MIN,
654  ArrayView<const T>(values, N),
655  mpi_communicator,
656  ArrayView<T>(minima, N));
657  }
658 
659  template <typename T>
660  std::map<unsigned int, T>
661  some_to_some(const MPI_Comm & comm,
662  const std::map<unsigned int, T> &objects_to_send)
663  {
664 # ifndef DEAL_II_WITH_MPI
665  (void)comm;
666  Assert(objects_to_send.size() == 0,
667  ExcMessage("Cannot send to more than one processor."));
668  Assert(objects_to_send.find(0) != objects_to_send.end() ||
669  objects_to_send.size() == 0,
670  ExcMessage("Can only send to myself or to nobody."));
671  return objects_to_send;
672 # else
673 
674  std::vector<unsigned int> send_to(objects_to_send.size());
675  {
676  unsigned int i = 0;
677  for (const auto &m : objects_to_send)
678  send_to[i++] = m.first;
679  }
680  AssertDimension(send_to.size(), objects_to_send.size());
681 
682  const auto receive_from =
684  send_to);
685 
686  // Sending buffers
687  std::vector<std::vector<char>> buffers_to_send(send_to.size());
688  std::vector<MPI_Request> buffer_send_requests(send_to.size());
689  {
690  unsigned int i = 0;
691  for (const auto &rank_obj : objects_to_send)
692  {
693  const auto &rank = rank_obj.first;
694  buffers_to_send[i] = Utilities::pack(rank_obj.second);
695  const int ierr = MPI_Isend(buffers_to_send[i].data(),
696  buffers_to_send[i].size(),
697  MPI_CHAR,
698  rank,
699  21,
700  comm,
701  &buffer_send_requests[i]);
702  AssertThrowMPI(ierr);
703  ++i;
704  }
705  }
706 
707  // Receiving buffers
708  std::map<unsigned int, T> received_objects;
709  {
710  std::vector<char> buffer;
711  // We do this on a first come/first served basis
712  for (unsigned int i = 0; i < receive_from.size(); ++i)
713  {
714  // Probe what's going on. Take data from the first available sender
715  MPI_Status status;
716  int ierr = MPI_Probe(MPI_ANY_SOURCE, 21, comm, &status);
717  AssertThrowMPI(ierr);
718 
719  // Length of the message
720  int len;
721  ierr = MPI_Get_count(&status, MPI_CHAR, &len);
722  AssertThrowMPI(ierr);
723  buffer.resize(len);
724 
725  // Source rank
726  const unsigned int rank = status.MPI_SOURCE;
727 
728  // Actually receive the message
729  ierr = MPI_Recv(
730  buffer.data(), len, MPI_CHAR, rank, 21, comm, MPI_STATUS_IGNORE);
731  AssertThrowMPI(ierr);
732  Assert(received_objects.find(rank) == received_objects.end(),
734  "I should not receive again from this rank"));
735  received_objects[rank] = Utilities::unpack<T>(buffer);
736  }
737  }
738 
739  // Wait to have sent all objects.
740  MPI_Waitall(send_to.size(),
741  buffer_send_requests.data(),
742  MPI_STATUSES_IGNORE);
743 
744  return received_objects;
745 # endif // deal.II with MPI
746  }
747 
748  template <typename T>
749  std::vector<T>
750  all_gather(const MPI_Comm &comm, const T &object)
751  {
752 # ifndef DEAL_II_WITH_MPI
753  (void)comm;
754  std::vector<T> v(1, object);
755  return v;
756 # else
757  const auto n_procs = ::Utilities::MPI::n_mpi_processes(comm);
758 
759  std::vector<char> buffer = Utilities::pack(object);
760 
761  int n_local_data = buffer.size();
762 
763  // Vector to store the size of loc_data_array for every process
764  std::vector<int> size_all_data(n_procs, 0);
765 
766  // Exchanging the size of each buffer
767  MPI_Allgather(
768  &n_local_data, 1, MPI_INT, &(size_all_data[0]), 1, MPI_INT, comm);
769 
770  // Now computing the displacement, relative to recvbuf,
771  // at which to store the incoming buffer
772  std::vector<int> rdispls(n_procs);
773  rdispls[0] = 0;
774  for (unsigned int i = 1; i < n_procs; ++i)
775  rdispls[i] = rdispls[i - 1] + size_all_data[i - 1];
776 
777  // Step 3: exchange the buffer:
778  std::vector<char> received_unrolled_buffer(rdispls.back() +
779  size_all_data.back());
780 
781  MPI_Allgatherv(buffer.data(),
782  n_local_data,
783  MPI_CHAR,
784  received_unrolled_buffer.data(),
785  size_all_data.data(),
786  rdispls.data(),
787  MPI_CHAR,
788  comm);
789 
790  std::vector<T> received_objects(n_procs);
791  for (unsigned int i = 0; i < n_procs; ++i)
792  {
793  std::vector<char> local_buffer(received_unrolled_buffer.begin() +
794  rdispls[i],
795  received_unrolled_buffer.begin() +
796  rdispls[i] + size_all_data[i]);
797  received_objects[i] = Utilities::unpack<T>(local_buffer);
798  }
799 
800  return received_objects;
801 # endif
802  }
803 
804  template <typename T>
805  std::vector<T>
806  gather(const MPI_Comm & comm,
807  const T & object_to_send,
808  const unsigned int root_process)
809  {
810 # ifndef DEAL_II_WITH_MPI
811  (void)comm;
812  (void)root_process;
813  std::vector<T> v(1, object_to_send);
814  return v;
815 # else
816  const auto n_procs = ::Utilities::MPI::n_mpi_processes(comm);
817  const auto my_rank = ::Utilities::MPI::this_mpi_process(comm);
818 
819  Assert(root_process < n_procs, ExcIndexRange(root_process, 0, n_procs));
820 
821  std::vector<char> buffer = Utilities::pack(object_to_send);
822  int n_local_data = buffer.size();
823 
824  // Vector to store the size of loc_data_array for every process
825  // only the root process needs to allocate memory for that purpose
826  std::vector<int> size_all_data;
827  if (my_rank == root_process)
828  size_all_data.resize(n_procs, 0);
829 
830  // Exchanging the size of each buffer
831  int ierr = MPI_Gather(&n_local_data,
832  1,
833  MPI_INT,
834  size_all_data.data(),
835  1,
836  MPI_INT,
837  root_process,
838  comm);
839  AssertThrowMPI(ierr);
840 
841  // Now computing the displacement, relative to recvbuf,
842  // at which to store the incoming buffer; only for root
843  std::vector<int> rdispls;
844  if (my_rank == root_process)
845  {
846  rdispls.resize(n_procs, 0);
847  for (unsigned int i = 1; i < n_procs; ++i)
848  rdispls[i] = rdispls[i - 1] + size_all_data[i - 1];
849  }
850  // exchange the buffer:
851  std::vector<char> received_unrolled_buffer;
852  if (my_rank == root_process)
853  received_unrolled_buffer.resize(rdispls.back() + size_all_data.back());
854 
855  ierr = MPI_Gatherv(buffer.data(),
856  n_local_data,
857  MPI_CHAR,
858  received_unrolled_buffer.data(),
859  size_all_data.data(),
860  rdispls.data(),
861  MPI_CHAR,
862  root_process,
863  comm);
864  AssertThrowMPI(ierr);
865 
866  std::vector<T> received_objects;
867 
868  if (my_rank == root_process)
869  {
870  received_objects.resize(n_procs);
871 
872  for (unsigned int i = 0; i < n_procs; ++i)
873  {
874  const std::vector<char> local_buffer(
875  received_unrolled_buffer.begin() + rdispls[i],
876  received_unrolled_buffer.begin() + rdispls[i] +
877  size_all_data[i]);
878  received_objects[i] = Utilities::unpack<T>(local_buffer);
879  }
880  }
881  return received_objects;
882 # endif
883  }
884 
885 #endif
886  } // end of namespace MPI
887 } // end of namespace Utilities
888 
889 
890 DEAL_II_NAMESPACE_CLOSE
891 
892 #endif
static const unsigned int invalid_unsigned_int
Definition: types.h:173
#define AssertDimension(dim1, dim2)
Definition: exceptions.h:1366
MPI_InitFinalize(int &argc, char **&argv, const unsigned int max_num_threads=numbers::invalid_unsigned_int)
Definition: mpi.cc:400
static::ExceptionBase & ExcIndexRange(int arg1, int arg2, int arg3)
static::ExceptionBase & ExcMessage(std::string arg1)
T sum(const T &t, const MPI_Comm &mpi_communicator)
#define Assert(cond, exc)
Definition: exceptions.h:1227
std::vector< T > gather(const MPI_Comm &comm, const T &object_to_send, const unsigned int root_process=0)
int create_group(const MPI_Comm &comm, const MPI_Group &group, const int tag, MPI_Comm *new_comm)
Definition: mpi.cc:102
size_t pack(const T &object, std::vector< char > &dest_buffer, const bool allow_compression=true)
Definition: utilities.h:1019
unsigned int n_mpi_processes(const MPI_Comm &mpi_communicator)
Definition: mpi.cc:69
Definition: cuda.h:32
#define AssertThrowMPI(error_code)
Definition: exceptions.h:1438
MPI_Comm duplicate_communicator(const MPI_Comm &mpi_communicator)
Definition: mpi.cc:91
Definition: mpi.h:55
T min(const T &t, const MPI_Comm &mpi_communicator)
unsigned int this_mpi_process(const MPI_Comm &mpi_communicator)
Definition: mpi.cc:80
std::vector< T > all_gather(const MPI_Comm &comm, const T &object_to_send)
MinMaxAvg min_max_avg(const double my_value, const MPI_Comm &mpi_communicator)
Definition: mpi.cc:293
std::vector< unsigned int > compute_point_to_point_communication_pattern(const MPI_Comm &mpi_comm, const std::vector< unsigned int > &destinations)
Definition: mpi.cc:184
bool job_supports_mpi()
Definition: mpi.cc:642
std::map< unsigned int, T > some_to_some(const MPI_Comm &comm, const std::map< unsigned int, T > &objects_to_send)
unsigned int min_index
Definition: mpi.h:400
T max(const T &t, const MPI_Comm &mpi_communicator)
unsigned int max_index
Definition: mpi.h:410
static::ExceptionBase & ExcInternalError()