Reference documentation for deal.II version Git 2276b94619 2020-02-28 10:11:55 +0100
\(\newcommand{\vcentcolon}{\mathrel{\mathop{:}}}\) \(\newcommand{\dealcoloneq}{\vcentcolon\mathrel{\mkern-1.2mu}=}\) \(\newcommand{\jump}[1]{\left[\!\left[ #1 \right]\!\right]}\) \(\newcommand{\average}[1]{\left\{\!\left\{ #1 \right\}\!\right\}}\)
partitioner.h
1 // ---------------------------------------------------------------------
2 //
3 // Copyright (C) 2011 - 2019 by the deal.II authors
4 //
5 // This file is part of the deal.II library.
6 //
7 // The deal.II library is free software; you can use it, redistribute
8 // it, and/or modify it under the terms of the GNU Lesser General
9 // Public License as published by the Free Software Foundation; either
10 // version 2.1 of the License, or (at your option) any later version.
11 // The full text of the license can be found in the file LICENSE.md at
12 // the top level directory of deal.II.
13 //
14 // ---------------------------------------------------------------------
15 
16 #ifndef dealii_partitioner_h
17 #define dealii_partitioner_h
18 
19 #include <deal.II/base/config.h>
20 
21 #include <deal.II/base/array_view.h>
22 #include <deal.II/base/index_set.h>
23 #include <deal.II/base/memory_consumption.h>
24 #include <deal.II/base/memory_space.h>
25 #include <deal.II/base/mpi.h>
26 #include <deal.II/base/types.h>
27 #include <deal.II/base/utilities.h>
28 
29 #include <deal.II/lac/communication_pattern_base.h>
30 #include <deal.II/lac/vector.h>
31 #include <deal.II/lac/vector_operation.h>
32 
33 #include <limits>
34 
35 
36 DEAL_II_NAMESPACE_OPEN
37 
38 namespace Utilities
39 {
40  namespace MPI
41  {
196  {
197  public:
201  Partitioner();
202 
207  Partitioner(const unsigned int size);
208 
216  Partitioner(const IndexSet &locally_owned_indices,
217  const IndexSet &ghost_indices_in,
218  const MPI_Comm communicator_in);
219 
227  Partitioner(const IndexSet &locally_owned_indices,
228  const MPI_Comm communicator_in);
229 
237  virtual void
238  reinit(const IndexSet &vector_space_vector_index_set,
239  const IndexSet &read_write_vector_index_set,
240  const MPI_Comm &communicator) override;
241 
245  void
246  set_owned_indices(const IndexSet &locally_owned_indices);
247 
258  void
260  const IndexSet &larger_ghost_index_set = IndexSet());
261 
266  size() const;
267 
272  unsigned int
273  local_size() const;
274 
281  const IndexSet &
282  locally_owned_range() const;
283 
289  std::pair<types::global_dof_index, types::global_dof_index>
290  local_range() const;
291 
296  bool
297  in_local_range(const types::global_dof_index global_index) const;
298 
309  unsigned int
310  global_to_local(const types::global_dof_index global_index) const;
311 
320  local_to_global(const unsigned int local_index) const;
321 
327  bool
328  is_ghost_entry(const types::global_dof_index global_index) const;
329 
333  const IndexSet &
334  ghost_indices() const;
335 
340  unsigned int
341  n_ghost_indices() const;
342 
353  const std::vector<std::pair<unsigned int, unsigned int>> &
355 
361  const std::vector<std::pair<unsigned int, unsigned int>> &
362  ghost_targets() const;
363 
372  const std::vector<std::pair<unsigned int, unsigned int>> &
373  import_indices() const;
374 
379  unsigned int
380  n_import_indices() const;
381 
391  const std::vector<std::pair<unsigned int, unsigned int>> &
392  import_targets() const;
393 
404  bool
405  is_compatible(const Partitioner &part) const;
406 
423  bool
424  is_globally_compatible(const Partitioner &part) const;
425 
430  unsigned int
431  this_mpi_process() const;
432 
437  unsigned int
438  n_mpi_processes() const;
439 
443  DEAL_II_DEPRECATED
444  const MPI_Comm &
445  get_communicator() const;
446 
450  virtual const MPI_Comm &
451  get_mpi_communicator() const override;
452 
458  bool
460 
461 #ifdef DEAL_II_WITH_MPI
462 
497  template <typename Number, typename MemorySpaceType = MemorySpace::Host>
498  void
500  const unsigned int communication_channel,
501  const ArrayView<const Number, MemorySpaceType> &locally_owned_array,
502  const ArrayView<Number, MemorySpaceType> & temporary_storage,
503  const ArrayView<Number, MemorySpaceType> & ghost_array,
504  std::vector<MPI_Request> & requests) const;
505 
523  template <typename Number, typename MemorySpaceType = MemorySpace::Host>
524  void
526  const ArrayView<Number, MemorySpaceType> &ghost_array,
527  std::vector<MPI_Request> & requests) const;
528 
566  template <typename Number, typename MemorySpaceType = MemorySpace::Host>
567  void
569  const VectorOperation::values vector_operation,
570  const unsigned int communication_channel,
571  const ArrayView<Number, MemorySpaceType> &ghost_array,
572  const ArrayView<Number, MemorySpaceType> &temporary_storage,
573  std::vector<MPI_Request> & requests) const;
574 
609  template <typename Number, typename MemorySpaceType = MemorySpace::Host>
610  void
612  const VectorOperation::values vector_operation,
613  const ArrayView<const Number, MemorySpaceType> &temporary_storage,
614  const ArrayView<Number, MemorySpaceType> & locally_owned_storage,
615  const ArrayView<Number, MemorySpaceType> & ghost_array,
616  std::vector<MPI_Request> & requests) const;
617 #endif
618 
622  std::size_t
623  memory_consumption() const;
624 
630  unsigned int,
631  << "Global index " << arg1
632  << " neither owned nor ghost on proc " << arg2 << ".");
633 
638  unsigned int,
639  unsigned int,
640  unsigned int,
641  << "The size of the ghost index array (" << arg1
642  << ") must either equal the number of ghost in the "
643  << "partitioner (" << arg2
644  << ") or be equal in size to a more comprehensive index"
645  << "set which contains " << arg3
646  << " elements for this partitioner.");
647 
648  private:
653  void
655 
660 
665 
670  std::pair<types::global_dof_index, types::global_dof_index>
672 
678 
683  unsigned int n_ghost_indices_data;
684 
689  std::vector<std::pair<unsigned int, unsigned int>> ghost_targets_data;
690 
697  std::vector<std::pair<unsigned int, unsigned int>> import_indices_data;
698 
705  // The variable is mutable to enable lazy initialization in
706  // export_to_ghosted_array_start(). This way partitioner does not have to
707  // be templated on the MemorySpaceType.
708  mutable std::vector<
709  std::pair<std::unique_ptr<unsigned int[], void (*)(unsigned int *)>,
710  unsigned int>>
712 
717  unsigned int n_import_indices_data;
718 
723  std::vector<std::pair<unsigned int, unsigned int>> import_targets_data;
724 
729  std::vector<unsigned int> import_indices_chunks_by_rank_data;
730 
736 
741  std::vector<unsigned int> ghost_indices_subset_chunks_by_rank_data;
742 
748  std::vector<std::pair<unsigned int, unsigned int>>
750 
754  unsigned int my_pid;
755 
759  unsigned int n_procs;
760 
764  MPI_Comm communicator;
765 
770  };
771 
772 
773 
774  /*--------------------- Inline functions --------------------------------*/
775 
776 #ifndef DOXYGEN
777 
779  Partitioner::size() const
780  {
781  return global_size;
782  }
783 
784 
785 
786  inline const IndexSet &
788  {
790  }
791 
792 
793 
794  inline std::pair<types::global_dof_index, types::global_dof_index>
796  {
797  return local_range_data;
798  }
799 
800 
801 
802  inline unsigned int
804  {
806  local_range_data.second - local_range_data.first;
807  Assert(size <= std::numeric_limits<unsigned int>::max(),
809  return static_cast<unsigned int>(size);
810  }
811 
812 
813 
814  inline bool
816  const types::global_dof_index global_index) const
817  {
818  return (local_range_data.first <= global_index &&
819  global_index < local_range_data.second);
820  }
821 
822 
823 
824  inline bool
826  const types::global_dof_index global_index) const
827  {
828  // if the index is in the global range, it is trivially not a ghost
829  if (in_local_range(global_index) == true)
830  return false;
831  else
832  return ghost_indices().is_element(global_index);
833  }
834 
835 
836 
837  inline unsigned int
839  const types::global_dof_index global_index) const
840  {
841  Assert(in_local_range(global_index) || is_ghost_entry(global_index),
842  ExcIndexNotPresent(global_index, my_pid));
843  if (in_local_range(global_index))
844  return static_cast<unsigned int>(global_index - local_range_data.first);
845  else if (is_ghost_entry(global_index))
846  return (local_size() +
847  static_cast<unsigned int>(
848  ghost_indices_data.index_within_set(global_index)));
849  else
850  // should only end up here in optimized mode, when we use this large
851  // number to trigger a segfault when using this method for array
852  // access
854  }
855 
856 
857 
859  Partitioner::local_to_global(const unsigned int local_index) const
860  {
862  if (local_index < local_size())
863  return local_range_data.first + types::global_dof_index(local_index);
864  else
865  return ghost_indices_data.nth_index_in_set(local_index - local_size());
866  }
867 
868 
869 
870  inline const IndexSet &
872  {
873  return ghost_indices_data;
874  }
875 
876 
877 
878  inline unsigned int
880  {
881  return n_ghost_indices_data;
882  }
883 
884 
885 
886  inline const std::vector<std::pair<unsigned int, unsigned int>> &
888  {
890  }
891 
892 
893 
894  inline const std::vector<std::pair<unsigned int, unsigned int>> &
896  {
897  return ghost_targets_data;
898  }
899 
900 
901  inline const std::vector<std::pair<unsigned int, unsigned int>> &
903  {
904  return import_indices_data;
905  }
906 
907 
908 
909  inline unsigned int
911  {
912  return n_import_indices_data;
913  }
914 
915 
916 
917  inline const std::vector<std::pair<unsigned int, unsigned int>> &
919  {
920  return import_targets_data;
921  }
922 
923 
924 
925  inline unsigned int
927  {
928  // return the id from the variable stored in this class instead of
929  // Utilities::MPI::this_mpi_process() in order to make this query also
930  // work when MPI is not initialized.
931  return my_pid;
932  }
933 
934 
935 
936  inline unsigned int
938  {
939  // return the number of MPI processes from the variable stored in this
940  // class instead of Utilities::MPI::n_mpi_processes() in order to make
941  // this query also work when MPI is not initialized.
942  return n_procs;
943  }
944 
945 
946 
947  inline const MPI_Comm &
949  {
950  return communicator;
951  }
952 
953 
954 
955  inline const MPI_Comm &
957  {
958  return communicator;
959  }
960 
961 
962 
963  inline bool
965  {
966  return have_ghost_indices;
967  }
968 
969 #endif // ifndef DOXYGEN
970 
971  } // end of namespace MPI
972 
973 } // end of namespace Utilities
974 
975 
976 DEAL_II_NAMESPACE_CLOSE
977 
978 #endif
std::vector< std::pair< unsigned int, unsigned int > > import_indices_data
Definition: partitioner.h:697
const IndexSet & locally_owned_range() const
bool is_globally_compatible(const Partitioner &part) const
Definition: partitioner.cc:462
static const unsigned int invalid_unsigned_int
Definition: types.h:187
#define DeclException2(Exception2, type1, type2, outsequence)
Definition: exceptions.h:541
bool is_compatible(const Partitioner &part) const
Definition: partitioner.cc:435
static ::ExceptionBase & ExcIndexNotPresent(types::global_dof_index arg1, unsigned int arg2)
unsigned int local_size() const
const std::vector< std::pair< unsigned int, unsigned int > > & import_targets() const
void import_from_ghosted_array_finish(const VectorOperation::values vector_operation, const ArrayView< const Number, MemorySpaceType > &temporary_storage, const ArrayView< Number, MemorySpaceType > &locally_owned_storage, const ArrayView< Number, MemorySpaceType > &ghost_array, std::vector< MPI_Request > &requests) const
virtual const MPI_Comm & get_mpi_communicator() const override
const std::vector< std::pair< unsigned int, unsigned int > > & import_indices() const
void import_from_ghosted_array_start(const VectorOperation::values vector_operation, const unsigned int communication_channel, const ArrayView< Number, MemorySpaceType > &ghost_array, const ArrayView< Number, MemorySpaceType > &temporary_storage, std::vector< MPI_Request > &requests) const
types::global_dof_index global_size
Definition: partitioner.h:659
types::global_dof_index local_to_global(const unsigned int local_index) const
#define AssertIndexRange(index, range)
Definition: exceptions.h:1641
unsigned int n_ghost_indices_data
Definition: partitioner.h:683
const std::vector< std::pair< unsigned int, unsigned int > > & ghost_indices_within_larger_ghost_set() const
unsigned int global_to_local(const types::global_dof_index global_index) const
bool in_local_range(const types::global_dof_index global_index) const
void set_owned_indices(const IndexSet &locally_owned_indices)
Definition: partitioner.cc:111
std::vector< unsigned int > ghost_indices_subset_chunks_by_rank_data
Definition: partitioner.h:741
std::vector< unsigned int > import_indices_chunks_by_rank_data
Definition: partitioner.h:729
bool ghost_indices_initialized() const
unsigned int this_mpi_process() const
const IndexSet & ghost_indices() const
const MPI_Comm & get_communicator() const
unsigned int n_ghost_indices() const
std::pair< types::global_dof_index, types::global_dof_index > local_range() const
#define Assert(cond, exc)
Definition: exceptions.h:1411
size_type index_within_set(const size_type global_index) const
Definition: index_set.h:1922
unsigned int n_mpi_processes() const
std::vector< std::pair< unsigned int, unsigned int > > ghost_indices_subset_data
Definition: partitioner.h:749
const std::vector< std::pair< unsigned int, unsigned int > > & ghost_targets() const
unsigned int global_dof_index
Definition: types.h:89
Definition: cuda.h:31
types::global_dof_index size() const
std::size_t memory_consumption() const
Definition: partitioner.cc:471
void initialize_import_indices_plain_dev() const
std::vector< std::pair< unsigned int, unsigned int > > import_targets_data
Definition: partitioner.h:723
std::pair< types::global_dof_index, types::global_dof_index > local_range_data
Definition: partitioner.h:671
std::vector< std::pair< unsigned int, unsigned int > > ghost_targets_data
Definition: partitioner.h:689
void export_to_ghosted_array_start(const unsigned int communication_channel, const ArrayView< const Number, MemorySpaceType > &locally_owned_array, const ArrayView< Number, MemorySpaceType > &temporary_storage, const ArrayView< Number, MemorySpaceType > &ghost_array, std::vector< MPI_Request > &requests) const
void export_to_ghosted_array_finish(const ArrayView< Number, MemorySpaceType > &ghost_array, std::vector< MPI_Request > &requests) const
static ::ExceptionBase & ExcNotImplemented()
bool is_element(const size_type index) const
Definition: index_set.h:1766
#define DeclException3(Exception3, type1, type2, type3, outsequence)
Definition: exceptions.h:564
std::vector< std::pair< std::unique_ptr< unsigned int[], void(*)(unsigned int *)>, unsigned int > > import_indices_plain_dev
Definition: partitioner.h:711
unsigned int n_import_indices() const
unsigned int n_import_indices_data
Definition: partitioner.h:717
virtual void reinit(const IndexSet &vector_space_vector_index_set, const IndexSet &read_write_vector_index_set, const MPI_Comm &communicator) override
Definition: partitioner.cc:98
bool is_ghost_entry(const types::global_dof_index global_index) const
size_type nth_index_in_set(const size_type local_index) const
Definition: index_set.h:1881
static ::ExceptionBase & ExcGhostIndexArrayHasWrongSize(unsigned int arg1, unsigned int arg2, unsigned int arg3)
unsigned int n_ghost_indices_in_larger_set
Definition: partitioner.h:735
void set_ghost_indices(const IndexSet &ghost_indices, const IndexSet &larger_ghost_index_set=IndexSet())
Definition: partitioner.cc:152