Reference documentation for deal.II version Git 4abc4a1666 2020-07-04 19:58:34 +0200
\(\newcommand{\dealvcentcolon}{\mathrel{\mathop{:}}}\) \(\newcommand{\dealcoloneq}{\dealvcentcolon\mathrel{\mkern-1.2mu}=}\) \(\newcommand{\jump}[1]{\left[\!\left[ #1 \right]\!\right]}\) \(\newcommand{\average}[1]{\left\{\!\left\{ #1 \right\}\!\right\}}\)
partitioner.h
Go to the documentation of this file.
1 // ---------------------------------------------------------------------
2 //
3 // Copyright (C) 2011 - 2020 by the deal.II authors
4 //
5 // This file is part of the deal.II library.
6 //
7 // The deal.II library is free software; you can use it, redistribute
8 // it, and/or modify it under the terms of the GNU Lesser General
9 // Public License as published by the Free Software Foundation; either
10 // version 2.1 of the License, or (at your option) any later version.
11 // The full text of the license can be found in the file LICENSE.md at
12 // the top level directory of deal.II.
13 //
14 // ---------------------------------------------------------------------
15 
16 #ifndef dealii_partitioner_h
17 #define dealii_partitioner_h
18 
19 #include <deal.II/base/config.h>
20 
22 #include <deal.II/base/index_set.h>
25 #include <deal.II/base/mpi.h>
26 #include <deal.II/base/types.h>
27 #include <deal.II/base/utilities.h>
28 
30 #include <deal.II/lac/vector.h>
32 
33 #include <limits>
34 
35 
37 
38 namespace Utilities
39 {
40  namespace MPI
41  {
194  {
195  public:
199  Partitioner();
200 
205  Partitioner(const unsigned int size);
206 
214  Partitioner(const IndexSet &locally_owned_indices,
215  const IndexSet &ghost_indices_in,
216  const MPI_Comm communicator_in);
217 
225  Partitioner(const IndexSet &locally_owned_indices,
226  const MPI_Comm communicator_in);
227 
235  virtual void
236  reinit(const IndexSet &vector_space_vector_index_set,
237  const IndexSet &read_write_vector_index_set,
238  const MPI_Comm &communicator) override;
239 
243  void
244  set_owned_indices(const IndexSet &locally_owned_indices);
245 
256  void
258  const IndexSet &larger_ghost_index_set = IndexSet());
259 
264  size() const;
265 
270  unsigned int
271  local_size() const;
272 
279  const IndexSet &
280  locally_owned_range() const;
281 
287  std::pair<types::global_dof_index, types::global_dof_index>
288  local_range() const;
289 
294  bool
296 
307  unsigned int
308  global_to_local(const types::global_dof_index global_index) const;
309 
318  local_to_global(const unsigned int local_index) const;
319 
325  bool
326  is_ghost_entry(const types::global_dof_index global_index) const;
327 
331  const IndexSet &
332  ghost_indices() const;
333 
338  unsigned int
339  n_ghost_indices() const;
340 
351  const std::vector<std::pair<unsigned int, unsigned int>> &
353 
359  const std::vector<std::pair<unsigned int, unsigned int>> &
360  ghost_targets() const;
361 
370  const std::vector<std::pair<unsigned int, unsigned int>> &
371  import_indices() const;
372 
377  unsigned int
378  n_import_indices() const;
379 
389  const std::vector<std::pair<unsigned int, unsigned int>> &
390  import_targets() const;
391 
402  bool
403  is_compatible(const Partitioner &part) const;
404 
421  bool
422  is_globally_compatible(const Partitioner &part) const;
423 
428  unsigned int
429  this_mpi_process() const;
430 
435  unsigned int
436  n_mpi_processes() const;
437 
441  virtual const MPI_Comm &
442  get_mpi_communicator() const override;
443 
449  bool
451 
452 #ifdef DEAL_II_WITH_MPI
453 
488  template <typename Number, typename MemorySpaceType = MemorySpace::Host>
489  void
491  const unsigned int communication_channel,
492  const ArrayView<const Number, MemorySpaceType> &locally_owned_array,
493  const ArrayView<Number, MemorySpaceType> & temporary_storage,
494  const ArrayView<Number, MemorySpaceType> & ghost_array,
495  std::vector<MPI_Request> & requests) const;
496 
514  template <typename Number, typename MemorySpaceType = MemorySpace::Host>
515  void
517  const ArrayView<Number, MemorySpaceType> &ghost_array,
518  std::vector<MPI_Request> & requests) const;
519 
557  template <typename Number, typename MemorySpaceType = MemorySpace::Host>
558  void
560  const VectorOperation::values vector_operation,
561  const unsigned int communication_channel,
562  const ArrayView<Number, MemorySpaceType> &ghost_array,
563  const ArrayView<Number, MemorySpaceType> &temporary_storage,
564  std::vector<MPI_Request> & requests) const;
565 
600  template <typename Number, typename MemorySpaceType = MemorySpace::Host>
601  void
603  const VectorOperation::values vector_operation,
604  const ArrayView<const Number, MemorySpaceType> &temporary_storage,
605  const ArrayView<Number, MemorySpaceType> & locally_owned_storage,
606  const ArrayView<Number, MemorySpaceType> & ghost_array,
607  std::vector<MPI_Request> & requests) const;
608 #endif
609 
613  std::size_t
614  memory_consumption() const;
615 
621  unsigned int,
622  << "Global index " << arg1
623  << " neither owned nor ghost on proc " << arg2 << ".");
624 
629  unsigned int,
630  unsigned int,
631  unsigned int,
632  << "The size of the ghost index array (" << arg1
633  << ") must either equal the number of ghost in the "
634  << "partitioner (" << arg2
635  << ") or be equal in size to a more comprehensive index"
636  << "set which contains " << arg3
637  << " elements for this partitioner.");
638 
639  private:
644  void
646 
651 
656 
661  std::pair<types::global_dof_index, types::global_dof_index>
663 
669 
674  unsigned int n_ghost_indices_data;
675 
680  std::vector<std::pair<unsigned int, unsigned int>> ghost_targets_data;
681 
688  std::vector<std::pair<unsigned int, unsigned int>> import_indices_data;
689 
696  // The variable is mutable to enable lazy initialization in
697  // export_to_ghosted_array_start(). This way partitioner does not have to
698  // be templated on the MemorySpaceType.
699  mutable std::vector<
700  std::pair<std::unique_ptr<unsigned int[], void (*)(unsigned int *)>,
701  unsigned int>>
703 
708  unsigned int n_import_indices_data;
709 
714  std::vector<std::pair<unsigned int, unsigned int>> import_targets_data;
715 
720  std::vector<unsigned int> import_indices_chunks_by_rank_data;
721 
727 
732  std::vector<unsigned int> ghost_indices_subset_chunks_by_rank_data;
733 
739  std::vector<std::pair<unsigned int, unsigned int>>
741 
745  unsigned int my_pid;
746 
750  unsigned int n_procs;
751 
755  MPI_Comm communicator;
756 
761  };
762 
763 
764 
765  /*--------------------- Inline functions --------------------------------*/
766 
767 #ifndef DOXYGEN
768 
770  Partitioner::size() const
771  {
772  return global_size;
773  }
774 
775 
776 
777  inline const IndexSet &
779  {
781  }
782 
783 
784 
785  inline std::pair<types::global_dof_index, types::global_dof_index>
787  {
788  return local_range_data;
789  }
790 
791 
792 
793  inline unsigned int
795  {
797  local_range_data.second - local_range_data.first;
800  return static_cast<unsigned int>(size);
801  }
802 
803 
804 
805  inline bool
808  {
809  return (local_range_data.first <= global_index &&
810  global_index < local_range_data.second);
811  }
812 
813 
814 
815  inline bool
817  const types::global_dof_index global_index) const
818  {
819  // if the index is in the global range, it is trivially not a ghost
820  if (in_local_range(global_index) == true)
821  return false;
822  else
823  return ghost_indices().is_element(global_index);
824  }
825 
826 
827 
828  inline unsigned int
830  const types::global_dof_index global_index) const
831  {
832  Assert(in_local_range(global_index) || is_ghost_entry(global_index),
833  ExcIndexNotPresent(global_index, my_pid));
834  if (in_local_range(global_index))
835  return static_cast<unsigned int>(global_index - local_range_data.first);
836  else if (is_ghost_entry(global_index))
837  return (local_size() +
838  static_cast<unsigned int>(
839  ghost_indices_data.index_within_set(global_index)));
840  else
841  // should only end up here in optimized mode, when we use this large
842  // number to trigger a segfault when using this method for array
843  // access
845  }
846 
847 
848 
850  Partitioner::local_to_global(const unsigned int local_index) const
851  {
853  if (local_index < local_size())
854  return local_range_data.first + types::global_dof_index(local_index);
855  else
856  return ghost_indices_data.nth_index_in_set(local_index - local_size());
857  }
858 
859 
860 
861  inline const IndexSet &
863  {
864  return ghost_indices_data;
865  }
866 
867 
868 
869  inline unsigned int
871  {
872  return n_ghost_indices_data;
873  }
874 
875 
876 
877  inline const std::vector<std::pair<unsigned int, unsigned int>> &
879  {
881  }
882 
883 
884 
885  inline const std::vector<std::pair<unsigned int, unsigned int>> &
887  {
888  return ghost_targets_data;
889  }
890 
891 
892  inline const std::vector<std::pair<unsigned int, unsigned int>> &
894  {
895  return import_indices_data;
896  }
897 
898 
899 
900  inline unsigned int
902  {
903  return n_import_indices_data;
904  }
905 
906 
907 
908  inline const std::vector<std::pair<unsigned int, unsigned int>> &
910  {
911  return import_targets_data;
912  }
913 
914 
915 
916  inline unsigned int
918  {
919  // return the id from the variable stored in this class instead of
920  // Utilities::MPI::this_mpi_process() in order to make this query also
921  // work when MPI is not initialized.
922  return my_pid;
923  }
924 
925 
926 
927  inline unsigned int
929  {
930  // return the number of MPI processes from the variable stored in this
931  // class instead of Utilities::MPI::n_mpi_processes() in order to make
932  // this query also work when MPI is not initialized.
933  return n_procs;
934  }
935 
936 
937 
938  inline const MPI_Comm &
940  {
941  return communicator;
942  }
943 
944 
945 
946  inline bool
948  {
949  return have_ghost_indices;
950  }
951 
952 #endif // ifndef DOXYGEN
953 
954  } // end of namespace MPI
955 
956 } // end of namespace Utilities
957 
958 
960 
961 #endif
std::vector< std::pair< unsigned int, unsigned int > > import_indices_data
Definition: partitioner.h:688
const IndexSet & locally_owned_range() const
bool is_globally_compatible(const Partitioner &part) const
Definition: partitioner.cc:462
static const unsigned int invalid_unsigned_int
Definition: types.h:191
#define DeclException2(Exception2, type1, type2, outsequence)
Definition: exceptions.h:538
bool is_compatible(const Partitioner &part) const
Definition: partitioner.cc:435
static ::ExceptionBase & ExcIndexNotPresent(types::global_dof_index arg1, unsigned int arg2)
unsigned int local_size() const
const std::vector< std::pair< unsigned int, unsigned int > > & import_targets() const
void import_from_ghosted_array_finish(const VectorOperation::values vector_operation, const ArrayView< const Number, MemorySpaceType > &temporary_storage, const ArrayView< Number, MemorySpaceType > &locally_owned_storage, const ArrayView< Number, MemorySpaceType > &ghost_array, std::vector< MPI_Request > &requests) const
virtual const MPI_Comm & get_mpi_communicator() const override
const std::vector< std::pair< unsigned int, unsigned int > > & import_indices() const
void import_from_ghosted_array_start(const VectorOperation::values vector_operation, const unsigned int communication_channel, const ArrayView< Number, MemorySpaceType > &ghost_array, const ArrayView< Number, MemorySpaceType > &temporary_storage, std::vector< MPI_Request > &requests) const
types::global_dof_index global_size
Definition: partitioner.h:650
types::global_dof_index local_to_global(const unsigned int local_index) const
#define AssertIndexRange(index, range)
Definition: exceptions.h:1628
unsigned int n_ghost_indices_data
Definition: partitioner.h:674
const std::vector< std::pair< unsigned int, unsigned int > > & ghost_indices_within_larger_ghost_set() const
unsigned int global_to_local(const types::global_dof_index global_index) const
bool in_local_range(const types::global_dof_index global_index) const
void set_owned_indices(const IndexSet &locally_owned_indices)
Definition: partitioner.cc:111
std::vector< unsigned int > ghost_indices_subset_chunks_by_rank_data
Definition: partitioner.h:732
std::vector< unsigned int > import_indices_chunks_by_rank_data
Definition: partitioner.h:720
bool ghost_indices_initialized() const
unsigned int this_mpi_process() const
const IndexSet & ghost_indices() const
unsigned int n_ghost_indices() const
std::pair< types::global_dof_index, types::global_dof_index > local_range() const
#define Assert(cond, exc)
Definition: exceptions.h:1403
size_type index_within_set(const size_type global_index) const
Definition: index_set.h:1919
unsigned int n_mpi_processes() const
#define DEAL_II_NAMESPACE_CLOSE
Definition: config.h:363
std::vector< std::pair< unsigned int, unsigned int > > ghost_indices_subset_data
Definition: partitioner.h:740
const std::vector< std::pair< unsigned int, unsigned int > > & ghost_targets() const
unsigned int global_dof_index
Definition: types.h:76
Definition: cuda.h:31
types::global_dof_index size() const
std::size_t memory_consumption() const
Definition: partitioner.cc:471
void initialize_import_indices_plain_dev() const
#define DEAL_II_NAMESPACE_OPEN
Definition: config.h:362
std::vector< std::pair< unsigned int, unsigned int > > import_targets_data
Definition: partitioner.h:714
std::pair< types::global_dof_index, types::global_dof_index > local_range_data
Definition: partitioner.h:662
std::vector< std::pair< unsigned int, unsigned int > > ghost_targets_data
Definition: partitioner.h:680
void export_to_ghosted_array_start(const unsigned int communication_channel, const ArrayView< const Number, MemorySpaceType > &locally_owned_array, const ArrayView< Number, MemorySpaceType > &temporary_storage, const ArrayView< Number, MemorySpaceType > &ghost_array, std::vector< MPI_Request > &requests) const
void export_to_ghosted_array_finish(const ArrayView< Number, MemorySpaceType > &ghost_array, std::vector< MPI_Request > &requests) const
static ::ExceptionBase & ExcNotImplemented()
TrilinosWrappers::types::int_type global_index(const Epetra_BlockMap &map, const ::types::global_dof_index i)
bool is_element(const size_type index) const
Definition: index_set.h:1763
#define DeclException3(Exception3, type1, type2, type3, outsequence)
Definition: exceptions.h:561
std::vector< std::pair< std::unique_ptr< unsigned int[], void(*)(unsigned int *)>, unsigned int > > import_indices_plain_dev
Definition: partitioner.h:702
unsigned int n_import_indices() const
unsigned int n_import_indices_data
Definition: partitioner.h:708
virtual void reinit(const IndexSet &vector_space_vector_index_set, const IndexSet &read_write_vector_index_set, const MPI_Comm &communicator) override
Definition: partitioner.cc:98
bool is_ghost_entry(const types::global_dof_index global_index) const
size_type nth_index_in_set(const size_type local_index) const
Definition: index_set.h:1878
T max(const T &t, const MPI_Comm &mpi_communicator)
static ::ExceptionBase & ExcGhostIndexArrayHasWrongSize(unsigned int arg1, unsigned int arg2, unsigned int arg3)
unsigned int n_ghost_indices_in_larger_set
Definition: partitioner.h:726
void set_ghost_indices(const IndexSet &ghost_indices, const IndexSet &larger_ghost_index_set=IndexSet())
Definition: partitioner.cc:152