Reference documentation for deal.II version Git cb0bd54b52 2019-09-21 16:31:22 -0400
\(\newcommand{\dealcoloneq}{\mathrel{\vcenter{:}}=}\)
partitioner.h
1 // ---------------------------------------------------------------------
2 //
3 // Copyright (C) 2011 - 2019 by the deal.II authors
4 //
5 // This file is part of the deal.II library.
6 //
7 // The deal.II library is free software; you can use it, redistribute
8 // it, and/or modify it under the terms of the GNU Lesser General
9 // Public License as published by the Free Software Foundation; either
10 // version 2.1 of the License, or (at your option) any later version.
11 // The full text of the license can be found in the file LICENSE.md at
12 // the top level directory of deal.II.
13 //
14 // ---------------------------------------------------------------------
15 
16 #ifndef dealii_partitioner_h
17 #define dealii_partitioner_h
18 
19 #include <deal.II/base/config.h>
20 
21 #include <deal.II/base/array_view.h>
22 #include <deal.II/base/index_set.h>
23 #include <deal.II/base/memory_consumption.h>
24 #include <deal.II/base/memory_space.h>
25 #include <deal.II/base/mpi.h>
26 #include <deal.II/base/types.h>
27 #include <deal.II/base/utilities.h>
28 
29 #include <deal.II/lac/communication_pattern_base.h>
30 #include <deal.II/lac/vector.h>
31 #include <deal.II/lac/vector_operation.h>
32 
33 #include <limits>
34 
35 
36 DEAL_II_NAMESPACE_OPEN
37 
38 namespace Utilities
39 {
40  namespace MPI
41  {
128  {
129  public:
133  Partitioner();
134 
139  Partitioner(const unsigned int size);
140 
148  Partitioner(const IndexSet &locally_owned_indices,
149  const IndexSet &ghost_indices_in,
150  const MPI_Comm communicator_in);
151 
159  Partitioner(const IndexSet &locally_owned_indices,
160  const MPI_Comm communicator_in);
161 
169  virtual void
170  reinit(const IndexSet &vector_space_vector_index_set,
171  const IndexSet &read_write_vector_index_set,
172  const MPI_Comm &communicator) override;
173 
177  void
178  set_owned_indices(const IndexSet &locally_owned_indices);
179 
190  void
192  const IndexSet &larger_ghost_index_set = IndexSet());
193 
198  size() const;
199 
204  unsigned int
205  local_size() const;
206 
213  const IndexSet &
214  locally_owned_range() const;
215 
221  std::pair<types::global_dof_index, types::global_dof_index>
222  local_range() const;
223 
228  bool
229  in_local_range(const types::global_dof_index global_index) const;
230 
240  unsigned int
241  global_to_local(const types::global_dof_index global_index) const;
242 
251  local_to_global(const unsigned int local_index) const;
252 
258  bool
259  is_ghost_entry(const types::global_dof_index global_index) const;
260 
264  const IndexSet &
265  ghost_indices() const;
266 
271  unsigned int
272  n_ghost_indices() const;
273 
284  const std::vector<std::pair<unsigned int, unsigned int>> &
286 
292  const std::vector<std::pair<unsigned int, unsigned int>> &
293  ghost_targets() const;
294 
303  const std::vector<std::pair<unsigned int, unsigned int>> &
304  import_indices() const;
305 
310  unsigned int
311  n_import_indices() const;
312 
322  const std::vector<std::pair<unsigned int, unsigned int>> &
323  import_targets() const;
324 
334  bool
335  is_compatible(const Partitioner &part) const;
336 
350  bool
351  is_globally_compatible(const Partitioner &part) const;
352 
357  unsigned int
358  this_mpi_process() const;
359 
364  unsigned int
365  n_mpi_processes() const;
366 
370  DEAL_II_DEPRECATED
371  const MPI_Comm &
372  get_communicator() const;
373 
377  virtual const MPI_Comm &
378  get_mpi_communicator() const override;
379 
385  bool
387 
388 #ifdef DEAL_II_WITH_MPI
389 
423  template <typename Number, typename MemorySpaceType = MemorySpace::Host>
424  void
426  const unsigned int communication_channel,
427  const ArrayView<const Number, MemorySpaceType> &locally_owned_array,
428  const ArrayView<Number, MemorySpaceType> & temporary_storage,
429  const ArrayView<Number, MemorySpaceType> & ghost_array,
430  std::vector<MPI_Request> & requests) const;
431 
449  template <typename Number, typename MemorySpaceType = MemorySpace::Host>
450  void
452  const ArrayView<Number, MemorySpaceType> &ghost_array,
453  std::vector<MPI_Request> & requests) const;
454 
491  template <typename Number, typename MemorySpaceType = MemorySpace::Host>
492  void
494  const VectorOperation::values vector_operation,
495  const unsigned int communication_channel,
496  const ArrayView<Number, MemorySpaceType> &ghost_array,
497  const ArrayView<Number, MemorySpaceType> &temporary_storage,
498  std::vector<MPI_Request> & requests) const;
499 
534  template <typename Number, typename MemorySpaceType = MemorySpace::Host>
535  void
537  const VectorOperation::values vector_operation,
538  const ArrayView<const Number, MemorySpaceType> &temporary_storage,
539  const ArrayView<Number, MemorySpaceType> & locally_owned_storage,
540  const ArrayView<Number, MemorySpaceType> & ghost_array,
541  std::vector<MPI_Request> & requests) const;
542 #endif
543 
547  std::size_t
548  memory_consumption() const;
549 
555  unsigned int,
556  << "Global index " << arg1
557  << " neither owned nor ghost on proc " << arg2 << ".");
558 
563  unsigned int,
564  unsigned int,
565  unsigned int,
566  << "The size of the ghost index array (" << arg1
567  << ") must either equal the number of ghost in the "
568  << "partitioner (" << arg2
569  << ") or be equal in size to a more comprehensive index"
570  << "set which contains " << arg3
571  << " elements for this partitioner.");
572 
573  private:
578  void
580 
585 
590 
595  std::pair<types::global_dof_index, types::global_dof_index>
597 
603 
608  unsigned int n_ghost_indices_data;
609 
614  std::vector<std::pair<unsigned int, unsigned int>> ghost_targets_data;
615 
622  std::vector<std::pair<unsigned int, unsigned int>> import_indices_data;
623 
630  // The variable is mutable to enable lazy initialization in
631  // export_to_ghosted_array_start(). This way partitioner does not have to
632  // be templated on the MemorySpaceType.
633  mutable std::vector<
634  std::pair<std::unique_ptr<unsigned int[], void (*)(unsigned int *)>,
635  unsigned int>>
637 
642  unsigned int n_import_indices_data;
643 
648  std::vector<std::pair<unsigned int, unsigned int>> import_targets_data;
649 
654  std::vector<unsigned int> import_indices_chunks_by_rank_data;
655 
661 
666  std::vector<unsigned int> ghost_indices_subset_chunks_by_rank_data;
667 
673  std::vector<std::pair<unsigned int, unsigned int>>
675 
679  unsigned int my_pid;
680 
684  unsigned int n_procs;
685 
689  MPI_Comm communicator;
690 
695  };
696 
697 
698 
699  /*--------------------- Inline functions --------------------------------*/
700 
701 #ifndef DOXYGEN
702 
704  Partitioner::size() const
705  {
706  return global_size;
707  }
708 
709 
710 
711  inline const IndexSet &
713  {
715  }
716 
717 
718 
719  inline std::pair<types::global_dof_index, types::global_dof_index>
721  {
722  return local_range_data;
723  }
724 
725 
726 
727  inline unsigned int
729  {
731  local_range_data.second - local_range_data.first;
732  Assert(size <= std::numeric_limits<unsigned int>::max(),
734  return static_cast<unsigned int>(size);
735  }
736 
737 
738 
739  inline bool
741  const types::global_dof_index global_index) const
742  {
743  return (local_range_data.first <= global_index &&
744  global_index < local_range_data.second);
745  }
746 
747 
748 
749  inline bool
751  const types::global_dof_index global_index) const
752  {
753  // if the index is in the global range, it is trivially not a ghost
754  if (in_local_range(global_index) == true)
755  return false;
756  else
757  return ghost_indices().is_element(global_index);
758  }
759 
760 
761 
762  inline unsigned int
764  const types::global_dof_index global_index) const
765  {
766  Assert(in_local_range(global_index) || is_ghost_entry(global_index),
767  ExcIndexNotPresent(global_index, my_pid));
768  if (in_local_range(global_index))
769  return static_cast<unsigned int>(global_index - local_range_data.first);
770  else if (is_ghost_entry(global_index))
771  return (local_size() +
772  static_cast<unsigned int>(
773  ghost_indices_data.index_within_set(global_index)));
774  else
775  // should only end up here in optimized mode, when we use this large
776  // number to trigger a segfault when using this method for array
777  // access
779  }
780 
781 
782 
784  Partitioner::local_to_global(const unsigned int local_index) const
785  {
787  if (local_index < local_size())
788  return local_range_data.first + types::global_dof_index(local_index);
789  else
790  return ghost_indices_data.nth_index_in_set(local_index - local_size());
791  }
792 
793 
794 
795  inline const IndexSet &
797  {
798  return ghost_indices_data;
799  }
800 
801 
802 
803  inline unsigned int
805  {
806  return n_ghost_indices_data;
807  }
808 
809 
810 
811  inline const std::vector<std::pair<unsigned int, unsigned int>> &
813  {
815  }
816 
817 
818 
819  inline const std::vector<std::pair<unsigned int, unsigned int>> &
821  {
822  return ghost_targets_data;
823  }
824 
825 
826  inline const std::vector<std::pair<unsigned int, unsigned int>> &
828  {
829  return import_indices_data;
830  }
831 
832 
833 
834  inline unsigned int
836  {
837  return n_import_indices_data;
838  }
839 
840 
841 
842  inline const std::vector<std::pair<unsigned int, unsigned int>> &
844  {
845  return import_targets_data;
846  }
847 
848 
849 
850  inline unsigned int
852  {
853  // return the id from the variable stored in this class instead of
854  // Utilities::MPI::this_mpi_process() in order to make this query also
855  // work when MPI is not initialized.
856  return my_pid;
857  }
858 
859 
860 
861  inline unsigned int
863  {
864  // return the number of MPI processes from the variable stored in this
865  // class instead of Utilities::MPI::n_mpi_processes() in order to make
866  // this query also work when MPI is not initialized.
867  return n_procs;
868  }
869 
870 
871 
872  inline const MPI_Comm &
874  {
875  return communicator;
876  }
877 
878 
879 
880  inline const MPI_Comm &
882  {
883  return communicator;
884  }
885 
886 
887 
888  inline bool
890  {
891  return have_ghost_indices;
892  }
893 
894 #endif // ifndef DOXYGEN
895 
896  } // end of namespace MPI
897 
898 } // end of namespace Utilities
899 
900 
901 DEAL_II_NAMESPACE_CLOSE
902 
903 #endif
std::vector< std::pair< unsigned int, unsigned int > > import_indices_data
Definition: partitioner.h:622
const IndexSet & locally_owned_range() const
bool is_globally_compatible(const Partitioner &part) const
Definition: partitioner.cc:449
static const unsigned int invalid_unsigned_int
Definition: types.h:187
#define DeclException2(Exception2, type1, type2, outsequence)
Definition: exceptions.h:541
bool is_compatible(const Partitioner &part) const
Definition: partitioner.cc:422
static ::ExceptionBase & ExcIndexNotPresent(types::global_dof_index arg1, unsigned int arg2)
unsigned int local_size() const
const std::vector< std::pair< unsigned int, unsigned int > > & import_targets() const
size_type nth_index_in_set(const unsigned int local_index) const
Definition: index_set.h:1847
void import_from_ghosted_array_finish(const VectorOperation::values vector_operation, const ArrayView< const Number, MemorySpaceType > &temporary_storage, const ArrayView< Number, MemorySpaceType > &locally_owned_storage, const ArrayView< Number, MemorySpaceType > &ghost_array, std::vector< MPI_Request > &requests) const
virtual const MPI_Comm & get_mpi_communicator() const override
const std::vector< std::pair< unsigned int, unsigned int > > & import_indices() const
void import_from_ghosted_array_start(const VectorOperation::values vector_operation, const unsigned int communication_channel, const ArrayView< Number, MemorySpaceType > &ghost_array, const ArrayView< Number, MemorySpaceType > &temporary_storage, std::vector< MPI_Request > &requests) const
types::global_dof_index global_size
Definition: partitioner.h:584
types::global_dof_index local_to_global(const unsigned int local_index) const
#define AssertIndexRange(index, range)
Definition: exceptions.h:1637
unsigned int n_ghost_indices_data
Definition: partitioner.h:608
const std::vector< std::pair< unsigned int, unsigned int > > & ghost_indices_within_larger_ghost_set() const
unsigned int global_to_local(const types::global_dof_index global_index) const
bool in_local_range(const types::global_dof_index global_index) const
void set_owned_indices(const IndexSet &locally_owned_indices)
Definition: partitioner.cc:111
std::vector< unsigned int > ghost_indices_subset_chunks_by_rank_data
Definition: partitioner.h:666
std::vector< unsigned int > import_indices_chunks_by_rank_data
Definition: partitioner.h:654
bool ghost_indices_initialized() const
unsigned int this_mpi_process() const
const IndexSet & ghost_indices() const
const MPI_Comm & get_communicator() const
unsigned int n_ghost_indices() const
std::pair< types::global_dof_index, types::global_dof_index > local_range() const
#define Assert(cond, exc)
Definition: exceptions.h:1407
size_type index_within_set(const size_type global_index) const
Definition: index_set.h:1888
unsigned int n_mpi_processes() const
std::vector< std::pair< unsigned int, unsigned int > > ghost_indices_subset_data
Definition: partitioner.h:674
const std::vector< std::pair< unsigned int, unsigned int > > & ghost_targets() const
unsigned int global_dof_index
Definition: types.h:89
Definition: cuda.h:31
types::global_dof_index size() const
std::size_t memory_consumption() const
Definition: partitioner.cc:458
void initialize_import_indices_plain_dev() const
std::vector< std::pair< unsigned int, unsigned int > > import_targets_data
Definition: partitioner.h:648
std::pair< types::global_dof_index, types::global_dof_index > local_range_data
Definition: partitioner.h:596
std::vector< std::pair< unsigned int, unsigned int > > ghost_targets_data
Definition: partitioner.h:614
void export_to_ghosted_array_start(const unsigned int communication_channel, const ArrayView< const Number, MemorySpaceType > &locally_owned_array, const ArrayView< Number, MemorySpaceType > &temporary_storage, const ArrayView< Number, MemorySpaceType > &ghost_array, std::vector< MPI_Request > &requests) const
void export_to_ghosted_array_finish(const ArrayView< Number, MemorySpaceType > &ghost_array, std::vector< MPI_Request > &requests) const
static ::ExceptionBase & ExcNotImplemented()
bool is_element(const size_type index) const
Definition: index_set.h:1732
#define DeclException3(Exception3, type1, type2, type3, outsequence)
Definition: exceptions.h:564
std::vector< std::pair< std::unique_ptr< unsigned int[], void(*)(unsigned int *)>, unsigned int > > import_indices_plain_dev
Definition: partitioner.h:636
unsigned int n_import_indices() const
unsigned int n_import_indices_data
Definition: partitioner.h:642
virtual void reinit(const IndexSet &vector_space_vector_index_set, const IndexSet &read_write_vector_index_set, const MPI_Comm &communicator) override
Definition: partitioner.cc:98
bool is_ghost_entry(const types::global_dof_index global_index) const
static ::ExceptionBase & ExcGhostIndexArrayHasWrongSize(unsigned int arg1, unsigned int arg2, unsigned int arg3)
unsigned int n_ghost_indices_in_larger_set
Definition: partitioner.h:660
void set_ghost_indices(const IndexSet &ghost_indices, const IndexSet &larger_ghost_index_set=IndexSet())
Definition: partitioner.cc:152