Reference documentation for deal.II version 9.4.0
\(\newcommand{\dealvcentcolon}{\mathrel{\mathop{:}}}\) \(\newcommand{\dealcoloneq}{\dealvcentcolon\mathrel{\mkern-1.2mu}=}\) \(\newcommand{\jump}[1]{\left[\!\left[ #1 \right]\!\right]}\) \(\newcommand{\average}[1]{\left\{\!\left\{ #1 \right\}\!\right\}}\)
Public Types | Public Member Functions | List of all members
LinearAlgebra::distributed::Vector< Number, MemorySpace > Class Template Reference

#include <deal.II/lac/la_parallel_vector.h>

Inheritance diagram for LinearAlgebra::distributed::Vector< Number, MemorySpace >:

Public Types

using memory_space = MemorySpace
using value_type = Number
using pointer = value_type *
using const_pointer = const value_type *
using iterator = value_type *
using const_iterator = const value_type *
using reference = value_type &
using const_reference = const value_type &
using size_type = types::global_dof_index
using real_type = typename numbers::NumberTraits< Number >::real_type

Public Member Functions

template<typename number >
Vectoroperator= (const ::Vector< number > &v)
virtual void compress (VectorOperation::values)
1: Basic Object-handling
 Vector ()
 Vector (const Vector< Number, MemorySpace > &in_vector)
 Vector (Vector< Number, MemorySpace > &&in_vector)
 Vector (const size_type size)
 Vector (const IndexSet &local_range, const IndexSet &ghost_indices, const MPI_Comm &communicator)
 Vector (const IndexSet &local_range, const MPI_Comm &communicator)
 Vector (const std::shared_ptr< const Utilities::MPI::Partitioner > &partitioner)
virtual ~Vector () override
void reinit (const size_type size, const bool omit_zeroing_entries=false)
template<typename Number2 >
void reinit (const Vector< Number2, MemorySpace > &in_vector, const bool omit_zeroing_entries=false)
void reinit (const IndexSet &local_range, const IndexSet &ghost_indices, const MPI_Comm &communicator)
void reinit (const IndexSet &local_range, const MPI_Comm &communicator)
void reinit (const std::shared_ptr< const Utilities::MPI::Partitioner > &partitioner, const MPI_Comm &comm_sm=MPI_COMM_SELF)
void reinit (const types::global_dof_index local_size, const types::global_dof_index ghost_size, const MPI_Comm &comm, const MPI_Comm &comm_sm=MPI_COMM_SELF)
void swap (Vector< Number, MemorySpace > &v)
Vector< Number, MemorySpace > & operator= (const Vector< Number, MemorySpace > &in_vector)
template<typename Number2 >
Vector< Number, MemorySpace > & operator= (const Vector< Number2, MemorySpace > &in_vector)
2: Parallel data exchange
virtual void compress (::VectorOperation::values operation) override
void update_ghost_values () const
void compress_start (const unsigned int communication_channel=0, ::VectorOperation::values operation=VectorOperation::add)
void compress_finish (::VectorOperation::values operation)
void update_ghost_values_start (const unsigned int communication_channel=0) const
void update_ghost_values_finish () const
void zero_out_ghosts () const
void zero_out_ghost_values () const
bool has_ghost_elements () const
template<typename Number2 >
void copy_locally_owned_data_from (const Vector< Number2, MemorySpace > &src)
template<typename MemorySpace2 >
void import (const Vector< Number, MemorySpace2 > &src, VectorOperation::values operation)
3: Implementation of VectorSpaceVector
virtual void reinit (const VectorSpaceVector< Number > &V, const bool omit_zeroing_entries=false) override
virtual Vector< Number, MemorySpace > & operator*= (const Number factor) override
virtual Vector< Number, MemorySpace > & operator/= (const Number factor) override
virtual Vector< Number, MemorySpace > & operator+= (const VectorSpaceVector< Number > &V) override
virtual Vector< Number, MemorySpace > & operator-= (const VectorSpaceVector< Number > &V) override
virtual void import (const LinearAlgebra::ReadWriteVector< Number > &V, VectorOperation::values operation, std::shared_ptr< const Utilities::MPI::CommunicationPatternBase > communication_pattern={}) override
virtual Number operator* (const VectorSpaceVector< Number > &V) const override
virtual void add (const Number a) override
virtual void add (const Number a, const VectorSpaceVector< Number > &V) override
virtual void add (const Number a, const VectorSpaceVector< Number > &V, const Number b, const VectorSpaceVector< Number > &W) override
virtual void add (const std::vector< size_type > &indices, const std::vector< Number > &values)
virtual void sadd (const Number s, const Number a, const VectorSpaceVector< Number > &V) override
virtual void scale (const VectorSpaceVector< Number > &scaling_factors) override
virtual void equ (const Number a, const VectorSpaceVector< Number > &V) override
virtual real_type l1_norm () const override
virtual real_type l2_norm () const override
real_type norm_sqr () const
virtual real_type linfty_norm () const override
virtual Number add_and_dot (const Number a, const VectorSpaceVector< Number > &V, const VectorSpaceVector< Number > &W) override
virtual size_type size () const override
virtual ::IndexSet locally_owned_elements () const override
virtual void print (std::ostream &out, const unsigned int precision=3, const bool scientific=true, const bool across=true) const override
virtual std::size_t memory_consumption () const override
4: Other vector operations not included in VectorSpaceVector
virtual Vector< Number, MemorySpace > & operator= (const Number s) override
template<typename OtherNumber >
void add (const std::vector< size_type > &indices, const ::Vector< OtherNumber > &values)
template<typename OtherNumber >
void add (const size_type n_elements, const size_type *indices, const OtherNumber *values)
void sadd (const Number s, const Vector< Number, MemorySpace > &V)
5: Entry access and local data representation
size_type local_size () const
size_type locally_owned_size () const
bool in_local_range (const size_type global_index) const
iterator begin ()
const_iterator begin () const
iterator end ()
const_iterator end () const
Number operator() (const size_type global_index) const
Number & operator() (const size_type global_index)
Number operator[] (const size_type global_index) const
Number & operator[] (const size_type global_index)
Number local_element (const size_type local_index) const
Number & local_element (const size_type local_index)
Number * get_values () const
template<typename OtherNumber >
void extract_subvector_to (const std::vector< size_type > &indices, std::vector< OtherNumber > &values) const
template<typename ForwardIterator , typename OutputIterator >
void extract_subvector_to (ForwardIterator indices_begin, const ForwardIterator indices_end, OutputIterator values_begin) const
virtual bool all_zero () const override
virtual Number mean_value () const override
real_type lp_norm (const real_type p) const

6: Mixed stuff

std::shared_ptr< const Utilities::MPI::Partitionerpartitioner
size_type allocated_size
mutable ::MemorySpace::MemorySpaceData< Number, MemorySpace > data
mutable ::MemorySpace::MemorySpaceData< Number, MemorySpace > import_data
bool vector_is_ghosted
std::vector< MPI_Request > compress_requests
std::vector< MPI_Request > update_ghost_values_requests
std::mutex mutex
MPI_Comm comm_sm
template<typename Number2 , typename MemorySpace2 >
class Vector
template<typename Number2 >
class BlockVector
const MPI_Commget_mpi_communicator () const
const std::shared_ptr< const Utilities::MPI::Partitioner > & get_partitioner () const
bool partitioners_are_compatible (const Utilities::MPI::Partitioner &part) const
bool partitioners_are_globally_compatible (const Utilities::MPI::Partitioner &part) const
void set_ghost_state (const bool ghosted) const
const std::vector< ArrayView< const Number > > & shared_vector_data () const
static ::ExceptionBaseExcVectorTypeNotCompatible ()
static ::ExceptionBaseExcNotAllowedForCuda ()
static ::ExceptionBaseExcNonMatchingElements (Number arg1, Number arg2, unsigned int arg3)
static ::ExceptionBaseExcAccessToNonLocalElement (size_type arg1, size_type arg2, size_type arg3, size_type arg4)
void add_local (const Number a, const VectorSpaceVector< Number > &V)
void sadd_local (const Number s, const Number a, const VectorSpaceVector< Number > &V)
template<typename Number2 >
Number inner_product_local (const Vector< Number2, MemorySpace > &V) const
real_type norm_sqr_local () const
Number mean_value_local () const
real_type l1_norm_local () const
real_type lp_norm_local (const real_type p) const
real_type linfty_norm_local () const
Number add_and_dot_local (const Number a, const Vector< Number, MemorySpace > &V, const Vector< Number, MemorySpace > &W)
void clear_mpi_requests ()
void resize_val (const size_type new_allocated_size, const MPI_Comm &comm_sm=MPI_COMM_SELF)

Subscriptor functionality

Classes derived from Subscriptor provide a facility to subscribe to this object. This is mostly used by the SmartPointer class.

std::atomic< unsigned intcounter
std::map< std::string, unsigned intcounter_map
std::vector< std::atomic< bool > * > validity_pointers
const std::type_info * object_info
void subscribe (std::atomic< bool > *const validity, const std::string &identifier="") const
void unsubscribe (std::atomic< bool > *const validity, const std::string &identifier="") const
unsigned int n_subscriptions () const
template<typename StreamType >
void list_subscribers (StreamType &stream) const
void list_subscribers () const
template<class Archive >
void serialize (Archive &ar, const unsigned int version)
static ::ExceptionBaseExcInUse (int arg1, std::string arg2, std::string arg3)
static ::ExceptionBaseExcNoSubscriber (std::string arg1, std::string arg2)
void check_no_subscribers () const noexcept
using map_value_type = decltype(counter_map)::value_type
using map_iterator = decltype(counter_map)::iterator

Detailed Description

template<typename Number, typename MemorySpace = MemorySpace::Host>
class LinearAlgebra::distributed::Vector< Number, MemorySpace >

Implementation of a parallel vector class. The design of this class is similar to the standard Vector class in deal.II, with the exception that storage is distributed with MPI.

The vector is designed for the following scheme of parallel partitioning:

Functions related to parallel functionality:

This vector can take two different states with respect to ghost elements:

This vector uses the facilities of the class ::Vector<Number> for implementing the operations on the local range of the vector. In particular, it also inherits thread parallelism that splits most vector-vector operations into smaller chunks if the program uses multiple threads. This may or may not be desired when working also with MPI.

Limitations regarding the vector size

This vector class is based on two different number types for indexing. The so-called global index type encodes the overall size of the vector. Its type is types::global_dof_index. The largest possible value is 2^32-1 or approximately 4 billion in case 64 bit integers are disabled at configuration of deal.II (default case) or 2^64-1 or approximately 10^19 if 64 bit integers are enabled (see the glossary entry on When to use types::global_dof_index instead of unsigned int for further information).

The second relevant index type is the local index used within one MPI rank. As opposed to the global index, the implementation assumes 32-bit unsigned integers unconditionally. In other words, to actually use a vector with more than four billion entries, you need to use MPI with more than one rank (which in general is a safe assumption since four billion entries consume at least 16 GB of memory for floats or 32 GB of memory for doubles) and enable 64-bit indices. If more than 4 billion local elements are present, the implementation tries to detect that, which triggers an exception and aborts the code. Note, however, that the detection of overflow is tricky and the detection mechanism might fail in some circumstances. Therefore, it is strongly recommended to not rely on this class to automatically detect the unsupported case.

CUDA support

This vector class supports two different memory spaces: Host and CUDA. By default, the memory space is Host and all the data are allocated on the CPU. When the memory space is CUDA, all the data is allocated on the GPU. The operations on the vector are performed on the chosen memory space. * From the host, there are two methods to access the elements of the Vector when using the CUDA memory space:

The import method is a lot safer and will perform an MPI communication if necessary. Since an MPI communication may be performed, import needs to be called on all the processors.

By default, all the ranks will try to access the device 0. This is fine is if you have one rank per node and one gpu per node. If you have multiple GPUs on one node, we need each process to access a different GPU. If each node has the same number of GPUs, this can be done as follows: int n_devices = 0; cudaGetDeviceCount(&n_devices); int device_id = my_rank % n_devices; cudaSetDevice(device_id);

MPI-3 shared-memory support

In Host mode, this class allows to use MPI-3 shared-memory features by providing a separate MPI communicator that consists of processes on the same shared-memory domain. By calling vector.shared_vector_data();, users have read-only access to both locally-owned and ghost values of processes combined in the shared-memory communicator (comm_sm in reinit()).

For this to work, you have to call the constructor or one of the reinit() functions of this class with a non-default value for the comm_sm argument, where the argument corresponds to a communicator consisting of all processes on the same shared-memory domain. This kind of communicator can be created using the following code snippet:

MPI_Comm_split_type(comm, MPI_COMM_TYPE_SHARED, rank, MPI_INFO_NULL,
See also

Definition at line 250 of file la_parallel_vector.h.

Member Function Documentation

◆ operator=()

template<typename Number , typename MemorySpace = MemorySpace::Host>
template<typename number >
Vector & LinearAlgebra::distributed::Vector< Number, MemorySpace >::operator= ( const ::Vector< number > &  v)

Definition at line 496 of file

The documentation for this class was generated from the following files: