Reference documentation for deal.II version Git 83f3a2a 2017-12-11 23:13:16 +0100
Public Types | Public Member Functions | Private Member Functions | Private Attributes | List of all members
ScaLAPACKMatrix< NumberType > Class Template Reference

#include <deal.II/lac/scalapack.h>

Inheritance diagram for ScaLAPACKMatrix< NumberType >:
[legend]

Public Types

typedef unsigned int size_type
 

Public Member Functions

 ScaLAPACKMatrix (const size_type n_rows, const size_type n_columns, const std::shared_ptr< const Utilities::MPI::ProcessGrid > process_grid, const size_type row_block_size=32, const size_type column_block_size=32, const LAPACKSupport::Property property=LAPACKSupport::Property::general)
 
 ScaLAPACKMatrix (const size_type size, const std::shared_ptr< const Utilities::MPI::ProcessGrid > process_grid, const size_type block_size=32, const LAPACKSupport::Property property=LAPACKSupport::Property::symmetric)
 
 ~ScaLAPACKMatrix ()=default
 
void set_property (const LAPACKSupport::Property property)
 
ScaLAPACKMatrix< NumberType > & operator= (const FullMatrix< NumberType > &)
 
void copy_to (FullMatrix< NumberType > &matrix) const
 
void compute_cholesky_factorization ()
 
void invert ()
 
std::vector< NumberType > eigenvalues_symmetric ()
 
std::vector< NumberType > eigenpairs_symmetric ()
 
NumberType reciprocal_condition_number (const NumberType a_norm) const
 
NumberType l1_norm () const
 
NumberType linfty_norm () const
 
NumberType frobenius_norm () const
 
size_type m () const
 
size_type n () const
 
unsigned int local_m () const
 
unsigned int local_n () const
 
unsigned int global_row (const unsigned int loc_row) const
 
unsigned int global_column (const unsigned int loc_column) const
 
NumberType local_el (const unsigned int loc_row, const unsigned int loc_column) const
 
NumberType & local_el (const unsigned int loc_row, const unsigned int loc_column)
 

Private Member Functions

NumberType norm (const char type) const
 

Private Attributes

LAPACKSupport::State state
 
LAPACKSupport::Property property
 
std::shared_ptr< const Utilities::MPI::ProcessGridgrid
 
int n_rows
 
int n_columns
 
int row_block_size
 
int column_block_size
 
int n_local_rows
 
int n_local_columns
 
int descriptor [9]
 
std::vector< NumberType > work
 
std::vector< int > iwork
 
const char uplo
 
const int first_process_row
 
const int first_process_column
 
const int submatrix_row
 
const int submatrix_column
 
Threads::Mutex mutex
 

Additional Inherited Members

- Protected Types inherited from TransposeTable< NumberType >
typedef TableBase< 2, NumberType >::size_type size_type
 
- Protected Types inherited from TableBase< 2, NumberType >
typedef AlignedVector< NumberType >::size_type size_type
 
- Protected Member Functions inherited from TransposeTable< NumberType >
AlignedVector< NumberType >::reference el (const unsigned int i, const unsigned int j)
 
AlignedVector< NumberType >::const_reference el (const unsigned int i, const unsigned int j) const
 
 TransposeTable ()=default
 
 TransposeTable (const unsigned int size1, const unsigned int size2)
 
void reinit (const unsigned int size1, const unsigned int size2, const bool omit_default_initialization=false)
 
AlignedVector< NumberType >::const_reference operator() (const unsigned int i, const unsigned int j) const
 
AlignedVector< NumberType >::reference operator() (const unsigned int i, const unsigned int j)
 
unsigned int n_rows () const
 
unsigned int n_cols () const
 
- Protected Member Functions inherited from TableBase< 2, NumberType >
size_type position (const TableIndices< N > &indices) const
 
AlignedVector< NumberType >::reference el (const TableIndices< N > &indices)
 
AlignedVector< NumberType >::const_reference el (const TableIndices< N > &indices) const
 
 TableBase ()=default
 
 TableBase (const TableIndices< N > &sizes)
 
 TableBase (const TableIndices< N > &sizes, InputIterator entries, const bool C_style_indexing=true)
 
 TableBase (const TableBase< N, NumberType > &src)
 
 TableBase (const TableBase< N, T2 > &src)
 
 TableBase (TableBase< N, NumberType > &&src)
 
 ~TableBase ()=default
 
TableBase< N, NumberType > & operator= (const TableBase< N, NumberType > &src)
 
TableBase< N, NumberType > & operator= (const TableBase< N, T2 > &src)
 
TableBase< N, NumberType > & operator= (TableBase< N, NumberType > &&src)
 
bool operator== (const TableBase< N, NumberType > &T2) const
 
void reset_values ()
 
void reinit (const TableIndices< N > &new_size, const bool omit_default_initialization=false)
 
unsigned int size (const unsigned int i) const
 
const TableIndices< N > & size () const
 
size_type n_elements () const
 
bool empty () const
 
void fill (InputIterator entries, const bool C_style_indexing=true)
 
void fill (const NumberType &value)
 
AlignedVector< NumberType >::reference operator() (const TableIndices< N > &indices)
 
AlignedVector< NumberType >::const_reference operator() (const TableIndices< N > &indices) const
 
void swap (TableBase< N, NumberType > &v)
 
std::size_t memory_consumption () const
 
void serialize (Archive &ar, const unsigned int version)
 
- Protected Member Functions inherited from Subscriptor
 Subscriptor ()
 
 Subscriptor (const Subscriptor &)
 
 Subscriptor (Subscriptor &&)
 
virtual ~Subscriptor ()
 
Subscriptoroperator= (const Subscriptor &)
 
Subscriptoroperator= (Subscriptor &&)
 
void subscribe (const char *identifier=nullptr) const
 
void unsubscribe (const char *identifier=nullptr) const
 
unsigned int n_subscriptions () const
 
void list_subscribers () const
 
template<class Archive >
void serialize (Archive &ar, const unsigned int version)
 
- Static Protected Member Functions inherited from Subscriptor
static::ExceptionBase & ExcInUse (int arg1, char *arg2, std::string &arg3)
 
static::ExceptionBase & ExcNoSubscriber (char *arg1, char *arg2)
 
- Protected Attributes inherited from TableBase< 2, NumberType >
AlignedVector< NumberType > values
 
TableIndices< N > table_size
 

Detailed Description

template<typename NumberType>
class ScaLAPACKMatrix< NumberType >

A wrapper class around ScaLAPACK parallel dense linear algebra.

ScaLAPACK assumes that matrices are distributed according to the block-cyclic decomposition scheme. An \(M\) by \(N\) matrix is first decomposed into \(MB\) by \(NB\) blocks which are then uniformly distributed across the 2D process grid \(p*q \le Np\), where \(p,q\) are grid dimensions and \(Np\) is the total number of processes.

For example, a global real symmetric matrix of size \(9\times 9\) is stored in upper storage mode with block sizes 4 × 4:

0 1 2
┌ ┐
| -6.0 0.0 0.0 0.0 | 0.0 -2.0 -2.0 0.0 | -2.0 |
| . -6.0 -2.0 0.0 | -2.0 -4.0 0.0 -4.0 | -2.0 |
0 | . . -6.0 -2.0 | -2.0 0.0 2.0 0.0 | 6.0 |
| . . . -6.0 | 2.0 0.0 2.0 0.0 | 2.0 |
| ---------------------|-----------------------|------- |
| . . . . | -8.0 -4.0 0.0 -2.0 | 0.0 |
| . . . . | . -6.0 0.0 -4.0 | -6.0 |
1 | . . . . | . . -4.0 0.0 | 0.0 |
| . . . . | . . . -4.0 | -4.0 |
| ---------------------|-----------------------|------- |
2 | . . . . | . . . . | -16.0 |
└ ┘

may be distributed using the 2x2 process grid:

| 0 2 | 1
-----| ------- |-----
0 | P00 | P01
2 | |
-----| ------- |-----
1 | P10 | P11

with the following local arrays:

p,q | 0 | 1
-----|----------------------------|----------------------
| -6.0 0.0 0.0 0.0 -2.0 | 0.0 -2.0 -2.0 0.0
| . -6.0 -2.0 0.0 -2.0 | -2.0 -4.0 0.0 -4.0
0 | . . -6.0 -2.0 6.0 | -2.0 0.0 2.0 0.0
| . . . -6.0 2.0 | 2.0 0.0 2.0 0.0
| . . . . -16.0 | . . . .
-----|----------------------------|----------------------
| . . . . 0.0 | -8.0 -4.0 0.0 -2.0
| . . . . -6.0 | . -6.0 0.0 -4.0
1 | . . . . 0.0 | . . -4.0 0.0
| . . . . -4.0 | . . . -4.0

Note how processes \((0,0)\) and \((1,0)\) of the process grid store an extra column to represent the last column of the original matrix that did not fit the decomposition into \(4\times 4\) sub-blocks.

The choice of the block size is a compromise between a sufficiently large size for efficient local/serial BLAS, but one that is also small enough to achieve good parallel load balance.

Below we show a strong scaling example of ScaLAPACKMatrix::invert() on up to 5 nodes each composed of two Intel Xeon 2660v2 IvyBridge sockets 2.20GHz, 10 cores/socket. Calculations are performed on square processor grids 1x1, 2x2, 3x3, 4x4, 5x5, 6x6, 7x7, 8x8, 9x9, 10x10.

scalapack_invert.png
Author
Denis Davydov, Benjamin Brands, 2017

Definition at line 29 of file process_grid.h.

Member Typedef Documentation

template<typename NumberType>
typedef unsigned int ScaLAPACKMatrix< NumberType >::size_type

Declare the type for container size.

Definition at line 112 of file scalapack.h.

Constructor & Destructor Documentation

template<typename NumberType >
ScaLAPACKMatrix< NumberType >::ScaLAPACKMatrix ( const size_type  n_rows,
const size_type  n_columns,
const std::shared_ptr< const Utilities::MPI::ProcessGrid process_grid,
const size_type  row_block_size = 32,
const size_type  column_block_size = 32,
const LAPACKSupport::Property  property = LAPACKSupport::Property::general 
)

Constructor for a rectangular matrix with n_rows and n_cols and distributed using the grid process_grid.

Definition at line 28 of file scalapack.cc.

template<typename NumberType >
ScaLAPACKMatrix< NumberType >::ScaLAPACKMatrix ( const size_type  size,
const std::shared_ptr< const Utilities::MPI::ProcessGrid process_grid,
const size_type  block_size = 32,
const LAPACKSupport::Property  property = LAPACKSupport::Property::symmetric 
)

Constructor for a square matrix of size size, and distributed using the process grid in process_grid.

Definition at line 89 of file scalapack.cc.

template<typename NumberType>
ScaLAPACKMatrix< NumberType >::~ScaLAPACKMatrix ( )
default

Destructor

Member Function Documentation

template<typename NumberType >
void ScaLAPACKMatrix< NumberType >::set_property ( const LAPACKSupport::Property  property)

Assign property to this matrix.

Definition at line 106 of file scalapack.cc.

template<typename NumberType >
ScaLAPACKMatrix< NumberType > & ScaLAPACKMatrix< NumberType >::operator= ( const FullMatrix< NumberType > &  matrix)

Assignment operator from a regular FullMatrix.

Note
This function should only be used for relatively small matrix dimensions. It is primarily intended for debugging purposes.

Definition at line 115 of file scalapack.cc.

template<typename NumberType >
void ScaLAPACKMatrix< NumberType >::copy_to ( FullMatrix< NumberType > &  matrix) const

Copy the contents of the distributed matrix into matrix.

Note
This function should only be used for relatively small matrix dimensions. It is primarily intended for debugging purposes.

Definition at line 169 of file scalapack.cc.

template<typename NumberType >
void ScaLAPACKMatrix< NumberType >::compute_cholesky_factorization ( )

Compute the Cholesky factorization of the matrix using ScaLAPACK function pXpotrf. The result of the factorization is stored in this object.

Definition at line 209 of file scalapack.cc.

template<typename NumberType >
void ScaLAPACKMatrix< NumberType >::invert ( )

Invert the matrix by first computing a Cholesky factorization and then building the actual inverse using pXpotri. The inverse is stored in this object.

Definition at line 228 of file scalapack.cc.

template<typename NumberType >
std::vector< NumberType > ScaLAPACKMatrix< NumberType >::eigenvalues_symmetric ( )

Compute all eigenvalues of a real symmetric matrix using pXsyev. If successful, the computed eigenvalues are arranged in ascending order. After this function is called, the content of the matrix is overwritten making it unusable.

Definition at line 246 of file scalapack.cc.

template<typename NumberType >
std::vector< NumberType > ScaLAPACKMatrix< NumberType >::eigenpairs_symmetric ( )

Compute all eigenpairs of a real symmetric matrix using pXsyev. If successful, the computed eigenvalues are arranged in ascending order. The eigenvectors are stored in the columns of the matrix, thereby overwriting the original content of the matrix.

Definition at line 299 of file scalapack.cc.

template<typename NumberType >
NumberType ScaLAPACKMatrix< NumberType >::reciprocal_condition_number ( const NumberType  a_norm) const

Estimate the the condition number of a SPD matrix in the \(l_1\)-norm. The matrix has to be in the Cholesky state (see compute_cholesky_factorization()). The reciprocal of the condition number is returned in order to avoid the possibility of overflow when the condition number is very large.

a_norm must contain the \(l_1\)-norm of the matrix prior to calling Cholesky factorization (see l1_norm()).

Note
An alternative is to compute the inverse of the matrix explicitly and manually construct \(k_1 = ||A||_1 ||A^{-1}||_1\).

Definition at line 362 of file scalapack.cc.

template<typename NumberType >
NumberType ScaLAPACKMatrix< NumberType >::l1_norm ( ) const

Compute the \(l_1\)-norm of the matrix.

Definition at line 388 of file scalapack.cc.

template<typename NumberType >
NumberType ScaLAPACKMatrix< NumberType >::linfty_norm ( ) const

Compute the \(l_{\infty}\) norm of the matrix.

Definition at line 397 of file scalapack.cc.

template<typename NumberType >
NumberType ScaLAPACKMatrix< NumberType >::frobenius_norm ( ) const

Compute the Frobenius norm of the matrix.

Definition at line 406 of file scalapack.cc.

template<typename NumberType>
size_type ScaLAPACKMatrix< NumberType >::m ( ) const

Number of rows of the \(M \times N\) matrix.

template<typename NumberType>
size_type ScaLAPACKMatrix< NumberType >::n ( ) const

Number of columns of the \(M \times N\) matrix.

template<typename NumberType>
unsigned int ScaLAPACKMatrix< NumberType >::local_m ( ) const

Number of local rows on this MPI processes.

template<typename NumberType>
unsigned int ScaLAPACKMatrix< NumberType >::local_n ( ) const

Number of local columns on this MPI process.

template<typename NumberType >
unsigned int ScaLAPACKMatrix< NumberType >::global_row ( const unsigned int  loc_row) const

Return the global row number for the given local row loc_row .

Definition at line 145 of file scalapack.cc.

template<typename NumberType >
unsigned int ScaLAPACKMatrix< NumberType >::global_column ( const unsigned int  loc_column) const

Return the global column number for the given local column loc_column.

Definition at line 157 of file scalapack.cc.

template<typename NumberType>
NumberType ScaLAPACKMatrix< NumberType >::local_el ( const unsigned int  loc_row,
const unsigned int  loc_column 
) const

Read access to local element.

template<typename NumberType>
NumberType& ScaLAPACKMatrix< NumberType >::local_el ( const unsigned int  loc_row,
const unsigned int  loc_column 
)

Write access to local element.

template<typename NumberType >
NumberType ScaLAPACKMatrix< NumberType >::norm ( const char  type) const
private

Calculate the norm of a distributed dense matrix using ScaLAPACK's internal function.

Definition at line 415 of file scalapack.cc.

Member Data Documentation

template<typename NumberType>
LAPACKSupport::State ScaLAPACKMatrix< NumberType >::state
private

Since ScaLAPACK operations notoriously change the meaning of the matrix entries, we record the current state after the last operation here.

Definition at line 272 of file scalapack.h.

template<typename NumberType>
LAPACKSupport::Property ScaLAPACKMatrix< NumberType >::property
private

Additional property of the matrix which may help to select more efficient ScaLAPACK functions.

Definition at line 278 of file scalapack.h.

template<typename NumberType>
std::shared_ptr<const Utilities::MPI::ProcessGrid> ScaLAPACKMatrix< NumberType >::grid
private

A shared pointer to a Utilities::MPI::ProcessGrid object which contains a BLACS context and a MPI communicator, as well as other necessary data structures.

Definition at line 284 of file scalapack.h.

template<typename NumberType>
int ScaLAPACKMatrix< NumberType >::n_rows
private

Number of rows in the matrix.

Definition at line 289 of file scalapack.h.

template<typename NumberType>
int ScaLAPACKMatrix< NumberType >::n_columns
private

Number of columns in the matrix.

Definition at line 294 of file scalapack.h.

template<typename NumberType>
int ScaLAPACKMatrix< NumberType >::row_block_size
private

Row block size.

Definition at line 299 of file scalapack.h.

template<typename NumberType>
int ScaLAPACKMatrix< NumberType >::column_block_size
private

Column block size.

Definition at line 304 of file scalapack.h.

template<typename NumberType>
int ScaLAPACKMatrix< NumberType >::n_local_rows
private

Number of rows in the matrix owned by the current process.

Definition at line 309 of file scalapack.h.

template<typename NumberType>
int ScaLAPACKMatrix< NumberType >::n_local_columns
private

Number of columns in the matrix owned by the current process.

Definition at line 314 of file scalapack.h.

template<typename NumberType>
int ScaLAPACKMatrix< NumberType >::descriptor[9]
private

ScaLAPACK description vector.

Definition at line 319 of file scalapack.h.

template<typename NumberType>
std::vector<NumberType> ScaLAPACKMatrix< NumberType >::work
mutableprivate

Workspace array.

Definition at line 324 of file scalapack.h.

template<typename NumberType>
std::vector<int> ScaLAPACKMatrix< NumberType >::iwork
mutableprivate

Integer workspace array.

Definition at line 329 of file scalapack.h.

template<typename NumberType>
const char ScaLAPACKMatrix< NumberType >::uplo
private

A character to define where elements are stored in case ScaLAPACK operations support this.

Definition at line 335 of file scalapack.h.

template<typename NumberType>
const int ScaLAPACKMatrix< NumberType >::first_process_row
private

The process row of the process grid over which the first row of the global matrix is distributed.

Definition at line 341 of file scalapack.h.

template<typename NumberType>
const int ScaLAPACKMatrix< NumberType >::first_process_column
private

The process column of the process grid over which the first column of the global matrix is distributed.

Definition at line 347 of file scalapack.h.

template<typename NumberType>
const int ScaLAPACKMatrix< NumberType >::submatrix_row
private

Global row index that determines where to start a submatrix. Currently this equals unity, as we don't use submatrices.

Definition at line 353 of file scalapack.h.

template<typename NumberType>
const int ScaLAPACKMatrix< NumberType >::submatrix_column
private

Global column index that determines where to start a submatrix. Currently this equals unity, as we don't use submatrices.

Definition at line 359 of file scalapack.h.

template<typename NumberType>
Threads::Mutex ScaLAPACKMatrix< NumberType >::mutex
mutableprivate

Thread mutex.

Definition at line 364 of file scalapack.h.


The documentation for this class was generated from the following files: