17 #ifndef dealii_aligned_vector_h
18 #define dealii_aligned_vector_h
30 #include <boost/version.hpp>
31 #if BOOST_VERSION >= 106400
32 # include <boost/serialization/array_wrapper.hpp>
34 # include <boost/serialization/array.hpp>
36 #include <boost/serialization/split_member.hpp>
40 #include <type_traits>
240 template <
typename ForwardIterator>
356 const unsigned int root_process);
444 template <
class Archive>
446 save(Archive &ar,
const unsigned int version)
const;
453 template <
class Archive>
455 load(Archive &ar,
const unsigned int version);
463 template <
class Archive>
465 serialize(Archive &archive,
const unsigned int version);
469 BOOST_SERIALIZATION_SPLIT_MEMBER()
479 "Changing the vector after a call to "
480 "replicate_across_communicator() is not allowed.");
489 const size_t new_size,
490 const size_t new_allocated_size);
593 #ifdef DEAL_II_WITH_MPI
602 const bool is_shmem_root,
603 T *aligned_shmem_pointer,
604 MPI_Comm shmem_group_communicator,
605 MPI_Win shmem_window);
647 #ifdef DEAL_II_WITH_MPI
747 template <
typename T>
752 160000 /
sizeof(
T) + 1;
765 const T *
const source_end,
766 T *
const destination)
771 Assert(source_end == source_begin || destination !=
nullptr,
773 const std::size_t size = source_end - source_begin;
786 const std::size_t
end)
const override
795 if (std::is_trivial_v<T> ==
true)
800 for (std::size_t i =
begin; i <
end; ++i)
816 template <
typename T>
821 160000 /
sizeof(
T) + 1;
835 T *
const destination)
840 Assert(source_end == source_begin || destination !=
nullptr,
842 const std::size_t size = source_end - source_begin;
855 const std::size_t
end)
const override
863 if (std::is_trivial_v<T> ==
true)
870 for (std::size_t i =
begin; i <
end; ++i)
897 template <
typename T,
bool initialize_memory>
901 160000 /
sizeof(
T) + 1;
910 T *
const destination)
922 if (std::is_trivial_v<T> ==
true &&
923 std::is_same_v<T, long double> ==
false)
925 const unsigned char zero[
sizeof(
T)] = {};
929 if (std::memcmp(
zero,
930 static_cast<const void *
>(&element),
945 const std::size_t
end)
const override
958 std::bool_constant<initialize_memory>());
969 const std::size_t
end,
970 std::bool_constant<false>)
const
972 for (std::size_t i =
begin; i <
end; ++i)
979 const std::size_t
end,
980 std::bool_constant<true>)
const
982 for (std::size_t i =
begin; i <
end; ++i)
1001 template <
typename T,
bool initialize_memory>
1006 160000 /
sizeof(
T) + 1;
1031 const std::size_t
end)
const override
1037 if (std::is_trivial_v<T> ==
true)
1044 std::bool_constant<initialize_memory>());
1053 const std::size_t
end,
1054 std::bool_constant<false>)
const
1056 for (std::size_t i =
begin; i <
end; ++i)
1063 const std::size_t
end,
1064 std::bool_constant<true>)
const
1066 for (std::size_t i =
begin; i <
end; ++i)
1078 template <
typename T>
1080 : deleter_action_object(nullptr)
1081 , owning_aligned_vector(owning_object)
1085 # ifdef DEAL_II_WITH_MPI
1087 template <
typename T>
1089 const bool is_shmem_root,
1090 T *aligned_shmem_pointer,
1091 MPI_Comm shmem_group_communicator,
1092 MPI_Win shmem_window)
1093 : deleter_action_object(
1094 std::make_unique<MPISharedMemDeleterAction>(is_shmem_root,
1095 aligned_shmem_pointer,
1096 shmem_group_communicator,
1098 , owning_aligned_vector(owning_object)
1103 template <
typename T>
1109 if (deleter_action_object ==
nullptr)
1113 Assert(owning_aligned_vector->used_elements_end !=
nullptr,
1116 if (std::is_trivial_v<T> ==
false)
1117 for (
T *p = owning_aligned_vector->used_elements_end - 1; p >= ptr;
1126 deleter_action_object->delete_array(owning_aligned_vector, ptr);
1131 template <
typename T>
1136 owning_aligned_vector = new_aligned_vector_ptr;
1140 # ifdef DEAL_II_WITH_MPI
1142 template <
typename T>
1145 T *aligned_shmem_pointer,
1146 MPI_Comm shmem_group_communicator,
1147 MPI_Win shmem_window)
1148 : is_shmem_root(is_shmem_root)
1149 , aligned_shmem_pointer(aligned_shmem_pointer)
1150 , shmem_group_communicator(shmem_group_communicator)
1151 , shmem_window(shmem_window)
1156 template <
typename T>
1173 if (std::is_trivial_v<T> ==
false)
1178 ierr = MPI_Win_free(&shmem_window);
1189 : elements(nullptr, Deleter(this))
1190 , used_elements_end(nullptr)
1191 , allocated_elements_end(nullptr)
1193 , replicated_across_communicator(false)
1201 : elements(nullptr, Deleter(this))
1202 , used_elements_end(nullptr)
1203 , allocated_elements_end(nullptr)
1205 , replicated_across_communicator(false)
1216 : elements(nullptr, Deleter(this))
1217 , used_elements_end(nullptr)
1218 , allocated_elements_end(nullptr)
1220 , replicated_across_communicator(false)
1224 reserve(vec.
size());
1225 used_elements_end = allocated_elements_end;
1238 *
this = std::move(vec);
1261 used_elements_end = elements.get() + new_size;
1280 elements = std::move(vec.
elements);
1281 elements.get_deleter().reset_owning_object(
this);
1303 else if (new_size == old_size)
1306 else if (new_size < old_size)
1311 if (std::is_trivial_v<T> ==
false)
1312 for (
T *p = used_elements_end - 1; p >= elements.get() + new_size; --p)
1314 used_elements_end = elements.get() + new_size;
1320 used_elements_end = elements.get() + new_size;
1324 if (std::is_trivial_v<T> ==
false)
1326 new_size - old_size, elements.get() + old_size);
1340 else if (new_size == old_size)
1343 else if (new_size < old_size)
1348 if (std::is_trivial_v<T> ==
false)
1349 for (
T *p = used_elements_end - 1; p >= elements.get() + new_size; --p)
1351 used_elements_end = elements.get() + new_size;
1357 used_elements_end = elements.get() + new_size;
1361 new_size - old_size, elements.get() + old_size);
1375 else if (new_size == old_size)
1378 else if (new_size < old_size)
1383 if (std::is_trivial_v<T> ==
false)
1384 for (
T *p = used_elements_end - 1; p >= elements.get() + new_size; --p)
1386 used_elements_end = elements.get() + new_size;
1392 used_elements_end = elements.get() + new_size;
1396 new_size - old_size, init, elements.get() + old_size);
1405 const size_t new_size,
1406 const size_t new_allocated_size)
1413 new_size *
sizeof(
T));
1420 Deleter deleter(
this);
1423 if (new_allocated_size > 0)
1425 elements.get() + old_size,
1437 elements = decltype(elements)(new_data_ptr, std::move(deleter));
1438 used_elements_end = elements.get() + old_size;
1439 allocated_elements_end = elements.get() + new_size;
1448 const size_type old_size = used_elements_end - elements.get();
1449 const size_type old_allocated_size = allocated_elements_end - elements.get();
1450 if (new_allocated_size > old_allocated_size)
1456 std::max(new_allocated_size, 2 * old_allocated_size);
1458 allocate_and_move(old_size, new_size, new_allocated_size);
1460 else if (new_allocated_size == 0)
1474 Assert(replicated_across_communicator ==
false,
1475 ExcAlignedVectorChangeAfterReplication());
1477 const size_type used_size = used_elements_end - elements.get();
1478 const size_type allocated_size = allocated_elements_end - elements.get();
1479 if (allocated_size > used_size)
1480 allocate_and_move(used_size, used_size, used_size);
1497 used_elements_end =
nullptr;
1498 allocated_elements_end =
nullptr;
1508 if (used_elements_end == allocated_elements_end)
1510 if (std::is_trivial_v<T> ==
false)
1511 new (used_elements_end++)
T(in_data);
1513 *used_elements_end++ = in_data;
1523 T *field = used_elements_end - 1;
1534 const T *field = used_elements_end - 1;
1541 template <
typename ForwardIterator>
1549 if (std::is_trivial_v<T> ==
false)
1550 new (used_elements_end)
T;
1551 *used_elements_end = *
begin;
1581 const unsigned int root_process)
1583 # ifdef DEAL_II_WITH_MPI
1604 used_elements_end =
nullptr;
1605 allocated_elements_end =
nullptr;
1627 MPI_Comm shmem_group_communicator;
1629 MPI_Comm shmem_group_communicator_temp;
1630 int ierr = MPI_Comm_split_type(communicator,
1631 MPI_COMM_TYPE_SHARED,
1634 &shmem_group_communicator_temp);
1639 ierr = MPI_Comm_split(shmem_group_communicator_temp,
1642 &shmem_group_communicator);
1653 const bool is_shmem_root =
1682 MPI_Comm shmem_roots_communicator;
1687 const int ierr = MPI_Comm_split(communicator,
1689 (is_shmem_root ? 0 : 1),
1691 &shmem_roots_communicator);
1700 const unsigned int shmem_roots_root_rank = 0;
1701 const bool is_shmem_roots_root =
1703 shmem_roots_communicator) == shmem_roots_root_rank));
1710 if (std::is_trivial_v<T>)
1722 shmem_roots_root_rank);
1723 if (is_shmem_roots_root ==
false)
1727 int ierr = MPI_Bcast(elements.get(),
1728 sizeof(
T) * new_size,
1730 shmem_roots_root_rank,
1731 shmem_roots_communicator);
1748 shmem_roots_root_rank);
1752 shmem_roots_root_rank);
1788 MPI_Win shmem_window;
1790 const MPI_Aint align_by = 64;
1791 const MPI_Aint alloc_size =
1793 (size() *
sizeof(
T) + (align_by - 1)),
1800 ierr = MPI_Info_create(&mpi_info);
1802 ierr = MPI_Info_set(mpi_info,
1803 "mpi_minimum_memory_alignment",
1806 ierr = MPI_Win_allocate_shared((is_shmem_root ? alloc_size : 0),
1809 shmem_group_communicator,
1814 ierr = MPI_Info_free(&mpi_info);
1837 if (is_shmem_root ==
false)
1840 MPI_Aint alloc_size;
1841 const int ierr = MPI_Win_shared_query(
1842 shmem_window, MPI_PROC_NULL, &alloc_size, &disp_unit, &base_ptr);
1862 std::size_t available_space = alloc_size;
1863 void *base_ptr_backup = base_ptr;
1864 T *aligned_shmem_pointer =
static_cast<T *
>(
1865 std::align(align_by, new_size *
sizeof(
T), base_ptr, available_space));
1895 aligned_shmem_pointer =
static_cast<T *
>(base_ptr_backup);
1903 if (std::is_trivial_v<T> ==
true)
1904 std::memcpy(aligned_shmem_pointer, elements.get(),
sizeof(
T) * size());
1906 for (std::size_t i = 0; i < size(); ++i)
1907 new (&aligned_shmem_pointer[i])
T(std::move(elements[i]));
1912 const int ierr = MPI_Barrier(shmem_group_communicator);
1932 elements = decltype(elements)(aligned_shmem_pointer,
1935 aligned_shmem_pointer,
1936 shmem_group_communicator,
1943 used_elements_end = elements.get() + new_size;
1944 allocated_elements_end = used_elements_end;
1950 replicated_across_communicator =
true;
1953 std::accumulate(packed_data.begin(), packed_data.end(),
int(0));
1973 elements.get_deleter().reset_owning_object(
this);
1974 vec.
elements.get_deleter().reset_owning_object(&vec);
1987 return used_elements_end == elements.get();
1996 return used_elements_end - elements.get();
2005 return allocated_elements_end - elements.get();
2015 return elements[
index];
2025 return elements[
index];
2030 template <
typename T>
2034 return elements.get();
2039 template <
typename T>
2043 return elements.get();
2052 return elements.get();
2061 return used_elements_end;
2070 return elements.get();
2079 return used_elements_end;
2085 template <
class Archive>
2092 ar &boost::serialization::make_array(elements.get(), vec_size);
2098 template <
class Archive>
2108 ar &boost::serialization::make_array(elements.get(), vec_size);
2109 used_elements_end = elements.get() + vec_size;
2120 for (
const T *t = elements.get(); t != used_elements_end; ++t)
2122 memory +=
sizeof(
T) * (allocated_elements_end - used_elements_end);
bool operator==(const AlignedVector< T > &lhs, const AlignedVector< T > &rhs)
virtual ~DeleterActionBase()=default
virtual void delete_array(const AlignedVector< T > *owning_aligned_vector, T *ptr)=0
MPI_Comm shmem_group_communicator
virtual void delete_array(const AlignedVector< T > *aligned_vector, T *ptr)
MPISharedMemDeleterAction(const bool is_shmem_root, T *aligned_shmem_pointer, MPI_Comm shmem_group_communicator, MPI_Win shmem_window)
T * aligned_shmem_pointer
Deleter(AlignedVector< T > *owning_object, const bool is_shmem_root, T *aligned_shmem_pointer, MPI_Comm shmem_group_communicator, MPI_Win shmem_window)
Deleter(AlignedVector< T > *owning_object)
std::unique_ptr< DeleterActionBase > deleter_action_object
void reset_owning_object(const AlignedVector< T > *new_aligned_vector_ptr)
const AlignedVector< T > * owning_aligned_vector
void replicate_across_communicator(const MPI_Comm communicator, const unsigned int root_process)
size_type memory_consumption() const
void resize_fast(const size_type new_size)
std::unique_ptr< T[], Deleter > elements
reference operator[](const size_type index)
void fill(const T &element)
AlignedVector & operator=(AlignedVector< T > &&vec) noexcept
const_iterator end() const
AlignedVector(AlignedVector< T > &&vec) noexcept
void reserve(const size_type new_allocated_size)
void serialize(Archive &archive, const unsigned int version)
bool operator!=(const AlignedVector< T > &lhs, const AlignedVector< T > &rhs)
const_reference operator[](const size_type index) const
void swap(AlignedVector< T > &vec)
void resize(const size_type new_size, const T &init)
size_type capacity() const
const value_type * const_pointer
bool replicated_across_communicator
void push_back(const T in_data)
AlignedVector & operator=(const AlignedVector< T > &vec)
const_iterator begin() const
AlignedVector(const size_type size, const T &init=T())
T * allocated_elements_end
AlignedVector(const AlignedVector< T > &vec)
const_reference back() const
bool operator==(const AlignedVector< T > &lhs, const AlignedVector< T > &rhs)
const value_type * const_iterator
void resize(const size_type new_size)
void load(Archive &ar, const unsigned int version)
void allocate_and_move(const size_t old_size, const size_t new_size, const size_t new_allocated_size)
void save(Archive &ar, const unsigned int version) const
void insert_back(ForwardIterator begin, ForwardIterator end)
const value_type & const_reference
const_pointer data() const
virtual void apply_to_subrange(const std::size_t begin, const std::size_t end) const override
AlignedVectorCopyConstruct(const T *const source_begin, const T *const source_end, T *const destination)
static const std::size_t minimum_parallel_grain_size
AlignedVectorDefaultInitialize(const std::size_t size, T *const destination)
static const std::size_t minimum_parallel_grain_size
virtual void apply_to_subrange(const std::size_t begin, const std::size_t end) const override
void default_construct_or_assign(const std::size_t begin, const std::size_t end, std::bool_constant< false >) const
void default_construct_or_assign(const std::size_t begin, const std::size_t end, std::bool_constant< true >) const
void copy_construct_or_assign(const std::size_t begin, const std::size_t end, std::bool_constant< false >) const
static const std::size_t minimum_parallel_grain_size
virtual void apply_to_subrange(const std::size_t begin, const std::size_t end) const override
void copy_construct_or_assign(const std::size_t begin, const std::size_t end, std::bool_constant< true >) const
AlignedVectorInitialize(const std::size_t size, const T &element, T *const destination)
static const std::size_t minimum_parallel_grain_size
AlignedVectorMoveConstruct(T *const source_begin, T *const source_end, T *const destination)
virtual void apply_to_subrange(const std::size_t begin, const std::size_t end) const override
#define DEAL_II_NAMESPACE_OPEN
#define DEAL_II_NAMESPACE_CLOSE
static ::ExceptionBase & ExcAlignedVectorChangeAfterReplication()
static ::ExceptionBase & ExcInternalError()
#define Assert(cond, exc)
#define AssertThrowMPI(error_code)
#define AssertIndexRange(index, range)
#define DeclExceptionMsg(Exception, defaulttext)
static const types::blas_int zero
types::global_dof_index size_type
std::enable_if_t< std::is_fundamental_v< T >, std::size_t > memory_consumption(const T &t)
void swap(MemorySpaceData< T, MemorySpace > &u, MemorySpaceData< T, MemorySpace > &v)
VectorType::value_type * begin(VectorType &V)
VectorType::value_type * end(VectorType &V)
std::enable_if_t< is_mpi_type< T >==false, T > broadcast(const MPI_Comm comm, const T &object_to_send, const unsigned int root_process=0)
T max(const T &t, const MPI_Comm mpi_communicator)
T min(const T &t, const MPI_Comm mpi_communicator)
unsigned int this_mpi_process(const MPI_Comm mpi_communicator)
void free_communicator(MPI_Comm mpi_communicator)
void posix_memalign(void **memptr, std::size_t alignment, std::size_t size)
size_t pack(const T &object, std::vector< char > &dest_buffer, const bool allow_compression=true)
void apply_parallel(const std::size_t begin, const std::size_t end, const std::size_t minimum_parallel_grain_size) const