17 #ifndef dealii_aligned_vector_h
18 #define dealii_aligned_vector_h
30 #include <boost/version.hpp>
31 #if BOOST_VERSION >= 106400
32 # include <boost/serialization/array_wrapper.hpp>
34 # include <boost/serialization/array.hpp>
36 #include <boost/serialization/split_member.hpp>
40 #include <type_traits>
234 template <
typename ForwardIterator>
350 const unsigned int root_process);
438 template <
class Archive>
440 save(Archive &ar,
const unsigned int version)
const;
447 template <
class Archive>
449 load(Archive &ar,
const unsigned int version);
457 template <
class Archive>
459 serialize(Archive &archive,
const unsigned int version);
463 BOOST_SERIALIZATION_SPLIT_MEMBER()
568 #ifdef DEAL_II_WITH_MPI
577 const bool is_shmem_root,
578 T * aligned_shmem_pointer,
579 MPI_Comm shmem_group_communicator,
580 MPI_Win shmem_window);
622 #ifdef DEAL_II_WITH_MPI
717 template <
typename T>
722 160000 /
sizeof(
T) + 1;
735 const T *
const source_end,
736 T *
const destination)
741 Assert(source_end == source_begin || destination !=
nullptr,
743 const std::size_t size = source_end - source_begin;
756 const std::size_t
end)
const override
765 if (std::is_trivial<T>::value ==
true)
770 for (std::size_t i =
begin; i <
end; ++i)
786 template <
typename T>
791 160000 /
sizeof(
T) + 1;
805 T *
const destination)
810 Assert(source_end == source_begin || destination !=
nullptr,
812 const std::size_t size = source_end - source_begin;
825 const std::size_t
end)
const override
833 if (std::is_trivial<T>::value ==
true)
840 for (std::size_t i =
begin; i <
end; ++i)
867 template <
typename T,
bool initialize_memory>
871 160000 /
sizeof(
T) + 1;
880 T *
const destination)
892 if (std::is_trivial<T>::value ==
true &&
893 std::is_same<T, long double>::value ==
false)
895 const unsigned char zero[
sizeof(
T)] = {};
899 if (std::memcmp(
zero,
900 static_cast<const void *
>(&element),
915 const std::size_t
end)
const override
927 begin,
end, std::integral_constant<bool, initialize_memory>());
938 const std::size_t
end,
939 std::integral_constant<bool, false>)
const
941 for (std::size_t i =
begin; i <
end; ++i)
948 const std::size_t
end,
949 std::integral_constant<bool, true>)
const
951 for (std::size_t i =
begin; i <
end; ++i)
970 template <
typename T,
bool initialize_memory>
975 160000 /
sizeof(
T) + 1;
1000 const std::size_t
end)
const override
1006 if (std::is_trivial<T>::value ==
true)
1012 begin,
end, std::integral_constant<bool, initialize_memory>());
1021 const std::size_t
end,
1022 std::integral_constant<bool, false>)
const
1024 for (std::size_t i =
begin; i <
end; ++i)
1031 const std::size_t
end,
1032 std::integral_constant<bool, true>)
const
1034 for (std::size_t i =
begin; i <
end; ++i)
1046 template <
typename T>
1048 : deleter_action_object(nullptr)
1049 , owning_aligned_vector(owning_object)
1053 # ifdef DEAL_II_WITH_MPI
1055 template <
typename T>
1057 const bool is_shmem_root,
1058 T * aligned_shmem_pointer,
1059 MPI_Comm shmem_group_communicator,
1060 MPI_Win shmem_window)
1061 : deleter_action_object(
1062 std::make_unique<MPISharedMemDeleterAction>(is_shmem_root,
1063 aligned_shmem_pointer,
1064 shmem_group_communicator,
1066 , owning_aligned_vector(owning_object)
1071 template <
typename T>
1077 if (deleter_action_object ==
nullptr)
1081 Assert(owning_aligned_vector->used_elements_end !=
nullptr,
1084 if (std::is_trivial<T>::value ==
false)
1085 for (
T *p = owning_aligned_vector->used_elements_end - 1; p >= ptr;
1094 deleter_action_object->delete_array(owning_aligned_vector, ptr);
1099 template <
typename T>
1104 owning_aligned_vector = new_aligned_vector_ptr;
1108 # ifdef DEAL_II_WITH_MPI
1110 template <
typename T>
1113 T * aligned_shmem_pointer,
1114 MPI_Comm shmem_group_communicator,
1115 MPI_Win shmem_window)
1116 : is_shmem_root(is_shmem_root)
1117 , aligned_shmem_pointer(aligned_shmem_pointer)
1118 , shmem_group_communicator(shmem_group_communicator)
1119 , shmem_window(shmem_window)
1124 template <
typename T>
1134 if (std::is_trivial<T>::value ==
false)
1136 p >= aligned_vector->
elements.get();
1141 ierr = MPI_Win_free(&shmem_window);
1152 : elements(nullptr, Deleter(this))
1153 , used_elements_end(nullptr)
1154 , allocated_elements_end(nullptr)
1161 : elements(nullptr, Deleter(this))
1162 , used_elements_end(nullptr)
1163 , allocated_elements_end(nullptr)
1173 : elements(nullptr, Deleter(this))
1174 , used_elements_end(nullptr)
1175 , allocated_elements_end(nullptr)
1178 reserve(vec.
size());
1179 used_elements_end = allocated_elements_end;
1192 *
this = std::move(vec);
1215 used_elements_end = elements.get() + new_size;
1234 elements = std::move(vec.
elements);
1235 elements.get_deleter().reset_owning_object(
this);
1257 else if (new_size == old_size)
1259 else if (new_size < old_size)
1264 if (std::is_trivial<T>::value ==
false)
1265 for (
T *p = used_elements_end - 1; p >= elements.get() + new_size; --p)
1267 used_elements_end = elements.get() + new_size;
1273 used_elements_end = elements.get() + new_size;
1277 if (std::is_trivial<T>::value ==
false)
1279 new_size - old_size, elements.get() + old_size);
1293 else if (new_size == old_size)
1295 else if (new_size < old_size)
1300 if (std::is_trivial<T>::value ==
false)
1301 for (
T *p = used_elements_end - 1; p >= elements.get() + new_size; --p)
1303 used_elements_end = elements.get() + new_size;
1309 used_elements_end = elements.get() + new_size;
1313 new_size - old_size, elements.get() + old_size);
1327 else if (new_size == old_size)
1329 else if (new_size < old_size)
1334 if (std::is_trivial<T>::value ==
false)
1335 for (
T *p = used_elements_end - 1; p >= elements.get() + new_size; --p)
1337 used_elements_end = elements.get() + new_size;
1343 used_elements_end = elements.get() + new_size;
1347 new_size - old_size, init, elements.get() + old_size);
1357 const size_type old_size = used_elements_end - elements.get();
1358 const size_type old_allocated_size = allocated_elements_end - elements.get();
1359 if (new_allocated_size > old_allocated_size)
1365 std::max(new_allocated_size, 2 * old_allocated_size);
1371 reinterpret_cast<void **
>(&new_data_ptr), 64, new_size *
sizeof(
T));
1378 Deleter deleter(
this);
1381 if (new_allocated_size > 0)
1383 elements.get(), elements.get() + old_size, new_data_ptr);
1394 elements = decltype(elements)(new_data_ptr, std::move(deleter));
1395 used_elements_end = elements.get() + old_size;
1396 allocated_elements_end = elements.get() + new_size;
1398 else if (new_allocated_size == 0)
1418 used_elements_end =
nullptr;
1419 allocated_elements_end =
nullptr;
1429 if (used_elements_end == allocated_elements_end)
1431 if (std::is_trivial<T>::value ==
false)
1432 new (used_elements_end++)
T(in_data);
1434 *used_elements_end++ = in_data;
1444 T *field = used_elements_end - 1;
1455 const T *field = used_elements_end - 1;
1462 template <
typename ForwardIterator>
1470 if (std::is_trivial<T>::value ==
false)
1471 new (used_elements_end)
T;
1472 *used_elements_end = *
begin;
1502 const unsigned int root_process)
1504 # ifdef DEAL_II_WITH_MPI
1525 used_elements_end =
nullptr;
1526 allocated_elements_end =
nullptr;
1548 MPI_Comm shmem_group_communicator;
1550 MPI_Comm shmem_group_communicator_temp;
1551 int ierr = MPI_Comm_split_type(communicator,
1552 MPI_COMM_TYPE_SHARED,
1555 &shmem_group_communicator_temp);
1560 ierr = MPI_Comm_split(shmem_group_communicator_temp,
1563 &shmem_group_communicator);
1574 const bool is_shmem_root =
1603 MPI_Comm shmem_roots_communicator;
1608 const int ierr = MPI_Comm_split(communicator,
1610 (is_shmem_root ? 0 : 1),
1612 &shmem_roots_communicator);
1621 const unsigned int shmem_roots_root_rank = 0;
1622 const bool is_shmem_roots_root =
1624 shmem_roots_communicator) == shmem_roots_root_rank));
1631 if (std::is_trivial<T>::value)
1643 shmem_roots_root_rank);
1644 if (is_shmem_roots_root ==
false)
1648 int ierr = MPI_Bcast(elements.get(),
1649 sizeof(
T) * new_size,
1651 shmem_roots_root_rank,
1652 shmem_roots_communicator);
1669 shmem_roots_root_rank);
1673 shmem_roots_root_rank);
1709 MPI_Win shmem_window;
1711 const MPI_Aint align_by = 64;
1712 const MPI_Aint alloc_size =
1714 (size() *
sizeof(
T) + (align_by - 1)),
1721 ierr = MPI_Info_create(&mpi_info);
1723 ierr = MPI_Info_set(mpi_info,
1724 "mpi_minimum_memory_alignment",
1727 ierr = MPI_Win_allocate_shared((is_shmem_root ? alloc_size : 0),
1730 shmem_group_communicator,
1735 ierr = MPI_Info_free(&mpi_info);
1758 if (is_shmem_root ==
false)
1761 MPI_Aint alloc_size;
1762 const int ierr = MPI_Win_shared_query(
1763 shmem_window, MPI_PROC_NULL, &alloc_size, &disp_unit, &base_ptr);
1783 std::size_t available_space = alloc_size;
1784 void * base_ptr_backup = base_ptr;
1785 T * aligned_shmem_pointer =
static_cast<T *
>(
1786 std::align(align_by, new_size *
sizeof(
T), base_ptr, available_space));
1816 aligned_shmem_pointer =
static_cast<T *
>(base_ptr_backup);
1824 if (std::is_trivial<T>::value ==
true)
1825 std::memcpy(aligned_shmem_pointer, elements.get(),
sizeof(
T) * size());
1827 for (std::size_t i = 0; i < size(); ++i)
1828 new (&aligned_shmem_pointer[i])
T(std::move(elements[i]));
1833 const int ierr = MPI_Barrier(shmem_group_communicator);
1853 elements = decltype(elements)(aligned_shmem_pointer,
1856 aligned_shmem_pointer,
1857 shmem_group_communicator,
1864 used_elements_end = elements.get() + new_size;
1865 allocated_elements_end = used_elements_end;
1873 std::accumulate(packed_data.begin(), packed_data.end(),
int(0));
1893 elements.get_deleter().reset_owning_object(
this);
1894 vec.
elements.get_deleter().reset_owning_object(&vec);
1907 return used_elements_end == elements.get();
1916 return used_elements_end - elements.get();
1925 return allocated_elements_end - elements.get();
1935 return elements[
index];
1945 return elements[
index];
1950 template <
typename T>
1954 return elements.get();
1959 template <
typename T>
1963 return elements.get();
1972 return elements.get();
1981 return used_elements_end;
1990 return elements.get();
1999 return used_elements_end;
2005 template <
class Archive>
2012 ar &boost::serialization::make_array(elements.get(), vec_size);
2018 template <
class Archive>
2028 ar &boost::serialization::make_array(elements.get(), vec_size);
2029 used_elements_end = elements.get() + vec_size;
2040 for (
const T *t = elements.get(); t != used_elements_end; ++t)
2042 memory +=
sizeof(
T) * (allocated_elements_end - used_elements_end);
bool operator==(const AlignedVector< T > &lhs, const AlignedVector< T > &rhs)
virtual ~DeleterActionBase()=default
virtual void delete_array(const AlignedVector< T > *owning_aligned_vector, T *ptr)=0
MPI_Comm shmem_group_communicator
virtual void delete_array(const AlignedVector< T > *aligned_vector, T *ptr)
MPISharedMemDeleterAction(const bool is_shmem_root, T *aligned_shmem_pointer, MPI_Comm shmem_group_communicator, MPI_Win shmem_window)
T * aligned_shmem_pointer
Deleter(AlignedVector< T > *owning_object, const bool is_shmem_root, T *aligned_shmem_pointer, MPI_Comm shmem_group_communicator, MPI_Win shmem_window)
Deleter(AlignedVector< T > *owning_object)
std::unique_ptr< DeleterActionBase > deleter_action_object
void reset_owning_object(const AlignedVector< T > *new_aligned_vector_ptr)
const AlignedVector< T > * owning_aligned_vector
size_type memory_consumption() const
void resize_fast(const size_type new_size)
std::unique_ptr< T[], Deleter > elements
reference operator[](const size_type index)
void fill(const T &element)
AlignedVector & operator=(AlignedVector< T > &&vec) noexcept
const_iterator end() const
AlignedVector(AlignedVector< T > &&vec) noexcept
void reserve(const size_type new_allocated_size)
void serialize(Archive &archive, const unsigned int version)
bool operator!=(const AlignedVector< T > &lhs, const AlignedVector< T > &rhs)
const_reference operator[](const size_type index) const
void swap(AlignedVector< T > &vec)
void resize(const size_type new_size, const T &init)
size_type capacity() const
const value_type * const_pointer
void push_back(const T in_data)
AlignedVector & operator=(const AlignedVector< T > &vec)
const_iterator begin() const
AlignedVector(const size_type size, const T &init=T())
T * allocated_elements_end
AlignedVector(const AlignedVector< T > &vec)
const_reference back() const
bool operator==(const AlignedVector< T > &lhs, const AlignedVector< T > &rhs)
const value_type * const_iterator
void replicate_across_communicator(const MPI_Comm &communicator, const unsigned int root_process)
void resize(const size_type new_size)
void load(Archive &ar, const unsigned int version)
void save(Archive &ar, const unsigned int version) const
void insert_back(ForwardIterator begin, ForwardIterator end)
const value_type & const_reference
const_pointer data() const
virtual void apply_to_subrange(const std::size_t begin, const std::size_t end) const override
AlignedVectorCopyConstruct(const T *const source_begin, const T *const source_end, T *const destination)
static const std::size_t minimum_parallel_grain_size
void default_construct_or_assign(const std::size_t begin, const std::size_t end, std::integral_constant< bool, true >) const
void default_construct_or_assign(const std::size_t begin, const std::size_t end, std::integral_constant< bool, false >) const
AlignedVectorDefaultInitialize(const std::size_t size, T *const destination)
static const std::size_t minimum_parallel_grain_size
virtual void apply_to_subrange(const std::size_t begin, const std::size_t end) const override
void copy_construct_or_assign(const std::size_t begin, const std::size_t end, std::integral_constant< bool, true >) const
static const std::size_t minimum_parallel_grain_size
virtual void apply_to_subrange(const std::size_t begin, const std::size_t end) const override
void copy_construct_or_assign(const std::size_t begin, const std::size_t end, std::integral_constant< bool, false >) const
AlignedVectorInitialize(const std::size_t size, const T &element, T *const destination)
static const std::size_t minimum_parallel_grain_size
AlignedVectorMoveConstruct(T *const source_begin, T *const source_end, T *const destination)
virtual void apply_to_subrange(const std::size_t begin, const std::size_t end) const override
#define DEAL_II_NAMESPACE_OPEN
#define DEAL_II_NAMESPACE_CLOSE
static ::ExceptionBase & ExcInternalError()
#define Assert(cond, exc)
std::string to_string(const T &t)
#define AssertThrowMPI(error_code)
#define AssertIndexRange(index, range)
static const types::blas_int zero
types::global_dof_index size_type
std::enable_if< std::is_fundamental< T >::value, std::size_t >::type memory_consumption(const T &t)
void swap(MemorySpaceData< Number, MemorySpace > &, MemorySpaceData< Number, MemorySpace > &)
VectorType::value_type * begin(VectorType &V)
VectorType::value_type * end(VectorType &V)
void free_communicator(MPI_Comm &mpi_communicator)
unsigned int this_mpi_process(const MPI_Comm &mpi_communicator)
T min(const T &t, const MPI_Comm &mpi_communicator)
T max(const T &t, const MPI_Comm &mpi_communicator)
std::enable_if< is_mpi_type< T >==false, T >::type broadcast(const MPI_Comm &comm, const T &object_to_send, const unsigned int root_process=0)
void posix_memalign(void **memptr, std::size_t alignment, std::size_t size)
size_t pack(const T &object, std::vector< char > &dest_buffer, const bool allow_compression=true)
void apply_parallel(const std::size_t begin, const std::size_t end, const std::size_t minimum_parallel_grain_size) const