16#ifndef dealii_matrix_free_vector_access_internal_h
17#define dealii_matrix_free_vector_access_internal_h
28# include <boost/algorithm/string/join.hpp>
43 std::enable_if_t<!has_local_element<VectorType>, VectorType> * =
nullptr>
44 inline typename VectorType::value_type
57 std::enable_if_t<!has_local_element<VectorType>, VectorType> * =
nullptr>
58 inline typename VectorType::value_type &
71 std::enable_if_t<has_local_element<VectorType>, VectorType> * =
nullptr>
72 inline typename VectorType::value_type &
75 return vec.local_element(entry);
83 std::enable_if_t<has_local_element<VectorType>, VectorType> * =
nullptr>
84 inline typename VectorType::value_type
85 vector_access(
const VectorType &vec,
const unsigned int entry)
87 return vec.local_element(entry);
94 std::enable_if_t<has_add_local_element<VectorType>, VectorType> * =
nullptr>
97 const unsigned int entry,
98 const typename VectorType::value_type &val)
100 vec.add_local_element(entry, val);
105 template <
typename VectorType,
106 std::enable_if_t<!has_add_local_element<VectorType>, VectorType> * =
110 const unsigned int entry,
111 const typename VectorType::value_type &val)
120 std::enable_if_t<has_add_local_element<VectorType>, VectorType> * =
nullptr>
124 const typename VectorType::value_type &val)
131 template <
typename VectorType,
132 std::enable_if_t<!has_add_local_element<VectorType>, VectorType> * =
137 const typename VectorType::value_type &val)
146 std::enable_if_t<has_set_local_element<VectorType>, VectorType> * =
nullptr>
149 const unsigned int entry,
150 const typename VectorType::value_type &val)
152 vec.set_local_element(entry, val);
157 template <
typename VectorType,
158 std::enable_if_t<!has_set_local_element<VectorType>, VectorType> * =
162 const unsigned int entry,
163 const typename VectorType::value_type &val)
176 typename VectorizedArrayType,
178 std::enable_if_t<!has_partitioners_are_compatible<VectorType>,
179 VectorType> * =
nullptr>
182 const VectorType &vec,
198 typename VectorizedArrayType,
200 std::enable_if_t<has_partitioners_are_compatible<VectorType>,
201 VectorType> * =
nullptr>
204 const VectorType &vec,
217 for (
unsigned int i = 0; i < matrix_free.
n_components(); ++i)
226 std::vector<std::string> dof_indices_with_compatible_partitioners;
228 for (
unsigned int i = 0; i < matrix_free.
n_components(); ++i)
229 if (vec.partitioners_are_compatible(
231 dof_indices_with_compatible_partitioners.push_back(
234 if (dof_indices_with_compatible_partitioners.empty())
237 ExcMessage(
"The parallel layout of the given vector is "
238 "compatible neither with the Partitioner of the "
239 "current FEEvaluation with dof_handler_index=" +
240 std::to_string(dof_index) +
241 " nor with any Partitioner in MatrixFree. A "
242 "potential reason is that you did not use "
243 "MatrixFree::initialize_dof_vector() to get a "
244 "compatible vector."));
251 "The parallel layout of the given vector is "
252 "not compatible with the Partitioner of the "
253 "current FEEvaluation with dof_handler_index=" +
254 std::to_string(dof_index) +
255 ". However, the underlying "
256 "MatrixFree contains Partitioner objects that are compatible. "
257 "They have the following dof_handler_index values: " +
258 boost::algorithm::join(dof_indices_with_compatible_partitioners,
260 ". Did you want to pass any of these values to the "
261 "constructor of the current FEEvaluation object or "
262 "did you not use MatrixFree::initialize_dof_vector() "
263 "with dof_handler_index=" +
264 std::to_string(dof_index) +
266 "compatible vector?"));
280 template <
typename Number,
typename VectorizedArrayType>
283 template <
typename VectorType>
286 const VectorType &vec,
294 template <
typename VectorNumberType>
303 template <
typename VectorType>
306 const unsigned int dof_index,
308 VectorizedArrayType *dof_values,
309 std::bool_constant<true>)
const
315 dofs_per_cell, dof_index, vec, dof_values, std::bool_constant<false>());
317 const Number *vec_ptr = vec.begin() + dof_index;
318 for (
unsigned int i = 0; i < dofs_per_cell;
319 ++i, vec_ptr += VectorizedArrayType::size())
320 dof_values[i].load(vec_ptr);
326 template <
typename VectorType>
329 const unsigned int dof_index,
330 const VectorType &vec,
331 VectorizedArrayType *dof_values,
332 std::bool_constant<false>)
const
334 for (
unsigned int i = 0; i < dofs_per_cell; ++i)
335 for (
unsigned int v = 0; v < VectorizedArrayType::size(); ++v)
337 vector_access(vec, dof_index + v + i * VectorizedArrayType::size());
342 template <
typename VectorType>
345 const unsigned int *dof_indices,
347 const unsigned int constant_offset,
348 VectorizedArrayType *dof_values,
349 std::bool_constant<true>)
const
352 vec.begin() + constant_offset,
359 template <
typename VectorType>
362 const unsigned int *dof_indices,
363 const VectorType &vec,
364 const unsigned int constant_offset,
365 VectorizedArrayType *dof_values,
366 std::bool_constant<false>)
const
368 for (
unsigned int d = 0; d < dofs_per_cell; ++d)
369 for (
unsigned int v = 0; v < VectorizedArrayType::size(); ++v)
376 template <
typename VectorType>
379 const unsigned int *dof_indices,
381 VectorizedArrayType *dof_values,
382 std::bool_constant<true> type)
const
385 dofs_per_cell, dof_indices, vec, 0, dof_values, type);
390 template <
typename VectorType>
393 const unsigned int *dof_indices,
394 const VectorType &vec,
395 VectorizedArrayType *dof_values,
396 std::bool_constant<false> type)
const
399 dofs_per_cell, dof_indices, vec, 0, dof_values, type);
404 template <
typename Number2>
407 const unsigned int dofs_per_cell,
408 const std::array<Number2 *, VectorizedArrayType::size()> &global_ptr,
409 VectorizedArrayType *dof_values,
410 std::bool_constant<true>)
const
419 template <
typename Number2>
423 const std::array<Number2 *, VectorizedArrayType::size()> &,
424 VectorizedArrayType *,
425 std::bool_constant<false>)
const
434 template <
typename VectorType>
438 const unsigned int constant_offset,
439 typename VectorType::value_type *vec_ptr,
440 VectorizedArrayType &res,
441 std::bool_constant<true>)
const
443 (void)constant_offset;
455 std::bool_constant<false>());
457 res.gather(vec_ptr, indices);
465 template <
typename VectorType>
468 const VectorType &vec,
469 const unsigned int constant_offset,
470 typename VectorType::value_type *,
471 VectorizedArrayType &res,
472 std::bool_constant<false>)
const
474 for (
unsigned int v = 0; v < VectorizedArrayType::size(); ++v)
480 template <
typename VectorType>
483 const VectorType &vec,
499 template <
typename VectorType>
503 const VectorType &vec,
522 res = VectorizedArrayType();
530 template <
typename Number,
typename VectorizedArrayType>
533 template <
typename VectorType>
541 template <
typename VectorNumberType>
550 template <
typename VectorType>
553 const unsigned int dof_index,
555 VectorizedArrayType *dof_values,
556 std::bool_constant<true>)
const
558 Number *vec_ptr = vec.begin() + dof_index;
559 for (
unsigned int i = 0; i < dofs_per_cell;
560 ++i, vec_ptr += VectorizedArrayType::size())
562 VectorizedArrayType tmp;
564 tmp += dof_values[i];
571 template <
typename VectorType>
574 const unsigned int dof_index,
576 VectorizedArrayType *dof_values,
577 std::bool_constant<false>)
const
579 for (
unsigned int i = 0; i < dofs_per_cell; ++i)
580 for (
unsigned int v = 0; v < VectorizedArrayType::size(); ++v)
582 dof_index + v + i * VectorizedArrayType::size(),
588 template <
typename VectorType>
591 const unsigned int *dof_indices,
593 const unsigned int constant_offset,
594 VectorizedArrayType *dof_values,
595 std::bool_constant<true>)
const
601 vec.begin() + constant_offset);
606 template <
typename VectorType>
609 const unsigned int *dof_indices,
611 const unsigned int constant_offset,
612 VectorizedArrayType *dof_values,
613 std::bool_constant<false>)
const
615 for (
unsigned int d = 0; d < dofs_per_cell; ++d)
616 for (
unsigned int v = 0; v < VectorizedArrayType::size(); ++v)
618 dof_indices[v] + constant_offset + d,
624 template <
typename VectorType>
627 const unsigned int *dof_indices,
629 VectorizedArrayType *dof_values,
630 std::bool_constant<true> type)
const
633 dofs_per_cell, dof_indices, vec, 0, dof_values, type);
638 template <
typename VectorType>
641 const unsigned int *dof_indices,
643 VectorizedArrayType *dof_values,
644 std::bool_constant<false> type)
const
647 dofs_per_cell, dof_indices, vec, 0, dof_values, type);
652 template <
typename Number2>
655 const unsigned int dofs_per_cell,
656 std::array<Number2 *, VectorizedArrayType::size()> &global_ptr,
657 VectorizedArrayType *dof_values,
658 std::bool_constant<true>)
const
668 template <
typename Number2>
672 std::array<Number2 *, VectorizedArrayType::size()> &,
673 VectorizedArrayType *,
674 std::bool_constant<false>)
const
683 template <
typename VectorType>
687 const unsigned int constant_offset,
688 typename VectorType::value_type *vec_ptr,
689 VectorizedArrayType &res,
690 std::bool_constant<true>)
const
692 (void)constant_offset;
696#if DEAL_II_VECTORIZATION_WIDTH_IN_BITS < 512
697 for (
unsigned int v = 0; v < VectorizedArrayType::size(); ++v)
701 VectorizedArrayType tmp;
702 tmp.gather(vec_ptr, indices);
704 tmp.scatter(indices, vec_ptr);
712 template <
typename VectorType>
716 const unsigned int constant_offset,
717 typename VectorType::value_type *,
718 VectorizedArrayType &res,
719 std::bool_constant<false>)
const
721 for (
unsigned int v = 0; v < VectorizedArrayType::size(); ++v)
727 template <
typename VectorType>
746 template <
typename VectorType>
772 template <
typename Number,
typename VectorizedArrayType>
775 template <
typename VectorType>
784 template <
typename VectorNumberType>
793 template <
typename VectorType>
796 const unsigned int dof_index,
798 VectorizedArrayType *dof_values,
799 std::bool_constant<true>)
const
801 Number *vec_ptr = vec.begin() + dof_index;
802 for (
unsigned int i = 0; i < dofs_per_cell;
803 ++i, vec_ptr += VectorizedArrayType::size())
804 dof_values[i].store(vec_ptr);
809 template <
typename VectorType>
812 const unsigned int dof_index,
814 VectorizedArrayType *dof_values,
815 std::bool_constant<false>)
const
817 for (
unsigned int i = 0; i < dofs_per_cell; ++i)
818 for (
unsigned int v = 0; v < VectorizedArrayType::size(); ++v)
819 vector_access(vec, dof_index + v + i * VectorizedArrayType::size()) =
825 template <
typename VectorType>
828 const unsigned int *dof_indices,
830 const unsigned int constant_offset,
831 VectorizedArrayType *dof_values,
832 std::bool_constant<true>)
const
838 vec.begin() + constant_offset);
843 template <
typename VectorType,
bool booltype>
846 const unsigned int *dof_indices,
848 const unsigned int constant_offset,
849 VectorizedArrayType *dof_values,
850 std::bool_constant<false>)
const
852 for (
unsigned int i = 0; i < dofs_per_cell; ++i)
853 for (
unsigned int v = 0; v < VectorizedArrayType::size(); ++v)
860 template <
typename VectorType>
863 const unsigned int *dof_indices,
865 VectorizedArrayType *dof_values,
866 std::bool_constant<true> type)
const
869 dofs_per_cell, dof_indices, vec, 0, dof_values, type);
874 template <
typename VectorType,
bool booltype>
877 const unsigned int *dof_indices,
879 VectorizedArrayType *dof_values,
880 std::bool_constant<false> type)
const
883 dofs_per_cell, dof_indices, vec, 0, dof_values, type);
888 template <
typename Number2>
891 const unsigned int dofs_per_cell,
892 std::array<Number2 *, VectorizedArrayType::size()> &global_ptr,
893 VectorizedArrayType *dof_values,
894 std::bool_constant<true>)
const
904 template <
typename Number2>
908 std::array<Number2 *, VectorizedArrayType::size()> &,
909 VectorizedArrayType *,
910 std::bool_constant<false>)
const
917 template <
typename VectorType>
921 const unsigned int constant_offset,
922 typename VectorType::value_type *vec_ptr,
923 VectorizedArrayType &res,
924 std::bool_constant<true>)
const
927 (void)constant_offset;
930 res.scatter(indices, vec_ptr);
935 template <
typename VectorType>
939 const unsigned int constant_offset,
940 typename VectorType::value_type *,
941 VectorizedArrayType &res,
942 std::bool_constant<false>)
const
944 for (
unsigned int v = 0; v < VectorizedArrayType::size(); ++v)
950 template <
typename VectorType>
967 template <
typename VectorType>
const internal::MatrixFreeFunctions::DoFInfo & get_dof_info(const unsigned int dof_handler_index_component=0) const
unsigned int n_components() const
#define DEAL_II_NAMESPACE_OPEN
#define DEAL_II_NAMESPACE_CLOSE
#define Assert(cond, exc)
#define AssertDimension(dim1, dim2)
static ::ExceptionBase & ExcInternalError()
static ::ExceptionBase & ExcMessage(std::string arg1)
#define DEAL_II_NOT_IMPLEMENTED()
void vector_access_add(VectorType &vec, const unsigned int entry, const typename VectorType::value_type &val)
void check_vector_compatibility(const VectorType &vec, const MatrixFree< dim, Number, VectorizedArrayType > &matrix_free, const internal::MatrixFreeFunctions::DoFInfo &dof_info)
void vector_access_add_global(VectorType &vec, const types::global_dof_index entry, const typename VectorType::value_type &val)
VectorType::value_type vector_access(const VectorType &vec, const unsigned int entry)
void vector_access_set(VectorType &vec, const unsigned int entry, const typename VectorType::value_type &val)
static const unsigned int invalid_unsigned_int
std::shared_ptr< const Utilities::MPI::Partitioner > vector_partitioner
void process_dof_gather(const unsigned int *indices, VectorType &vec, const unsigned int constant_offset, typename VectorType::value_type *vec_ptr, VectorizedArrayType &res, std::bool_constant< true >) const
void process_dofs_vectorized_transpose(const unsigned int dofs_per_cell, const unsigned int *dof_indices, VectorType &vec, VectorizedArrayType *dof_values, std::bool_constant< false > type) const
void pre_constraints(const Number &input, Number &res) const
void process_empty(VectorizedArrayType &) const
void process_dof(const unsigned int index, VectorType &vec, Number &res) const
void process_dofs_vectorized_transpose(const unsigned int dofs_per_cell, const unsigned int *dof_indices, VectorType &vec, const unsigned int constant_offset, VectorizedArrayType *dof_values, std::bool_constant< false >) const
void post_constraints(const Number &, Number &) const
void process_dofs_vectorized(const unsigned int dofs_per_cell, const unsigned int dof_index, VectorType &vec, VectorizedArrayType *dof_values, std::bool_constant< true >) const
void process_dofs_vectorized(const unsigned int dofs_per_cell, const unsigned int dof_index, VectorType &vec, VectorizedArrayType *dof_values, std::bool_constant< false >) const
void process_dofs_vectorized_transpose(const unsigned int dofs_per_cell, const unsigned int *dof_indices, VectorType &vec, const unsigned int constant_offset, VectorizedArrayType *dof_values, std::bool_constant< true >) const
void process_dofs_vectorized_transpose(const unsigned int dofs_per_cell, const unsigned int *dof_indices, VectorType &vec, VectorizedArrayType *dof_values, std::bool_constant< true > type) const
void process_dofs_vectorized_transpose(const unsigned int, std::array< Number2 *, VectorizedArrayType::size()> &, VectorizedArrayType *, std::bool_constant< false >) const
void process_dof_gather(const unsigned int *indices, VectorType &vec, const unsigned int constant_offset, typename VectorType::value_type *, VectorizedArrayType &res, std::bool_constant< false >) const
void process_dofs_vectorized_transpose(const unsigned int dofs_per_cell, std::array< Number2 *, VectorizedArrayType::size()> &global_ptr, VectorizedArrayType *dof_values, std::bool_constant< true >) const
void process_dof(VectorNumberType &global, Number &local) const
void process_constraint(const unsigned int index, const Number weight, VectorType &vec, Number &res) const
void process_dof_global(const types::global_dof_index index, VectorType &vec, Number &res) const
void pre_constraints(const Number &, Number &res) const
void post_constraints(const Number &sum, Number &write_pos) const
void process_dofs_vectorized_transpose(const unsigned int dofs_per_cell, const unsigned int *dof_indices, VectorType &vec, const unsigned int constant_offset, VectorizedArrayType *dof_values, std::bool_constant< true >) const
void process_dof_global(const types::global_dof_index index, const VectorType &vec, Number &res) const
void process_dofs_vectorized(const unsigned int dofs_per_cell, const unsigned int dof_index, const VectorType &vec, VectorizedArrayType *dof_values, std::bool_constant< false >) const
void process_dofs_vectorized_transpose(const unsigned int dofs_per_cell, const unsigned int *dof_indices, const VectorType &vec, const unsigned int constant_offset, VectorizedArrayType *dof_values, std::bool_constant< false >) const
void process_constraint(const unsigned int index, const Number weight, const VectorType &vec, Number &res) const
void process_dofs_vectorized_transpose(const unsigned int dofs_per_cell, const unsigned int *dof_indices, VectorType &vec, VectorizedArrayType *dof_values, std::bool_constant< true > type) const
void process_dof_gather(const unsigned int *indices, const VectorType &vec, const unsigned int constant_offset, typename VectorType::value_type *, VectorizedArrayType &res, std::bool_constant< false >) const
void process_dofs_vectorized(const unsigned int dofs_per_cell, const unsigned int dof_index, VectorType &vec, VectorizedArrayType *dof_values, std::bool_constant< true >) const
void process_dof(const VectorNumberType &global, Number &local) const
void process_empty(VectorizedArrayType &res) const
void process_dof(const unsigned int index, const VectorType &vec, Number &res) const
void process_dofs_vectorized_transpose(const unsigned int, const std::array< Number2 *, VectorizedArrayType::size()> &, VectorizedArrayType *, std::bool_constant< false >) const
void process_dofs_vectorized_transpose(const unsigned int dofs_per_cell, const unsigned int *dof_indices, const VectorType &vec, VectorizedArrayType *dof_values, std::bool_constant< false > type) const
void process_dofs_vectorized_transpose(const unsigned int dofs_per_cell, const std::array< Number2 *, VectorizedArrayType::size()> &global_ptr, VectorizedArrayType *dof_values, std::bool_constant< true >) const
void process_dof_gather(const unsigned int *indices, VectorType &vec, const unsigned int constant_offset, typename VectorType::value_type *vec_ptr, VectorizedArrayType &res, std::bool_constant< true >) const
void process_dofs_vectorized(const unsigned int dofs_per_cell, const unsigned int dof_index, VectorType &vec, VectorizedArrayType *dof_values, std::bool_constant< false >) const
void process_dofs_vectorized_transpose(const unsigned int dofs_per_cell, std::array< Number2 *, VectorizedArrayType::size()> &global_ptr, VectorizedArrayType *dof_values, std::bool_constant< true >) const
void process_empty(VectorizedArrayType &) const
void process_dofs_vectorized_transpose(const unsigned int dofs_per_cell, const unsigned int *dof_indices, VectorType &vec, VectorizedArrayType *dof_values, std::bool_constant< true > type) const
void process_dofs_vectorized_transpose(const unsigned int dofs_per_cell, const unsigned int *dof_indices, VectorType &vec, const unsigned int constant_offset, VectorizedArrayType *dof_values, std::bool_constant< false >) const
void process_dofs_vectorized_transpose(const unsigned int dofs_per_cell, const unsigned int *dof_indices, VectorType &vec, VectorizedArrayType *dof_values, std::bool_constant< false > type) const
void process_dofs_vectorized_transpose(const unsigned int dofs_per_cell, const unsigned int *dof_indices, VectorType &vec, const unsigned int constant_offset, VectorizedArrayType *dof_values, std::bool_constant< true >) const
void process_dof_global(const types::global_dof_index index, VectorType &vec, Number &res) const
void pre_constraints(const Number &, Number &) const
void process_dof_gather(const unsigned int *indices, VectorType &vec, const unsigned int constant_offset, typename VectorType::value_type *vec_ptr, VectorizedArrayType &res, std::bool_constant< true >) const
void process_dofs_vectorized(const unsigned int dofs_per_cell, const unsigned int dof_index, VectorType &vec, VectorizedArrayType *dof_values, std::bool_constant< true >) const
void process_dofs_vectorized_transpose(const unsigned int, std::array< Number2 *, VectorizedArrayType::size()> &, VectorizedArrayType *, std::bool_constant< false >) const
void process_constraint(const unsigned int, const Number, VectorType &, Number &) const
void process_dof(const unsigned int index, VectorType &vec, Number &res) const
void post_constraints(const Number &, Number &) const
void process_dof(VectorNumberType &global, Number &local) const
void process_dof_gather(const unsigned int *indices, VectorType &vec, const unsigned int constant_offset, typename VectorType::value_type *, VectorizedArrayType &res, std::bool_constant< false >) const
void vectorized_load_and_transpose(const unsigned int n_entries, const Number *in, const unsigned int *offsets, VectorizedArray< Number, width > *out)
void vectorized_transpose_and_store(const bool add_into, const unsigned int n_entries, const VectorizedArray< Number, width > *in, const unsigned int *offsets, Number *out)