17 #ifndef dealii_matrix_free_h
18 #define dealii_matrix_free_h
115 std::is_same<Number, typename VectorizedArrayType::value_type>::value,
116 "Type of Number and of VectorizedArrayType do not match.");
587 template <
typename QuadratureType,
typename number2,
typename MappingType>
592 const QuadratureType & quad,
616 template <
typename QuadratureType,
typename number2,
typename MappingType>
621 const std::vector<QuadratureType> & quad,
631 template <
typename QuadratureType,
typename number2,
typename MappingType>
636 const QuadratureType & quad,
793 template <
typename OutVector,
typename InVector>
799 const std::pair<unsigned int, unsigned int> &)> &cell_operation,
801 const InVector & src,
802 const bool zero_dst_vector =
false)
const;
850 template <
typename CLASS,
typename OutVector,
typename InVector>
856 const std::pair<unsigned int, unsigned int> &)
const,
857 const CLASS * owning_class,
860 const bool zero_dst_vector =
false)
const;
865 template <
typename CLASS,
typename OutVector,
typename InVector>
871 const std::pair<unsigned int, unsigned int> &),
872 CLASS * owning_class,
875 const bool zero_dst_vector =
false)
const;
961 template <
typename CLASS,
typename OutVector,
typename InVector>
967 const std::pair<unsigned int, unsigned int> &)
const,
968 const CLASS * owning_class,
971 const std::function<
void(
const unsigned int,
const unsigned int)>
972 &operation_before_loop,
973 const std::function<
void(
const unsigned int,
const unsigned int)>
974 & operation_after_loop,
975 const unsigned int dof_handler_index_pre_post = 0)
const;
980 template <
typename CLASS,
typename OutVector,
typename InVector>
986 const std::pair<unsigned int, unsigned int> &),
987 CLASS * owning_class,
990 const std::function<
void(
const unsigned int,
const unsigned int)>
991 &operation_before_loop,
992 const std::function<
void(
const unsigned int,
const unsigned int)>
993 & operation_after_loop,
994 const unsigned int dof_handler_index_pre_post = 0)
const;
1000 template <
typename OutVector,
typename InVector>
1006 const std::pair<unsigned int, unsigned int> &)> &cell_operation,
1008 const InVector & src,
1009 const std::function<
void(
const unsigned int,
const unsigned int)>
1010 &operation_before_loop,
1011 const std::function<
void(
const unsigned int,
const unsigned int)>
1012 & operation_after_loop,
1013 const unsigned int dof_handler_index_pre_post = 0)
const;
1090 template <
typename OutVector,
typename InVector>
1096 const std::pair<unsigned int, unsigned int> &)> &cell_operation,
1097 const std::function<
1101 const std::pair<unsigned int, unsigned int> &)> &face_operation,
1102 const std::function<
void(
1106 const std::pair<unsigned int, unsigned int> &)> &boundary_operation,
1108 const InVector & src,
1109 const bool zero_dst_vector =
false,
1200 template <
typename CLASS,
typename OutVector,
typename InVector>
1203 void (CLASS::*cell_operation)(
const MatrixFree &,
1206 const std::pair<unsigned int, unsigned int> &)
1208 void (CLASS::*face_operation)(
const MatrixFree &,
1211 const std::pair<unsigned int, unsigned int> &)
1213 void (CLASS::*boundary_operation)(
1217 const std::pair<unsigned int, unsigned int> &)
const,
1218 const CLASS * owning_class,
1220 const InVector & src,
1221 const bool zero_dst_vector =
false,
1230 template <
typename CLASS,
typename OutVector,
typename InVector>
1232 loop(
void (CLASS::*cell_operation)(
1236 const std::pair<unsigned int, unsigned int> &),
1237 void (CLASS::*face_operation)(
1241 const std::pair<unsigned int, unsigned int> &),
1242 void (CLASS::*boundary_operation)(
1246 const std::pair<unsigned int, unsigned int> &),
1247 CLASS * owning_class,
1249 const InVector & src,
1250 const bool zero_dst_vector =
false,
1375 template <
typename CLASS,
typename OutVector,
typename InVector>
1378 void (CLASS::*cell_operation)(
const MatrixFree &,
1381 const std::pair<unsigned int, unsigned int> &)
1383 void (CLASS::*face_operation)(
const MatrixFree &,
1386 const std::pair<unsigned int, unsigned int> &)
1388 void (CLASS::*boundary_operation)(
1392 const std::pair<unsigned int, unsigned int> &)
const,
1393 const CLASS * owning_class,
1395 const InVector &src,
1396 const std::function<
void(
const unsigned int,
const unsigned int)>
1397 &operation_before_loop,
1398 const std::function<
void(
const unsigned int,
const unsigned int)>
1399 & operation_after_loop,
1400 const unsigned int dof_handler_index_pre_post = 0,
1409 template <
typename CLASS,
typename OutVector,
typename InVector>
1411 loop(
void (CLASS::*cell_operation)(
1415 const std::pair<unsigned int, unsigned int> &),
1416 void (CLASS::*face_operation)(
1420 const std::pair<unsigned int, unsigned int> &),
1421 void (CLASS::*boundary_operation)(
1425 const std::pair<unsigned int, unsigned int> &),
1426 const CLASS * owning_class,
1428 const InVector &src,
1429 const std::function<
void(
const unsigned int,
const unsigned int)>
1430 &operation_before_loop,
1431 const std::function<
void(
const unsigned int,
const unsigned int)>
1432 & operation_after_loop,
1433 const unsigned int dof_handler_index_pre_post = 0,
1444 template <
typename OutVector,
typename InVector>
1450 const std::pair<unsigned int, unsigned int> &)> &cell_operation,
1451 const std::function<
1455 const std::pair<unsigned int, unsigned int> &)> &face_operation,
1456 const std::function<
void(
1460 const std::pair<unsigned int, unsigned int> &)> &boundary_operation,
1462 const InVector & src,
1463 const std::function<
void(
const unsigned int,
const unsigned int)>
1464 &operation_before_loop,
1465 const std::function<
void(
const unsigned int,
const unsigned int)>
1466 & operation_after_loop,
1467 const unsigned int dof_handler_index_pre_post = 0,
1537 template <
typename CLASS,
typename OutVector,
typename InVector>
1543 const std::pair<unsigned int, unsigned int> &)
const,
1544 const CLASS * owning_class,
1546 const InVector & src,
1547 const bool zero_dst_vector =
false,
1554 template <
typename CLASS,
typename OutVector,
typename InVector>
1560 const std::pair<unsigned int, unsigned int> &),
1561 CLASS * owning_class,
1563 const InVector & src,
1564 const bool zero_dst_vector =
false,
1571 template <
typename OutVector,
typename InVector>
1577 const std::pair<unsigned int, unsigned int> &)>
1580 const InVector & src,
1581 const bool zero_dst_vector =
false,
1592 std::pair<unsigned int, unsigned int>
1594 const unsigned int fe_degree,
1595 const unsigned int dof_handler_index = 0)
const;
1603 std::pair<unsigned int, unsigned int>
1605 const std::pair<unsigned int, unsigned int> &range,
1607 const unsigned int dof_handler_index = 0)
const;
1620 const std::pair<unsigned int, unsigned int> range)
const;
1627 const bool is_interior_face =
true)
const;
1640 template <
typename T>
1649 template <
typename T>
1666 template <
typename VectorType>
1669 const unsigned int dof_handler_index = 0)
const;
1688 template <
typename Number2>
1691 const unsigned int dof_handler_index = 0)
const;
1703 const std::shared_ptr<const Utilities::MPI::Partitioner> &
1727 const std::vector<unsigned int> &
1742 const unsigned int dof_handler_index = 0);
1753 template <
int spacedim>
1852 const unsigned int face_number)
const;
1884 const unsigned int lane_index,
1885 const unsigned int dof_handler_index = 0)
const;
1894 const unsigned int lane_index)
const;
1918 std::pair<typename DoFHandler<dim>::cell_iterator,
unsigned int>
1920 const unsigned int lane_index,
1921 const bool interior =
true,
1922 const unsigned int fe_component = 0)
const;
1968 const unsigned int hp_active_fe_index = 0)
const;
1975 const unsigned int hp_active_fe_index = 0)
const;
1983 const unsigned int hp_active_fe_index = 0)
const;
1991 const unsigned int hp_active_fe_index = 0)
const;
1998 const unsigned int hp_active_fe_index = 0)
const;
2005 const unsigned int hp_active_fe_index = 0)
const;
2021 const std::pair<unsigned int, unsigned int> cell_batch_range)
const;
2027 std::pair<unsigned int, unsigned int>
2029 const std::pair<unsigned int, unsigned int> face_batch_range)
const;
2049 std::pair<unsigned int, unsigned int>
2083 template <
typename StreamType>
2111 const internal::MatrixFreeFunctions::
2112 MappingInfo<dim, Number, VectorizedArrayType> &
2147 const unsigned int quad_index = 0,
2148 const unsigned int fe_base_element = 0,
2149 const unsigned int hp_active_fe_index = 0,
2150 const unsigned int hp_active_quad_index = 0)
const;
2156 VectorizedArrayType::size()> &
2216 template <
typename number2,
int q_dim>
2222 const std::vector<IndexSet> & locally_owned_set,
2232 template <
typename number2>
2236 const std::vector<IndexSet> & locally_owned_set,
2264 std::vector<internal::MatrixFreeFunctions::DoFInfo>
dof_info;
2347 std::list<std::pair<bool, AlignedVector<VectorizedArrayType>>>>
2354 mutable std::list<std::pair<bool, AlignedVector<Number>>>
2371 template <
int dim,
typename Number,
typename VectorizedArrayType>
2372 template <
typename T>
2382 template <
int dim,
typename Number,
typename VectorizedArrayType>
2383 template <
typename T>
2394 template <
int dim,
typename Number,
typename VectorizedArrayType>
2395 template <
typename VectorType>
2399 const unsigned int comp)
const
2402 "This function is not supported for block vectors.");
2405 ExcMessage(
"This function can only be used in serial."));
2408 vec.reinit(
dof_info[comp].vector_partitioner->size());
2413 template <
int dim,
typename Number,
typename VectorizedArrayType>
2414 template <
typename Number2>
2418 const unsigned int comp)
const
2426 template <
int dim,
typename Number,
typename VectorizedArrayType>
2427 inline const std::shared_ptr<const Utilities::MPI::Partitioner> &
2429 const unsigned int comp)
const
2432 return dof_info[comp].vector_partitioner;
2437 template <
int dim,
typename Number,
typename VectorizedArrayType>
2438 inline const std::vector<unsigned int> &
2440 const unsigned int comp)
const
2443 return dof_info[comp].constrained_dofs;
2448 template <
int dim,
typename Number,
typename VectorizedArrayType>
2458 template <
int dim,
typename Number,
typename VectorizedArrayType>
2461 const unsigned int dof_no)
const
2465 return dof_handlers[dof_no]->get_fe().n_base_elements();
2470 template <
int dim,
typename Number,
typename VectorizedArrayType>
2479 template <
int dim,
typename Number,
typename VectorizedArrayType>
2488 template <
int dim,
typename Number,
typename VectorizedArrayType>
2497 template <
int dim,
typename Number,
typename VectorizedArrayType>
2507 template <
int dim,
typename Number,
typename VectorizedArrayType>
2518 template <
int dim,
typename Number,
typename VectorizedArrayType>
2530 template <
int dim,
typename Number,
typename VectorizedArrayType>
2541 template <
int dim,
typename Number,
typename VectorizedArrayType>
2544 const unsigned int face_batch_index)
const
2556 template <
int dim,
typename Number,
typename VectorizedArrayType>
2559 const unsigned int cell_batch_index,
2560 const unsigned int face_number)
const
2568 for (
unsigned int v = 0;
2578 template <
int dim,
typename Number,
typename VectorizedArrayType>
2579 inline const internal::MatrixFreeFunctions::
2580 MappingInfo<dim, Number, VectorizedArrayType> &
2588 template <
int dim,
typename Number,
typename VectorizedArrayType>
2591 const unsigned int dof_index)
const
2599 template <
int dim,
typename Number,
typename VectorizedArrayType>
2608 template <
int dim,
typename Number,
typename VectorizedArrayType>
2609 inline const Number *
2611 const unsigned int row)
const
2621 template <
int dim,
typename Number,
typename VectorizedArrayType>
2622 inline const Number *
2624 const unsigned int row)
const
2634 template <
int dim,
typename Number,
typename VectorizedArrayType>
2635 inline std::pair<unsigned int, unsigned int>
2637 const std::pair<unsigned int, unsigned int> &range,
2638 const unsigned int degree,
2639 const unsigned int dof_handler_component)
const
2641 if (
dof_info[dof_handler_component].cell_active_fe_index.empty())
2644 dof_info[dof_handler_component].fe_index_conversion.size(), 1);
2646 dof_info[dof_handler_component].fe_index_conversion[0].size(), 1);
2647 if (
dof_info[dof_handler_component].fe_index_conversion[0][0] == degree)
2650 return {range.second, range.second};
2654 dof_info[dof_handler_component].fe_index_from_degree(0, degree);
2656 return {range.second, range.second};
2660 dof_handler_component);
2665 template <
int dim,
typename Number,
typename VectorizedArrayType>
2668 const unsigned int cell_batch_index)
const
2671 return VectorizedArrayType::size() > 1 &&
2674 VectorizedArrayType::size() -
2680 template <
int dim,
typename Number,
typename VectorizedArrayType>
2688 template <
int dim,
typename Number,
typename VectorizedArrayType>
2691 const std::pair<unsigned int, unsigned int> range)
const
2693 const auto &fe_indices =
dof_info[0].cell_active_fe_index;
2695 if (fe_indices.empty() ==
true)
2698 const auto index = fe_indices[range.first];
2700 for (
unsigned int i = range.first; i < range.second; ++i)
2708 template <
int dim,
typename Number,
typename VectorizedArrayType>
2711 const std::pair<unsigned int, unsigned int> range,
2712 const bool is_interior_face)
const
2714 const auto &fe_indices =
dof_info[0].cell_active_fe_index;
2716 if (fe_indices.empty() ==
true)
2719 if (is_interior_face)
2721 const unsigned int index =
2723 VectorizedArrayType::size()];
2725 for (
unsigned int i = range.first; i < range.second; ++i)
2728 VectorizedArrayType::size()]);
2734 const unsigned int index =
2736 VectorizedArrayType::size()];
2738 for (
unsigned int i = range.first; i < range.second; ++i)
2741 VectorizedArrayType::size()]);
2749 template <
int dim,
typename Number,
typename VectorizedArrayType>
2752 const unsigned int cell_batch_index)
const
2756 const std::vector<unsigned char> &n_lanes_filled =
2757 dof_info[0].n_vectorization_lanes_filled
2761 return n_lanes_filled[cell_batch_index];
2766 template <
int dim,
typename Number,
typename VectorizedArrayType>
2769 const unsigned int face_batch_index)
const
2773 const std::vector<unsigned char> &n_lanes_filled =
2774 dof_info[0].n_vectorization_lanes_filled
2777 return n_lanes_filled[face_batch_index];
2782 template <
int dim,
typename Number,
typename VectorizedArrayType>
2785 const unsigned int dof_handler_index,
2786 const unsigned int active_fe_index)
const
2788 return dof_info[dof_handler_index].dofs_per_cell[active_fe_index];
2793 template <
int dim,
typename Number,
typename VectorizedArrayType>
2796 const unsigned int quad_index,
2797 const unsigned int active_fe_index)
const
2801 .descriptor[active_fe_index]
2807 template <
int dim,
typename Number,
typename VectorizedArrayType>
2810 const unsigned int dof_handler_index,
2811 const unsigned int active_fe_index)
const
2813 return dof_info[dof_handler_index].dofs_per_face[active_fe_index];
2818 template <
int dim,
typename Number,
typename VectorizedArrayType>
2821 const unsigned int quad_index,
2822 const unsigned int active_fe_index)
const
2826 .descriptor[active_fe_index]
2832 template <
int dim,
typename Number,
typename VectorizedArrayType>
2835 const unsigned int dof_handler_index)
const
2837 return dof_info[dof_handler_index].vector_partitioner->locally_owned_range();
2842 template <
int dim,
typename Number,
typename VectorizedArrayType>
2845 const unsigned int dof_handler_index)
const
2847 return dof_info[dof_handler_index].vector_partitioner->ghost_indices();
2852 template <
int dim,
typename Number,
typename VectorizedArrayType>
2855 const unsigned int dof_handler_index,
2856 const unsigned int index_quad,
2857 const unsigned int index_fe,
2858 const unsigned int active_fe_index,
2859 const unsigned int active_quad_index)
const
2862 const unsigned int ind =
2863 dof_info[dof_handler_index].global_base_element_offset + index_fe;
2868 return shape_info(ind, index_quad, active_fe_index, active_quad_index);
2873 template <
int dim,
typename Number,
typename VectorizedArrayType>
2875 VectorizedArrayType::size()> &
2877 const unsigned int face_batch_index)
const
2885 template <
int dim,
typename Number,
typename VectorizedArrayType>
2895 template <
int dim,
typename Number,
typename VectorizedArrayType>
2898 const unsigned int quad_index,
2899 const unsigned int active_fe_index)
const
2903 .descriptor[active_fe_index]
2909 template <
int dim,
typename Number,
typename VectorizedArrayType>
2912 const unsigned int quad_index,
2913 const unsigned int active_fe_index)
const
2917 .descriptor[active_fe_index]
2923 template <
int dim,
typename Number,
typename VectorizedArrayType>
2926 const std::pair<unsigned int, unsigned int> range)
const
2930 for (
unsigned int i = range.first; i < range.second; ++i)
2938 template <
int dim,
typename Number,
typename VectorizedArrayType>
2939 inline std::pair<unsigned int, unsigned int>
2941 const std::pair<unsigned int, unsigned int> range)
const
2945 for (
unsigned int i = range.first; i < range.second; ++i)
2956 template <
int dim,
typename Number,
typename VectorizedArrayType>
2959 const unsigned int cell_batch_index)
const
2963 if (
dof_info[0].cell_active_fe_index.empty())
2966 return dof_info[0].cell_active_fe_index[cell_batch_index];
2971 template <
int dim,
typename Number,
typename VectorizedArrayType>
2972 inline std::pair<unsigned int, unsigned int>
2974 const unsigned int face_batch_index)
const
2977 if (
dof_info[0].cell_active_fe_index.empty())
2978 return std::make_pair(0
U, 0
U);
2980 std::pair<unsigned int, unsigned int> result = std::make_pair(0
U, 0
U);
2981 for (
unsigned int v = 0;
2982 v < VectorizedArrayType::size() &&
2989 .cells_interior[v] /
2990 VectorizedArrayType::size()]);
2993 for (
unsigned int v = 0;
2994 v < VectorizedArrayType::size() &&
3001 .cells_exterior[v] /
3002 VectorizedArrayType::size()]);
3010 template <
int dim,
typename Number,
typename VectorizedArrayType>
3019 template <
int dim,
typename Number,
typename VectorizedArrayType>
3027 template <
int dim,
typename Number,
typename VectorizedArrayType>
3036 template <
int dim,
typename Number,
typename VectorizedArrayType>
3041 std::list<std::pair<bool, AlignedVector<VectorizedArrayType>>>;
3043 for (
typename list_type::iterator it = data.begin(); it != data.end(); ++it)
3044 if (it->first ==
false)
3050 return &data.front().second;
3055 template <
int dim,
typename Number,
typename VectorizedArrayType>
3061 std::list<std::pair<bool, AlignedVector<VectorizedArrayType>>>;
3063 for (
typename list_type::iterator it = data.begin(); it != data.end(); ++it)
3064 if (&it->second == scratch)
3075 template <
int dim,
typename Number,
typename VectorizedArrayType>
3084 if (it->first ==
false)
3096 template <
int dim,
typename Number,
typename VectorizedArrayType>
3106 if (&it->second == scratch)
3121 namespace MatrixFreeImplementation
3123 template <
int dim,
int spacedim>
3124 inline std::vector<IndexSet>
3125 extract_locally_owned_index_sets(
3127 const unsigned int level)
3129 std::vector<IndexSet> locally_owned_set;
3130 locally_owned_set.reserve(dofh.size());
3131 for (
unsigned int j = 0; j < dofh.size(); ++j)
3133 locally_owned_set.push_back(dofh[j]->locally_owned_dofs());
3135 locally_owned_set.push_back(dofh[j]->locally_owned_mg_dofs(
level));
3136 return locally_owned_set;
3143 template <
int dim,
typename Number,
typename VectorizedArrayType>
3144 template <
typename QuadratureType,
typename number2,
typename MappingType>
3147 const MappingType & mapping,
3150 const QuadratureType & quad,
3154 std::vector<const DoFHandler<dim, dim> *>
dof_handlers;
3155 std::vector<const AffineConstraints<number2> *> constraints;
3158 constraints.push_back(&constraints_in);
3160 std::vector<IndexSet> locally_owned_sets =
3161 internal::MatrixFreeImplementation::extract_locally_owned_index_sets(
3164 std::vector<hp::QCollection<dim>> quad_hp;
3165 quad_hp.emplace_back(quad);
3177 template <
int dim,
typename Number,
typename VectorizedArrayType>
3178 template <
typename QuadratureType,
typename number2,
typename MappingType>
3181 const MappingType & mapping,
3184 const QuadratureType & quad,
3188 std::vector<IndexSet> locally_owned_set =
3189 internal::MatrixFreeImplementation::extract_locally_owned_index_sets(
3190 dof_handler, additional_data.
mg_level);
3191 std::vector<hp::QCollection<dim>> quad_hp;
3192 quad_hp.emplace_back(quad);
3204 template <
int dim,
typename Number,
typename VectorizedArrayType>
3205 template <
typename QuadratureType,
typename number2,
typename MappingType>
3208 const MappingType & mapping,
3211 const std::vector<QuadratureType> & quad,
3215 std::vector<IndexSet> locally_owned_set =
3216 internal::MatrixFreeImplementation::extract_locally_owned_index_sets(
3217 dof_handler, additional_data.
mg_level);
3218 std::vector<hp::QCollection<dim>> quad_hp;
3219 for (
unsigned int q = 0; q < quad.size(); ++q)
3220 quad_hp.emplace_back(quad[q]);
3247 template <
int dim,
typename Number,
typename VectorizedArrayType>
3248 struct VectorDataExchange
3255 static constexpr
unsigned int channel_shift = 20;
3264 const ::MatrixFree<dim, Number, VectorizedArrayType> &matrix_free,
3265 const typename ::MatrixFree<dim, Number, VectorizedArrayType>::
3266 DataAccessOnFaces vector_face_access,
3267 const unsigned int n_components)
3268 : matrix_free(matrix_free)
3269 , vector_face_access(
3270 matrix_free.get_task_info().face_partition_data.empty() ?
3271 ::
MatrixFree<dim, Number, VectorizedArrayType>::
3272 DataAccessOnFaces::unspecified :
3274 , ghosts_were_set(false)
3275 # ifdef DEAL_II_WITH_MPI
3276 , tmp_data(n_components)
3277 , requests(n_components)
3281 if (this->vector_face_access !=
3283 DataAccessOnFaces::unspecified)
3284 for (
unsigned int c = 0; c < matrix_free.n_components(); ++c)
3286 matrix_free.get_dof_info(c).vector_exchanger_face_variants.size(),
3295 ~VectorDataExchange()
3297 # ifdef DEAL_II_WITH_MPI
3298 for (
unsigned int i = 0; i < tmp_data.size(); ++i)
3299 if (tmp_data[i] !=
nullptr)
3300 matrix_free.release_scratch_data_non_threadsafe(tmp_data[i]);
3310 template <
typename VectorType>
3312 find_vector_in_mf(
const VectorType &vec,
3313 const bool check_global_compatibility =
true)
const
3316 for (
unsigned int c = 0; c < matrix_free.n_components(); ++c)
3317 if (vec.get_partitioner().get() ==
3318 matrix_free.get_dof_info(c).vector_partitioner.get())
3322 for (
unsigned int c = 0; c < matrix_free.n_components(); ++c)
3323 if (check_global_compatibility ?
3324 vec.get_partitioner()->is_globally_compatible(
3325 *matrix_free.get_dof_info(c).vector_partitioner) :
3326 vec.get_partitioner()->is_compatible(
3327 *matrix_free.get_dof_info(c).vector_partitioner))
3343 get_partitioner(
const unsigned int mf_component)
const
3346 .vector_exchanger_face_variants.size(),
3348 if (vector_face_access ==
3351 return *matrix_free.get_dof_info(mf_component)
3352 .vector_exchanger_face_variants[0];
3353 else if (vector_face_access ==
3356 return *matrix_free.get_dof_info(mf_component)
3357 .vector_exchanger_face_variants[1];
3358 else if (vector_face_access ==
3361 return *matrix_free.get_dof_info(mf_component)
3362 .vector_exchanger_face_variants[2];
3363 else if (vector_face_access ==
3365 DataAccessOnFaces::values_all_faces)
3366 return *matrix_free.get_dof_info(mf_component)
3367 .vector_exchanger_face_variants[3];
3368 else if (vector_face_access ==
3370 DataAccessOnFaces::gradients_all_faces)
3371 return *matrix_free.get_dof_info(mf_component)
3372 .vector_exchanger_face_variants[4];
3374 return *matrix_free.get_dof_info(mf_component).vector_exchanger.get();
3382 template <
typename VectorType,
3383 std::enable_if_t<is_not_parallel_vector<VectorType>, VectorType>
3386 update_ghost_values_start(
const unsigned int ,
3387 const VectorType & )
3395 template <
typename VectorType,
3396 std::enable_if_t<!has_update_ghost_values_start<VectorType> &&
3397 !is_not_parallel_vector<VectorType>,
3398 VectorType> * =
nullptr>
3400 update_ghost_values_start(
const unsigned int component_in_block_vector,
3401 const VectorType & vec)
3403 (void)component_in_block_vector;
3404 bool ghosts_set = vec.has_ghost_elements();
3406 Assert(matrix_free.get_task_info().allow_ghosted_vectors_in_loops ||
3407 ghosts_set ==
false,
3411 ghosts_were_set =
true;
3413 vec.update_ghost_values();
3423 template <
typename VectorType,
3424 std::enable_if_t<has_update_ghost_values_start<VectorType> &&
3425 !has_exchange_on_subset<VectorType>,
3426 VectorType> * =
nullptr>
3428 update_ghost_values_start(
const unsigned int component_in_block_vector,
3429 const VectorType & vec)
3431 (void)component_in_block_vector;
3432 bool ghosts_set = vec.has_ghost_elements();
3434 Assert(matrix_free.get_task_info().allow_ghosted_vectors_in_loops ||
3435 ghosts_set ==
false,
3439 ghosts_were_set =
true;
3441 vec.update_ghost_values_start(component_in_block_vector + channel_shift);
3452 template <
typename VectorType,
3453 std::enable_if_t<has_update_ghost_values_start<VectorType> &&
3454 has_exchange_on_subset<VectorType>,
3455 VectorType> * =
nullptr>
3457 update_ghost_values_start(
const unsigned int component_in_block_vector,
3458 const VectorType & vec)
3461 std::is_same<Number, typename VectorType::value_type>::value,
3462 "Type mismatch between VectorType and VectorDataExchange");
3463 (void)component_in_block_vector;
3464 bool ghosts_set = vec.has_ghost_elements();
3466 Assert(matrix_free.get_task_info().allow_ghosted_vectors_in_loops ||
3467 ghosts_set ==
false,
3471 ghosts_were_set =
true;
3473 if (vec.size() != 0)
3475 # ifdef DEAL_II_WITH_MPI
3476 const unsigned int mf_component = find_vector_in_mf(vec);
3478 const auto &part = get_partitioner(mf_component);
3480 if (part.n_ghost_indices() == 0 && part.n_import_indices() == 0 &&
3481 part.n_import_sm_procs() == 0)
3484 tmp_data[component_in_block_vector] =
3485 matrix_free.acquire_scratch_data_non_threadsafe();
3486 tmp_data[component_in_block_vector]->resize_fast(
3487 part.n_import_indices());
3490 part.export_to_ghosted_array_start(
3491 component_in_block_vector * 2 + channel_shift,
3493 vec.shared_vector_data(),
3495 part.locally_owned_size(),
3496 matrix_free.get_dof_info(mf_component)
3497 .vector_partitioner->n_ghost_indices()),
3499 part.n_import_indices()),
3500 this->requests[component_in_block_vector]);
3511 template <
typename VectorType,
3512 std::enable_if_t<!has_update_ghost_values_start<VectorType>,
3513 VectorType> * =
nullptr>
3515 update_ghost_values_finish(
const unsigned int ,
3516 const VectorType & )
3526 template <
typename VectorType,
3527 std::enable_if_t<has_update_ghost_values_start<VectorType> &&
3528 !has_exchange_on_subset<VectorType>,
3529 VectorType> * =
nullptr>
3531 update_ghost_values_finish(
const unsigned int component_in_block_vector,
3532 const VectorType & vec)
3534 (void)component_in_block_vector;
3535 vec.update_ghost_values_finish();
3546 template <
typename VectorType,
3547 std::enable_if_t<has_update_ghost_values_start<VectorType> &&
3548 has_exchange_on_subset<VectorType>,
3549 VectorType> * =
nullptr>
3551 update_ghost_values_finish(
const unsigned int component_in_block_vector,
3552 const VectorType & vec)
3555 std::is_same<Number, typename VectorType::value_type>::value,
3556 "Type mismatch between VectorType and VectorDataExchange");
3557 (void)component_in_block_vector;
3559 if (vec.size() != 0)
3561 # ifdef DEAL_II_WITH_MPI
3565 const unsigned int mf_component = find_vector_in_mf(vec);
3567 const auto &part = get_partitioner(mf_component);
3569 if (part.n_ghost_indices() != 0 || part.n_import_indices() != 0 ||
3570 part.n_import_sm_procs() != 0)
3572 part.export_to_ghosted_array_finish(
3574 vec.shared_vector_data(),
3576 part.locally_owned_size(),
3577 matrix_free.get_dof_info(mf_component)
3578 .vector_partitioner->n_ghost_indices()),
3579 this->requests[component_in_block_vector]);
3581 matrix_free.release_scratch_data_non_threadsafe(
3582 tmp_data[component_in_block_vector]);
3583 tmp_data[component_in_block_vector] =
nullptr;
3589 vec.set_ghost_state(
true);
3597 template <
typename VectorType,
3598 std::enable_if_t<is_not_parallel_vector<VectorType>, VectorType>
3601 compress_start(
const unsigned int ,
3611 template <
typename VectorType,
3612 std::enable_if_t<!has_compress_start<VectorType> &&
3613 !is_not_parallel_vector<VectorType>,
3614 VectorType> * =
nullptr>
3616 compress_start(
const unsigned int component_in_block_vector,
3619 (void)component_in_block_vector;
3631 template <
typename VectorType,
3632 std::enable_if_t<has_compress_start<VectorType> &&
3633 !has_exchange_on_subset<VectorType>,
3634 VectorType> * =
nullptr>
3636 compress_start(
const unsigned int component_in_block_vector,
3639 (void)component_in_block_vector;
3641 vec.compress_start(component_in_block_vector + channel_shift);
3652 template <
typename VectorType,
3653 std::enable_if_t<has_compress_start<VectorType> &&
3654 has_exchange_on_subset<VectorType>,
3655 VectorType> * =
nullptr>
3657 compress_start(
const unsigned int component_in_block_vector,
3661 std::is_same<Number, typename VectorType::value_type>::value,
3662 "Type mismatch between VectorType and VectorDataExchange");
3663 (void)component_in_block_vector;
3666 if (vec.size() != 0)
3668 # ifdef DEAL_II_WITH_MPI
3669 const unsigned int mf_component = find_vector_in_mf(vec);
3671 const auto &part = get_partitioner(mf_component);
3673 if (part.n_ghost_indices() == 0 && part.n_import_indices() == 0 &&
3674 part.n_import_sm_procs() == 0)
3677 tmp_data[component_in_block_vector] =
3678 matrix_free.acquire_scratch_data_non_threadsafe();
3679 tmp_data[component_in_block_vector]->resize_fast(
3680 part.n_import_indices());
3683 part.import_from_ghosted_array_start(
3685 component_in_block_vector * 2 + channel_shift,
3687 vec.shared_vector_data(),
3689 matrix_free.get_dof_info(mf_component)
3690 .vector_partitioner->n_ghost_indices()),
3692 part.n_import_indices()),
3693 this->requests[component_in_block_vector]);
3705 typename VectorType,
3706 std::enable_if_t<!has_compress_start<VectorType>, VectorType> * =
nullptr>
3708 compress_finish(
const unsigned int ,
3719 template <
typename VectorType,
3720 std::enable_if_t<has_compress_start<VectorType> &&
3721 !has_exchange_on_subset<VectorType>,
3722 VectorType> * =
nullptr>
3724 compress_finish(
const unsigned int component_in_block_vector,
3727 (void)component_in_block_vector;
3739 template <
typename VectorType,
3740 std::enable_if_t<has_compress_start<VectorType> &&
3741 has_exchange_on_subset<VectorType>,
3742 VectorType> * =
nullptr>
3744 compress_finish(
const unsigned int component_in_block_vector,
3748 std::is_same<Number, typename VectorType::value_type>::value,
3749 "Type mismatch between VectorType and VectorDataExchange");
3750 (void)component_in_block_vector;
3751 if (vec.size() != 0)
3753 # ifdef DEAL_II_WITH_MPI
3757 const unsigned int mf_component = find_vector_in_mf(vec);
3759 const auto &part = get_partitioner(mf_component);
3761 if (part.n_ghost_indices() != 0 || part.n_import_indices() != 0 ||
3762 part.n_import_sm_procs() != 0)
3764 part.import_from_ghosted_array_finish(
3767 vec.shared_vector_data(),
3769 matrix_free.get_dof_info(mf_component)
3770 .vector_partitioner->n_ghost_indices()),
3772 tmp_data[component_in_block_vector]->begin(),
3773 part.n_import_indices()),
3774 this->requests[component_in_block_vector]);
3776 matrix_free.release_scratch_data_non_threadsafe(
3777 tmp_data[component_in_block_vector]);
3778 tmp_data[component_in_block_vector] =
nullptr;
3784 MPI_Barrier(matrix_free.get_task_info().communicator_sm);
3796 template <
typename VectorType,
3797 std::enable_if_t<is_not_parallel_vector<VectorType>, VectorType>
3800 reset_ghost_values(
const VectorType & )
const
3809 template <
typename VectorType,
3810 std::enable_if_t<!has_exchange_on_subset<VectorType> &&
3811 !is_not_parallel_vector<VectorType>,
3812 VectorType> * =
nullptr>
3814 reset_ghost_values(
const VectorType &vec)
const
3816 if (ghosts_were_set ==
true)
3819 vec.zero_out_ghost_values();
3829 template <
typename VectorType,
3830 std::enable_if_t<has_exchange_on_subset<VectorType>, VectorType>
3833 reset_ghost_values(
const VectorType &vec)
const
3836 std::is_same<Number, typename VectorType::value_type>::value,
3837 "Type mismatch between VectorType and VectorDataExchange");
3838 if (ghosts_were_set ==
true)
3841 if (vec.size() != 0)
3843 # ifdef DEAL_II_WITH_MPI
3846 const unsigned int mf_component = find_vector_in_mf(vec);
3848 const auto &part = get_partitioner(mf_component);
3850 if (part.n_ghost_indices() > 0)
3855 part.locally_owned_size(),
3856 matrix_free.get_dof_info(mf_component)
3857 .vector_partitioner->n_ghost_indices()));
3863 vec.set_ghost_state(
false);
3873 template <
typename VectorType,
3874 std::enable_if_t<has_exchange_on_subset<VectorType>, VectorType>
3877 zero_vector_region(
const unsigned int range_index, VectorType &vec)
const
3880 std::is_same<Number, typename VectorType::value_type>::value,
3881 "Type mismatch between VectorType and VectorDataExchange");
3886 const unsigned int mf_component = find_vector_in_mf(vec,
false);
3888 matrix_free.get_dof_info(mf_component);
3896 for (
unsigned int id =
3915 template <
typename VectorType,
3916 std::enable_if_t<!has_exchange_on_subset<VectorType>, VectorType>
3918 typename VectorType::value_type * =
nullptr>
3920 zero_vector_region(
const unsigned int range_index, VectorType &vec)
const
3923 vec =
typename VectorType::value_type();
3933 zero_vector_region(
const unsigned int, ...)
const
3937 "which provide VectorType::value_type"));
3942 const ::MatrixFree<dim, Number, VectorizedArrayType> &matrix_free;
3943 const typename ::MatrixFree<dim, Number, VectorizedArrayType>::
3944 DataAccessOnFaces vector_face_access;
3945 bool ghosts_were_set;
3946 # ifdef DEAL_II_WITH_MPI
3947 std::vector<AlignedVector<Number> *> tmp_data;
3948 std::vector<std::vector<MPI_Request>> requests;
3952 template <
typename VectorStruct>
3954 n_components(
const VectorStruct &vec);
3956 template <
typename VectorStruct>
3958 n_components_block(
const VectorStruct &vec,
3959 std::integral_constant<bool, true>)
3961 unsigned int components = 0;
3962 for (
unsigned int bl = 0; bl < vec.n_blocks(); ++bl)
3963 components += n_components(vec.block(bl));
3967 template <
typename VectorStruct>
3969 n_components_block(
const VectorStruct &, std::integral_constant<bool, false>)
3974 template <
typename VectorStruct>
3976 n_components(
const VectorStruct &vec)
3978 return n_components_block(
3982 template <
typename VectorStruct>
3984 n_components(
const std::vector<VectorStruct> &vec)
3986 unsigned int components = 0;
3987 for (
unsigned int comp = 0; comp < vec.size(); ++comp)
3988 components += n_components_block(
3994 template <
typename VectorStruct>
3996 n_components(
const std::vector<VectorStruct *> &vec)
3998 unsigned int components = 0;
3999 for (
unsigned int comp = 0; comp < vec.size(); ++comp)
4000 components += n_components_block(
4013 template <
typename VectorStruct,
4014 std::enable_if_t<!has_communication_block_size<VectorStruct>,
4015 VectorStruct> * =
nullptr>
4016 constexpr
unsigned int
4017 get_communication_block_size(
const VectorStruct &)
4024 template <
typename VectorStruct,
4025 std::enable_if_t<has_communication_block_size<VectorStruct>,
4026 VectorStruct> * =
nullptr>
4027 constexpr
unsigned int
4028 get_communication_block_size(
const VectorStruct &)
4030 return VectorStruct::communication_block_size;
4035 template <
typename VectorType,
4036 std::enable_if_t<is_not_parallel_vector<VectorType>, VectorType> * =
4039 has_ghost_elements(
const VectorType &vec)
4047 template <
typename VectorType,
4048 std::enable_if_t<!is_not_parallel_vector<VectorType>, VectorType>
4051 has_ghost_elements(
const VectorType &vec)
4053 return vec.has_ghost_elements();
4068 typename VectorStruct,
4070 typename VectorizedArrayType,
4071 std::enable_if_t<IsBlockVector<VectorStruct>::value, VectorStruct>
4074 update_ghost_values_start(
4075 const VectorStruct & vec,
4076 VectorDataExchange<dim, Number, VectorizedArrayType> &exchanger,
4077 const unsigned int channel = 0)
4079 if (get_communication_block_size(vec) < vec.n_blocks())
4083 exchanger.ghosts_were_set = vec.has_ghost_elements();
4084 vec.update_ghost_values();
4088 for (
unsigned int i = 0; i < vec.n_blocks(); ++i)
4089 update_ghost_values_start(vec.block(i), exchanger, channel + i);
4097 typename VectorStruct,
4099 typename VectorizedArrayType,
4100 std::enable_if_t<!IsBlockVector<VectorStruct>::value, VectorStruct>
4103 update_ghost_values_start(
4104 const VectorStruct & vec,
4105 VectorDataExchange<dim, Number, VectorizedArrayType> &exchanger,
4106 const unsigned int channel = 0)
4108 exchanger.update_ghost_values_start(channel, vec);
4115 typename VectorStruct,
4117 typename VectorizedArrayType>
4119 update_ghost_values_start(
4120 const std::vector<VectorStruct> & vec,
4121 VectorDataExchange<dim, Number, VectorizedArrayType> &exchanger)
4123 unsigned int component_index = 0;
4124 for (
unsigned int comp = 0; comp < vec.size(); ++comp)
4126 update_ghost_values_start(vec[comp], exchanger, component_index);
4127 component_index += n_components(vec[comp]);
4135 typename VectorStruct,
4137 typename VectorizedArrayType>
4139 update_ghost_values_start(
4140 const std::vector<VectorStruct *> & vec,
4141 VectorDataExchange<dim, Number, VectorizedArrayType> &exchanger)
4143 unsigned int component_index = 0;
4144 for (
unsigned int comp = 0; comp < vec.size(); ++comp)
4146 update_ghost_values_start(*vec[comp], exchanger, component_index);
4147 component_index += n_components(*vec[comp]);
4159 typename VectorStruct,
4161 typename VectorizedArrayType,
4162 std::enable_if_t<IsBlockVector<VectorStruct>::value, VectorStruct>
4165 update_ghost_values_finish(
4166 const VectorStruct & vec,
4167 VectorDataExchange<dim, Number, VectorizedArrayType> &exchanger,
4168 const unsigned int channel = 0)
4170 if (get_communication_block_size(vec) < vec.n_blocks())
4176 for (
unsigned int i = 0; i < vec.n_blocks(); ++i)
4177 update_ghost_values_finish(vec.block(i), exchanger, channel + i);
4184 typename VectorStruct,
4186 typename VectorizedArrayType,
4187 std::enable_if_t<!IsBlockVector<VectorStruct>::value, VectorStruct>
4190 update_ghost_values_finish(
4191 const VectorStruct & vec,
4192 VectorDataExchange<dim, Number, VectorizedArrayType> &exchanger,
4193 const unsigned int channel = 0)
4195 exchanger.update_ghost_values_finish(channel, vec);
4202 typename VectorStruct,
4204 typename VectorizedArrayType>
4206 update_ghost_values_finish(
4207 const std::vector<VectorStruct> & vec,
4208 VectorDataExchange<dim, Number, VectorizedArrayType> &exchanger)
4210 unsigned int component_index = 0;
4211 for (
unsigned int comp = 0; comp < vec.size(); ++comp)
4213 update_ghost_values_finish(vec[comp], exchanger, component_index);
4214 component_index += n_components(vec[comp]);
4222 typename VectorStruct,
4224 typename VectorizedArrayType>
4226 update_ghost_values_finish(
4227 const std::vector<VectorStruct *> & vec,
4228 VectorDataExchange<dim, Number, VectorizedArrayType> &exchanger)
4230 unsigned int component_index = 0;
4231 for (
unsigned int comp = 0; comp < vec.size(); ++comp)
4233 update_ghost_values_finish(*vec[comp], exchanger, component_index);
4234 component_index += n_components(*vec[comp]);
4246 typename VectorStruct,
4248 typename VectorizedArrayType,
4249 std::enable_if_t<IsBlockVector<VectorStruct>::value, VectorStruct>
4254 VectorDataExchange<dim, Number, VectorizedArrayType> &exchanger,
4255 const unsigned int channel = 0)
4257 if (get_communication_block_size(vec) < vec.n_blocks())
4260 for (
unsigned int i = 0; i < vec.n_blocks(); ++i)
4261 compress_start(vec.block(i), exchanger, channel + i);
4268 typename VectorStruct,
4270 typename VectorizedArrayType,
4271 std::enable_if_t<!IsBlockVector<VectorStruct>::value, VectorStruct>
4276 VectorDataExchange<dim, Number, VectorizedArrayType> &exchanger,
4277 const unsigned int channel = 0)
4279 exchanger.compress_start(channel, vec);
4286 typename VectorStruct,
4288 typename VectorizedArrayType>
4291 std::vector<VectorStruct> & vec,
4292 VectorDataExchange<dim, Number, VectorizedArrayType> &exchanger)
4294 unsigned int component_index = 0;
4295 for (
unsigned int comp = 0; comp < vec.size(); ++comp)
4297 compress_start(vec[comp], exchanger, component_index);
4298 component_index += n_components(vec[comp]);
4306 typename VectorStruct,
4308 typename VectorizedArrayType>
4311 std::vector<VectorStruct *> & vec,
4312 VectorDataExchange<dim, Number, VectorizedArrayType> &exchanger)
4314 unsigned int component_index = 0;
4315 for (
unsigned int comp = 0; comp < vec.size(); ++comp)
4317 compress_start(*vec[comp], exchanger, component_index);
4318 component_index += n_components(*vec[comp]);
4330 typename VectorStruct,
4332 typename VectorizedArrayType,
4333 std::enable_if_t<IsBlockVector<VectorStruct>::value, VectorStruct>
4338 VectorDataExchange<dim, Number, VectorizedArrayType> &exchanger,
4339 const unsigned int channel = 0)
4341 if (get_communication_block_size(vec) < vec.n_blocks())
4347 for (
unsigned int i = 0; i < vec.n_blocks(); ++i)
4348 compress_finish(vec.block(i), exchanger, channel + i);
4355 typename VectorStruct,
4357 typename VectorizedArrayType,
4358 std::enable_if_t<!IsBlockVector<VectorStruct>::value, VectorStruct>
4363 VectorDataExchange<dim, Number, VectorizedArrayType> &exchanger,
4364 const unsigned int channel = 0)
4366 exchanger.compress_finish(channel, vec);
4373 typename VectorStruct,
4375 typename VectorizedArrayType>
4378 std::vector<VectorStruct> & vec,
4379 VectorDataExchange<dim, Number, VectorizedArrayType> &exchanger)
4381 unsigned int component_index = 0;
4382 for (
unsigned int comp = 0; comp < vec.size(); ++comp)
4384 compress_finish(vec[comp], exchanger, component_index);
4385 component_index += n_components(vec[comp]);
4393 typename VectorStruct,
4395 typename VectorizedArrayType>
4398 std::vector<VectorStruct *> & vec,
4399 VectorDataExchange<dim, Number, VectorizedArrayType> &exchanger)
4401 unsigned int component_index = 0;
4402 for (
unsigned int comp = 0; comp < vec.size(); ++comp)
4404 compress_finish(*vec[comp], exchanger, component_index);
4405 component_index += n_components(*vec[comp]);
4421 typename VectorStruct,
4423 typename VectorizedArrayType,
4424 std::enable_if_t<IsBlockVector<VectorStruct>::value, VectorStruct>
4428 const VectorStruct & vec,
4429 VectorDataExchange<dim, Number, VectorizedArrayType> &exchanger)
4432 if (exchanger.ghosts_were_set ==
true)
4435 for (
unsigned int i = 0; i < vec.n_blocks(); ++i)
4436 reset_ghost_values(vec.block(i), exchanger);
4443 typename VectorStruct,
4445 typename VectorizedArrayType,
4446 std::enable_if_t<!IsBlockVector<VectorStruct>::value, VectorStruct>
4450 const VectorStruct & vec,
4451 VectorDataExchange<dim, Number, VectorizedArrayType> &exchanger)
4453 exchanger.reset_ghost_values(vec);
4460 typename VectorStruct,
4462 typename VectorizedArrayType>
4465 const std::vector<VectorStruct> & vec,
4466 VectorDataExchange<dim, Number, VectorizedArrayType> &exchanger)
4469 if (exchanger.ghosts_were_set ==
true)
4472 for (
unsigned int comp = 0; comp < vec.size(); ++comp)
4473 reset_ghost_values(vec[comp], exchanger);
4480 typename VectorStruct,
4482 typename VectorizedArrayType>
4485 const std::vector<VectorStruct *> & vec,
4486 VectorDataExchange<dim, Number, VectorizedArrayType> &exchanger)
4489 if (exchanger.ghosts_were_set ==
true)
4492 for (
unsigned int comp = 0; comp < vec.size(); ++comp)
4493 reset_ghost_values(*vec[comp], exchanger);
4504 typename VectorStruct,
4506 typename VectorizedArrayType,
4507 std::enable_if_t<IsBlockVector<VectorStruct>::value, VectorStruct>
4511 const unsigned int range_index,
4513 VectorDataExchange<dim, Number, VectorizedArrayType> &exchanger)
4515 for (
unsigned int i = 0; i < vec.n_blocks(); ++i)
4516 exchanger.zero_vector_region(range_index, vec.block(i));
4523 typename VectorStruct,
4525 typename VectorizedArrayType,
4526 std::enable_if_t<!IsBlockVector<VectorStruct>::value, VectorStruct>
4530 const unsigned int range_index,
4532 VectorDataExchange<dim, Number, VectorizedArrayType> &exchanger)
4534 exchanger.zero_vector_region(range_index, vec);
4541 typename VectorStruct,
4543 typename VectorizedArrayType>
4546 const unsigned int range_index,
4547 std::vector<VectorStruct> & vec,
4548 VectorDataExchange<dim, Number, VectorizedArrayType> &exchanger)
4550 for (
unsigned int comp = 0; comp < vec.size(); ++comp)
4551 zero_vector_region(range_index, vec[comp], exchanger);
4558 typename VectorStruct,
4560 typename VectorizedArrayType>
4563 const unsigned int range_index,
4564 std::vector<VectorStruct *> & vec,
4565 VectorDataExchange<dim, Number, VectorizedArrayType> &exchanger)
4567 for (
unsigned int comp = 0; comp < vec.size(); ++comp)
4568 zero_vector_region(range_index, *vec[comp], exchanger);
4576 template <
typename VectorStruct1,
typename VectorStruct2>
4578 apply_operation_to_constrained_dofs(
const std::vector<unsigned int> &,
4579 const VectorStruct1 &,
4583 template <
typename Number>
4585 apply_operation_to_constrained_dofs(
4586 const std::vector<unsigned int> & constrained_dofs,
4590 for (
const unsigned int i : constrained_dofs)
4595 namespace MatrixFreeFunctions
4599 template <
typename,
typename,
typename,
typename,
bool>
4600 struct InterfaceSelector
4604 template <
typename MF,
4608 struct InterfaceSelector<MF, InVector, OutVector, Container, true>
4610 using function_type = void (Container::*)(
4614 const std::pair<unsigned int, unsigned int> &)
const;
4618 template <
typename MF,
4622 struct InterfaceSelector<MF, InVector, OutVector, Container, false>
4624 using function_type =
4625 void (Container::*)(
const MF &,
4628 const std::pair<unsigned int, unsigned int> &);
4636 template <
typename MF,
4641 class MFWorker :
public MFWorkerInterface
4645 using function_type =
typename MatrixFreeFunctions::
4646 InterfaceSelector<MF, InVector, OutVector, Container, is_constant>::
4650 MFWorker(
const MF & matrix_free,
4651 const InVector & src,
4653 const bool zero_dst_vector_setting,
4654 const Container & container,
4655 function_type cell_function,
4656 function_type face_function,
4657 function_type boundary_function,
4658 const typename MF::DataAccessOnFaces src_vector_face_access =
4660 const typename MF::DataAccessOnFaces dst_vector_face_access =
4662 const std::function<
void(
const unsigned int,
const unsigned int)>
4663 &operation_before_loop = {},
4664 const std::function<void(
const unsigned int,
const unsigned int)>
4665 & operation_after_loop = {},
4666 const unsigned int dof_handler_index_pre_post = 0)
4667 : matrix_free(matrix_free)
4668 , container(
const_cast<Container &
>(container))
4669 , cell_function(cell_function)
4670 , face_function(face_function)
4671 , boundary_function(boundary_function)
4674 , src_data_exchanger(matrix_free,
4675 src_vector_face_access,
4677 , dst_data_exchanger(matrix_free,
4678 dst_vector_face_access,
4681 , zero_dst_vector_setting(zero_dst_vector_setting &&
4682 !src_and_dst_are_same)
4683 , operation_before_loop(operation_before_loop)
4684 , operation_after_loop(operation_after_loop)
4685 , dof_handler_index_pre_post(dof_handler_index_pre_post)
4687 Assert(!has_ghost_elements(dst),
4688 ExcMessage(
"The destination vector passed to the matrix-free "
4689 "loop is ghosted. This is not allowed."));
4694 cell(
const std::pair<unsigned int, unsigned int> &cell_range)
override
4696 if (cell_function !=
nullptr && cell_range.second > cell_range.first)
4697 for (
unsigned int i = 0; i < matrix_free.n_active_fe_indices(); ++i)
4699 const auto cell_subrange =
4700 matrix_free.create_cell_subrange_hp_by_index(cell_range, i);
4702 if (cell_subrange.second <= cell_subrange.first)
4706 cell_function)(matrix_free, this->dst, this->src, cell_subrange);
4711 cell(
const unsigned int range_index)
override
4713 process_range(cell_function,
4714 matrix_free.get_task_info().cell_partition_data_hp_ptr,
4715 matrix_free.get_task_info().cell_partition_data_hp,
4720 face(
const unsigned int range_index)
override
4722 process_range(face_function,
4723 matrix_free.get_task_info().face_partition_data_hp_ptr,
4724 matrix_free.get_task_info().face_partition_data_hp,
4729 boundary(
const unsigned int range_index)
override
4731 process_range(boundary_function,
4732 matrix_free.get_task_info().boundary_partition_data_hp_ptr,
4733 matrix_free.get_task_info().boundary_partition_data_hp,
4739 process_range(
const function_type & fu,
4740 const std::vector<unsigned int> &ptr,
4741 const std::vector<unsigned int> &data,
4742 const unsigned int range_index)
4748 for (
unsigned int i = ptr[range_index]; i < ptr[range_index + 1]; ++i)
4751 (container.*fu)(matrix_free,
4754 std::make_pair(data[2 * i], data[2 * i + 1]));
4766 vector_update_ghosts_start()
override
4768 if (!src_and_dst_are_same)
4769 internal::update_ghost_values_start(src, src_data_exchanger);
4774 vector_update_ghosts_finish()
override
4776 if (!src_and_dst_are_same)
4777 internal::update_ghost_values_finish(src, src_data_exchanger);
4782 vector_compress_start()
override
4784 internal::compress_start(dst, dst_data_exchanger);
4789 vector_compress_finish()
override
4791 internal::compress_finish(dst, dst_data_exchanger);
4792 if (!src_and_dst_are_same)
4793 internal::reset_ghost_values(src, src_data_exchanger);
4798 zero_dst_vector_range(
const unsigned int range_index)
override
4800 if (zero_dst_vector_setting)
4801 internal::zero_vector_region(range_index, dst, dst_data_exchanger);
4805 cell_loop_pre_range(
const unsigned int range_index)
override
4807 if (operation_before_loop)
4810 matrix_free.get_dof_info(dof_handler_index_pre_post);
4817 operation_before_loop,
4824 for (
unsigned int id =
4835 cell_loop_post_range(
const unsigned int range_index)
override
4837 if (operation_after_loop)
4841 const std::vector<unsigned int> &partition_row_index =
4842 matrix_free.get_task_info().partition_row_index;
4844 partition_row_index[partition_row_index.size() - 2] - 1)
4845 apply_operation_to_constrained_dofs(
4846 matrix_free.get_constrained_dofs(dof_handler_index_pre_post),
4851 matrix_free.get_dof_info(dof_handler_index_pre_post);
4858 operation_after_loop,
4865 for (
unsigned int id =
4876 const MF & matrix_free;
4877 Container & container;
4878 function_type cell_function;
4879 function_type face_function;
4880 function_type boundary_function;
4882 const InVector &src;
4884 VectorDataExchange<MF::dimension,
4885 typename MF::value_type,
4886 typename MF::vectorized_value_type>
4888 VectorDataExchange<MF::dimension,
4889 typename MF::value_type,
4890 typename MF::vectorized_value_type>
4892 const bool src_and_dst_are_same;
4893 const bool zero_dst_vector_setting;
4894 const std::function<void(
const unsigned int,
const unsigned int)>
4895 operation_before_loop;
4896 const std::function<void(
const unsigned int,
const unsigned int)>
4897 operation_after_loop;
4898 const unsigned int dof_handler_index_pre_post;
4907 template <
class MF,
typename InVector,
typename OutVector>
4908 struct MFClassWrapper
4910 using function_type =
4911 std::function<void(
const MF &,
4914 const std::pair<unsigned int, unsigned int> &)>;
4916 MFClassWrapper(
const function_type cell,
4917 const function_type face,
4918 const function_type boundary)
4921 , boundary(boundary)
4925 cell_integrator(
const MF & mf,
4927 const InVector & src,
4928 const std::pair<unsigned int, unsigned int> &range)
const
4931 cell(mf, dst, src, range);
4935 face_integrator(
const MF & mf,
4937 const InVector & src,
4938 const std::pair<unsigned int, unsigned int> &range)
const
4941 face(mf, dst, src, range);
4945 boundary_integrator(
4948 const InVector & src,
4949 const std::pair<unsigned int, unsigned int> &range)
const
4952 boundary(mf, dst, src, range);
4955 const function_type cell;
4956 const function_type face;
4957 const function_type boundary;
4964 template <
int dim,
typename Number,
typename VectorizedArrayType>
4965 template <
typename OutVector,
typename InVector>
4971 const std::pair<unsigned int, unsigned int> &)>
4974 const InVector &src,
4975 const bool zero_dst_vector)
const
4978 internal::MFClassWrapper<MatrixFree<dim, Number, VectorizedArrayType>,
4981 Wrapper wrap(cell_operation,
nullptr,
nullptr);
4982 internal::MFWorker<MatrixFree<dim, Number, VectorizedArrayType>,
4992 &Wrapper::cell_integrator,
4993 &Wrapper::face_integrator,
4994 &Wrapper::boundary_integrator);
5001 template <
int dim,
typename Number,
typename VectorizedArrayType>
5002 template <
typename OutVector,
typename InVector>
5008 const std::pair<unsigned int, unsigned int> &)>
5011 const InVector &src,
5012 const std::function<
void(
const unsigned int,
const unsigned int)>
5013 &operation_before_loop,
5014 const std::function<
void(
const unsigned int,
const unsigned int)>
5015 & operation_after_loop,
5016 const unsigned int dof_handler_index_pre_post)
const
5019 internal::MFClassWrapper<MatrixFree<dim, Number, VectorizedArrayType>,
5022 Wrapper wrap(cell_operation,
nullptr,
nullptr);
5023 internal::MFWorker<MatrixFree<dim, Number, VectorizedArrayType>,
5033 &Wrapper::cell_integrator,
5034 &Wrapper::face_integrator,
5035 &Wrapper::boundary_integrator,
5038 operation_before_loop,
5039 operation_after_loop,
5040 dof_handler_index_pre_post);
5047 template <
int dim,
typename Number,
typename VectorizedArrayType>
5048 template <
typename OutVector,
typename InVector>
5054 const std::pair<unsigned int, unsigned int> &)>
5059 const std::pair<unsigned int, unsigned int> &)>
5064 const std::pair<unsigned int, unsigned int> &)>
5065 & boundary_operation,
5067 const InVector & src,
5068 const bool zero_dst_vector,
5069 const DataAccessOnFaces dst_vector_face_access,
5070 const DataAccessOnFaces src_vector_face_access)
const
5073 internal::MFClassWrapper<MatrixFree<dim, Number, VectorizedArrayType>,
5076 Wrapper wrap(cell_operation, face_operation, boundary_operation);
5077 internal::MFWorker<MatrixFree<dim, Number, VectorizedArrayType>,
5087 &Wrapper::cell_integrator,
5088 &Wrapper::face_integrator,
5089 &Wrapper::boundary_integrator,
5090 src_vector_face_access,
5091 dst_vector_face_access);
5098 template <
int dim,
typename Number,
typename VectorizedArrayType>
5099 template <
typename CLASS,
typename OutVector,
typename InVector>
5102 void (CLASS::*function_pointer)(
5106 const std::pair<unsigned int, unsigned int> &)
const,
5107 const CLASS * owning_class,
5109 const InVector &src,
5110 const bool zero_dst_vector)
const
5112 internal::MFWorker<MatrixFree<dim, Number, VectorizedArrayType>,
5130 template <
int dim,
typename Number,
typename VectorizedArrayType>
5131 template <
typename CLASS,
typename OutVector,
typename InVector>
5134 void (CLASS::*function_pointer)(
5138 const std::pair<unsigned int, unsigned int> &)
const,
5139 const CLASS * owning_class,
5141 const InVector &src,
5142 const std::function<
void(
const unsigned int,
const unsigned int)>
5143 &operation_before_loop,
5144 const std::function<
void(
const unsigned int,
const unsigned int)>
5145 & operation_after_loop,
5146 const unsigned int dof_handler_index_pre_post)
const
5148 internal::MFWorker<MatrixFree<dim, Number, VectorizedArrayType>,
5163 operation_before_loop,
5164 operation_after_loop,
5165 dof_handler_index_pre_post);
5171 template <
int dim,
typename Number,
typename VectorizedArrayType>
5172 template <
typename CLASS,
typename OutVector,
typename InVector>
5175 void (CLASS::*cell_operation)(
5179 const std::pair<unsigned int, unsigned int> &)
const,
5180 void (CLASS::*face_operation)(
5184 const std::pair<unsigned int, unsigned int> &)
const,
5185 void (CLASS::*boundary_operation)(
5189 const std::pair<unsigned int, unsigned int> &)
const,
5190 const CLASS * owning_class,
5192 const InVector & src,
5193 const bool zero_dst_vector,
5194 const DataAccessOnFaces dst_vector_face_access,
5195 const DataAccessOnFaces src_vector_face_access)
const
5197 internal::MFWorker<MatrixFree<dim, Number, VectorizedArrayType>,
5210 src_vector_face_access,
5211 dst_vector_face_access);
5217 template <
int dim,
typename Number,
typename VectorizedArrayType>
5218 template <
typename CLASS,
typename OutVector,
typename InVector>
5221 void (CLASS::*function_pointer)(
5225 const std::pair<unsigned int, unsigned int> &),
5226 CLASS * owning_class,
5228 const InVector &src,
5229 const bool zero_dst_vector)
const
5231 internal::MFWorker<MatrixFree<dim, Number, VectorizedArrayType>,
5249 template <
int dim,
typename Number,
typename VectorizedArrayType>
5250 template <
typename CLASS,
typename OutVector,
typename InVector>
5253 void (CLASS::*function_pointer)(
5257 const std::pair<unsigned int, unsigned int> &),
5258 CLASS * owning_class,
5260 const InVector &src,
5261 const std::function<
void(
const unsigned int,
const unsigned int)>
5262 &operation_before_loop,
5263 const std::function<
void(
const unsigned int,
const unsigned int)>
5264 & operation_after_loop,
5265 const unsigned int dof_handler_index_pre_post)
const
5267 internal::MFWorker<MatrixFree<dim, Number, VectorizedArrayType>,
5282 operation_before_loop,
5283 operation_after_loop,
5284 dof_handler_index_pre_post);
5290 template <
int dim,
typename Number,
typename VectorizedArrayType>
5291 template <
typename CLASS,
typename OutVector,
typename InVector>
5294 void (CLASS::*cell_operation)(
5298 const std::pair<unsigned int, unsigned int> &),
5299 void (CLASS::*face_operation)(
5303 const std::pair<unsigned int, unsigned int> &),
5304 void (CLASS::*boundary_operation)(
5308 const std::pair<unsigned int, unsigned int> &),
5309 CLASS * owning_class,
5311 const InVector & src,
5312 const bool zero_dst_vector,
5313 const DataAccessOnFaces dst_vector_face_access,
5314 const DataAccessOnFaces src_vector_face_access)
const
5316 internal::MFWorker<MatrixFree<dim, Number, VectorizedArrayType>,
5329 src_vector_face_access,
5330 dst_vector_face_access);
5336 template <
int dim,
typename Number,
typename VectorizedArrayType>
5337 template <
typename OutVector,
typename InVector>
5343 const std::pair<unsigned int, unsigned int> &)>
5348 const std::pair<unsigned int, unsigned int> &)>
5353 const std::pair<unsigned int, unsigned int> &)>
5354 & boundary_operation,
5356 const InVector &src,
5357 const std::function<
void(
const unsigned int,
const unsigned int)>
5358 &operation_before_loop,
5359 const std::function<
void(
const unsigned int,
const unsigned int)>
5360 & operation_after_loop,
5361 const unsigned int dof_handler_index_pre_post,
5362 const DataAccessOnFaces dst_vector_face_access,
5363 const DataAccessOnFaces src_vector_face_access)
const
5366 internal::MFClassWrapper<MatrixFree<dim, Number, VectorizedArrayType>,
5369 Wrapper wrap(cell_operation, face_operation, boundary_operation);
5370 internal::MFWorker<MatrixFree<dim, Number, VectorizedArrayType>,
5380 &Wrapper::cell_integrator,
5381 &Wrapper::face_integrator,
5382 &Wrapper::boundary_integrator,
5383 src_vector_face_access,
5384 dst_vector_face_access,
5385 operation_before_loop,
5386 operation_after_loop,
5387 dof_handler_index_pre_post);
5394 template <
int dim,
typename Number,
typename VectorizedArrayType>
5395 template <
typename CLASS,
typename OutVector,
typename InVector>
5398 void (CLASS::*cell_operation)(
const MatrixFree &,
5401 const std::pair<unsigned int, unsigned int> &)
5403 void (CLASS::*face_operation)(
const MatrixFree &,
5406 const std::pair<unsigned int, unsigned int> &)
5408 void (CLASS::*boundary_operation)(
5412 const std::pair<unsigned int, unsigned int> &)
const,
5413 const CLASS * owning_class,
5415 const InVector &src,
5416 const std::function<
void(
const unsigned int,
const unsigned int)>
5417 &operation_before_loop,
5418 const std::function<
void(
const unsigned int,
const unsigned int)>
5419 & operation_after_loop,
5420 const unsigned int dof_handler_index_pre_post,
5421 const DataAccessOnFaces dst_vector_face_access,
5422 const DataAccessOnFaces src_vector_face_access)
const
5424 internal::MFWorker<MatrixFree<dim, Number, VectorizedArrayType>,
5437 src_vector_face_access,
5438 dst_vector_face_access,
5439 operation_before_loop,
5440 operation_after_loop,
5441 dof_handler_index_pre_post);
5447 template <
int dim,
typename Number,
typename VectorizedArrayType>
5448 template <
typename CLASS,
typename OutVector,
typename InVector>
5451 void (CLASS::*cell_operation)(
const MatrixFree &,
5454 const std::pair<unsigned int, unsigned int> &),
5455 void (CLASS::*face_operation)(
const MatrixFree &,
5458 const std::pair<unsigned int, unsigned int> &),
5459 void (CLASS::*boundary_operation)(
5463 const std::pair<unsigned int, unsigned int> &),
5464 const CLASS * owning_class,
5466 const InVector &src,
5467 const std::function<
void(
const unsigned int,
const unsigned int)>
5468 &operation_before_loop,
5469 const std::function<
void(
const unsigned int,
const unsigned int)>
5470 & operation_after_loop,
5471 const unsigned int dof_handler_index_pre_post,
5472 const DataAccessOnFaces dst_vector_face_access,
5473 const DataAccessOnFaces src_vector_face_access)
const
5475 internal::MFWorker<MatrixFree<dim, Number, VectorizedArrayType>,
5488 src_vector_face_access,
5489 dst_vector_face_access,
5490 operation_before_loop,
5491 operation_after_loop,
5492 dof_handler_index_pre_post);
5498 template <
int dim,
typename Number,
typename VectorizedArrayType>
5499 template <
typename CLASS,
typename OutVector,
typename InVector>
5502 void (CLASS::*function_pointer)(
5506 const std::pair<unsigned int, unsigned int> &)
const,
5507 const CLASS * owning_class,
5509 const InVector & src,
5510 const bool zero_dst_vector,
5511 const DataAccessOnFaces src_vector_face_access)
const
5513 auto src_vector_face_access_temp = src_vector_face_access;
5519 internal::MFWorker<MatrixFree<dim, Number, VectorizedArrayType>,
5532 src_vector_face_access_temp,
5539 template <
int dim,
typename Number,
typename VectorizedArrayType>
5540 template <
typename CLASS,
typename OutVector,
typename InVector>
5543 void (CLASS::*function_pointer)(
5547 const std::pair<unsigned int, unsigned int> &),
5548 CLASS * owning_class,
5550 const InVector & src,
5551 const bool zero_dst_vector,
5552 const DataAccessOnFaces src_vector_face_access)
const
5554 auto src_vector_face_access_temp = src_vector_face_access;
5560 internal::MFWorker<MatrixFree<dim, Number, VectorizedArrayType>,
5573 src_vector_face_access_temp,
5580 template <
int dim,
typename Number,
typename VectorizedArrayType>
5581 template <
typename OutVector,
typename InVector>
5587 const std::pair<unsigned int, unsigned int> &)>
5590 const InVector & src,
5591 const bool zero_dst_vector,
5592 const DataAccessOnFaces src_vector_face_access)
const
5594 auto src_vector_face_access_temp = src_vector_face_access;
5601 internal::MFClassWrapper<MatrixFree<dim, Number, VectorizedArrayType>,
5604 Wrapper wrap(cell_operation,
nullptr,
nullptr);
5606 internal::MFWorker<MatrixFree<dim, Number, VectorizedArrayType>,
5616 &Wrapper::cell_integrator,
5617 &Wrapper::face_integrator,
5618 &Wrapper::boundary_integrator,
5619 src_vector_face_access_temp,
void resize(const size_type new_size)
Number local_element(const size_type local_index) const
void reinit(const size_type size, const bool omit_zeroing_entries=false)
Abstract base class for mapping classes.
unsigned int n_active_fe_indices() const
void loop_cell_centric(void(CLASS::*cell_operation)(const MatrixFree &, OutVector &, const InVector &, const std::pair< unsigned int, unsigned int > &), CLASS *owning_class, OutVector &dst, const InVector &src, const bool zero_dst_vector=false, const DataAccessOnFaces src_vector_face_access=DataAccessOnFaces::unspecified) const
const Table< 3, unsigned int > & get_cell_and_face_to_plain_faces() const
internal::MatrixFreeFunctions::MappingInfo< dim, Number, VectorizedArrayType > mapping_info
unsigned int get_cell_active_fe_index(const std::pair< unsigned int, unsigned int > range) const
AlignedVector< Number > * acquire_scratch_data_non_threadsafe() const
unsigned int n_ghost_cell_batches() const
void loop(const std::function< void(const MatrixFree< dim, Number, VectorizedArrayType > &, OutVector &, const InVector &, const std::pair< unsigned int, unsigned int > &)> &cell_operation, const std::function< void(const MatrixFree< dim, Number, VectorizedArrayType > &, OutVector &, const InVector &, const std::pair< unsigned int, unsigned int > &)> &face_operation, const std::function< void(const MatrixFree< dim, Number, VectorizedArrayType > &, OutVector &, const InVector &, const std::pair< unsigned int, unsigned int > &)> &boundary_operation, OutVector &dst, const InVector &src, const bool zero_dst_vector=false, const DataAccessOnFaces dst_vector_face_access=DataAccessOnFaces::unspecified, const DataAccessOnFaces src_vector_face_access=DataAccessOnFaces::unspecified) const
void initialize_dof_handlers(const std::vector< const DoFHandler< dim, dim > * > &dof_handlers, const AdditionalData &additional_data)
types::boundary_id get_boundary_id(const unsigned int face_batch_index) const
Table< 4, internal::MatrixFreeFunctions::ShapeInfo< VectorizedArrayType > > shape_info
const Quadrature< dim > & get_quadrature(const unsigned int quad_index=0, const unsigned int hp_active_fe_index=0) const
std::pair< unsigned int, unsigned int > create_cell_subrange_hp(const std::pair< unsigned int, unsigned int > &range, const unsigned int fe_degree, const unsigned int dof_handler_index=0) const
const IndexSet & get_ghost_set(const unsigned int dof_handler_index=0) const
void print(std::ostream &out) const
const Quadrature< dim - 1 > & get_face_quadrature(const unsigned int quad_index=0, const unsigned int hp_active_fe_index=0) const
unsigned int n_inner_face_batches() const
void internal_reinit(const std::shared_ptr< hp::MappingCollection< dim >> &mapping, const std::vector< const DoFHandler< dim, dim > * > &dof_handlers, const std::vector< const AffineConstraints< number2 > * > &constraint, const std::vector< IndexSet > &locally_owned_set, const std::vector< hp::QCollection< q_dim >> &quad, const AdditionalData &additional_data)
unsigned int n_active_entries_per_cell_batch(const unsigned int cell_batch_index) const
void update_mapping(const Mapping< dim > &mapping)
unsigned int get_mg_level() const
void print_memory_consumption(StreamType &out) const
void update_mapping(const std::shared_ptr< hp::MappingCollection< dim >> &mapping)
bool mapping_initialized() const
void cell_loop(const std::function< void(const MatrixFree< dim, Number, VectorizedArrayType > &, OutVector &, const InVector &, const std::pair< unsigned int, unsigned int > &)> &cell_operation, OutVector &dst, const InVector &src, const std::function< void(const unsigned int, const unsigned int)> &operation_before_loop, const std::function< void(const unsigned int, const unsigned int)> &operation_after_loop, const unsigned int dof_handler_index_pre_post=0) const
const Number * constraint_pool_begin(const unsigned int pool_index) const
unsigned int get_n_q_points_face(const unsigned int quad_index=0, const unsigned int hp_active_fe_index=0) const
const internal::MatrixFreeFunctions::ShapeInfo< VectorizedArrayType > & get_shape_info(const unsigned int dof_handler_index_component=0, const unsigned int quad_index=0, const unsigned int fe_base_element=0, const unsigned int hp_active_fe_index=0, const unsigned int hp_active_quad_index=0) const
std::pair< unsigned int, unsigned int > create_cell_subrange_hp_by_index(const std::pair< unsigned int, unsigned int > &range, const unsigned int fe_index, const unsigned int dof_handler_index=0) const
bool at_irregular_cell(const unsigned int cell_batch_index) const
~MatrixFree() override=default
void copy_from(const MatrixFree< dim, Number, VectorizedArrayType > &matrix_free_base)
void initialize_face_data_vector(AlignedVector< T > &vec) const
unsigned int get_dofs_per_cell(const unsigned int dof_handler_index=0, const unsigned int hp_active_fe_index=0) const
unsigned int n_constraint_pool_entries() const
void cell_loop(void(CLASS::*cell_operation)(const MatrixFree &, OutVector &, const InVector &, const std::pair< unsigned int, unsigned int > &), CLASS *owning_class, OutVector &dst, const InVector &src, const std::function< void(const unsigned int, const unsigned int)> &operation_before_loop, const std::function< void(const unsigned int, const unsigned int)> &operation_after_loop, const unsigned int dof_handler_index_pre_post=0) const
std::pair< int, int > get_cell_level_and_index(const unsigned int cell_batch_index, const unsigned int lane_index) const
std::pair< unsigned int, unsigned int > get_face_range_category(const std::pair< unsigned int, unsigned int > face_batch_range) const
const std::vector< unsigned int > & get_constrained_dofs(const unsigned int dof_handler_index=0) const
std::pair< unsigned int, unsigned int > get_face_category(const unsigned int face_batch_index) const
const internal::MatrixFreeFunctions::DoFInfo & get_dof_info(const unsigned int dof_handler_index_component=0) const
unsigned int get_dofs_per_face(const unsigned int dof_handler_index=0, const unsigned int hp_active_fe_index=0) const
const Number * constraint_pool_end(const unsigned int pool_index) const
internal::MatrixFreeFunctions::TaskInfo task_info
void release_scratch_data(const AlignedVector< VectorizedArrayType > *memory) const
void loop(void(CLASS::*cell_operation)(const MatrixFree &, OutVector &, const InVector &, const std::pair< unsigned int, unsigned int > &) const, void(CLASS::*face_operation)(const MatrixFree &, OutVector &, const InVector &, const std::pair< unsigned int, unsigned int > &) const, void(CLASS::*boundary_operation)(const MatrixFree &, OutVector &, const InVector &, const std::pair< unsigned int, unsigned int > &) const, const CLASS *owning_class, OutVector &dst, const InVector &src, const bool zero_dst_vector=false, const DataAccessOnFaces dst_vector_face_access=DataAccessOnFaces::unspecified, const DataAccessOnFaces src_vector_face_access=DataAccessOnFaces::unspecified) const
std::pair< typename DoFHandler< dim >::cell_iterator, unsigned int > get_face_iterator(const unsigned int face_batch_index, const unsigned int lane_index, const bool interior=true, const unsigned int fe_component=0) const
bool mapping_is_initialized
void loop_cell_centric(void(CLASS::*cell_operation)(const MatrixFree &, OutVector &, const InVector &, const std::pair< unsigned int, unsigned int > &) const, const CLASS *owning_class, OutVector &dst, const InVector &src, const bool zero_dst_vector=false, const DataAccessOnFaces src_vector_face_access=DataAccessOnFaces::unspecified) const
void initialize_dof_vector(LinearAlgebra::distributed::Vector< Number2 > &vec, const unsigned int dof_handler_index=0) const
MatrixFree(const MatrixFree< dim, Number, VectorizedArrayType > &other)
void cell_loop(void(CLASS::*cell_operation)(const MatrixFree &, OutVector &, const InVector &, const std::pair< unsigned int, unsigned int > &) const, const CLASS *owning_class, OutVector &dst, const InVector &src, const std::function< void(const unsigned int, const unsigned int)> &operation_before_loop, const std::function< void(const unsigned int, const unsigned int)> &operation_after_loop, const unsigned int dof_handler_index_pre_post=0) const
std::vector< std::pair< unsigned int, unsigned int > > cell_level_index
std::array< types::boundary_id, VectorizedArrayType::size()> get_faces_by_cells_boundary_id(const unsigned int cell_batch_index, const unsigned int face_number) const
std::vector< Number > constraint_pool_data
VectorizedArrayType vectorized_value_type
unsigned int n_cell_batches() const
bool indices_initialized() const
const internal::MatrixFreeFunctions::MappingInfo< dim, Number, VectorizedArrayType > & get_mapping_info() const
unsigned int get_cell_category(const unsigned int cell_batch_index) const
Threads::ThreadLocalStorage< std::list< std::pair< bool, AlignedVector< VectorizedArrayType > > > > scratch_pad
const internal::MatrixFreeFunctions::FaceToCellTopology< VectorizedArrayType::size()> & get_face_info(const unsigned int face_batch_index) const
unsigned int get_matrix_free_cell_index(const typename Triangulation< dim >::cell_iterator &cell) const
std::size_t memory_consumption() const
const AffineConstraints< Number > & get_affine_constraints(const unsigned int dof_handler_index=0) const
unsigned int n_boundary_face_batches() const
void cell_loop(void(CLASS::*cell_operation)(const MatrixFree &, OutVector &, const InVector &, const std::pair< unsigned int, unsigned int > &) const, const CLASS *owning_class, OutVector &dst, const InVector &src, const bool zero_dst_vector=false) const
std::list< std::pair< bool, AlignedVector< Number > > > scratch_pad_non_threadsafe
DoFHandler< dim >::cell_iterator get_cell_iterator(const unsigned int cell_batch_index, const unsigned int lane_index, const unsigned int dof_handler_index=0) const
void loop_cell_centric(const std::function< void(const MatrixFree &, OutVector &, const InVector &, const std::pair< unsigned int, unsigned int > &)> &cell_operation, OutVector &dst, const InVector &src, const bool zero_dst_vector=false, const DataAccessOnFaces src_vector_face_access=DataAccessOnFaces::unspecified) const
std::vector< SmartPointer< const DoFHandler< dim > > > dof_handlers
void initialize_indices(const std::vector< const AffineConstraints< number2 > * > &constraint, const std::vector< IndexSet > &locally_owned_set, const AdditionalData &additional_data)
std::vector< SmartPointer< const AffineConstraints< Number > > > affine_constraints
unsigned int n_components() const
unsigned int n_physical_cells() const
void initialize_dof_vector(VectorType &vec, const unsigned int dof_handler_index=0) const
unsigned int get_n_q_points(const unsigned int quad_index=0, const unsigned int hp_active_fe_index=0) const
void reinit(const MappingType &mapping, const std::vector< const DoFHandler< dim > * > &dof_handler, const std::vector< const AffineConstraints< number2 > * > &constraint, const QuadratureType &quad, const AdditionalData &additional_data=AdditionalData())
std::vector< unsigned int > mf_cell_indices
void cell_loop(const std::function< void(const MatrixFree< dim, Number, VectorizedArrayType > &, OutVector &, const InVector &, const std::pair< unsigned int, unsigned int > &)> &cell_operation, OutVector &dst, const InVector &src, const bool zero_dst_vector=false) const
void loop(void(CLASS::*cell_operation)(const MatrixFree &, OutVector &, const InVector &, const std::pair< unsigned int, unsigned int > &), void(CLASS::*face_operation)(const MatrixFree &, OutVector &, const InVector &, const std::pair< unsigned int, unsigned int > &), void(CLASS::*boundary_operation)(const MatrixFree &, OutVector &, const InVector &, const std::pair< unsigned int, unsigned int > &), const CLASS *owning_class, OutVector &dst, const InVector &src, const std::function< void(const unsigned int, const unsigned int)> &operation_before_loop, const std::function< void(const unsigned int, const unsigned int)> &operation_after_loop, const unsigned int dof_handler_index_pre_post=0, const DataAccessOnFaces dst_vector_face_access=DataAccessOnFaces::unspecified, const DataAccessOnFaces src_vector_face_access=DataAccessOnFaces::unspecified) const
void cell_loop(void(CLASS::*cell_operation)(const MatrixFree &, OutVector &, const InVector &, const std::pair< unsigned int, unsigned int > &), CLASS *owning_class, OutVector &dst, const InVector &src, const bool zero_dst_vector=false) const
std::vector< internal::MatrixFreeFunctions::DoFInfo > dof_info
void initialize_cell_data_vector(AlignedVector< T > &vec) const
unsigned int n_ghost_inner_face_batches() const
AlignedVector< VectorizedArrayType > * acquire_scratch_data() const
void release_scratch_data_non_threadsafe(const AlignedVector< Number > *memory) const
void renumber_dofs(std::vector< types::global_dof_index > &renumbering, const unsigned int dof_handler_index=0)
unsigned int get_face_active_fe_index(const std::pair< unsigned int, unsigned int > range, const bool is_interior_face=true) const
unsigned int n_active_entries_per_face_batch(const unsigned int face_batch_index) const
bool indices_are_initialized
internal::MatrixFreeFunctions::FaceInfo< VectorizedArrayType::size()> face_info
void reinit(const MappingType &mapping, const DoFHandler< dim > &dof_handler, const AffineConstraints< number2 > &constraint, const QuadratureType &quad, const AdditionalData &additional_data=AdditionalData())
unsigned int get_cell_range_category(const std::pair< unsigned int, unsigned int > cell_batch_range) const
void loop(void(CLASS::*cell_operation)(const MatrixFree &, OutVector &, const InVector &, const std::pair< unsigned int, unsigned int > &), void(CLASS::*face_operation)(const MatrixFree &, OutVector &, const InVector &, const std::pair< unsigned int, unsigned int > &), void(CLASS::*boundary_operation)(const MatrixFree &, OutVector &, const InVector &, const std::pair< unsigned int, unsigned int > &), CLASS *owning_class, OutVector &dst, const InVector &src, const bool zero_dst_vector=false, const DataAccessOnFaces dst_vector_face_access=DataAccessOnFaces::unspecified, const DataAccessOnFaces src_vector_face_access=DataAccessOnFaces::unspecified) const
void reinit(const MappingType &mapping, const std::vector< const DoFHandler< dim > * > &dof_handler, const std::vector< const AffineConstraints< number2 > * > &constraint, const std::vector< QuadratureType > &quad, const AdditionalData &additional_data=AdditionalData())
std::vector< unsigned int > constraint_pool_row_index
const std::shared_ptr< const Utilities::MPI::Partitioner > & get_vector_partitioner(const unsigned int dof_handler_index=0) const
void loop(const std::function< void(const MatrixFree< dim, Number, VectorizedArrayType > &, OutVector &, const InVector &, const std::pair< unsigned int, unsigned int > &)> &cell_operation, const std::function< void(const MatrixFree< dim, Number, VectorizedArrayType > &, OutVector &, const InVector &, const std::pair< unsigned int, unsigned int > &)> &face_operation, const std::function< void(const MatrixFree< dim, Number, VectorizedArrayType > &, OutVector &, const InVector &, const std::pair< unsigned int, unsigned int > &)> &boundary_operation, OutVector &dst, const InVector &src, const std::function< void(const unsigned int, const unsigned int)> &operation_before_loop, const std::function< void(const unsigned int, const unsigned int)> &operation_after_loop, const unsigned int dof_handler_index_pre_post=0, const DataAccessOnFaces dst_vector_face_access=DataAccessOnFaces::unspecified, const DataAccessOnFaces src_vector_face_access=DataAccessOnFaces::unspecified) const
unsigned int cell_level_index_end_local
unsigned int n_base_elements(const unsigned int dof_handler_index) const
void loop(void(CLASS::*cell_operation)(const MatrixFree &, OutVector &, const InVector &, const std::pair< unsigned int, unsigned int > &) const, void(CLASS::*face_operation)(const MatrixFree &, OutVector &, const InVector &, const std::pair< unsigned int, unsigned int > &) const, void(CLASS::*boundary_operation)(const MatrixFree &, OutVector &, const InVector &, const std::pair< unsigned int, unsigned int > &) const, const CLASS *owning_class, OutVector &dst, const InVector &src, const std::function< void(const unsigned int, const unsigned int)> &operation_before_loop, const std::function< void(const unsigned int, const unsigned int)> &operation_after_loop, const unsigned int dof_handler_index_pre_post=0, const DataAccessOnFaces dst_vector_face_access=DataAccessOnFaces::unspecified, const DataAccessOnFaces src_vector_face_access=DataAccessOnFaces::unspecified) const
const DoFHandler< dim > & get_dof_handler(const unsigned int dof_handler_index=0) const
const internal::MatrixFreeFunctions::TaskInfo & get_task_info() const
const IndexSet & get_locally_owned_set(const unsigned int dof_handler_index=0) const
static bool is_supported(const FiniteElement< dim, spacedim > &fe)
static constexpr unsigned int dimension
A class that provides a separate storage location on each thread that accesses the object.
#define DEAL_II_NAMESPACE_OPEN
#define DEAL_II_NAMESPACE_CLOSE
static ::ExceptionBase & ExcInternalError()
static ::ExceptionBase & ExcNotInitialized()
#define Assert(cond, exc)
static ::ExceptionBase & ExcNotImplemented()
#define AssertDimension(dim1, dim2)
#define AssertThrowMPI(error_code)
#define AssertIndexRange(index, range)
static ::ExceptionBase & ExcIndexRange(std::size_t arg1, std::size_t arg2, std::size_t arg3)
static ::ExceptionBase & ExcMessage(std::string arg1)
#define AssertThrow(cond, exc)
typename ActiveSelector::cell_iterator cell_iterator
@ update_JxW_values
Transformed quadrature weights.
@ update_gradients
Shape function gradients.
@ update_default
No update.
VectorType::value_type * begin(VectorType &V)
unsigned int minimum_parallel_grain_size
const types::boundary_id invalid_boundary_id
static const unsigned int invalid_unsigned_int
void apply_to_subranges(const tbb::blocked_range< Iterator > &range, const Function &f)
unsigned short int fe_index
TasksParallelScheme tasks_parallel_scheme
bool hold_all_faces_to_owned_cells
UpdateFlags mapping_update_flags_inner_faces
AdditionalData(const TasksParallelScheme tasks_parallel_scheme=partition_partition, const unsigned int tasks_block_size=0, const UpdateFlags mapping_update_flags=update_gradients|update_JxW_values, const UpdateFlags mapping_update_flags_boundary_faces=update_default, const UpdateFlags mapping_update_flags_inner_faces=update_default, const UpdateFlags mapping_update_flags_faces_by_cells=update_default, const unsigned int mg_level=numbers::invalid_unsigned_int, const bool store_plain_indices=true, const bool initialize_indices=true, const bool initialize_mapping=true, const bool overlap_communication_computation=true, const bool hold_all_faces_to_owned_cells=false, const bool cell_vectorization_categories_strict=false, const bool allow_ghosted_vectors_in_loops=true)
AdditionalData & operator=(const AdditionalData &other)
std::vector< unsigned int > cell_vectorization_category
bool cell_vectorization_categories_strict
UpdateFlags mapping_update_flags_boundary_faces
UpdateFlags mapping_update_flags
UpdateFlags mapping_update_flags_faces_by_cells
AdditionalData(const AdditionalData &other)
bool allow_ghosted_vectors_in_loops
unsigned int tasks_block_size
bool overlap_communication_computation
static bool equal(const T *p1, const T *p2)
@ dof_access_face_interior
std::vector< unsigned int > cell_loop_pre_list_index
std::vector< unsigned int > cell_loop_post_list_index
std::vector< std::pair< unsigned int, unsigned int > > vector_zero_range_list
std::shared_ptr< const Utilities::MPI::Partitioner > vector_partitioner
std::vector< std::pair< unsigned int, unsigned int > > cell_loop_pre_list
std::vector< unsigned int > vector_zero_range_list_index
std::vector< std::pair< unsigned int, unsigned int > > cell_loop_post_list
::Table< 3, unsigned int > cell_and_face_to_plain_faces
std::vector< FaceToCellTopology< vectorization_width > > faces
::Table< 3, types::boundary_id > cell_and_face_boundary_id
std::vector< unsigned int > boundary_partition_data
void loop(MFWorkerInterface &worker) const
unsigned int n_active_cells
std::vector< unsigned int > cell_partition_data
std::vector< unsigned int > face_partition_data