17 #ifndef dealii_matrix_free_evaluation_kernels_face_h
18 #define dealii_matrix_free_evaluation_kernels_face_h
39 template <
bool symmetric_evaluate,
65 const unsigned int subface_index,
66 const unsigned int direction)
68 if (symmetric_evaluate)
82 const unsigned int index =
83 direction == 0 ? subface_index % 2 : subface_index / 2;
94 const unsigned int n_components,
99 Number *gradients_quad,
100 Number *hessians_quad,
101 Number *scratch_data,
102 const unsigned int subface_index)
107 const std::size_t n_dofs = fe_degree > -1 ?
110 const std::size_t n_q_points =
115 Number *values_dofs_ptr = values_dofs;
119 for (
unsigned int c = 0; c < n_components; ++c)
124 eval0.template values<0, true, false>(values_dofs,
126 eval1.template values<1, true, false>(values_quad,
130 eval0.template values<0, true, false>(values_dofs,
134 values_quad[0] = values_dofs[0];
141 values_dofs += 3 * n_dofs;
142 values_quad += n_q_points;
145 for (
unsigned int c = 0; c < n_components; ++c)
150 if (symmetric_evaluate &&
153 eval0.template values<0, true, false>(values_dofs,
155 eval0.template values<1, true, false>(values_quad,
164 eval_grad.template gradients<0, true, false, 3>(
165 values_quad, gradients_quad);
166 eval_grad.template gradients<1, true, false, 3>(
167 values_quad, gradients_quad + 1);
172 eval0.template gradients<0, true, false>(values_dofs,
174 eval1.template values<1, true, false, 3>(scratch_data,
178 eval0.template values<0, true, false>(values_dofs,
180 eval1.template gradients<1, true, false, 3>(
181 scratch_data, gradients_quad + 1);
184 eval1.template values<1, true, false>(scratch_data,
188 eval0.template values<0, true, false>(values_dofs + n_dofs,
190 eval1.template values<1, true, false, 3>(scratch_data,
195 eval0.template values<0, true, false, 2>(values_dofs + n_dofs,
197 eval0.template gradients<0, true, false, 2>(values_dofs,
200 eval0.template values<0, true, false>(values_dofs,
204 values_quad[0] = values_dofs[0];
205 gradients_quad[0] = values_dofs[1];
210 values_dofs += 3 * n_dofs;
211 values_quad += n_q_points;
212 gradients_quad += dim * n_q_points;
217 values_dofs = values_dofs_ptr;
218 for (
unsigned int c = 0; c < n_components; ++c)
224 eval0.template hessians<0, true, false>(values_dofs,
226 eval1.template values<1, true, false>(scratch_data,
230 eval0.template values<0, true, false>(values_dofs,
232 eval1.template hessians<1, true, false>(scratch_data,
237 eval0.template values<0, true, false>(values_dofs +
240 eval1.template values<1, true, false>(scratch_data,
245 eval0.template gradients<0, true, false>(values_dofs,
247 eval1.template gradients<1, true, false>(scratch_data,
252 eval0.template gradients<0, true, false>(values_dofs +
255 eval1.template values<1, true, false>(scratch_data,
260 eval0.template values<0, true, false>(values_dofs + n_dofs,
262 eval1.template gradients<1, true, false>(scratch_data,
269 eval0.template hessians<0, true, false>(values_dofs,
272 eval0.template values<0, true, false>(
273 values_dofs + 2 * n_dofs, hessians_quad + n_q_points);
275 eval0.template gradients<0, true, false>(
276 values_dofs + n_dofs, hessians_quad + 2 * n_q_points);
279 hessians_quad[0] = values_dofs[2];
284 values_dofs += 3 * n_dofs;
285 hessians_quad += dim * (dim + 1) / 2 * n_q_points;
292 const unsigned int n_components,
297 Number *gradients_quad,
298 Number *hessians_quad,
299 Number *scratch_data,
300 const unsigned int subface_index)
305 const std::size_t n_dofs =
309 const std::size_t n_q_points =
314 Number *values_dofs_ptr = values_dofs;
318 for (
unsigned int c = 0; c < n_components; ++c)
323 eval1.template values<1, false, false>(values_quad,
325 eval0.template values<0, false, false>(values_quad,
329 eval0.template values<0, false, false>(values_quad,
333 values_dofs[0] = values_quad[0];
338 values_dofs += 3 * n_dofs;
339 values_quad += n_q_points;
342 for (
unsigned int c = 0; c < n_components; ++c)
348 eval1.template values<1, false, false, 3>(gradients_quad + 2,
350 eval0.template values<0, false, false>(scratch_data,
351 values_dofs + n_dofs);
352 if (symmetric_evaluate &&
363 eval_grad.template gradients<1, false, true, 3>(
364 gradients_quad + 1, values_quad);
366 eval_grad.template gradients<1, false, false, 3>(
367 gradients_quad + 1, values_quad);
368 eval_grad.template gradients<0, false, true, 3>(
369 gradients_quad, values_quad);
370 eval0.template values<1, false, false>(values_quad,
372 eval0.template values<0, false, false>(values_quad,
379 eval1.template values<1, false, false>(values_quad,
381 eval1.template gradients<1, false, true, 3>(
382 gradients_quad + 1, scratch_data);
385 eval1.template gradients<1, false, false, 3>(
386 gradients_quad + 1, scratch_data);
389 eval0.template values<0, false, false>(scratch_data,
393 eval1.template values<1, false, false, 3>(gradients_quad,
395 eval0.template gradients<0, false, true>(scratch_data,
400 eval0.template values<0, false, false, 2>(gradients_quad + 1,
403 eval0.template gradients<0, false, false, 2>(gradients_quad,
406 eval0.template values<0, false, true>(values_quad,
410 values_dofs[0] = values_quad[0];
411 values_dofs[1] = gradients_quad[0];
416 values_dofs += 3 * n_dofs;
417 values_quad += n_q_points;
418 gradients_quad += dim * n_q_points;
423 values_dofs = values_dofs_ptr;
424 for (
unsigned int c = 0; c < n_components; ++c)
430 eval1.template values<1, false, false>(hessians_quad,
434 eval0.template hessians<0, false, true>(scratch_data,
437 eval0.template hessians<0, false, false>(scratch_data,
441 eval1.template hessians<1, false, false>(hessians_quad +
444 eval0.template values<0, false, true>(scratch_data,
448 eval1.template values<1, false, false>(hessians_quad +
451 eval0.template values<0, false, false>(scratch_data,
456 eval1.template gradients<1, false, false>(hessians_quad +
459 eval0.template gradients<0, false, true>(scratch_data,
463 eval1.template values<1, false, false>(hessians_quad +
467 eval0.template gradients<0, false, true>(scratch_data,
471 eval0.template gradients<0, false, false>(scratch_data,
476 eval1.template gradients<1, false, false>(hessians_quad +
479 eval0.template values<0, false, true>(scratch_data,
480 values_dofs + n_dofs);
487 eval0.template hessians<0, false, true>(hessians_quad,
490 eval0.template hessians<0, false, false>(hessians_quad,
494 eval0.template values<0, false, false>(
495 hessians_quad + n_q_points, values_dofs + 2 * n_dofs);
498 eval0.template gradients<0, false, true>(
499 hessians_quad + 2 * n_q_points, values_dofs + n_dofs);
501 eval0.template gradients<0, false, false>(
502 hessians_quad + 2 * n_q_points, values_dofs + n_dofs);
505 values_dofs[2] = hessians_quad[0];
514 values_dofs += 3 * n_dofs;
515 hessians_quad += dim * (dim + 1) / 2 * n_q_points;
523 template <
int dim,
int fe_degree,
int n_q_po
ints_1d,
typename Number>
533 template <
bool do_
integrate>
539 Number *values_dofs_in,
542 Number *scratch_data,
543 const unsigned int subface_index,
544 const unsigned int face_direction)
548 const int degree = fe_degree != -1 ? fe_degree : shape_data[0].fe_degree;
549 const int n_rows_n = degree + 1;
550 const int n_rows_t = degree;
551 const ::ndarray<int, 3, 3> dofs_per_direction{
552 {{{n_rows_n, n_rows_t, n_rows_t}},
553 {{n_rows_t, n_rows_n, n_rows_t}},
554 {{n_rows_t, n_rows_t, n_rows_n}}}};
565 (fe_degree > 0 ? fe_degree : 0),
570 std::array<int, dim> values_dofs_offsets = {};
571 for (
unsigned int comp = 0; comp < dim - 1; ++comp)
574 values_dofs_offsets[comp + 1] =
575 values_dofs_offsets[comp] +
576 3 * dofs_per_direction[comp][(face_direction + 1) % dim];
578 values_dofs_offsets[comp + 1] =
579 values_dofs_offsets[comp] +
580 3 * dofs_per_direction[comp][(face_direction + 1) % dim] *
581 dofs_per_direction[comp][(face_direction + 2) % dim];
587 std::array<unsigned int, dim> components;
588 for (
unsigned int comp = 0; comp < dim; ++comp)
589 components[comp] = (face_direction + comp + 1) % dim;
591 for (
const unsigned int comp : components)
593 Number *values_dofs = values_dofs_in + values_dofs_offsets[comp];
596 {dofs_per_direction[comp][(face_direction + 1) % dim],
597 (dim > 2 ? dofs_per_direction[comp][(face_direction + 2) % dim] :
600 if constexpr (dim == 3)
609 shape_data[0].shape_gradients_collocation_eo.data(),
621 eval.template normal<0>(shape_data[0],
624 eval.template tangential<1, 0>(shape_data[1],
630 eval.template normal<0>(shape_data[0],
634 eval.template tangential<1, 0, dim>(shape_data[1],
641 eval.template normal<1>(shape_data[0],
644 eval.template tangential<0, 1>(shape_data[1],
650 eval.template normal<1>(shape_data[0],
654 eval.template tangential<0, 1, dim>(shape_data[1],
661 Eval eval(shape_data[1].shape_values_eo.data(), {}, {});
662 eval.template values<0, true, false>(values_dofs,
values);
666 eval.template values<0, true, false>(values_dofs +
670 eval.template values<1, true, false, dim>(
676 eval_g.template gradients<0, true, false, dim>(
values,
678 eval_g.template gradients<1, true, false, dim>(
values,
694 eval_g.template gradients<0, false, true, dim>(
697 eval_g.template gradients<0, false, false, dim>(
699 eval_g.template gradients<1, false, true, dim>(
gradients +
705 eval.template tangential<1, 0>(shape_data[1],
708 eval.template normal<0>(shape_data[0],
714 eval.template tangential<1, 0, dim>(shape_data[1],
717 eval.template normal<0>(shape_data[0],
725 eval.template tangential<0, 1>(shape_data[1],
728 eval.template normal<1>(shape_data[0],
734 eval.template tangential<0, 1, dim>(shape_data[1],
737 eval.template normal<1>(shape_data[0],
745 Eval eval_iso(shape_data[1].shape_values_eo.data(),
748 eval_iso.template values<1, false, false>(
values,
values);
749 eval_iso.template values<0, false, false>(
values,
753 eval_iso.template values<1, false, false, dim>(
755 eval_iso.template values<0, false, false>(
775 EvalN eval(shape_data[0].shape_values_eo,
776 shape_data[0].shape_gradients_eo,
778 eval.template values<0, true, false>(values_dofs,
values);
781 eval.template gradients<0, true, false, dim>(
783 eval.template values<0, true, false, dim>(
789 Eval eval(shape_data[1].shape_values_eo,
790 shape_data[1].shape_gradients_eo,
792 eval.template values<0, true, false>(values_dofs,
values);
795 eval.template gradients<0, true, false, dim>(
797 eval.template values<0, true, false, dim>(
807 EvalN eval(shape_data[0].shape_values_eo,
808 shape_data[0].shape_gradients_eo,
811 eval.template values<0, false, false>(
values,
816 eval.template gradients<0, false, true, dim>(
819 eval.template gradients<0, false, false, dim>(
821 eval.template values<0, false, false, dim>(
827 Eval eval(shape_data[1].shape_values_eo,
828 shape_data[1].shape_gradients_eo,
831 eval.template values<0, false, false>(
values,
836 eval.template gradients<0, false, true, dim>(
839 eval.template gradients<0, false, false, dim>(
841 eval.template values<0, false, false, dim>(
855 template <
int dim,
int fe_degree,
typename Number>
861 template <
bool do_evaluate,
bool add_
into_output>
868 const unsigned int face_no)
870 Assert(
static_cast<unsigned int>(fe_degree) ==
871 shape_info.
data.front().fe_degree ||
875 interpolate_raviart_thomas<do_evaluate, add_into_output>(
876 n_components, input, output, flags, face_no, shape_info);
878 interpolate_generic<do_evaluate, add_into_output>(
884 shape_info.
data.front().fe_degree + 1,
885 shape_info.
data.front().shape_data_on_face,
893 template <
bool do_evaluate,
bool add_
into_output>
896 const unsigned int n_components,
901 const unsigned int face_no)
903 Assert(
static_cast<unsigned int>(fe_degree + 1) ==
904 shape_info.
data.front().n_q_points_1d ||
908 interpolate_generic<do_evaluate, add_into_output>(
914 shape_info.
data.front().quadrature.size(),
915 shape_info.
data.front().quadrature_data_on_face,
921 template <
bool do_evaluate,
bool add_
into_output,
int face_direction = 0>
927 const unsigned int face_no,
928 const unsigned int n_points_1d,
930 const unsigned int dofs_per_component_on_cell,
931 const unsigned int dofs_per_component_on_face)
933 if (face_direction == face_no / 2)
935 constexpr
int stride_ =
Utilities::pow(fe_degree + 1, face_direction);
937 const int n_rows = fe_degree != -1 ? fe_degree + 1 : n_points_1d;
940 {(dim > 1 ? n_rows : 1), (dim > 2 ? n_rows : 1)}};
941 std::array<int, 2> steps;
942 if constexpr (face_direction == 0)
943 steps = {{n_rows, 0}};
944 else if constexpr (face_direction == 1 && dim == 2)
946 else if constexpr (face_direction == 1)
948 steps = {{n_rows * n_rows, -n_rows * n_rows * n_rows + 1}};
949 else if constexpr (face_direction == 2)
952 for (
unsigned int c = 0; c < n_components; ++c)
959 2>(shape_data[face_no % 2].begin(),
971 1>(shape_data[face_no % 2].begin(),
983 0>(shape_data[face_no % 2].begin(),
992 input += dofs_per_component_on_cell;
993 output += dofs_per_component_on_face;
997 output += dofs_per_component_on_cell;
998 input += dofs_per_component_on_face;
1002 else if (face_direction < dim)
1006 std::min(face_direction + 1, dim - 1)>(
1014 dofs_per_component_on_cell,
1015 dofs_per_component_on_face);
1019 template <
bool do_evaluate,
1020 bool add_into_output,
1021 int face_direction = 0,
1022 int max_derivative = 0>
1025 const unsigned int n_components,
1026 const Number *input,
1029 const unsigned int face_no,
1041 bool increase_max_der =
false;
1044 increase_max_der =
true;
1046 if (face_direction == face_no / 2 && !increase_max_der)
1048 constexpr
int stride1 =
Utilities::pow(fe_degree + 1, face_direction);
1049 constexpr
int stride0 =
Utilities::pow(fe_degree, face_direction);
1050 constexpr
int stride2 = fe_degree * (fe_degree + 1);
1053 fe_degree != -1 ? fe_degree : shape_info.
data[0].fe_degree;
1054 const int n_rows_n = degree + 1;
1055 const int n_rows_t = degree;
1057 std::array<int, 3> strides{{1, 1, 1}};
1058 if (face_direction > 0)
1062 strides[1] = n_rows_t * (face_direction == 3 ? n_rows_n : 1);
1065 const ::ndarray<int, 3, 3> dofs_per_direction{
1066 {{{n_rows_n, n_rows_t, n_rows_t}},
1067 {{n_rows_t, n_rows_n, n_rows_t}},
1068 {{n_rows_t, n_rows_t, n_rows_n}}}};
1070 std::array<int, 2> steps,
n_blocks;
1072 if constexpr (face_direction == 0)
1073 steps = {{degree + (face_direction == 0), 0}};
1074 else if constexpr (face_direction == 1 && dim == 2)
1076 else if constexpr (face_direction == 1)
1079 {n_rows_n * n_rows_t, -n_rows_n * n_rows_t * n_rows_t + 1}};
1080 else if constexpr (face_direction == 2)
1083 n_blocks[0] = dofs_per_direction[0][(face_direction + 1) % dim];
1085 dim > 2 ? dofs_per_direction[0][(face_direction + 2) % dim] : 1;
1088 (fe_degree != -1 ? (fe_degree + (face_direction == 0)) : 0),
1089 ((face_direction < 2) ? stride1 : stride2),
1092 max_derivative>(shape_info.
data[face_direction != 0]
1093 .shape_data_on_face[face_no % 2]
1099 degree + (face_direction == 0),
1114 if constexpr (face_direction == 0)
1115 steps = {{degree, 0}};
1117 n_blocks[0] = dofs_per_direction[1][(face_direction + 1) % dim];
1119 dim > 2 ? dofs_per_direction[1][(face_direction + 2) % dim] : 1;
1122 (fe_degree != -1 ? (fe_degree + (face_direction == 1)) : 0),
1123 ((face_direction < 2) ? stride0 : stride2),
1126 max_derivative>(shape_info.
data[face_direction != 1]
1127 .shape_data_on_face[face_no % 2]
1133 degree + (face_direction == 1),
1136 if constexpr (dim > 2)
1149 if constexpr (face_direction == 0)
1150 steps = {{degree, 0}};
1151 else if constexpr (face_direction == 1)
1154 {n_rows_t * n_rows_t, -n_rows_n * n_rows_t * n_rows_t + 1}};
1155 else if constexpr (face_direction == 2)
1158 n_blocks[0] = dofs_per_direction[2][(face_direction + 1) % dim];
1159 n_blocks[1] = dofs_per_direction[2][(face_direction + 2) % dim];
1162 (fe_degree != -1 ? (fe_degree + (face_direction == 2)) : 0),
1166 max_derivative>(shape_info.
data[face_direction != 2]
1167 .shape_data_on_face[face_no % 2]
1173 degree + (face_direction == 2),
1177 else if (face_direction == face_no / 2)
1184 n_components, input, output, flag, face_no, shape_info);
1186 else if (face_direction < dim)
1188 if (increase_max_der)
1192 std::min(face_direction + 1, dim - 1),
1194 n_components, input, output, flag, face_no, shape_info);
1200 std::min(face_direction + 1, dim - 1),
1202 n_components, input, output, flag, face_no, shape_info);
1211 template <
typename VectorizedArrayType,
typename Number2>
1215 for (
unsigned int v = 0; v < VectorizedArrayType::size(); ++v)
1216 dst[v] = src_ptr[v];
1223 template <
typename Number, std::
size_t w
idth>
1233 template <
typename VectorizedArrayType,
typename Number2>
1236 const unsigned int *indices,
1237 VectorizedArrayType &dst)
1239 for (
unsigned int v = 0; v < VectorizedArrayType::size(); ++v)
1240 dst[v] = src_ptr[indices[v]];
1247 template <
typename Number, std::
size_t w
idth>
1250 const unsigned int *indices,
1253 dst.
gather(src_ptr, indices);
1259 template <
typename VectorizedArrayType,
typename Number2>
1263 for (
unsigned int v = 0; v < VectorizedArrayType::size(); ++v)
1264 dst_ptr[v] += src[v];
1271 template <
typename Number, std::
size_t w
idth>
1277 (tmp + src).store(dst_ptr);
1283 template <
typename VectorizedArrayType,
typename Number2>
1286 const unsigned int *indices,
1289 for (
unsigned int v = 0; v < VectorizedArrayType::size(); ++v)
1290 dst_ptr[indices[v]] += src[v];
1297 template <
typename Number, std::
size_t w
idth>
1300 const unsigned int *indices,
1303 #if DEAL_II_VECTORIZATION_WIDTH_IN_BITS < 512
1304 for (
unsigned int v = 0; v < width; ++v)
1305 dst_ptr[indices[v]] += src[v];
1308 tmp.
gather(dst_ptr, indices);
1309 (tmp + src).
scatter(indices, dst_ptr);
1315 template <
typename Number>
1318 const unsigned int n_components,
1320 const unsigned int *orientation,
1321 const bool integrate,
1322 const std::size_t n_q_points,
1324 Number *values_quad,
1325 Number *gradients_quad,
1326 Number *hessians_quad)
1328 for (
unsigned int c = 0; c < n_components; ++c)
1333 for (
unsigned int q = 0; q < n_q_points; ++q)
1334 tmp_values[q] = values_quad[c * n_q_points + orientation[q]];
1336 for (
unsigned int q = 0; q < n_q_points; ++q)
1337 tmp_values[orientation[q]] = values_quad[c * n_q_points + q];
1338 for (
unsigned int q = 0; q < n_q_points; ++q)
1339 values_quad[c * n_q_points + q] = tmp_values[q];
1342 for (
unsigned int d = 0;
d < dim; ++
d)
1345 for (
unsigned int q = 0; q < n_q_points; ++q)
1347 gradients_quad[(c * n_q_points + orientation[q]) * dim +
d];
1349 for (
unsigned int q = 0; q < n_q_points; ++q)
1350 tmp_values[orientation[q]] =
1351 gradients_quad[(c * n_q_points + q) * dim +
d];
1352 for (
unsigned int q = 0; q < n_q_points; ++q)
1353 gradients_quad[(c * n_q_points + q) * dim +
d] = tmp_values[q];
1357 const unsigned int hdim = (dim * (dim + 1)) / 2;
1358 for (
unsigned int d = 0;
d < hdim; ++
d)
1361 for (
unsigned int q = 0; q < n_q_points; ++q)
1362 tmp_values[q] = hessians_quad[(c * hdim +
d) * n_q_points +
1365 for (
unsigned int q = 0; q < n_q_points; ++q)
1366 tmp_values[orientation[q]] =
1367 hessians_quad[(c * hdim +
d) * n_q_points + q];
1368 for (
unsigned int q = 0; q < n_q_points; ++q)
1369 hessians_quad[(c * hdim +
d) * n_q_points + q] =
1378 template <
typename Number,
typename VectorizedArrayType>
1381 const unsigned int dim,
1382 const unsigned int n_components,
1383 const unsigned int v,
1385 const unsigned int *orientation,
1386 const bool integrate,
1387 const std::size_t n_q_points,
1389 VectorizedArrayType *values_quad,
1390 VectorizedArrayType *gradients_quad =
nullptr,
1391 VectorizedArrayType *hessians_quad =
nullptr)
1393 for (
unsigned int c = 0; c < n_components; ++c)
1398 for (
unsigned int q = 0; q < n_q_points; ++q)
1399 tmp_values[q] = values_quad[c * n_q_points + orientation[q]][v];
1401 for (
unsigned int q = 0; q < n_q_points; ++q)
1402 tmp_values[orientation[q]] = values_quad[c * n_q_points + q][v];
1403 for (
unsigned int q = 0; q < n_q_points; ++q)
1404 values_quad[c * n_q_points + q][v] = tmp_values[q];
1407 for (
unsigned int d = 0;
d < dim; ++
d)
1411 for (
unsigned int q = 0; q < n_q_points; ++q)
1413 gradients_quad[(c * n_q_points + orientation[q]) * dim +
d]
1416 for (
unsigned int q = 0; q < n_q_points; ++q)
1417 tmp_values[orientation[q]] =
1418 gradients_quad[(c * n_q_points + q) * dim +
d][v];
1419 for (
unsigned int q = 0; q < n_q_points; ++q)
1420 gradients_quad[(c * n_q_points + q) * dim +
d][v] =
1426 const unsigned int hdim = (dim * (dim + 1)) / 2;
1427 for (
unsigned int d = 0;
d < hdim; ++
d)
1430 for (
unsigned int q = 0; q < n_q_points; ++q)
1431 tmp_values[q] = hessians_quad[(c * hdim +
d) * n_q_points +
1434 for (
unsigned int q = 0; q < n_q_points; ++q)
1435 tmp_values[orientation[q]] =
1436 hessians_quad[(c * hdim +
d) * n_q_points + q][v];
1437 for (
unsigned int q = 0; q < n_q_points; ++q)
1438 hessians_quad[(c * hdim +
d) * n_q_points + q][v] =
1447 template <
int dim,
typename Number>
1450 template <
int fe_degree,
int n_q_po
ints_1d>
1452 run(
const unsigned int n_components,
1454 const Number *values_dofs,
1458 const auto &shape_data = shape_info.
data.front();
1469 const unsigned int face_no = fe_eval.
get_face_no();
1471 const std::size_t n_dofs = shape_info.dofs_per_component_on_cell;
1472 const std::size_t n_q_points = shape_info.n_q_points_faces[face_no];
1479 const auto *
const shape_values =
1480 &shape_data.shape_values_face(face_no, face_orientation, 0);
1483 auto *values_dofs_actual_ptr = values_dofs;
1485 Eval eval(shape_values,
nullptr,
nullptr, n_dofs, n_q_points);
1486 for (
unsigned int c = 0; c < n_components; ++c)
1488 eval.template values<0, true, false>(values_dofs_actual_ptr,
1491 values_quad_ptr += n_q_points;
1492 values_dofs_actual_ptr += n_dofs;
1499 const auto *values_dofs_actual_ptr = values_dofs;
1501 std::array<const Number2 *, dim> shape_gradients;
1502 for (
unsigned int d = 0;
d < dim; ++
d)
1503 shape_gradients[
d] = &shape_data.shape_gradients_face(
1504 face_no, face_orientation,
d, 0);
1506 for (
unsigned int c = 0; c < n_components; ++c)
1508 for (
unsigned int d = 0;
d < dim; ++
d)
1516 eval.template gradients<0, true, false, dim>(
1517 values_dofs_actual_ptr, gradients_quad_ptr +
d);
1519 gradients_quad_ptr += n_q_points * dim;
1520 values_dofs_actual_ptr += n_dofs;
1530 const unsigned int dofs_per_face =
1537 Number *scratch_data = temp + 3 * n_components * dofs_per_face;
1539 bool use_vectorization =
true;
1544 for (
unsigned int v = 0; v < Number::size(); ++v)
1547 use_vectorization =
false;
1549 if (use_vectorization ==
false)
1551 for (
unsigned int v = 0; v < Number::size(); ++v)
1558 for (
unsigned int i = 0; i < 3 * n_components * dofs_per_face;
1565 template interpolate<true, false>(n_components,
1572 for (
unsigned int i = 0; i < 3 * n_components * dofs_per_face;
1574 temp[i][v] = scratch_data[i][v];
1579 template interpolate<true, false>(n_components,
1587 constexpr
unsigned int n_q_points_1d_actual =
1588 fe_degree > -1 ? n_q_points_1d : 0;
1594 n_q_points_1d_actual,
1596 template evaluate_or_integrate_in_face<false>(
1606 else if (fe_degree > -1 &&
1612 n_q_points_1d_actual,
1613 Number>::evaluate_in_face(n_components,
1627 n_q_points_1d_actual,
1628 Number>::evaluate_in_face(n_components,
1639 if (use_vectorization ==
false)
1641 for (
unsigned int v = 0; v < Number::size(); ++v)
1658 shape_info.n_q_points_face,
1673 shape_info.n_q_points_face,
1685 template <
int dim,
typename Number>
1688 template <
int fe_degree,
int n_q_po
ints_1d>
1690 run(
const unsigned int n_components,
1692 Number *values_dofs,
1696 const auto &shape_data = shape_info.
data.front();
1707 const unsigned int face_no = fe_eval.
get_face_no();
1709 const std::size_t n_dofs = shape_info.dofs_per_component_on_cell;
1710 const std::size_t n_q_points = shape_info.n_q_points_faces[face_no];
1717 const auto *
const shape_values =
1718 &shape_data.shape_values_face(face_no, face_orientation, 0);
1721 auto *values_dofs_actual_ptr = values_dofs;
1723 Eval eval(shape_values,
nullptr,
nullptr, n_dofs, n_q_points);
1724 for (
unsigned int c = 0; c < n_components; ++c)
1726 eval.template values<0, false, false>(values_quad_ptr,
1727 values_dofs_actual_ptr);
1729 values_quad_ptr += n_q_points;
1730 values_dofs_actual_ptr += n_dofs;
1737 auto *values_dofs_actual_ptr = values_dofs;
1739 std::array<const Number2 *, dim> shape_gradients;
1740 for (
unsigned int d = 0;
d < dim; ++
d)
1741 shape_gradients[
d] = &shape_data.shape_gradients_face(
1742 face_no, face_orientation,
d, 0);
1744 for (
unsigned int c = 0; c < n_components; ++c)
1746 for (
unsigned int d = 0;
d < dim; ++
d)
1756 eval.template gradients<0, false, false, dim>(
1757 gradients_quad_ptr +
d, values_dofs_actual_ptr);
1759 eval.template gradients<0, false, true, dim>(
1760 gradients_quad_ptr +
d, values_dofs_actual_ptr);
1762 gradients_quad_ptr += n_q_points * dim;
1763 values_dofs_actual_ptr += n_dofs;
1773 const unsigned int dofs_per_face =
1778 Number *scratch_data = temp + 3 * n_components * dofs_per_face;
1780 bool use_vectorization =
true;
1789 [&](
const auto &v) {
1790 return v == fe_eval.get_cell_ids()[0] ||
1791 v == numbers::invalid_unsigned_int;
1794 if (use_vectorization ==
false)
1796 for (
unsigned int v = 0; v < Number::size(); ++v)
1813 shape_info.n_q_points_face,
1828 shape_info.n_q_points_face,
1834 const unsigned int n_q_points_1d_actual =
1835 fe_degree > -1 ? n_q_points_1d : 0;
1842 n_q_points_1d_actual,
1844 template evaluate_or_integrate_in_face<true>(
1854 else if (fe_degree > -1 &&
1862 n_q_points_1d_actual,
1863 Number>::integrate_in_face(n_components,
1877 n_q_points_1d_actual,
1878 Number>::integrate_in_face(n_components,
1888 if (use_vectorization ==
false)
1890 for (
unsigned int v = 0; v < Number::size(); ++v)
1899 template interpolate<false, false>(n_components,
1906 for (
unsigned int i = 0; i < 3 * n_components * dofs_per_face;
1908 temp[i][v] = scratch_data[i][v];
1913 template interpolate<false, false>(n_components,
1925 template <
int n_face_orientations,
1927 typename EvaluationData,
1928 const bool check_face_orientations =
false>
1932 const unsigned int n_components,
1934 typename Processor::Number2_ *global_vector_ptr,
1936 const EvaluationData &fe_eval,
1937 typename Processor::VectorizedArrayType_ *temp1)
1939 constexpr
int dim = Processor::dim_;
1940 constexpr
int fe_degree = Processor::fe_degree_;
1941 using VectorizedArrayType =
typename Processor::VectorizedArrayType_;
1942 constexpr
int n_lanes = VectorizedArrayType::size();
1944 using Number =
typename Processor::Number_;
1945 using Number2_ =
typename Processor::Number2_;
1947 const auto &shape_data = fe_eval.get_shape_info().data.front();
1948 constexpr
bool integrate = Processor::do_integrate;
1949 const unsigned int face_no = fe_eval.get_face_no();
1950 const auto &dof_info = fe_eval.get_dof_info();
1951 const unsigned int cell = fe_eval.get_cell_or_face_batch_id();
1953 fe_eval.get_dof_access_index();
1955 dof_info.index_storage_variants[dof_access_index].size());
1956 constexpr
unsigned int dofs_per_face =
1958 const unsigned int subface_index = fe_eval.get_subface_index();
1960 const unsigned int n_filled_lanes =
1961 dof_info.n_vectorization_lanes_filled[dof_access_index][cell];
1963 bool all_faces_are_same = n_filled_lanes == n_lanes;
1964 if (n_face_orientations == n_lanes)
1965 for (
unsigned int v = 1; v < n_lanes; ++v)
1966 if (fe_eval.get_face_no(v) != fe_eval.get_face_no(0) ||
1967 fe_eval.get_face_orientation(v) != fe_eval.get_face_orientation(0))
1969 all_faces_are_same =
false;
1974 std::array<const unsigned int *, n_face_orientations> orientation = {};
1976 if (dim == 3 && n_face_orientations == n_lanes && !all_faces_are_same &&
1977 fe_eval.is_interior_face() == 0)
1978 for (
unsigned int v = 0; v < n_lanes; ++v)
1986 if (shape_data.nodal_at_cell_boundaries &&
1987 fe_eval.get_face_orientation(v) != 0)
1992 check_face_orientations ==
false)
2006 orientation[v] = &fe_eval.get_shape_info().face_orientations_dofs(
2007 fe_eval.get_face_orientation(v), 0);
2010 else if (dim == 3 && fe_eval.get_face_orientation() != 0)
2014 check_face_orientations ==
false)
2028 for (
unsigned int v = 0; v < n_face_orientations; ++v)
2029 orientation[v] = &fe_eval.get_shape_info().face_orientations_dofs(
2030 fe_eval.get_face_orientation(), 0);
2036 VectorizedArrayType grad_weight =
2038 .shape_data_on_face[0][fe_degree + (integrate ? (2 - face_no % 2) :
2039 (1 + face_no % 2))];
2042 std::array<const unsigned int *, n_face_orientations> index_array_hermite =
2046 if (n_face_orientations == 1)
2047 index_array_hermite[0] =
2048 &fe_eval.get_shape_info().face_to_cell_index_hermite(face_no, 0);
2051 for (
unsigned int v = 0; v < n_lanes; ++v)
2056 const auto face_no = fe_eval.get_face_no(v);
2059 shape_data.shape_data_on_face[0][fe_degree +
2061 (2 - (face_no % 2)) :
2062 (1 + (face_no % 2)))][0];
2064 index_array_hermite[v] =
2065 &fe_eval.get_shape_info().face_to_cell_index_hermite(face_no,
2072 std::array<const unsigned int *, n_face_orientations> index_array_nodal =
2074 if (shape_data.nodal_at_cell_boundaries ==
true)
2076 if (n_face_orientations == 1)
2077 index_array_nodal[0] =
2078 &fe_eval.get_shape_info().face_to_cell_index_nodal(face_no, 0);
2081 for (
unsigned int v = 0; v < n_lanes; ++v)
2086 const auto face_no = fe_eval.get_face_no(v);
2088 index_array_nodal[v] =
2089 &fe_eval.get_shape_info().face_to_cell_index_nodal(face_no,
2096 const auto reorientate = [&](
const unsigned int v,
const unsigned int i) {
2097 return (!check_face_orientations || orientation[v] ==
nullptr) ?
2104 fe_eval.get_cell_ids()[0] :
2106 const unsigned int *dof_indices =
2107 &dof_info.dof_indices_contiguous[dof_access_index][
cell_index];
2109 for (
unsigned int comp = 0; comp < n_components; ++comp)
2111 const std::size_t index_offset =
2112 dof_info.component_dof_indices_offset
2113 [fe_eval.get_active_fe_index()]
2114 [fe_eval.get_first_selected_component()] +
2118 if (n_face_orientations == 1 &&
2119 dof_info.index_storage_variants[dof_access_index][cell] ==
2121 interleaved_contiguous)
2124 dof_info.n_vectorization_lanes_filled[dof_access_index][cell],
2126 Number2_ *vector_ptr =
2127 global_vector_ptr + dof_indices[0] + index_offset * n_lanes;
2131 for (
unsigned int i = 0; i < dofs_per_face; ++i)
2135 const unsigned int ind1 = index_array_hermite[0][2 * i];
2136 const unsigned int ind2 = index_array_hermite[0][2 * i + 1];
2137 const unsigned int i_ = reorientate(0, i);
2138 proc.hermite_grad_vectorized(temp1[i_],
2139 temp1[i_ + dofs_per_face],
2140 vector_ptr + ind1 * n_lanes,
2141 vector_ptr + ind2 * n_lanes,
2147 for (
unsigned int i = 0; i < dofs_per_face; ++i)
2151 const unsigned int i_ = reorientate(0, i);
2152 const unsigned int ind = index_array_nodal[0][i];
2153 proc.value_vectorized(temp1[i_],
2154 vector_ptr + ind * n_lanes);
2160 else if (n_face_orientations == 1 &&
2161 dof_info.index_storage_variants[dof_access_index][cell] ==
2163 interleaved_contiguous_strided)
2166 dof_info.n_vectorization_lanes_filled[dof_access_index][cell],
2168 Number2_ *vector_ptr = global_vector_ptr + index_offset * n_lanes;
2171 for (
unsigned int i = 0; i < dofs_per_face; ++i)
2175 const unsigned int i_ = reorientate(0, i);
2176 const unsigned int ind1 =
2177 index_array_hermite[0][2 * i] * n_lanes;
2178 const unsigned int ind2 =
2179 index_array_hermite[0][2 * i + 1] * n_lanes;
2180 proc.hermite_grad_vectorized_indexed(
2182 temp1[i_ + dofs_per_face],
2192 for (
unsigned int i = 0; i < dofs_per_face; ++i)
2196 const unsigned int i_ = reorientate(0, i);
2197 const unsigned int ind = index_array_nodal[0][i] * n_lanes;
2198 proc.value_vectorized_indexed(temp1[i_],
2206 else if (n_face_orientations == 1 &&
2207 dof_info.index_storage_variants[dof_access_index][cell] ==
2209 interleaved_contiguous_mixed_strides)
2211 const unsigned int *strides =
2212 &dof_info.dof_indices_interleave_strides[dof_access_index]
2214 unsigned int indices[n_lanes];
2215 for (
unsigned int v = 0; v < n_lanes; ++v)
2216 indices[v] = dof_indices[v] + index_offset * strides[v];
2217 const unsigned int n_filled_lanes =
2218 dof_info.n_vectorization_lanes_filled[dof_access_index][cell];
2222 if (n_filled_lanes == n_lanes)
2223 for (
unsigned int i = 0; i < dofs_per_face; ++i)
2227 const unsigned int i_ = reorientate(0, i);
2228 unsigned int ind1[n_lanes];
2230 for (
unsigned int v = 0; v < n_lanes; ++v)
2231 ind1[v] = indices[v] +
2232 index_array_hermite[0][2 * i] * strides[v];
2233 unsigned int ind2[n_lanes];
2235 for (
unsigned int v = 0; v < n_lanes; ++v)
2239 index_array_hermite[0][2 * i + 1] * strides[v];
2240 proc.hermite_grad_vectorized_indexed(
2242 temp1[i_ + dofs_per_face],
2251 if (integrate ==
false)
2252 for (
unsigned int i = 0; i < 2 * dofs_per_face; ++i)
2253 temp1[i] = VectorizedArrayType();
2255 for (
unsigned int v = 0; v < n_filled_lanes; ++v)
2256 for (
unsigned int i = 0; i < dofs_per_face; ++i)
2258 const unsigned int i_ =
2259 reorientate(n_face_orientations == 1 ? 0 : v, i);
2262 temp1[i_ + dofs_per_face][v],
2266 [n_face_orientations == 1 ? 0 : v][2 * i] *
2270 index_array_hermite[n_face_orientations == 1 ?
2274 grad_weight[n_face_orientations == 1 ? 0 : v]);
2280 if (n_filled_lanes == n_lanes)
2281 for (
unsigned int i = 0; i < dofs_per_face; ++i)
2284 unsigned int ind[n_lanes];
2286 for (
unsigned int v = 0; v < n_lanes; ++v)
2288 indices[v] + index_array_nodal[0][i] * strides[v];
2289 const unsigned int i_ = reorientate(0, i);
2290 proc.value_vectorized_indexed(temp1[i_],
2296 if (integrate ==
false)
2297 for (
unsigned int i = 0; i < dofs_per_face; ++i)
2298 temp1[i] = VectorizedArrayType();
2300 for (
unsigned int v = 0; v < n_filled_lanes; ++v)
2301 for (
unsigned int i = 0; i < dofs_per_face; ++i)
2303 temp1[reorientate(n_face_orientations == 1 ? 0 : v,
2307 index_array_nodal[n_face_orientations == 1 ? 0 : v]
2315 else if (n_face_orientations > 1 ||
2316 dof_info.index_storage_variants[dof_access_index][cell] ==
2320 Number2_ *vector_ptr = global_vector_ptr + index_offset;
2322 const bool vectorization_possible =
2323 all_faces_are_same && (sm_ptr ==
nullptr);
2325 std::array<Number2_ *, n_lanes> vector_ptrs;
2326 std::array<unsigned int, n_lanes> reordered_indices;
2328 if (vectorization_possible ==
false)
2331 if (n_face_orientations == 1)
2333 for (
unsigned int v = 0; v < n_filled_lanes; ++v)
2334 if (sm_ptr ==
nullptr)
2336 vector_ptrs[v] = vector_ptr + dof_indices[v];
2342 .dof_indices_contiguous_sm[dof_access_index]
2343 [cell * n_lanes + v];
2344 vector_ptrs[v] =
const_cast<Number2_ *
>(
2345 sm_ptr->operator[](temp.first).data() +
2346 temp.second + index_offset);
2349 else if (n_face_orientations == n_lanes)
2351 const auto &cells = fe_eval.get_cell_ids();
2352 for (
unsigned int v = 0; v < n_lanes; ++v)
2355 if (sm_ptr ==
nullptr)
2360 .dof_indices_contiguous[dof_access_index]
2367 .dof_indices_contiguous_sm[dof_access_index]
2369 vector_ptrs[v] =
const_cast<Number2_ *
>(
2370 sm_ptr->operator[](temp.first).data() +
2371 temp.second + index_offset);
2380 else if (n_face_orientations == n_lanes)
2382 for (
unsigned int v = 0; v < n_lanes; ++v)
2383 reordered_indices[v] =
2384 dof_info.dof_indices_contiguous[dof_access_index]
2385 [fe_eval.get_cell_ids()[v]];
2386 dof_indices = reordered_indices.data();
2391 if (vectorization_possible)
2392 for (
unsigned int i = 0; i < dofs_per_face; ++i)
2394 const unsigned int ind1 = index_array_hermite[0][2 * i];
2395 const unsigned int ind2 =
2396 index_array_hermite[0][2 * i + 1];
2397 const unsigned int i_ = reorientate(0, i);
2399 proc.hermite_grad_vectorized_indexed(
2401 temp1[i_ + dofs_per_face],
2408 else if (n_face_orientations == 1)
2409 for (
unsigned int i = 0; i < dofs_per_face; ++i)
2411 const unsigned int ind1 = index_array_hermite[0][2 * i];
2412 const unsigned int ind2 =
2413 index_array_hermite[0][2 * i + 1];
2414 const unsigned int i_ = reorientate(0, i);
2416 for (
unsigned int v = 0; v < n_filled_lanes; ++v)
2417 proc.hermite_grad(temp1[i_][v],
2418 temp1[i_ + dofs_per_face][v],
2419 vector_ptrs[v][ind1],
2420 vector_ptrs[v][ind2],
2423 if (integrate ==
false)
2424 for (
unsigned int v = n_filled_lanes; v < n_lanes; ++v)
2427 temp1[i + dofs_per_face][v] = 0.0;
2432 if (integrate ==
false && n_filled_lanes < n_lanes)
2433 for (
unsigned int i = 0; i < dofs_per_face; ++i)
2434 temp1[i] = temp1[i + dofs_per_face] = Number();
2436 for (
unsigned int v = 0; v < n_filled_lanes; ++v)
2437 for (
unsigned int i = 0; i < dofs_per_face; ++i)
2439 temp1[reorientate(v, i)][v],
2440 temp1[reorientate(v, i) + dofs_per_face][v],
2441 vector_ptrs[v][index_array_hermite[v][2 * i]],
2442 vector_ptrs[v][index_array_hermite[v][2 * i + 1]],
2448 if (vectorization_possible)
2449 for (
unsigned int i = 0; i < dofs_per_face; ++i)
2451 const unsigned int ind = index_array_nodal[0][i];
2452 const unsigned int i_ = reorientate(0, i);
2454 proc.value_vectorized_indexed(temp1[i_],
2458 else if (n_face_orientations == 1)
2459 for (
unsigned int i = 0; i < dofs_per_face; ++i)
2461 const unsigned int ind = index_array_nodal[0][i];
2462 const unsigned int i_ = reorientate(0, i);
2464 for (
unsigned int v = 0; v < n_filled_lanes; ++v)
2465 proc.value(temp1[i_][v], vector_ptrs[v][ind]);
2467 if (integrate ==
false)
2468 for (
unsigned int v = n_filled_lanes; v < n_lanes; ++v)
2473 if (integrate ==
false && n_filled_lanes < n_lanes)
2474 for (
unsigned int i = 0; i < dofs_per_face; ++i)
2475 temp1[i] = Number();
2477 for (
unsigned int v = 0; v < n_filled_lanes; ++v)
2478 for (
unsigned int i = 0; i < dofs_per_face; ++i)
2479 proc.value(temp1[reorientate(v, i)][v],
2480 vector_ptrs[v][index_array_nodal[v][i]]);
2490 temp1 += 3 * dofs_per_face;
2496 template <
int dim,
typename Number2,
typename VectorizedArrayType>
2499 using Number =
typename VectorizedArrayType::value_type;
2501 template <
int fe_degree,
int n_q_po
ints_1d>
2503 run(
const unsigned int n_components,
2505 const Number2 *src_ptr,
2514 const unsigned int dofs_per_face =
Utilities::pow(fe_degree + 1, dim - 1);
2517 VectorizedArrayType *scratch_data =
2518 temp + 3 * n_components * dofs_per_face;
2525 fe_face_evaluation_process_and_io<VectorizedArrayType::size()>(
2526 p, n_components, evaluation_flag, src_ptr, sm_ptr, fe_eval, temp);
2528 fe_face_evaluation_process_and_io<1>(
2529 p, n_components, evaluation_flag, src_ptr, sm_ptr, fe_eval, temp);
2538 VectorizedArrayType>::
2539 evaluate_in_face(n_components,
2553 VectorizedArrayType>::
2554 evaluate_in_face(n_components,
2571 for (
unsigned int v = 0; v < VectorizedArrayType::size(); ++v)
2614 template <
typename Number3>
2618 const Number2 *vector_ptr,
2621 const unsigned int fe_degree = shape_info.
data.front().fe_degree;
2622 if (fe_degree < 1 || !shape_info.
data.front().nodal_at_cell_boundaries ||
2625 shape_info.
data.front().element_type !=
2628 vector_ptr ==
nullptr ||
2629 shape_info.
data.front().element_type >
2639 template <
int fe_degree>
2649 template <
typename T0,
typename T1,
typename T2>
2655 const T2 &grad_weight)
2659 temp_2 = grad_weight * (temp_1 - temp_2);
2662 template <
typename T1,
typename T2>
2669 template <
typename T0,
typename T1,
typename T2,
typename T3>
2675 const T2 &grad_weight,
2676 const T3 &indices_1,
2677 const T3 &indices_2)
2681 temp_2 = grad_weight * (temp_1 - temp_2);
2684 template <
typename T0,
typename T1,
typename T2>
2691 template <
typename T0,
typename T1,
typename T2>
2695 const T1 &src_ptr_1,
2696 const T1 &src_ptr_2,
2697 const T2 &grad_weight)
2701 temp_2 = grad_weight * (temp_1 - src_ptr_2);
2704 template <
typename T1,
typename T2>
2716 template <
int dim,
typename Number2,
typename VectorizedArrayType>
2719 using Number =
typename VectorizedArrayType::value_type;
2721 template <
int fe_degree,
int n_q_po
ints_1d>
2723 run(
const unsigned int n_components,
2734 const unsigned int dofs_per_face =
Utilities::pow(fe_degree + 1, dim - 1);
2737 VectorizedArrayType *scratch_data =
2738 temp + 3 * n_components * dofs_per_face;
2748 for (
unsigned int v = 0; v < VectorizedArrayType::size(); ++v)
2792 VectorizedArrayType>::
2793 integrate_in_face(n_components,
2807 VectorizedArrayType>::
2808 integrate_in_face(n_components,
2823 fe_face_evaluation_process_and_io<VectorizedArrayType::size()>(
2824 p, n_components, integration_flag, dst_ptr, sm_ptr, fe_eval, temp);
2826 fe_face_evaluation_process_and_io<1>(
2827 p, n_components, integration_flag, dst_ptr, sm_ptr, fe_eval, temp);
2833 template <
int fe_degree>
2843 template <
typename T0,
typename T1,
typename T2,
typename T3,
typename T4>
2849 const T4 &grad_weight)
2852 const VectorizedArrayType val = temp_1 - grad_weight * temp_2;
2853 const VectorizedArrayType grad = grad_weight * temp_2;
2858 template <
typename T0,
typename T1>
2866 template <
typename T0,
typename T1,
typename T2,
typename T3>
2872 const T2 &grad_weight,
2873 const T3 &indices_1,
2874 const T3 &indices_2)
2877 const VectorizedArrayType val = temp_1 - grad_weight * temp_2;
2878 const VectorizedArrayType grad = grad_weight * temp_2;
2883 template <
typename T0,
typename T1,
typename T2>
2891 template <
typename T0,
typename T1,
typename T2>
2897 const T2 &grad_weight)
2900 const Number val = temp_1 - grad_weight * temp_2;
2901 const Number grad = grad_weight * temp_2;
2906 template <
typename T0,
typename T1>
std::uint8_t get_face_no(const unsigned int v=0) const
internal::MatrixFreeFunctions::DoFInfo::DoFAccessIndex get_dof_access_index() const
const Number * begin_values() const
const Number * begin_hessians() const
const std::array< unsigned int, n_lanes > & get_cell_ids() const
unsigned int get_subface_index() const
Number shape_info_number_type
const Number * begin_gradients() const
bool is_interior_face() const
ArrayView< Number > get_scratch_data() const
const ShapeInfoType & get_shape_info() const
std::uint8_t get_face_orientation(const unsigned int v=0) const
void gather(const Number *base_ptr, const unsigned int *offsets)
void load(const OtherNumber *ptr)
#define DEAL_II_OPENMP_SIMD_PRAGMA
#define DEAL_II_NAMESPACE_OPEN
#define DEAL_II_NAMESPACE_CLOSE
static ::ExceptionBase & ExcInternalError()
#define Assert(cond, exc)
static ::ExceptionBase & ExcNotImplemented()
#define AssertDimension(dim1, dim2)
#define AssertIndexRange(index, range)
#define AssertThrow(cond, exc)
@ tensor_symmetric_no_collocation
@ tensor_symmetric_hermite
EvaluationFlags
The EvaluationFlags enum.
std::enable_if_t< IsBlockVector< VectorType >::value, unsigned int > n_blocks(const VectorType &vector)
SymmetricTensor< 2, dim, Number > d(const Tensor< 2, dim, Number > &F, const Tensor< 2, dim, Number > &dF_dt)
T scatter(const MPI_Comm comm, const std::vector< T > &objects_to_send, const unsigned int root_process=0)
constexpr T pow(const T base, const int iexp)
void do_vectorized_add(const VectorizedArrayType src, Number2 *dst_ptr)
constexpr bool use_collocation_evaluation(const unsigned int fe_degree, const unsigned int n_q_points_1d)
void adjust_for_face_orientation_per_lane(const unsigned int dim, const unsigned int n_components, const unsigned int v, const EvaluationFlags::EvaluationFlags flag, const unsigned int *orientation, const bool integrate, const std::size_t n_q_points, Number *tmp_values, VectorizedArrayType *values_quad, VectorizedArrayType *gradients_quad=nullptr, VectorizedArrayType *hessians_quad=nullptr)
void do_vectorized_scatter_add(const VectorizedArrayType src, const unsigned int *indices, Number2 *dst_ptr)
std::enable_if_t< contract_onto_face, void > interpolate_to_face(const Number2 *shape_values, const std::array< int, 2 > &n_blocks, const std::array< int, 2 > &steps, const Number *input, Number *DEAL_II_RESTRICT output, const int n_rows_runtime=0, const int stride_runtime=1)
void do_vectorized_gather(const Number2 *src_ptr, const unsigned int *indices, VectorizedArrayType &dst)
void do_vectorized_read(const Number2 *src_ptr, VectorizedArrayType &dst)
void fe_face_evaluation_process_and_io(Processor &proc, const unsigned int n_components, const EvaluationFlags::EvaluationFlags evaluation_flag, typename Processor::Number2_ *global_vector_ptr, const std::vector< ArrayView< const typename Processor::Number2_ >> *sm_ptr, const EvaluationData &fe_eval, typename Processor::VectorizedArrayType_ *temp1)
void adjust_for_face_orientation(const unsigned int dim, const unsigned int n_components, const EvaluationFlags::EvaluationFlags flag, const unsigned int *orientation, const bool integrate, const std::size_t n_q_points, Number *tmp_values, Number *values_quad, Number *gradients_quad, Number *hessians_quad)
static const unsigned int invalid_unsigned_int
static bool run(const unsigned int n_components, const EvaluationFlags::EvaluationFlags evaluation_flag, const Number *values_dofs, FEEvaluationData< dim, Number, true > &fe_eval)
void hermite_grad(T0 &temp_1, T0 &temp_2, const T1 &src_ptr_1, const T1 &src_ptr_2, const T2 &grad_weight)
static const int fe_degree_
static const bool do_integrate
void value_vectorized(T1 &temp, const T2 src_ptr)
void value(T1 &temp, const T2 &src_ptr)
VectorizedArrayType VectorizedArrayType_
void value_vectorized_indexed(T0 &temp, const T1 src_ptr, const T2 &indices)
void hermite_grad_vectorized(T0 &temp_1, T0 &temp_2, const T1 src_ptr_1, const T1 src_ptr_2, const T2 &grad_weight)
void hermite_grad_vectorized_indexed(T0 &temp_1, T0 &temp_2, const T1 src_ptr_1, const T1 src_ptr_2, const T2 &grad_weight, const T3 &indices_1, const T3 &indices_2)
static bool run(const unsigned int n_components, const EvaluationFlags::EvaluationFlags evaluation_flag, const Number2 *src_ptr, const std::vector< ArrayView< const Number2 >> *sm_ptr, FEEvaluationData< dim, VectorizedArrayType, true > &fe_eval)
typename VectorizedArrayType::value_type Number
static bool supports(const EvaluationFlags::EvaluationFlags evaluation_flag, const MatrixFreeFunctions::ShapeInfo< Number3 > &shape_info, const Number2 *vector_ptr, MatrixFreeFunctions::DoFInfo::IndexStorageVariants storage)
static const int fe_degree_
void value(const T0 &temp, T1 &dst_ptr)
void hermite_grad_vectorized(const T0 &temp_1, const T1 &temp_2, T2 dst_ptr_1, T3 dst_ptr_2, const T4 &grad_weight)
void hermite_grad_vectorized_indexed(const T0 &temp_1, const T0 &temp_2, T1 dst_ptr_1, T1 dst_ptr_2, const T2 &grad_weight, const T3 &indices_1, const T3 &indices_2)
static const bool do_integrate
VectorizedArrayType VectorizedArrayType_
void hermite_grad(const T0 &temp_1, const T0 &temp_2, T1 &dst_ptr_1, T1 &dst_ptr_2, const T2 &grad_weight)
void value_vectorized(const T0 &temp, T1 dst_ptr)
void value_vectorized_indexed(const T0 &temp, T1 dst_ptr, const T2 &indices)
static bool run(const unsigned int n_components, const EvaluationFlags::EvaluationFlags integration_flag, Number2 *dst_ptr, const std::vector< ArrayView< const Number2 >> *sm_ptr, FEEvaluationData< dim, VectorizedArrayType, true > &fe_eval)
typename VectorizedArrayType::value_type Number
static bool run(const unsigned int n_components, const EvaluationFlags::EvaluationFlags integration_flag, Number *values_dofs, FEEvaluationData< dim, Number, true > &fe_eval)
typename FEEvaluationData< dim, Number, true >::shape_info_number_type Number2
static void evaluate_or_integrate_in_face(const EvaluationFlags::EvaluationFlags evaluation_flag, const std::vector< MatrixFreeFunctions::UnivariateShapeData< Number2 >> &shape_data, Number *values_dofs_in, Number *values, Number *gradients, Number *scratch_data, const unsigned int subface_index, const unsigned int face_direction)
EvaluatorTensorProduct< symmetric_evaluate ? evaluate_evenodd :evaluate_general, dim - 1, fe_degree+1, n_q_points_1d, Number, Number2 > Eval
static void evaluate_in_face(const unsigned int n_components, const EvaluationFlags::EvaluationFlags evaluation_flag, const MatrixFreeFunctions::UnivariateShapeData< Number2 > &data, Number *values_dofs, Number *values_quad, Number *gradients_quad, Number *hessians_quad, Number *scratch_data, const unsigned int subface_index)
static Eval create_evaluator_tensor_product(const MatrixFreeFunctions::UnivariateShapeData< Number2 > &data, const unsigned int subface_index, const unsigned int direction)
static void integrate_in_face(const unsigned int n_components, const EvaluationFlags::EvaluationFlags integration_flag, const MatrixFreeFunctions::UnivariateShapeData< Number2 > &data, Number *values_dofs, Number *values_quad, Number *gradients_quad, Number *hessians_quad, Number *scratch_data, const unsigned int subface_index)
typename FEEvaluationData< dim, Number, true >::shape_info_number_type Number2
typename FEEvaluationData< dim, Number, true >::shape_info_number_type Number2
static void interpolate_quadrature(const unsigned int n_components, const EvaluationFlags::EvaluationFlags flags, const MatrixFreeFunctions::ShapeInfo< Number2 > &shape_info, const Number *input, Number *output, const unsigned int face_no)
static void interpolate_generic(const unsigned int n_components, const Number *input, Number *output, const EvaluationFlags::EvaluationFlags flag, const unsigned int face_no, const unsigned int n_points_1d, const std::array< AlignedVector< Number2 >, 2 > &shape_data, const unsigned int dofs_per_component_on_cell, const unsigned int dofs_per_component_on_face)
static void interpolate(const unsigned int n_components, const EvaluationFlags::EvaluationFlags flags, const MatrixFreeFunctions::ShapeInfo< Number2 > &shape_info, const Number *input, Number *output, const unsigned int face_no)
static void interpolate_raviart_thomas(const unsigned int n_components, const Number *input, Number *output, const EvaluationFlags::EvaluationFlags flag, const unsigned int face_no, const MatrixFreeFunctions::ShapeInfo< Number2 > &shape_info)
unsigned int n_q_points_face
unsigned int dofs_per_component_on_cell
std::vector< UnivariateShapeData< Number > > data
::Table< 2, unsigned int > face_orientations_quad
unsigned int dofs_per_component_on_face
AlignedVector< Number > shape_values
AlignedVector< Number > shape_values_eo
AlignedVector< Number > shape_hessians_eo
AlignedVector< Number > shape_gradients_collocation_eo
unsigned int n_q_points_1d
AlignedVector< Number > shape_gradients_eo
AlignedVector< Number > shape_hessians
AlignedVector< Number > shape_gradients
std::array< AlignedVector< Number >, 2 > hessians_within_subface
std::array< AlignedVector< Number >, 2 > values_within_subface
std::array< AlignedVector< Number >, 2 > gradients_within_subface