Reference documentation for deal.II version GIT relicensing-214-g6e74dec06b 2024-03-27 18:10:01+00:00
\(\newcommand{\dealvcentcolon}{\mathrel{\mathop{:}}}\) \(\newcommand{\dealcoloneq}{\dealvcentcolon\mathrel{\mkern-1.2mu}=}\) \(\newcommand{\jump}[1]{\left[\!\left[ #1 \right]\!\right]}\) \(\newcommand{\average}[1]{\left\{\!\left\{ #1 \right\}\!\right\}}\)
Loading...
Searching...
No Matches
vector_operations_internal.h
Go to the documentation of this file.
1// ------------------------------------------------------------------------
2//
3// SPDX-License-Identifier: LGPL-2.1-or-later
4// Copyright (C) 2016 - 2023 by the deal.II authors
5//
6// This file is part of the deal.II library.
7//
8// Part of the source code is dual licensed under Apache-2.0 WITH
9// LLVM-exception OR LGPL-2.1-or-later. Detailed license information
10// governing the source code and code contributions can be found in
11// LICENSE.md and CONTRIBUTING.md at the top level directory of deal.II.
12//
13// ------------------------------------------------------------------------
14
15
16#ifndef dealii_vector_operations_internal_h
17#define dealii_vector_operations_internal_h
18
19#include <deal.II/base/config.h>
20
25#include <deal.II/base/types.h>
27
29
30#include <cstdio>
31#include <cstring>
32
34
35namespace internal
36{
37 namespace VectorOperations
38 {
40
41 template <typename T>
42 bool
43 is_non_negative(const T &t)
44 {
45 return t >= 0;
46 }
47
48
49 template <typename T>
50 bool
51 is_non_negative(const std::complex<T> &)
52 {
53 Assert(false, ExcMessage("Complex numbers do not have an ordering."));
54
55 return false;
56 }
57
58
59 // call std::copy, except for in
60 // the case where we want to copy
61 // from std::complex to a
62 // non-complex type
63 template <typename T, typename U>
64 void
65 copy(const T *begin, const T *end, U *dest)
66 {
67 std::copy(begin, end, dest);
68 }
69
70 template <typename T, typename U>
71 void
72 copy(const std::complex<T> *begin,
73 const std::complex<T> *end,
74 std::complex<U> *dest)
75 {
76 std::copy(begin, end, dest);
77 }
78
79 template <typename T, typename U>
80 void
81 copy(const std::complex<T> *, const std::complex<T> *, U *)
82 {
83 Assert(false,
84 ExcMessage("Can't convert a vector of complex numbers "
85 "into a vector of reals/doubles"));
86 }
87
88
89
90#ifdef DEAL_II_WITH_TBB
99 template <typename Functor>
101 {
103 const size_type start,
104 const size_type end)
106 , start(start)
107 , end(end)
108 {
109 const size_type vec_size = end - start;
110 // set chunk size for sub-tasks
111 const unsigned int gs =
113 n_chunks =
115 vec_size / gs);
116 chunk_size = vec_size / n_chunks;
117
118 // round to next multiple of 512 (or minimum grain size if that happens
119 // to be smaller). this is advantageous because our accumulation
120 // algorithms favor lengths of a power of 2 due to pairwise summation ->
121 // at most one 'oddly' sized chunk
122 if (chunk_size > 512)
123 chunk_size = ((chunk_size + 511) / 512) * 512;
124 n_chunks = (vec_size + chunk_size - 1) / chunk_size;
125 AssertIndexRange((n_chunks - 1) * chunk_size, vec_size);
126 AssertIndexRange(vec_size, n_chunks * chunk_size + 1);
127 }
128
129 void
130 operator()(const tbb::blocked_range<size_type> &range) const
131 {
132 const size_type r_begin = start + range.begin() * chunk_size;
133 const size_type r_end = std::min(start + range.end() * chunk_size, end);
134 functor(r_begin, r_end);
135 }
136
137 Functor &functor;
140 unsigned int n_chunks;
142 };
143#endif
144
145 template <typename Functor>
146 void
148 Functor &functor,
149 const size_type start,
150 const size_type end,
151 const std::shared_ptr<::parallel::internal::TBBPartitioner>
152 &partitioner)
153 {
154#ifdef DEAL_II_WITH_TBB
155 const size_type vec_size = end - start;
156 // only go to the parallel function in case there are at least 4 parallel
157 // items, otherwise the overhead is too large
158 if (vec_size >=
161 {
162 Assert(partitioner.get() != nullptr,
164 "Unexpected initialization of Vector that does "
165 "not set the TBB partitioner to a usable state."));
166 std::shared_ptr<tbb::affinity_partitioner> tbb_partitioner =
167 partitioner->acquire_one_partitioner();
168
169 TBBForFunctor<Functor> generic_functor(functor, start, end);
170 // We use a minimum grain size of 1 here since the grains at this
171 // stage of dividing the work refer to the number of vector chunks
172 // that are processed by (possibly different) threads in the
173 // parallelized for loop (i.e., they do not refer to individual
174 // vector entries). The number of chunks here is calculated inside
175 // TBBForFunctor. See also GitHub issue #2496 for further discussion
176 // of this strategy.
178 static_cast<size_type>(0),
179 static_cast<size_type>(generic_functor.n_chunks),
180 generic_functor,
181 1,
182 tbb_partitioner);
183 partitioner->release_one_partitioner(tbb_partitioner);
184 }
185 else if (vec_size > 0)
186 functor(start, end);
187#else
188 functor(start, end);
189 (void)partitioner;
190#endif
191 }
192
193
194 // Define the functors necessary to use SIMD with TBB. we also include the
195 // simple copy and set operations
196
197 template <typename Number>
199 {
200 Vector_set(const Number value, Number *const dst)
201 : value(value)
202 , dst(dst)
203 {
204 Assert(dst != nullptr, ExcInternalError());
205 }
206
207 void
208 operator()(const size_type begin, const size_type end) const
209 {
210 Assert(end >= begin, ExcInternalError());
211
212 if (value == Number())
213 {
214 if constexpr (std::is_trivial_v<Number>)
215 {
216 std::memset(dst + begin, 0, sizeof(Number) * (end - begin));
217 return;
218 }
219 }
220 std::fill(dst + begin, dst + end, value);
221 }
222
223 const Number value;
224 Number *const dst;
225 };
226
227 template <typename Number, typename OtherNumber>
229 {
230 Vector_copy(const OtherNumber *const src, Number *const dst)
231 : src(src)
232 , dst(dst)
233 {
234 Assert(src != nullptr, ExcInternalError());
235 Assert(dst != nullptr, ExcInternalError());
236 }
237
238 void
239 operator()(const size_type begin, const size_type end) const
240 {
241 Assert(end >= begin, ExcInternalError());
242
243 if constexpr (std::is_trivially_copyable<Number>() &&
244 std::is_same_v<Number, OtherNumber>)
245 std::memcpy(dst + begin, src + begin, (end - begin) * sizeof(Number));
246 else
247 {
249 for (size_type i = begin; i < end; ++i)
250 dst[i] = src[i];
251 }
252 }
253
254 const OtherNumber *const src;
255 Number *const dst;
256 };
257
258 template <typename Number>
260 {
261 Vectorization_multiply_factor(Number *const val, const Number factor)
262 : val(val)
263 , stored_factor(factor)
264 {}
265
266 void
267 operator()(const size_type begin, const size_type end) const
268 {
269 // create a local copy of the variable to help the compiler with the
270 // aliasing analysis
271 const Number factor = stored_factor;
272
274 {
276 for (size_type i = begin; i < end; ++i)
277 val[i] *= factor;
278 }
279 else
280 {
281 for (size_type i = begin; i < end; ++i)
282 val[i] *= factor;
283 }
284 }
285
286 Number *const val;
287 const Number stored_factor;
288 };
289
290 template <typename Number>
292 {
294 const Number *const v_val,
295 const Number factor)
296 : val(val)
297 , v_val(v_val)
298 , stored_factor(factor)
299 {}
300
301 void
302 operator()(const size_type begin, const size_type end) const
303 {
304 // create a local copy of the variable to help the compiler with the
305 // aliasing analysis
306 const Number factor = stored_factor;
308 {
310 for (size_type i = begin; i < end; ++i)
311 val[i] += factor * v_val[i];
312 }
313 else
314 {
315 for (size_type i = begin; i < end; ++i)
316 val[i] += factor * v_val[i];
317 }
318 }
319
320 Number *const val;
321 const Number *const v_val;
322 const Number stored_factor;
323 };
324
325 template <typename Number>
327 {
329 const Number *const v_val,
330 const Number a,
331 const Number x)
332 : val(val)
333 , v_val(v_val)
334 , stored_a(a)
335 , stored_x(x)
336 {}
337
338 void
339 operator()(const size_type begin, const size_type end) const
340 {
341 // create a local copy of the variable to help the compiler with the
342 // aliasing analysis
343 const Number x = stored_x, a = stored_a;
344
346 {
348 for (size_type i = begin; i < end; ++i)
349 val[i] = x * val[i] + a * v_val[i];
350 }
351 else
352 {
353 for (size_type i = begin; i < end; ++i)
354 val[i] = x * val[i] + a * v_val[i];
355 }
356 }
357
358 Number *const val;
359 const Number *const v_val;
360 const Number stored_a;
361 const Number stored_x;
362 };
363
364 template <typename Number>
366 {
367 Vectorization_subtract_v(Number *val, const Number *const v_val)
368 : val(val)
369 , v_val(v_val)
370 {}
371
372 void
373 operator()(const size_type begin, const size_type end) const
374 {
376 {
378 for (size_type i = begin; i < end; ++i)
379 val[i] -= v_val[i];
380 }
381 else
382 {
383 for (size_type i = begin; i < end; ++i)
384 val[i] -= v_val[i];
385 }
386 }
387
388 Number *const val;
389 const Number *const v_val;
390 };
391
392 template <typename Number>
394 {
395 Vectorization_add_factor(Number *const val, const Number factor)
396 : val(val)
397 , stored_factor(factor)
398 {}
399
400 void
401 operator()(const size_type begin, const size_type end) const
402 {
403 const Number factor = stored_factor;
404
406 {
408 for (size_type i = begin; i < end; ++i)
409 val[i] += factor;
410 }
411 else
412 {
413 for (size_type i = begin; i < end; ++i)
414 val[i] += factor;
415 }
416 }
417
418 Number *const val;
419 const Number stored_factor;
420 };
421
422 template <typename Number>
424 {
425 Vectorization_add_v(Number *const val, const Number *const v_val)
426 : val(val)
427 , v_val(v_val)
428 {}
429
430 void
431 operator()(const size_type begin, const size_type end) const
432 {
434 {
436 for (size_type i = begin; i < end; ++i)
437 val[i] += v_val[i];
438 }
439 else
440 {
441 for (size_type i = begin; i < end; ++i)
442 val[i] += v_val[i];
443 }
444 }
445
446 Number *const val;
447 const Number *const v_val;
448 };
449
450 template <typename Number>
452 {
454 const Number *const v_val,
455 const Number *const w_val,
456 const Number a,
457 const Number b)
458 : val(val)
459 , v_val(v_val)
460 , w_val(w_val)
461 , stored_a(a)
462 , stored_b(b)
463 {}
464
465 void
466 operator()(const size_type begin, const size_type end) const
467 {
468 const Number a = stored_a, b = stored_b;
469
471 {
473 for (size_type i = begin; i < end; ++i)
474 val[i] = val[i] + a * v_val[i] + b * w_val[i];
475 }
476 else
477 {
478 for (size_type i = begin; i < end; ++i)
479 val[i] = val[i] + a * v_val[i] + b * w_val[i];
480 }
481 }
482
483 Number *const val;
484 const Number *const v_val;
485 const Number *const w_val;
486 const Number stored_a;
487 const Number stored_b;
488 };
489
490 template <typename Number>
492 {
494 const Number *const v_val,
495 const Number x)
496 : val(val)
497 , v_val(v_val)
498 , stored_x(x)
499 {}
500
501 void
502 operator()(const size_type begin, const size_type end) const
503 {
504 const Number x = stored_x;
505
507 {
509 for (size_type i = begin; i < end; ++i)
510 val[i] = x * val[i] + v_val[i];
511 }
512 else
513 {
514 for (size_type i = begin; i < end; ++i)
515 val[i] = x * val[i] + v_val[i];
516 }
517 }
518
519 Number *const val;
520 const Number *const v_val;
521 const Number stored_x;
522 };
523
524 template <typename Number>
526 {
528 const Number *v_val,
529 const Number *w_val,
530 Number x,
531 Number a,
532 Number b)
533 : val(val)
534 , v_val(v_val)
535 , w_val(w_val)
536 , stored_x(x)
537 , stored_a(a)
538 , stored_b(b)
539 {}
540
541 void
542 operator()(const size_type begin, const size_type end) const
543 {
544 const Number x = stored_x, a = stored_a, b = stored_b;
545
547 {
549 for (size_type i = begin; i < end; ++i)
550 val[i] = x * val[i] + a * v_val[i] + b * w_val[i];
551 }
552 else
553 {
554 for (size_type i = begin; i < end; ++i)
555 val[i] = x * val[i] + a * v_val[i] + b * w_val[i];
556 }
557 }
558
559 Number *const val;
560 const Number *const v_val;
561 const Number *const w_val;
562 const Number stored_x;
563 const Number stored_a;
564 const Number stored_b;
565 };
566
567 template <typename Number>
569 {
570 Vectorization_scale(Number *const val, const Number *const v_val)
571 : val(val)
572 , v_val(v_val)
573 {}
574
575 void
576 operator()(const size_type begin, const size_type end) const
577 {
579 {
581 for (size_type i = begin; i < end; ++i)
582 val[i] *= v_val[i];
583 }
584 else
585 {
586 for (size_type i = begin; i < end; ++i)
587 val[i] *= v_val[i];
588 }
589 }
590
591 Number *const val;
592 const Number *const v_val;
593 };
594
595 template <typename Number>
597 {
599 const Number *const u_val,
600 const Number a)
601 : val(val)
602 , u_val(u_val)
603 , stored_a(a)
604 {}
605
606 void
607 operator()(const size_type begin, const size_type end) const
608 {
609 const Number a = stored_a;
610
612 {
614 for (size_type i = begin; i < end; ++i)
615 val[i] = a * u_val[i];
616 }
617 else
618 {
619 for (size_type i = begin; i < end; ++i)
620 val[i] = a * u_val[i];
621 }
622 }
623
624 Number *const val;
625 const Number *const u_val;
626 const Number stored_a;
627 };
628
629 template <typename Number>
631 {
633 const Number *const u_val,
634 const Number *const v_val,
635 const Number a,
636 const Number b)
637 : val(val)
638 , u_val(u_val)
639 , v_val(v_val)
640 , stored_a(a)
641 , stored_b(b)
642 {}
643
644 void
645 operator()(const size_type begin, const size_type end) const
646 {
647 const Number a = stored_a, b = stored_b;
648
650 {
652 for (size_type i = begin; i < end; ++i)
653 val[i] = a * u_val[i] + b * v_val[i];
654 }
655 else
656 {
657 for (size_type i = begin; i < end; ++i)
658 val[i] = a * u_val[i] + b * v_val[i];
659 }
660 }
661
662 Number *const val;
663 const Number *const u_val;
664 const Number *const v_val;
665 const Number stored_a;
666 const Number stored_b;
667 };
668
669 template <typename Number>
671 {
673 const Number *u_val,
674 const Number *v_val,
675 const Number *w_val,
676 const Number a,
677 const Number b,
678 const Number c)
679 : val(val)
680 , u_val(u_val)
681 , v_val(v_val)
682 , w_val(w_val)
683 , stored_a(a)
684 , stored_b(b)
685 , stored_c(c)
686 {}
687
688 void
689 operator()(const size_type begin, const size_type end) const
690 {
691 const Number a = stored_a, b = stored_b, c = stored_c;
692
694 {
696 for (size_type i = begin; i < end; ++i)
697 val[i] = a * u_val[i] + b * v_val[i] + c * w_val[i];
698 }
699 else
700 {
701 for (size_type i = begin; i < end; ++i)
702 val[i] = a * u_val[i] + b * v_val[i] + c * w_val[i];
703 }
704 }
705
706 Number *const val;
707 const Number *const u_val;
708 const Number *const v_val;
709 const Number *const w_val;
710 const Number stored_a;
711 const Number stored_b;
712 const Number stored_c;
713 };
714
715 template <typename Number>
717 {
718 Vectorization_ratio(Number *val, const Number *a_val, const Number *b_val)
719 : val(val)
720 , a_val(a_val)
721 , b_val(b_val)
722 {}
723
724 void
725 operator()(const size_type begin, const size_type end) const
726 {
728 {
730 for (size_type i = begin; i < end; ++i)
731 val[i] = a_val[i] / b_val[i];
732 }
733 else
734 {
735 for (size_type i = begin; i < end; ++i)
736 val[i] = a_val[i] / b_val[i];
737 }
738 }
739
740 Number *const val;
741 const Number *const a_val;
742 const Number *const b_val;
743 };
744
745
746
747 // All sums over all the vector entries (l2-norm, inner product, etc.) are
748 // performed with the same code, using a templated operation defined
749 // here. There are always two versions defined, a standard one that covers
750 // most cases and a vectorized one which is only for equal types and float
751 // and double.
752 template <typename Number, typename Number2>
753 struct Dot
754 {
755 static constexpr bool vectorizes = std::is_same_v<Number, Number2> &&
757
758 Dot(const Number *const X, const Number2 *const Y)
759 : X(X)
760 , Y(Y)
761 {}
762
763 Number
764 operator()(const size_type i) const
765 {
766 return X[i] * Number(numbers::NumberTraits<Number2>::conjugate(Y[i]));
767 }
768
771 {
773 x.load(X + i);
774 y.load(Y + i);
775
776 // the following operation in VectorizedArray does an element-wise
777 // scalar product without taking into account complex values and
778 // the need to take the complex-conjugate of one argument. this
779 // may be a bug, but because all VectorizedArray classes only
780 // work on real scalars, it doesn't really matter very much.
781 // in any case, assert that we really don't get here for
782 // complex-valued objects
783 static_assert(numbers::NumberTraits<Number>::is_complex == false,
784 "This operation is not correctly implemented for "
785 "complex-valued objects.");
786 return x * y;
787 }
788
789 const Number *const X;
790 const Number2 *const Y;
791 };
792
793 template <typename Number, typename RealType>
794 struct Norm2
795 {
796 static const bool vectorizes = VectorizedArray<Number>::size() > 1;
797
798 Norm2(const Number *const X)
799 : X(X)
800 {}
801
802 RealType
803 operator()(const size_type i) const
804 {
806 }
807
810 {
812 x.load(X + i);
813 return x * x;
814 }
815
816 const Number *const X;
817 };
818
819 template <typename Number, typename RealType>
820 struct Norm1
821 {
822 static const bool vectorizes = VectorizedArray<Number>::size() > 1;
823
824 Norm1(const Number *X)
825 : X(X)
826 {}
827
828 RealType
829 operator()(const size_type i) const
830 {
832 }
833
836 {
838 x.load(X + i);
839 return std::abs(x);
840 }
841
842 const Number *X;
843 };
844
845 template <typename Number, typename RealType>
846 struct NormP
847 {
848 static const bool vectorizes = VectorizedArray<Number>::size() > 1;
849
850 NormP(const Number *X, RealType p)
851 : X(X)
852 , p(p)
853 {}
854
855 RealType
856 operator()(const size_type i) const
857 {
859 }
860
863 {
865 x.load(X + i);
866 return std::pow(std::abs(x), p);
867 }
868
869 const Number *X;
870 const RealType p;
871 };
872
873 template <typename Number>
875 {
876 static const bool vectorizes = VectorizedArray<Number>::size() > 1;
877
878 MeanValue(const Number *X)
879 : X(X)
880 {}
881
882 Number
883 operator()(const size_type i) const
884 {
885 return X[i];
886 }
887
890 {
892 x.load(X + i);
893 return x;
894 }
895
896 const Number *X;
897 };
898
899 template <typename Number>
901 {
902 static const bool vectorizes = VectorizedArray<Number>::size() > 1;
903
904 AddAndDot(Number *const X,
905 const Number *const V,
906 const Number *const W,
907 const Number a)
908 : X(X)
909 , V(V)
910 , W(W)
911 , a(a)
912 {}
913
914 Number
915 operator()(const size_type i) const
916 {
917 X[i] += a * V[i];
918 return X[i] * Number(numbers::NumberTraits<Number>::conjugate(W[i]));
919 }
920
923 {
925 x.load(X + i);
926 v.load(V + i);
927 x += a * v;
928 x.store(X + i);
929 // may only load from W after storing in X because the pointers might
930 // point to the same memory
931 w.load(W + i);
932
933 // the following operation in VectorizedArray does an element-wise
934 // scalar product without taking into account complex values and
935 // the need to take the complex-conjugate of one argument. this
936 // may be a bug, but because all VectorizedArray classes only
937 // work on real scalars, it doesn't really matter very much.
938 // in any case, assert that we really don't get here for
939 // complex-valued objects
940 static_assert(numbers::NumberTraits<Number>::is_complex == false,
941 "This operation is not correctly implemented for "
942 "complex-valued objects.");
943 return x * w;
944 }
945
946 Number *const X;
947 const Number *const V;
948 const Number *const W;
949 const Number a;
950 };
951
952
953
954 // this is the main working loop for all vector sums using the templated
955 // operation above. it accumulates the sums using a block-wise summation
956 // algorithm with post-update. this blocked algorithm has been proposed in
957 // a similar form by Castaldo, Whaley and Chronopoulos (SIAM
958 // J. Sci. Comput. 31, 1156-1174, 2008) and we use the smallest possible
959 // block size, 2. Sometimes it is referred to as pairwise summation. The
960 // worst case error made by this algorithm is on the order O(eps *
961 // log2(vec_size)), whereas a naive summation is O(eps * vec_size). Even
962 // though the Kahan summation is even more accurate with an error O(eps)
963 // by carrying along remainders not captured by the main sum, that involves
964 // additional costs which are not worthwhile. See the Wikipedia article on
965 // the Kahan summation algorithm.
966
967 // The algorithm implemented here has the additional benefit that it is
968 // easily parallelized without changing the order of how the elements are
969 // added (floating point addition is not associative). For the same vector
970 // size and minimum_parallel_grainsize, the blocks are always the
971 // same and added pairwise.
972
973 // The depth of recursion is controlled by the 'magic' parameter
974 // vector_accumulation_recursion_threshold: If the length is below
975 // vector_accumulation_recursion_threshold * 32 (32 is the part of code we
976 // unroll), a straight loop instead of recursion will be used. At the
977 // innermost level, eight values are added consecutively in order to better
978 // balance multiplications and additions.
979
980 // Loops are unrolled as follows: the range [first,last) is broken into
981 // @p n_chunks each of size 32 plus the @p remainder.
982 // accumulate_regular() does the work on 32*n_chunks elements employing SIMD
983 // if possible and stores the result of the operation for each chunk in @p outer_results.
984
985 // The code returns the result as the last argument in order to make
986 // spawning tasks simpler and use automatic template deduction.
987
988
995
996 template <typename Operation, typename ResultType>
997 void
998 accumulate_recursive(const Operation &op,
999 const size_type first,
1000 const size_type last,
1001 ResultType &result)
1002 {
1003 if (first == last)
1004 {
1005 result = ResultType();
1006 return;
1007 }
1008
1009 const size_type vec_size = last - first;
1010 if (vec_size <= vector_accumulation_recursion_threshold * 32)
1011 {
1012 // The vector is short enough so we perform the summation. We store
1013 // the number of chunks (each 32 indices) for the given vector
1014 // length; all results are stored in outer_results[0,n_chunks). We
1015 // keep twice the number around to be able to do the pairwise
1016 // summation with a single for loop (see the loop over j below)
1017 ResultType outer_results[vector_accumulation_recursion_threshold * 2];
1018
1019 // Select between the regular version and vectorized version based
1020 // on the number types we are given. To choose the vectorized
1021 // version often enough, we need to have all tasks but the last one
1022 // to be divisible by the vectorization length
1023 size_type n_chunks =
1024 do_accumulate(op,
1025 vec_size,
1026 first,
1027 outer_results,
1028 std::bool_constant<Operation::vectorizes>());
1029
1030 AssertIndexRange(n_chunks,
1032
1033 // now sum the results from the chunks stored in
1034 // outer_results[0,n_chunks) recursively
1035 unsigned int j = 0;
1036 constexpr unsigned int n_lanes = VectorizedArray<ResultType>::size();
1037 for (; j + 2 * n_lanes - 1 < n_chunks;
1038 j += 2 * n_lanes, n_chunks += n_lanes)
1039 {
1041 a.load(outer_results + j);
1042 b.load(outer_results + j + n_lanes);
1043 a += b;
1044 a.store(outer_results + n_chunks);
1045 }
1046
1047 // In the vectorized case, we know the loop bounds and can do things
1048 // more efficiently
1049 if (Operation::vectorizes)
1050 {
1051 AssertDimension(j + n_lanes, n_chunks);
1052 AssertIndexRange(n_chunks,
1054 ResultType *result_ptr = outer_results + j;
1055 if (n_lanes >= 16)
1056 for (unsigned int i = 0; i < 8; ++i)
1057 result_ptr[i] = result_ptr[i] + result_ptr[i + 8];
1058 if (n_lanes >= 8)
1059 for (unsigned int i = 0; i < 4; ++i)
1060 result_ptr[i] = result_ptr[i] + result_ptr[i + 4];
1061 if (n_lanes >= 4)
1062 for (unsigned int i = 0; i < 2; ++i)
1063 result_ptr[i] = result_ptr[i] + result_ptr[i + 2];
1064 result = result_ptr[0] + result_ptr[1];
1065 }
1066 else
1067 {
1068 // Without vectorization, we do not know the exact bounds, so we
1069 // need to continue the variable-length pairwise summation loop
1070 // from above
1071 for (; j + 1 < n_chunks; j += 2, ++n_chunks)
1072 outer_results[n_chunks] =
1073 outer_results[j] + outer_results[j + 1];
1074
1075 AssertIndexRange(n_chunks,
1077 Assert(n_chunks > 0, ExcInternalError());
1078 result = outer_results[n_chunks - 1];
1079 }
1080 }
1081 else
1082 {
1083 // split vector into four pieces and work on the pieces
1084 // recursively. Make pieces (except last) divisible by one fourth the
1085 // recursion threshold.
1086 const size_type new_size =
1087 (vec_size / (vector_accumulation_recursion_threshold * 32)) *
1089 Assert(first + 3 * new_size < last, ExcInternalError());
1090 ResultType r0, r1, r2, r3;
1091 accumulate_recursive(op, first, first + new_size, r0);
1092 accumulate_recursive(op, first + new_size, first + 2 * new_size, r1);
1094 first + 2 * new_size,
1095 first + 3 * new_size,
1096 r2);
1097 accumulate_recursive(op, first + 3 * new_size, last, r3);
1098 result = (r0 + r1) + (r2 + r3);
1099 }
1100 }
1101
1102
1103 // this is the inner working routine for the accumulation loops below. We
1104 // pulled this part out of the regular accumulate routine because we might
1105 // do this thing vectorized (see specialized function below; this is the
1106 // un-vectorized version). As opposed to the vector add functions above,
1107 // we here pass the functor 'op' by value, because we cannot create a copy
1108 // of the scalar inline, and instead make sure that the numbers get local
1109 // (and thus definitely not aliased) for the compiler
1110 template <typename Operation, typename ResultType>
1111 size_type
1112 do_accumulate(const Operation op,
1113 const size_type vec_size,
1114 const size_type start_index,
1115 ResultType *outer_results,
1116 std::bool_constant<false>)
1117 {
1118 // Create local copy to indicate no aliasing to the compiler
1119 size_type index = start_index;
1120
1121 // choose each chunk to have a width of 32, thereby the index
1122 // is incremented by 4*8 for each @p i.
1123 size_type n_chunks = vec_size / 32;
1124 for (size_type i = 0; i < n_chunks; ++i)
1125 {
1126 ResultType r = {};
1127 for (unsigned int k = 0; k < 2; ++k)
1128 {
1129 ResultType r0 = op(index);
1130 ResultType r1 = op(index + 1);
1131 ResultType r2 = op(index + 2);
1132 ResultType r3 = op(index + 3);
1133 index += 4;
1134 for (size_type j = 1; j < 4; ++j, index += 4)
1135 {
1136 r0 += op(index);
1137 r1 += op(index + 1);
1138 r2 += op(index + 2);
1139 r3 += op(index + 3);
1140 }
1141 r += (r0 + r1) + (r2 + r3);
1142 }
1143 outer_results[i] = r;
1144 }
1145
1146 if (n_chunks * 32 < vec_size)
1147 {
1148 const size_type remainder = vec_size - n_chunks * 32;
1149 const size_type inner_chunks = remainder / 8;
1150 const size_type remainder_inner = remainder % 8;
1151 ResultType r0 = ResultType(), r1 = ResultType(), r2 = ResultType();
1152 switch (inner_chunks)
1153 {
1154 case 3:
1155 r2 = op(index++);
1156 for (size_type j = 1; j < 8; ++j)
1157 r2 += op(index++);
1159 case 2:
1160 r1 = op(index++);
1161 for (size_type j = 1; j < 8; ++j)
1162 r1 += op(index++);
1163 r1 += r2;
1165 case 1:
1166 r2 = op(index++);
1167 for (size_type j = 1; j < 8; ++j)
1168 r2 += op(index++);
1170 default:
1171 for (size_type j = 0; j < remainder_inner; ++j)
1172 r0 += op(index++);
1173 outer_results[n_chunks++] = (r0 + r2) + r1;
1174 break;
1175 }
1176 }
1177
1178 // make sure we worked through all indices
1179 AssertDimension(index, start_index + vec_size);
1180
1181 return n_chunks;
1182 }
1183
1184
1185
1186 // this is the inner working routine for the accumulation loops
1187 // below. This is the specialized case where we can vectorize. We request
1188 // the 'do_vectorized' routine of the operation instead of the regular one
1189 // which does several operations at once. As above, pass in the functor by
1190 // value to create a local copy of the scalar factors in the function (if
1191 // there are any).
1192 template <typename Operation, typename Number>
1193 size_type
1194 do_accumulate(const Operation op,
1195 const size_type vec_size,
1196 const size_type start_index,
1197 Number *outer_results,
1198 std::bool_constant<true>)
1199 {
1200 // Create local copy to indicate no aliasing to the compiler
1201 size_type index = start_index;
1202
1203 // we start from @p index and workout @p n_chunks each of size 32.
1204 // in order employ SIMD and work on @p nvecs at a time, we split this
1205 // loop yet again:
1206 // First we work on (n_chunks/nvecs) chunks, where each chunk processes
1207 // nvecs*(4*8) elements.
1208
1209 constexpr size_type n_lanes = VectorizedArray<Number>::size();
1210 const size_type regular_chunks = vec_size / (32 * n_lanes);
1211 for (size_type i = 0; i < regular_chunks; ++i)
1212 {
1214 for (unsigned int k = 0; k < 2; ++k)
1215 {
1216 VectorizedArray<Number> r0 = op.do_vectorized(index);
1217 VectorizedArray<Number> r1 = op.do_vectorized(index + n_lanes);
1219 op.do_vectorized(index + 2 * n_lanes);
1221 op.do_vectorized(index + 3 * n_lanes);
1222 index += n_lanes * 4;
1223 for (size_type j = 1; j < 4; ++j, index += n_lanes * 4)
1224 {
1225 r0 += op.do_vectorized(index);
1226 r1 += op.do_vectorized(index + n_lanes);
1227 r2 += op.do_vectorized(index + 2 * n_lanes);
1228 r3 += op.do_vectorized(index + 3 * n_lanes);
1229 }
1230 r += (r0 + r1) + (r2 + r3);
1231 }
1232 r.store(&outer_results[i * n_lanes]);
1233 }
1234
1235 // If we are treating a case where the vector length is not divisible by
1236 // the vectorization length, need a cleanup loop
1237 // The remaining chunks are processed one by one starting from
1238 // regular_chunks * n_lanes; We do as much as possible with 2 SIMD
1239 // operations within each chunk. Here we assume that n_lanes < 32/2 = 16
1240 // as well as 16 % n_lanes == 0.
1241 static_assert(n_lanes <= 16 && 16 % n_lanes == 0,
1242 "VectorizedArray::size() must be 1, 2, 4, 8, or 16");
1243 size_type n_chunks = regular_chunks * n_lanes;
1244 const size_type start_irregular = regular_chunks * n_lanes * 32;
1245 if (start_irregular < vec_size)
1246 {
1249 const size_type remainder = vec_size - start_irregular;
1250 const size_type loop_length = remainder / (2 * n_lanes);
1251 for (size_type j = 0; j < loop_length; ++j, index += 2 * n_lanes)
1252 {
1253 r0 += op.do_vectorized(index);
1254 r1 += op.do_vectorized(index + n_lanes);
1255 }
1256 Number scalar_part = Number();
1257 size_type last = remainder % (2 * n_lanes);
1258 if (last > 0)
1259 {
1260 if (last >= n_lanes)
1261 {
1262 r0 += op.do_vectorized(index);
1263 index += n_lanes;
1264 last -= n_lanes;
1265 }
1266 for (unsigned int i = 0; i < last; ++i)
1267 scalar_part += op(index++);
1268 }
1269
1270 r0 += r1;
1271 r0.store(&outer_results[n_chunks]);
1272 outer_results[n_chunks] += scalar_part;
1273
1274 // update n_chunks to denote range of entries to sum up in
1275 // outer_results[].
1276 n_chunks += n_lanes;
1277 }
1278
1279 // make sure we worked through all indices
1280 AssertDimension(index, start_index + vec_size);
1281
1282 return n_chunks;
1283 }
1284
1285
1286
1287#ifdef DEAL_II_WITH_TBB
1316 template <typename Operation, typename ResultType>
1318 {
1319 static const unsigned int threshold_array_allocate = 512;
1320
1321 TBBReduceFunctor(const Operation &op,
1322 const size_type start,
1323 const size_type end)
1324 : op(op)
1325 , start(start)
1326 , end(end)
1327 {
1328 const size_type vec_size = end - start;
1329 // set chunk size for sub-tasks
1330 const unsigned int gs =
1332 n_chunks =
1333 std::min(static_cast<size_type>(4 * MultithreadInfo::n_threads()),
1334 vec_size / gs);
1335 chunk_size = vec_size / n_chunks;
1336
1337 // round to next multiple of 512 (or leave it at the minimum grain size
1338 // if that happens to be smaller). this is advantageous because our
1339 // algorithm favors lengths of a power of 2 due to pairwise summation ->
1340 // at most one 'oddly' sized chunk
1341 if (chunk_size > 512)
1342 chunk_size = ((chunk_size + 511) / 512) * 512;
1343 n_chunks = (vec_size + chunk_size - 1) / chunk_size;
1344 AssertIndexRange((n_chunks - 1) * chunk_size, vec_size);
1345 AssertIndexRange(vec_size, n_chunks * chunk_size + 1);
1346
1348 {
1349 // make sure we allocate an even number of elements,
1350 // access to the new last element is needed in do_sum()
1351 large_array.resize(2 * ((n_chunks + 1) / 2));
1352 array_ptr = large_array.data();
1353 }
1354 else
1355 array_ptr = &small_array[0];
1356 }
1357
1362 void
1363 operator()(const tbb::blocked_range<size_type> &range) const
1364 {
1365 for (size_type i = range.begin(); i < range.end(); ++i)
1367 start + i * chunk_size,
1368 std::min(start + (i + 1) * chunk_size, end),
1369 array_ptr[i]);
1370 }
1371
1372 ResultType
1373 do_sum() const
1374 {
1375 while (n_chunks > 1)
1376 {
1377 if (n_chunks % 2 == 1)
1378 array_ptr[n_chunks++] = ResultType();
1379 for (size_type i = 0; i < n_chunks; i += 2)
1380 array_ptr[i / 2] = array_ptr[i] + array_ptr[i + 1];
1381 n_chunks /= 2;
1382 }
1383 return array_ptr[0];
1384 }
1385
1386 const Operation &op;
1389
1390 mutable unsigned int n_chunks;
1391 unsigned int chunk_size;
1393 std::vector<ResultType> large_array;
1394 // this variable either points to small_array or large_array depending on
1395 // the number of threads we want to feed
1396 mutable ResultType *array_ptr;
1397 };
1398#endif
1399
1400
1401
1406 template <typename Operation, typename ResultType>
1407#ifndef DEBUG
1409#endif
1410 inline void
1412 const Operation &op,
1413 const size_type start,
1414 const size_type end,
1415 ResultType &result,
1416 const std::shared_ptr<::parallel::internal::TBBPartitioner>
1417 &partitioner)
1418 {
1419#ifdef DEAL_II_WITH_TBB
1420 const size_type vec_size = end - start;
1421 // only go to the parallel function in case there are at least 4 parallel
1422 // items, otherwise the overhead is too large
1423 if (vec_size >=
1426 {
1427 Assert(partitioner.get() != nullptr,
1429 "Unexpected initialization of Vector that does "
1430 "not set the TBB partitioner to a usable state."));
1431 std::shared_ptr<tbb::affinity_partitioner> tbb_partitioner =
1432 partitioner->acquire_one_partitioner();
1433
1434 TBBReduceFunctor<Operation, ResultType> generic_functor(op,
1435 start,
1436 end);
1437 // We use a minimum grain size of 1 here since the grains at this
1438 // stage of dividing the work refer to the number of vector chunks
1439 // that are processed by (possibly different) threads in the
1440 // parallelized for loop (i.e., they do not refer to individual
1441 // vector entries). The number of chunks here is calculated inside
1442 // TBBForFunctor. See also GitHub issue #2496 for further discussion
1443 // of this strategy.
1445 static_cast<size_type>(0),
1446 static_cast<size_type>(generic_functor.n_chunks),
1447 generic_functor,
1448 1,
1449 tbb_partitioner);
1450 partitioner->release_one_partitioner(tbb_partitioner);
1451 result = generic_functor.do_sum();
1452 }
1453 else
1454 accumulate_recursive(op, start, end, result);
1455#else
1456 accumulate_recursive(op, start, end, result);
1457 (void)partitioner;
1458#endif
1459 }
1460
1461
1462 template <typename Number, typename Number2, typename MemorySpace>
1464 {
1465 static void
1467 const std::shared_ptr<::parallel::internal::TBBPartitioner> &
1468 /*thread_loop_partitioner*/,
1469 const size_type /*size*/,
1470 const ::MemorySpace::MemorySpaceData<Number2, MemorySpace>
1471 & /*v_data*/,
1473 {
1474 static_assert(
1475 std::is_same_v<MemorySpace, ::MemorySpace::Default> &&
1476 std::is_same_v<Number, Number2>,
1477 "For the Default MemorySpace Number and Number2 should be the same type");
1478 }
1479
1480 static void
1482 const std::shared_ptr<::parallel::internal::TBBPartitioner> &
1483 /*thread_loop_partitioner*/,
1484 const size_type /*size*/,
1485 const Number /*s*/,
1487 {}
1488
1489 static void
1491 const std::shared_ptr<::parallel::internal::TBBPartitioner> &
1492 /*thread_loop_partitioner*/,
1493 const size_type /*size*/,
1494 const ::MemorySpace::MemorySpaceData<Number, MemorySpace>
1495 & /*v_data*/,
1497 {}
1498
1499 static void
1501 const std::shared_ptr<::parallel::internal::TBBPartitioner> &
1502 /*thread_loop_partitioner*/,
1503 const size_type /*size*/,
1504 const ::MemorySpace::MemorySpaceData<Number, MemorySpace>
1505 & /*v_data*/,
1507 {}
1508
1509 static void
1511 const std::shared_ptr<::parallel::internal::TBBPartitioner> &
1512 /*thread_loop_partitioner*/,
1513 const size_type /*size*/,
1514 Number /*a*/,
1516 {}
1517
1518 static void
1520 const std::shared_ptr<::parallel::internal::TBBPartitioner> &
1521 /*thread_loop_partitioner*/,
1522 const size_type /*size*/,
1523 const Number /*a*/,
1524 const ::MemorySpace::MemorySpaceData<Number, MemorySpace>
1525 & /*v_data*/,
1527 {}
1528
1529 static void
1531 const std::shared_ptr<::parallel::internal::TBBPartitioner> &
1532 /*thread_loop_partitioner*/,
1533 const size_type /*size*/,
1534 const Number /*a*/,
1535 const Number /*b*/,
1536 const ::MemorySpace::MemorySpaceData<Number, MemorySpace>
1537 & /*v_data*/,
1538 const ::MemorySpace::MemorySpaceData<Number, MemorySpace>
1539 & /*w_data*/,
1541 {}
1542
1543 static void
1545 const std::shared_ptr<::parallel::internal::TBBPartitioner> &
1546 /*thread_loop_partitioner*/,
1547 const size_type /*size*/,
1548 const Number /*x*/,
1549 const ::MemorySpace::MemorySpaceData<Number, MemorySpace>
1550 & /*v_data*/,
1552 {}
1553
1554 static void
1556 const std::shared_ptr<::parallel::internal::TBBPartitioner> &
1557 /*thread_loop_partitioner*/,
1558 const size_type /*size*/,
1559 const Number /*x*/,
1560 const Number /*a*/,
1561 const ::MemorySpace::MemorySpaceData<Number, MemorySpace>
1562 & /*v_data*/,
1564 {}
1565
1566 static void
1568 const std::shared_ptr<::parallel::internal::TBBPartitioner> &
1569 /*thread_loop_partitioner*/,
1570 const size_type /*size*/,
1571 const Number /*x*/,
1572 const Number /*a*/,
1573 const Number /*b*/,
1574 const ::MemorySpace::MemorySpaceData<Number, MemorySpace>
1575 & /*v_data*/,
1576 const ::MemorySpace::MemorySpaceData<Number, MemorySpace>
1577 & /*w_data*/,
1579 {}
1580
1581 static void
1583 const std::shared_ptr<::parallel::internal::TBBPartitioner> &
1584 /*thread_loop_partitioner*/,
1585 const size_type /*size*/,
1586 const Number /*factor*/,
1588 {}
1589
1590 static void
1592 const std::shared_ptr<::parallel::internal::TBBPartitioner> &
1593 /*thread_loop_partitioner*/,
1594 const size_type /*size*/,
1595 const ::MemorySpace::MemorySpaceData<Number, MemorySpace>
1596 & /*v_data*/,
1598 {}
1599
1600 static void
1602 const std::shared_ptr<::parallel::internal::TBBPartitioner> &
1603 /*thread_loop_partitioner*/,
1604 const size_type /*size*/,
1605 const Number /*a*/,
1606 const ::MemorySpace::MemorySpaceData<Number, MemorySpace>
1607 & /*v_data*/,
1609 {}
1610
1611 static void
1613 const std::shared_ptr<::parallel::internal::TBBPartitioner> &
1614 /*thread_loop_partitioner*/,
1615 const size_type /*size*/,
1616 const Number /*a*/,
1617 const Number /*b*/,
1618 const ::MemorySpace::MemorySpaceData<Number, MemorySpace>
1619 & /*v_data*/,
1620 const ::MemorySpace::MemorySpaceData<Number, MemorySpace>
1621 & /*w_data*/,
1623 {}
1624
1625 static Number
1627 const std::shared_ptr<::parallel::internal::TBBPartitioner> &
1628 /*thread_loop_partitioner*/,
1629 const size_type /*size*/,
1630 const ::MemorySpace::MemorySpaceData<Number2, MemorySpace>
1631 & /*v_data*/,
1633 {
1634 return Number();
1635 }
1636
1637 template <typename real_type>
1638 static void
1640 const std::shared_ptr<::parallel::internal::TBBPartitioner> &
1641 /*thread_loop_partitioner*/,
1642 const size_type /*size*/,
1643 real_type & /*sum*/,
1644 const ::MemorySpace::MemorySpaceData<Number, MemorySpace>
1645 & /*v_data*/,
1647 {}
1648
1649 static Number
1651 const std::shared_ptr<::parallel::internal::TBBPartitioner> &
1652 /*thread_loop_partitioner*/,
1653 const size_type /*size*/,
1654 const ::MemorySpace::MemorySpaceData<Number, MemorySpace>
1655 & /*data*/)
1656 {
1657 return Number();
1658 }
1659
1660 template <typename real_type>
1661 static void
1663 const std::shared_ptr<::parallel::internal::TBBPartitioner> &
1664 /*thread_loop_partitioner*/,
1665 const size_type /*size*/,
1666 real_type & /*sum*/,
1667 Number * /*values*/,
1668 Number * /*values*/)
1669 {}
1670
1671 template <typename real_type>
1672 static void
1674 const std::shared_ptr<::parallel::internal::TBBPartitioner> &
1675 /*thread_loop_partitioner*/,
1676 const size_type /*size*/,
1677 real_type & /*sum*/,
1678 real_type /*p*/,
1680 {}
1681
1682 static Number
1684 const std::shared_ptr<::parallel::internal::TBBPartitioner> &
1685 /*thread_loop_partitioner*/,
1686 const size_type /*size*/,
1687 const Number /*a*/,
1688 const ::MemorySpace::MemorySpaceData<Number, MemorySpace>
1689 & /*v_data*/,
1690 const ::MemorySpace::MemorySpaceData<Number, MemorySpace>
1691 & /*w_data*/,
1693 {
1694 return Number();
1695 }
1696
1697 template <typename MemorySpace2>
1698 static void
1700 const std::shared_ptr<::parallel::internal::TBBPartitioner> &
1701 /*thread_loop_partitioner*/,
1702 const size_type /*size*/,
1703 VectorOperation::values /*operation*/,
1704 const ::MemorySpace::MemorySpaceData<Number, MemorySpace2>
1705 & /*v_data*/,
1707 {}
1708 };
1709
1710
1711
1712 template <typename Number, typename Number2>
1713 struct functions<Number, Number2, ::MemorySpace::Host>
1714 {
1715 static void
1716 copy(const std::shared_ptr<::parallel::internal::TBBPartitioner>
1717 &thread_loop_partitioner,
1718 const size_type size,
1719 const ::MemorySpace::
1720 MemorySpaceData<Number2, ::MemorySpace::Host> &v_data,
1721 ::MemorySpace::MemorySpaceData<Number,
1722 ::MemorySpace::Host>
1723 &data)
1724 {
1725 Vector_copy<Number, Number2> copier(v_data.values.data(),
1726 data.values.data());
1727 parallel_for(copier, 0, size, thread_loop_partitioner);
1728 }
1729
1730 static void
1731 set(const std::shared_ptr<::parallel::internal::TBBPartitioner>
1732 &thread_loop_partitioner,
1733 const size_type size,
1734 const Number s,
1737 &data)
1738 {
1739 Vector_set<Number> setter(s, data.values.data());
1740 parallel_for(setter, 0, size, thread_loop_partitioner);
1741 }
1742
1743 static void
1745 const std::shared_ptr<::parallel::internal::TBBPartitioner>
1746 &thread_loop_partitioner,
1747 const size_type size,
1748 const ::MemorySpace::
1749 MemorySpaceData<Number, ::MemorySpace::Host> &v_data,
1750 ::MemorySpace::MemorySpaceData<Number,
1751 ::MemorySpace::Host>
1752 &data)
1753 {
1754 Vectorization_add_v<Number> vector_add(data.values.data(),
1755 v_data.values.data());
1756 parallel_for(vector_add, 0, size, thread_loop_partitioner);
1757 }
1758
1759 static void
1761 const std::shared_ptr<::parallel::internal::TBBPartitioner>
1762 &thread_loop_partitioner,
1763 const size_type size,
1764 const ::MemorySpace::
1765 MemorySpaceData<Number, ::MemorySpace::Host> &v_data,
1766 ::MemorySpace::MemorySpaceData<Number,
1767 ::MemorySpace::Host>
1768 &data)
1769 {
1770 Vectorization_subtract_v<Number> vector_subtract(data.values.data(),
1771 v_data.values.data());
1772 parallel_for(vector_subtract, 0, size, thread_loop_partitioner);
1773 }
1774
1775 static void
1777 const std::shared_ptr<::parallel::internal::TBBPartitioner>
1778 &thread_loop_partitioner,
1779 const size_type size,
1780 Number a,
1783 &data)
1784 {
1785 Vectorization_add_factor<Number> vector_add(data.values.data(), a);
1786 parallel_for(vector_add, 0, size, thread_loop_partitioner);
1787 }
1788
1789 static void
1790 add_av(const std::shared_ptr<::parallel::internal::TBBPartitioner>
1791 &thread_loop_partitioner,
1792 const size_type size,
1793 const Number a,
1794 const ::MemorySpace::
1795 MemorySpaceData<Number, ::MemorySpace::Host> &v_data,
1796 ::MemorySpace::MemorySpaceData<Number,
1797 ::MemorySpace::Host>
1798 &data)
1799 {
1800 Vectorization_add_av<Number> vector_add(data.values.data(),
1801 v_data.values.data(),
1802 a);
1803 parallel_for(vector_add, 0, size, thread_loop_partitioner);
1804 }
1805
1806 static void
1808 const std::shared_ptr<::parallel::internal::TBBPartitioner>
1809 &thread_loop_partitioner,
1810 const size_type size,
1811 const Number a,
1812 const Number b,
1813 const ::MemorySpace::
1814 MemorySpaceData<Number, ::MemorySpace::Host> &v_data,
1815 const ::MemorySpace::
1816 MemorySpaceData<Number, ::MemorySpace::Host> &w_data,
1817 ::MemorySpace::MemorySpaceData<Number,
1818 ::MemorySpace::Host>
1819 &data)
1820 {
1822 data.values.data(), v_data.values.data(), w_data.values.data(), a, b);
1823 parallel_for(vector_add, 0, size, thread_loop_partitioner);
1824 }
1825
1826 static void
1828 const std::shared_ptr<::parallel::internal::TBBPartitioner>
1829 &thread_loop_partitioner,
1830 const size_type size,
1831 const Number x,
1832 const ::MemorySpace::
1833 MemorySpaceData<Number, ::MemorySpace::Host> &v_data,
1834 ::MemorySpace::MemorySpaceData<Number,
1835 ::MemorySpace::Host>
1836 &data)
1837 {
1838 Vectorization_sadd_xv<Number> vector_sadd(data.values.data(),
1839 v_data.values.data(),
1840 x);
1841 parallel_for(vector_sadd, 0, size, thread_loop_partitioner);
1842 }
1843
1844 static void
1846 const std::shared_ptr<::parallel::internal::TBBPartitioner>
1847 &thread_loop_partitioner,
1848 const size_type size,
1849 const Number x,
1850 const Number a,
1851 const ::MemorySpace::
1852 MemorySpaceData<Number, ::MemorySpace::Host> &v_data,
1853 ::MemorySpace::MemorySpaceData<Number,
1854 ::MemorySpace::Host>
1855 &data)
1856 {
1857 Vectorization_sadd_xav<Number> vector_sadd(data.values.data(),
1858 v_data.values.data(),
1859 a,
1860 x);
1861 parallel_for(vector_sadd, 0, size, thread_loop_partitioner);
1862 }
1863
1864 static void
1866 const std::shared_ptr<::parallel::internal::TBBPartitioner>
1867 &thread_loop_partitioner,
1868 const size_type size,
1869 const Number x,
1870 const Number a,
1871 const Number b,
1872 const ::MemorySpace::
1873 MemorySpaceData<Number, ::MemorySpace::Host> &v_data,
1874 const ::MemorySpace::
1875 MemorySpaceData<Number, ::MemorySpace::Host> &w_data,
1876 ::MemorySpace::MemorySpaceData<Number,
1877 ::MemorySpace::Host>
1878 &data)
1879 {
1880 Vectorization_sadd_xavbw<Number> vector_sadd(data.values.data(),
1881 v_data.values.data(),
1882 w_data.values.data(),
1883 x,
1884 a,
1885 b);
1886 parallel_for(vector_sadd, 0, size, thread_loop_partitioner);
1887 }
1888
1889 static void
1891 const std::shared_ptr<::parallel::internal::TBBPartitioner>
1892 &thread_loop_partitioner,
1893 const size_type size,
1894 const Number factor,
1897 &data)
1898 {
1900 data.values.data(), factor);
1901 parallel_for(vector_multiply, 0, size, thread_loop_partitioner);
1902 }
1903
1904 static void
1905 scale(const std::shared_ptr<::parallel::internal::TBBPartitioner>
1906 &thread_loop_partitioner,
1907 const size_type size,
1908 const ::MemorySpace::
1909 MemorySpaceData<Number, ::MemorySpace::Host> &v_data,
1910 ::MemorySpace::MemorySpaceData<Number,
1911 ::MemorySpace::Host>
1912 &data)
1913 {
1914 Vectorization_scale<Number> vector_scale(data.values.data(),
1915 v_data.values.data());
1916 parallel_for(vector_scale, 0, size, thread_loop_partitioner);
1917 }
1918
1919 static void
1920 equ_au(const std::shared_ptr<::parallel::internal::TBBPartitioner>
1921 &thread_loop_partitioner,
1922 const size_type size,
1923 const Number a,
1924 const ::MemorySpace::
1925 MemorySpaceData<Number, ::MemorySpace::Host> &v_data,
1926 ::MemorySpace::MemorySpaceData<Number,
1927 ::MemorySpace::Host>
1928 &data)
1929 {
1930 Vectorization_equ_au<Number> vector_equ(data.values.data(),
1931 v_data.values.data(),
1932 a);
1933 parallel_for(vector_equ, 0, size, thread_loop_partitioner);
1934 }
1935
1936 static void
1938 const std::shared_ptr<::parallel::internal::TBBPartitioner>
1939 &thread_loop_partitioner,
1940 const size_type size,
1941 const Number a,
1942 const Number b,
1943 const ::MemorySpace::
1944 MemorySpaceData<Number, ::MemorySpace::Host> &v_data,
1945 const ::MemorySpace::
1946 MemorySpaceData<Number, ::MemorySpace::Host> &w_data,
1947 ::MemorySpace::MemorySpaceData<Number,
1948 ::MemorySpace::Host>
1949 &data)
1950 {
1952 data.values.data(), v_data.values.data(), w_data.values.data(), a, b);
1953 parallel_for(vector_equ, 0, size, thread_loop_partitioner);
1954 }
1955
1956 static Number
1957 dot(const std::shared_ptr<::parallel::internal::TBBPartitioner>
1958 &thread_loop_partitioner,
1959 const size_type size,
1960 const ::MemorySpace::
1961 MemorySpaceData<Number2, ::MemorySpace::Host> &v_data,
1962 ::MemorySpace::MemorySpaceData<Number,
1963 ::MemorySpace::Host>
1964 &data)
1965 {
1966 Number sum;
1968 data.values.data(), v_data.values.data());
1970 dot, 0, size, sum, thread_loop_partitioner);
1971 AssertIsFinite(sum);
1972
1973 return sum;
1974 }
1975
1976 template <typename real_type>
1977 static void
1978 norm_2(const std::shared_ptr<::parallel::internal::TBBPartitioner>
1979 &thread_loop_partitioner,
1980 const size_type size,
1981 real_type &sum,
1984 &data)
1985 {
1986 Norm2<Number, real_type> norm2(data.values.data());
1987 parallel_reduce(norm2, 0, size, sum, thread_loop_partitioner);
1988 }
1989
1990 static Number
1992 const std::shared_ptr<::parallel::internal::TBBPartitioner>
1993 &thread_loop_partitioner,
1994 const size_type size,
1995 const ::MemorySpace::
1996 MemorySpaceData<Number, ::MemorySpace::Host> &data)
1997 {
1998 Number sum;
1999 MeanValue<Number> mean(data.values.data());
2000 parallel_reduce(mean, 0, size, sum, thread_loop_partitioner);
2001
2002 return sum;
2003 }
2004
2005 template <typename real_type>
2006 static void
2007 norm_1(const std::shared_ptr<::parallel::internal::TBBPartitioner>
2008 &thread_loop_partitioner,
2009 const size_type size,
2010 real_type &sum,
2013 &data)
2014 {
2015 Norm1<Number, real_type> norm1(data.values.data());
2016 parallel_reduce(norm1, 0, size, sum, thread_loop_partitioner);
2017 }
2018
2019 template <typename real_type>
2020 static void
2021 norm_p(const std::shared_ptr<::parallel::internal::TBBPartitioner>
2022 &thread_loop_partitioner,
2023 const size_type size,
2024 real_type &sum,
2025 const real_type p,
2028 &data)
2029 {
2030 NormP<Number, real_type> normp(data.values.data(), p);
2031 parallel_reduce(normp, 0, size, sum, thread_loop_partitioner);
2032 }
2033
2034 static Number
2036 const std::shared_ptr<::parallel::internal::TBBPartitioner>
2037 &thread_loop_partitioner,
2038 const size_type size,
2039 const Number a,
2040 const ::MemorySpace::
2041 MemorySpaceData<Number, ::MemorySpace::Host> &v_data,
2042 const ::MemorySpace::
2043 MemorySpaceData<Number, ::MemorySpace::Host> &w_data,
2044 ::MemorySpace::MemorySpaceData<Number,
2045 ::MemorySpace::Host>
2046 &data)
2047 {
2048 Number sum;
2049 AddAndDot<Number> adder(data.values.data(),
2050 v_data.values.data(),
2051 w_data.values.data(),
2052 a);
2053 parallel_reduce(adder, 0, size, sum, thread_loop_partitioner);
2054
2055 return sum;
2056 }
2057
2058 template <typename MemorySpace2>
2059 static void
2061 const std::shared_ptr<::parallel::internal::TBBPartitioner>
2062 &thread_loop_partitioner,
2063 const size_type size,
2064 VectorOperation::values operation,
2065 const ::MemorySpace::MemorySpaceData<Number, MemorySpace2>
2066 &v_data,
2069 &data,
2070 std::enable_if_t<
2071 std::is_same_v<MemorySpace2, ::MemorySpace::Host>,
2072 int> = 0)
2073 {
2074 if (operation == VectorOperation::insert)
2075 {
2076 copy(thread_loop_partitioner, size, v_data, data);
2077 }
2078 else if (operation == VectorOperation::add)
2079 {
2080 add_vector(thread_loop_partitioner, size, v_data, data);
2081 }
2082 else
2083 {
2085 }
2086 }
2087
2088 template <typename MemorySpace2>
2089 static void
2091 const std::shared_ptr<::parallel::internal::TBBPartitioner>
2092 & /*thread_loop_partitioner*/,
2093 const size_type size,
2094 VectorOperation::values operation,
2095 const ::MemorySpace::MemorySpaceData<Number, MemorySpace2>
2096 &v_data,
2099 &data,
2100 std::enable_if_t<
2101 std::is_same_v<MemorySpace2, ::MemorySpace::Default>,
2102 int> = 0)
2103 {
2104 if (operation == VectorOperation::insert)
2105 {
2106 Kokkos::deep_copy(
2107 Kokkos::subview(data.values,
2108 Kokkos::pair<size_type, size_type>(0, size)),
2109 Kokkos::subview(v_data.values,
2110 Kokkos::pair<size_type, size_type>(0, size)));
2111 }
2112 else
2113 {
2115 }
2116 }
2117 };
2118
2119
2120
2121 template <typename Number>
2122 struct functions<Number, Number, ::MemorySpace::Default>
2123 {
2124 static void
2126 const std::shared_ptr<::parallel::internal::TBBPartitioner> &,
2127 const size_type size,
2128 const ::MemorySpace::
2129 MemorySpaceData<Number, ::MemorySpace::Default> &v_data,
2130 ::MemorySpace::MemorySpaceData<Number,
2131 ::MemorySpace::Default>
2132 &data)
2133 {
2134 Kokkos::deep_copy(
2135 Kokkos::subview(data.values,
2136 Kokkos::pair<size_type, size_type>(0, size)),
2137 Kokkos::subview(v_data.values,
2138 Kokkos::pair<size_type, size_type>(0, size)));
2139 }
2140
2141 static void
2142 set(const std::shared_ptr<::parallel::internal::TBBPartitioner> &,
2143 const size_type size,
2144 const Number s,
2147 &data)
2148 {
2149 Kokkos::deep_copy(
2150 Kokkos::subview(data.values,
2151 Kokkos::pair<size_type, size_type>(0, size)),
2152 s);
2153 }
2154
2155 static void
2157 const std::shared_ptr<::parallel::internal::TBBPartitioner> &,
2158 const size_type size,
2159 const ::MemorySpace::
2160 MemorySpaceData<Number, ::MemorySpace::Default> &v_data,
2161 ::MemorySpace::MemorySpaceData<Number,
2162 ::MemorySpace::Default>
2163 &data)
2164 {
2165 auto exec = typename ::MemorySpace::Default::kokkos_space::
2166 execution_space{};
2167 Kokkos::parallel_for(
2168 "::add_vector",
2169 Kokkos::RangePolicy<
2170 ::MemorySpace::Default::kokkos_space::execution_space>(
2171 exec, 0, size),
2172 KOKKOS_LAMBDA(int i) { data.values(i) += v_data.values(i); });
2173 exec.fence();
2174 }
2175
2176 static void
2178 const std::shared_ptr<::parallel::internal::TBBPartitioner> &,
2179 const size_type size,
2180 const ::MemorySpace::
2181 MemorySpaceData<Number, ::MemorySpace::Default> &v_data,
2182 ::MemorySpace::MemorySpaceData<Number,
2183 ::MemorySpace::Default>
2184 &data)
2185 {
2186 auto exec = typename ::MemorySpace::Default::kokkos_space::
2187 execution_space{};
2188 Kokkos::parallel_for(
2189 "::subtract_vector",
2190 Kokkos::RangePolicy<
2191 ::MemorySpace::Default::kokkos_space::execution_space>(
2192 exec, 0, size),
2193 KOKKOS_LAMBDA(size_type i) { data.values(i) -= v_data.values(i); });
2194 exec.fence();
2195 }
2196
2197 static void
2199 const std::shared_ptr<::parallel::internal::TBBPartitioner> &,
2200 const size_type size,
2201 Number a,
2204 &data)
2205 {
2206 auto exec = typename ::MemorySpace::Default::kokkos_space::
2207 execution_space{};
2208 Kokkos::parallel_for(
2209 "::add_factor",
2210 Kokkos::RangePolicy<
2211 ::MemorySpace::Default::kokkos_space::execution_space>(
2212 exec, 0, size),
2213 KOKKOS_LAMBDA(size_type i) { data.values(i) += a; });
2214 exec.fence();
2215 }
2216
2217 static void
2219 const std::shared_ptr<::parallel::internal::TBBPartitioner> &,
2220 const size_type size,
2221 const Number a,
2222 const ::MemorySpace::
2223 MemorySpaceData<Number, ::MemorySpace::Default> &v_data,
2224 ::MemorySpace::MemorySpaceData<Number,
2225 ::MemorySpace::Default>
2226 &data)
2227 {
2228 auto exec = typename ::MemorySpace::Default::kokkos_space::
2229 execution_space{};
2230 Kokkos::parallel_for(
2231 "::add_av",
2232 Kokkos::RangePolicy<
2233 ::MemorySpace::Default::kokkos_space::execution_space>(
2234 exec, 0, size),
2235 KOKKOS_LAMBDA(size_type i) {
2236 data.values(i) += a * v_data.values(i);
2237 });
2238 exec.fence();
2239 }
2240
2241 static void
2243 const std::shared_ptr<::parallel::internal::TBBPartitioner> &,
2244 const size_type size,
2245 const Number a,
2246 const Number b,
2247 const ::MemorySpace::
2248 MemorySpaceData<Number, ::MemorySpace::Default> &v_data,
2249 const ::MemorySpace::
2250 MemorySpaceData<Number, ::MemorySpace::Default> &w_data,
2251 ::MemorySpace::MemorySpaceData<Number,
2252 ::MemorySpace::Default>
2253 &data)
2254 {
2255 auto exec = typename ::MemorySpace::Default::kokkos_space::
2256 execution_space{};
2257 Kokkos::parallel_for(
2258 "::add_avpbw",
2259 Kokkos::RangePolicy<
2260 ::MemorySpace::Default::kokkos_space::execution_space>(
2261 exec, 0, size),
2262 KOKKOS_LAMBDA(size_type i) {
2263 data.values(i) += a * v_data.values(i) + b * w_data.values(i);
2264 });
2265 exec.fence();
2266 }
2267
2268 static void
2270 const std::shared_ptr<::parallel::internal::TBBPartitioner> &,
2271 const size_type size,
2272 const Number x,
2273 const ::MemorySpace::
2274 MemorySpaceData<Number, ::MemorySpace::Default> &v_data,
2275 ::MemorySpace::MemorySpaceData<Number,
2276 ::MemorySpace::Default>
2277 &data)
2278 {
2279 auto exec = typename ::MemorySpace::Default::kokkos_space::
2280 execution_space{};
2281 Kokkos::parallel_for(
2282 "::sadd_xv",
2283 Kokkos::RangePolicy<
2284 ::MemorySpace::Default::kokkos_space::execution_space>(
2285 exec, 0, size),
2286 KOKKOS_LAMBDA(size_type i) {
2287 data.values(i) = x * data.values(i) + v_data.values(i);
2288 });
2289 exec.fence();
2290 }
2291
2292 static void
2294 const std::shared_ptr<::parallel::internal::TBBPartitioner> &,
2295 const size_type size,
2296 const Number x,
2297 const Number a,
2298 const ::MemorySpace::
2299 MemorySpaceData<Number, ::MemorySpace::Default> &v_data,
2300 ::MemorySpace::MemorySpaceData<Number,
2301 ::MemorySpace::Default>
2302 &data)
2303 {
2304 auto exec = typename ::MemorySpace::Default::kokkos_space::
2305 execution_space{};
2306 Kokkos::parallel_for(
2307 "::sadd_xav",
2308 Kokkos::RangePolicy<
2309 ::MemorySpace::Default::kokkos_space::execution_space>(
2310 exec, 0, size),
2311 KOKKOS_LAMBDA(size_type i) {
2312 data.values(i) = x * data.values(i) + a * v_data.values(i);
2313 });
2314 exec.fence();
2315 }
2316
2317 static void
2319 const std::shared_ptr<::parallel::internal::TBBPartitioner> &,
2320 const size_type size,
2321 const Number x,
2322 const Number a,
2323 const Number b,
2324 const ::MemorySpace::
2325 MemorySpaceData<Number, ::MemorySpace::Default> &v_data,
2326 const ::MemorySpace::
2327 MemorySpaceData<Number, ::MemorySpace::Default> &w_data,
2328 ::MemorySpace::MemorySpaceData<Number,
2329 ::MemorySpace::Default>
2330 &data)
2331 {
2332 auto exec = typename ::MemorySpace::Default::kokkos_space::
2333 execution_space{};
2334 Kokkos::parallel_for(
2335 "::sadd_xavbw",
2336 Kokkos::RangePolicy<
2337 ::MemorySpace::Default::kokkos_space::execution_space>(
2338 exec, 0, size),
2339 KOKKOS_LAMBDA(size_type i) {
2340 data.values(i) =
2341 x * data.values(i) + a * v_data.values(i) + b * w_data.values(i);
2342 });
2343 exec.fence();
2344 }
2345
2346 static void
2348 const std::shared_ptr<::parallel::internal::TBBPartitioner> &,
2349 const size_type size,
2350 const Number factor,
2353 &data)
2354 {
2355 auto exec = typename ::MemorySpace::Default::kokkos_space::
2356 execution_space{};
2357 Kokkos::parallel_for(
2358 "::multiply_factor",
2359 Kokkos::RangePolicy<
2360 ::MemorySpace::Default::kokkos_space::execution_space>(
2361 exec, 0, size),
2362 KOKKOS_LAMBDA(size_type i) { data.values(i) *= factor; });
2363 exec.fence();
2364 }
2365
2366 static void
2368 const std::shared_ptr<::parallel::internal::TBBPartitioner> &,
2369 const size_type size,
2370 const ::MemorySpace::
2371 MemorySpaceData<Number, ::MemorySpace::Default> &v_data,
2372 ::MemorySpace::MemorySpaceData<Number,
2373 ::MemorySpace::Default>
2374 &data)
2375 {
2376 auto exec = typename ::MemorySpace::Default::kokkos_space::
2377 execution_space{};
2378 Kokkos::parallel_for(
2379 "::scale",
2380 Kokkos::RangePolicy<
2381 ::MemorySpace::Default::kokkos_space::execution_space>(
2382 exec, 0, size),
2383 KOKKOS_LAMBDA(size_type i) { data.values(i) *= v_data.values(i); });
2384 exec.fence();
2385 }
2386
2387 static void
2389 const std::shared_ptr<::parallel::internal::TBBPartitioner> &,
2390 const size_type size,
2391 const Number a,
2392 const ::MemorySpace::
2393 MemorySpaceData<Number, ::MemorySpace::Default> &v_data,
2394 ::MemorySpace::MemorySpaceData<Number,
2395 ::MemorySpace::Default>
2396 &data)
2397 {
2398 auto exec = typename ::MemorySpace::Default::kokkos_space::
2399 execution_space{};
2400 Kokkos::parallel_for(
2401 "::equ_au",
2402 Kokkos::RangePolicy<
2403 ::MemorySpace::Default::kokkos_space::execution_space>(
2404 exec, 0, size),
2405 KOKKOS_LAMBDA(size_type i) {
2406 data.values(i) = a * v_data.values(i);
2407 });
2408 exec.fence();
2409 }
2410
2411 static void
2413 const std::shared_ptr<::parallel::internal::TBBPartitioner> &,
2414 const size_type size,
2415 const Number a,
2416 const Number b,
2417 const ::MemorySpace::
2418 MemorySpaceData<Number, ::MemorySpace::Default> &v_data,
2419 const ::MemorySpace::
2420 MemorySpaceData<Number, ::MemorySpace::Default> &w_data,
2421 ::MemorySpace::MemorySpaceData<Number,
2422 ::MemorySpace::Default>
2423 &data)
2424 {
2425 auto exec = typename ::MemorySpace::Default::kokkos_space::
2426 execution_space{};
2427 Kokkos::parallel_for(
2428 "::equ_aubv",
2429 Kokkos::RangePolicy<
2430 ::MemorySpace::Default::kokkos_space::execution_space>(
2431 exec, 0, size),
2432 KOKKOS_LAMBDA(size_type i) {
2433 data.values(i) = a * v_data.values(i) + b * w_data.values(i);
2434 });
2435 exec.fence();
2436 }
2437
2438 static Number
2439 dot(const std::shared_ptr<::parallel::internal::TBBPartitioner> &,
2440 const size_type size,
2441 const ::MemorySpace::
2442 MemorySpaceData<Number, ::MemorySpace::Default> &v_data,
2443 ::MemorySpace::MemorySpaceData<Number,
2444 ::MemorySpace::Default>
2445 &data)
2446 {
2447 Number result;
2448
2449 auto exec = typename ::MemorySpace::Default::kokkos_space::
2450 execution_space{};
2451 Kokkos::parallel_reduce(
2452 "::dot",
2453 Kokkos::RangePolicy<
2454 ::MemorySpace::Default::kokkos_space::execution_space>(
2455 exec, 0, size),
2456 KOKKOS_LAMBDA(size_type i, Number & update) {
2457 update += data.values(i) * v_data.values(i);
2458 },
2459 result);
2460
2461 AssertIsFinite(result);
2462 return result;
2463 }
2464
2465 template <typename real_type>
2466 static void
2467 norm_2(const std::shared_ptr<::parallel::internal::TBBPartitioner>
2468 &thread_loop_partitioner,
2469 const size_type size,
2470 real_type &sum,
2471 ::MemorySpace::
2472 MemorySpaceData<Number, ::MemorySpace::Default> &data)
2473 {
2474 sum = dot(thread_loop_partitioner, size, data, data);
2475 }
2476
2477 static Number
2479 const std::shared_ptr<::parallel::internal::TBBPartitioner> &,
2480 const size_type size,
2481 const ::MemorySpace::
2482 MemorySpaceData<Number, ::MemorySpace::Default> &data)
2483 {
2484 Number result;
2485
2486 auto exec = typename ::MemorySpace::Default::kokkos_space::
2487 execution_space{};
2488 Kokkos::parallel_reduce(
2489 "::mean_value",
2490 Kokkos::RangePolicy<
2491 ::MemorySpace::Default::kokkos_space::execution_space>(
2492 exec, 0, size),
2493 KOKKOS_LAMBDA(size_type i, Number & update) {
2494 update += data.values(i);
2495 },
2496 result);
2497
2498 AssertIsFinite(result);
2499 return result;
2500 }
2501
2502 template <typename real_type>
2503 static void
2505 const std::shared_ptr<::parallel::internal::TBBPartitioner> &,
2506 const size_type size,
2507 real_type &sum,
2510 &data)
2511 {
2512 auto exec = typename ::MemorySpace::Default::kokkos_space::
2513 execution_space{};
2514 Kokkos::parallel_reduce(
2515 "::norm_1",
2516 Kokkos::RangePolicy<
2517 ::MemorySpace::Default::kokkos_space::execution_space>(
2518 exec, 0, size),
2519 KOKKOS_LAMBDA(size_type i, Number & update) {
2520#if KOKKOS_VERSION < 30400
2521 update += std::abs(data.values(i));
2522#elif KOKKOS_VERSION < 30700
2523 update += Kokkos::Experimental::fabs(data.values(i));
2524#else
2525 update += Kokkos::abs(data.values(i));
2526#endif
2527 },
2528 sum);
2529 }
2530
2531 template <typename real_type>
2532 static void
2534 const std::shared_ptr<::parallel::internal::TBBPartitioner> &,
2535 const size_type size,
2536 real_type &sum,
2537 real_type exp,
2540 &data)
2541 {
2542 auto exec = typename ::MemorySpace::Default::kokkos_space::
2543 execution_space{};
2544 Kokkos::parallel_reduce(
2545 "::norm_p",
2546 Kokkos::RangePolicy<
2547 ::MemorySpace::Default::kokkos_space::execution_space>(
2548 exec, 0, size),
2549 KOKKOS_LAMBDA(size_type i, Number & update) {
2550#if KOKKOS_VERSION < 30400
2551 update += std::pow(fabs(data.values(i)), exp);
2552#elif KOKKOS_VERSION < 30700
2553 update += Kokkos::Experimental::pow(
2554 Kokkos::Experimental::fabs(data.values(i)), exp);
2555#else
2556 update += Kokkos::pow(Kokkos::abs(data.values(i)), exp);
2557#endif
2558 },
2559 sum);
2560 }
2561
2562 static Number
2564 const std::shared_ptr<::parallel::internal::TBBPartitioner> &,
2565 const size_type size,
2566 const Number a,
2567 const ::MemorySpace::
2568 MemorySpaceData<Number, ::MemorySpace::Default> &v_data,
2569 const ::MemorySpace::
2570 MemorySpaceData<Number, ::MemorySpace::Default> &w_data,
2571 ::MemorySpace::MemorySpaceData<Number,
2572 ::MemorySpace::Default>
2573 &data)
2574 {
2575 Number res;
2576
2577 auto exec = typename ::MemorySpace::Default::kokkos_space::
2578 execution_space{};
2579 Kokkos::parallel_reduce(
2580 "::add_and_dot",
2581 Kokkos::RangePolicy<
2582 ::MemorySpace::Default::kokkos_space::execution_space>(
2583 exec, 0, size),
2584 KOKKOS_LAMBDA(size_type i, Number & update) {
2585 data.values(i) += a * v_data.values(i);
2586 update +=
2587 data.values(i) * Number(numbers::NumberTraits<Number>::conjugate(
2588 w_data.values(i)));
2589 },
2590 res);
2591
2592 return res;
2593 }
2594
2595 template <typename MemorySpace2>
2596 static void
2598 const std::shared_ptr<::parallel::internal::TBBPartitioner>
2599 &thread_loop_partitioner,
2600 const size_type size,
2601 VectorOperation::values operation,
2602 const ::MemorySpace::MemorySpaceData<Number, MemorySpace2>
2603 &v_data,
2606 &data,
2607 std::enable_if_t<
2608 std::is_same_v<MemorySpace2, ::MemorySpace::Default>,
2609 int> = 0)
2610 {
2611 if (operation == VectorOperation::insert)
2612 {
2613 copy(thread_loop_partitioner, size, v_data, data);
2614 }
2615 else if (operation == VectorOperation::add)
2616 {
2617 add_vector(thread_loop_partitioner, size, v_data, data);
2618 }
2619 else
2620 {
2622 }
2623 }
2624
2625 template <typename MemorySpace2>
2626 static void
2628 const std::shared_ptr<::parallel::internal::TBBPartitioner>
2629 & /*thread_loop_partitioner*/,
2630 const size_type size,
2631 VectorOperation::values operation,
2632 const ::MemorySpace::MemorySpaceData<Number, MemorySpace2>
2633 &v_data,
2636 &data,
2637 std::enable_if_t<
2638 std::is_same_v<MemorySpace2, ::MemorySpace::Host>,
2639 int> = 0)
2640 {
2641 if (operation == VectorOperation::insert)
2642 {
2643 Kokkos::deep_copy(
2644 Kokkos::subview(data.values,
2645 Kokkos::pair<size_type, size_type>(0, size)),
2646 Kokkos::subview(v_data.values,
2647 Kokkos::pair<size_type, size_type>(0, size)));
2648 }
2649 else
2650 {
2652 }
2653 }
2654 };
2655 } // namespace VectorOperations
2656} // namespace internal
2657
2659
2660#endif
static unsigned int n_threads()
void store(OtherNumber *ptr) const
void load(const OtherNumber *ptr)
#define DEAL_II_ALWAYS_INLINE
Definition config.h:109
#define DEAL_II_OPENMP_SIMD_PRAGMA
Definition config.h:143
#define DEAL_II_NAMESPACE_OPEN
Definition config.h:502
#define DEAL_II_NAMESPACE_CLOSE
Definition config.h:503
#define DEAL_II_FALLTHROUGH
Definition config.h:235
Point< 2 > first
Definition grid_out.cc:4613
static ::ExceptionBase & ExcNotImplemented()
#define Assert(cond, exc)
#define AssertIsFinite(number)
#define AssertDimension(dim1, dim2)
#define AssertIndexRange(index, range)
static ::ExceptionBase & ExcInternalError()
static ::ExceptionBase & ExcMessage(std::string arg1)
#define AssertThrow(cond, exc)
unsigned int minimum_parallel_grain_size
Definition parallel.cc:33
void accumulate_recursive(const Operation &op, const size_type first, const size_type last, ResultType &result)
size_type do_accumulate(const Operation op, const size_type vec_size, const size_type start_index, ResultType *outer_results, std::bool_constant< false >)
void parallel_reduce(const Operation &op, const size_type start, const size_type end, ResultType &result, const std::shared_ptr<::parallel::internal::TBBPartitioner > &partitioner)
void copy(const T *begin, const T *end, U *dest)
void parallel_for(Functor &functor, const size_type start, const size_type end, const std::shared_ptr<::parallel::internal::TBBPartitioner > &partitioner)
const unsigned int vector_accumulation_recursion_threshold
void parallel_for(Iterator x_begin, Iterator x_end, const Functor &functor, const unsigned int grainsize)
Definition parallel.h:83
::VectorizedArray< Number, width > min(const ::VectorizedArray< Number, width > &, const ::VectorizedArray< Number, width > &)
::VectorizedArray< Number, width > pow(const ::VectorizedArray< Number, width > &, const Number p)
::VectorizedArray< Number, width > abs(const ::VectorizedArray< Number, width > &)
unsigned int global_dof_index
Definition types.h:81
AddAndDot(Number *const X, const Number *const V, const Number *const W, const Number a)
VectorizedArray< Number > do_vectorized(const size_type i) const
Dot(const Number *const X, const Number2 *const Y)
Number operator()(const size_type i) const
VectorizedArray< Number > do_vectorized(const size_type i) const
VectorizedArray< Number > do_vectorized(const size_type i) const
RealType operator()(const size_type i) const
VectorizedArray< Number > do_vectorized(const size_type i) const
RealType operator()(const size_type i) const
VectorizedArray< Number > do_vectorized(const size_type i) const
RealType operator()(const size_type i) const
VectorizedArray< Number > do_vectorized(const size_type i) const
void operator()(const tbb::blocked_range< size_type > &range) const
TBBForFunctor(Functor &functor, const size_type start, const size_type end)
TBBReduceFunctor(const Operation &op, const size_type start, const size_type end)
void operator()(const tbb::blocked_range< size_type > &range) const
Vector_copy(const OtherNumber *const src, Number *const dst)
void operator()(const size_type begin, const size_type end) const
Vector_set(const Number value, Number *const dst)
void operator()(const size_type begin, const size_type end) const
Vectorization_add_av(Number *const val, const Number *const v_val, const Number factor)
void operator()(const size_type begin, const size_type end) const
Vectorization_add_avpbw(Number *const val, const Number *const v_val, const Number *const w_val, const Number a, const Number b)
void operator()(const size_type begin, const size_type end) const
void operator()(const size_type begin, const size_type end) const
Vectorization_add_factor(Number *const val, const Number factor)
void operator()(const size_type begin, const size_type end) const
Vectorization_add_v(Number *const val, const Number *const v_val)
Vectorization_equ_au(Number *const val, const Number *const u_val, const Number a)
void operator()(const size_type begin, const size_type end) const
Vectorization_equ_aubv(Number *const val, const Number *const u_val, const Number *const v_val, const Number a, const Number b)
void operator()(const size_type begin, const size_type end) const
Vectorization_equ_aubvcw(Number *val, const Number *u_val, const Number *v_val, const Number *w_val, const Number a, const Number b, const Number c)
void operator()(const size_type begin, const size_type end) const
Vectorization_multiply_factor(Number *const val, const Number factor)
void operator()(const size_type begin, const size_type end) const
void operator()(const size_type begin, const size_type end) const
Vectorization_ratio(Number *val, const Number *a_val, const Number *b_val)
Vectorization_sadd_xav(Number *val, const Number *const v_val, const Number a, const Number x)
void operator()(const size_type begin, const size_type end) const
void operator()(const size_type begin, const size_type end) const
Vectorization_sadd_xavbw(Number *val, const Number *v_val, const Number *w_val, Number x, Number a, Number b)
Vectorization_sadd_xv(Number *const val, const Number *const v_val, const Number x)
void operator()(const size_type begin, const size_type end) const
void operator()(const size_type begin, const size_type end) const
Vectorization_scale(Number *const val, const Number *const v_val)
Vectorization_subtract_v(Number *val, const Number *const v_val)
void operator()(const size_type begin, const size_type end) const
static void set(const std::shared_ptr<::parallel::internal::TBBPartitioner > &thread_loop_partitioner, const size_type size, const Number s, ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Host > &data)
static void norm_2(const std::shared_ptr<::parallel::internal::TBBPartitioner > &thread_loop_partitioner, const size_type size, real_type &sum, ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Host > &data)
static void norm_p(const std::shared_ptr<::parallel::internal::TBBPartitioner > &thread_loop_partitioner, const size_type size, real_type &sum, const real_type p, ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Host > &data)
static void import_elements(const std::shared_ptr<::parallel::internal::TBBPartitioner > &, const size_type size, VectorOperation::values operation, const ::MemorySpace::MemorySpaceData< Number, MemorySpace2 > &v_data, ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Host > &data, std::enable_if_t< std::is_same_v< MemorySpace2, ::MemorySpace::Default >, int >=0)
static void add_avpbw(const std::shared_ptr<::parallel::internal::TBBPartitioner > &thread_loop_partitioner, const size_type size, const Number a, const Number b, const ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Host > &v_data, const ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Host > &w_data, ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Host > &data)
static void sadd_xav(const std::shared_ptr<::parallel::internal::TBBPartitioner > &thread_loop_partitioner, const size_type size, const Number x, const Number a, const ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Host > &v_data, ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Host > &data)
static void equ_au(const std::shared_ptr<::parallel::internal::TBBPartitioner > &thread_loop_partitioner, const size_type size, const Number a, const ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Host > &v_data, ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Host > &data)
static void scale(const std::shared_ptr<::parallel::internal::TBBPartitioner > &thread_loop_partitioner, const size_type size, const ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Host > &v_data, ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Host > &data)
static Number mean_value(const std::shared_ptr<::parallel::internal::TBBPartitioner > &thread_loop_partitioner, const size_type size, const ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Host > &data)
static void add_factor(const std::shared_ptr<::parallel::internal::TBBPartitioner > &thread_loop_partitioner, const size_type size, Number a, ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Host > &data)
static void sadd_xv(const std::shared_ptr<::parallel::internal::TBBPartitioner > &thread_loop_partitioner, const size_type size, const Number x, const ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Host > &v_data, ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Host > &data)
static void norm_1(const std::shared_ptr<::parallel::internal::TBBPartitioner > &thread_loop_partitioner, const size_type size, real_type &sum, ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Host > &data)
static void copy(const std::shared_ptr<::parallel::internal::TBBPartitioner > &thread_loop_partitioner, const size_type size, const ::MemorySpace::MemorySpaceData< Number2, ::MemorySpace::Host > &v_data, ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Host > &data)
static void import_elements(const std::shared_ptr<::parallel::internal::TBBPartitioner > &thread_loop_partitioner, const size_type size, VectorOperation::values operation, const ::MemorySpace::MemorySpaceData< Number, MemorySpace2 > &v_data, ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Host > &data, std::enable_if_t< std::is_same_v< MemorySpace2, ::MemorySpace::Host >, int >=0)
static void add_av(const std::shared_ptr<::parallel::internal::TBBPartitioner > &thread_loop_partitioner, const size_type size, const Number a, const ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Host > &v_data, ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Host > &data)
static void add_vector(const std::shared_ptr<::parallel::internal::TBBPartitioner > &thread_loop_partitioner, const size_type size, const ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Host > &v_data, ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Host > &data)
static void sadd_xavbw(const std::shared_ptr<::parallel::internal::TBBPartitioner > &thread_loop_partitioner, const size_type size, const Number x, const Number a, const Number b, const ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Host > &v_data, const ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Host > &w_data, ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Host > &data)
static void subtract_vector(const std::shared_ptr<::parallel::internal::TBBPartitioner > &thread_loop_partitioner, const size_type size, const ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Host > &v_data, ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Host > &data)
static Number add_and_dot(const std::shared_ptr<::parallel::internal::TBBPartitioner > &thread_loop_partitioner, const size_type size, const Number a, const ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Host > &v_data, const ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Host > &w_data, ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Host > &data)
static Number dot(const std::shared_ptr<::parallel::internal::TBBPartitioner > &thread_loop_partitioner, const size_type size, const ::MemorySpace::MemorySpaceData< Number2, ::MemorySpace::Host > &v_data, ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Host > &data)
static void equ_aubv(const std::shared_ptr<::parallel::internal::TBBPartitioner > &thread_loop_partitioner, const size_type size, const Number a, const Number b, const ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Host > &v_data, const ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Host > &w_data, ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Host > &data)
static void multiply_factor(const std::shared_ptr<::parallel::internal::TBBPartitioner > &thread_loop_partitioner, const size_type size, const Number factor, ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Host > &data)
static void subtract_vector(const std::shared_ptr<::parallel::internal::TBBPartitioner > &, const size_type size, const ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Default > &v_data, ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Default > &data)
static void sadd_xav(const std::shared_ptr<::parallel::internal::TBBPartitioner > &, const size_type size, const Number x, const Number a, const ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Default > &v_data, ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Default > &data)
static Number dot(const std::shared_ptr<::parallel::internal::TBBPartitioner > &, const size_type size, const ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Default > &v_data, ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Default > &data)
static void add_av(const std::shared_ptr<::parallel::internal::TBBPartitioner > &, const size_type size, const Number a, const ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Default > &v_data, ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Default > &data)
static void copy(const std::shared_ptr<::parallel::internal::TBBPartitioner > &, const size_type size, const ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Default > &v_data, ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Default > &data)
static Number add_and_dot(const std::shared_ptr<::parallel::internal::TBBPartitioner > &, const size_type size, const Number a, const ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Default > &v_data, const ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Default > &w_data, ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Default > &data)
static void sadd_xavbw(const std::shared_ptr<::parallel::internal::TBBPartitioner > &, const size_type size, const Number x, const Number a, const Number b, const ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Default > &v_data, const ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Default > &w_data, ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Default > &data)
static Number mean_value(const std::shared_ptr<::parallel::internal::TBBPartitioner > &, const size_type size, const ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Default > &data)
static void norm_p(const std::shared_ptr<::parallel::internal::TBBPartitioner > &, const size_type size, real_type &sum, real_type exp, ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Default > &data)
static void add_factor(const std::shared_ptr<::parallel::internal::TBBPartitioner > &, const size_type size, Number a, ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Default > &data)
static void add_vector(const std::shared_ptr<::parallel::internal::TBBPartitioner > &, const size_type size, const ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Default > &v_data, ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Default > &data)
static void add_avpbw(const std::shared_ptr<::parallel::internal::TBBPartitioner > &, const size_type size, const Number a, const Number b, const ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Default > &v_data, const ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Default > &w_data, ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Default > &data)
static void set(const std::shared_ptr<::parallel::internal::TBBPartitioner > &, const size_type size, const Number s, ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Default > &data)
static void norm_2(const std::shared_ptr<::parallel::internal::TBBPartitioner > &thread_loop_partitioner, const size_type size, real_type &sum, ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Default > &data)
static void import_elements(const std::shared_ptr<::parallel::internal::TBBPartitioner > &, const size_type size, VectorOperation::values operation, const ::MemorySpace::MemorySpaceData< Number, MemorySpace2 > &v_data, ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Default > &data, std::enable_if_t< std::is_same_v< MemorySpace2, ::MemorySpace::Host >, int >=0)
static void sadd_xv(const std::shared_ptr<::parallel::internal::TBBPartitioner > &, const size_type size, const Number x, const ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Default > &v_data, ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Default > &data)
static void multiply_factor(const std::shared_ptr<::parallel::internal::TBBPartitioner > &, const size_type size, const Number factor, ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Default > &data)
static void scale(const std::shared_ptr<::parallel::internal::TBBPartitioner > &, const size_type size, const ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Default > &v_data, ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Default > &data)
static void equ_aubv(const std::shared_ptr<::parallel::internal::TBBPartitioner > &, const size_type size, const Number a, const Number b, const ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Default > &v_data, const ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Default > &w_data, ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Default > &data)
static void import_elements(const std::shared_ptr<::parallel::internal::TBBPartitioner > &thread_loop_partitioner, const size_type size, VectorOperation::values operation, const ::MemorySpace::MemorySpaceData< Number, MemorySpace2 > &v_data, ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Default > &data, std::enable_if_t< std::is_same_v< MemorySpace2, ::MemorySpace::Default >, int >=0)
static void norm_1(const std::shared_ptr<::parallel::internal::TBBPartitioner > &, const size_type size, real_type &sum, ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Default > &data)
static void equ_au(const std::shared_ptr<::parallel::internal::TBBPartitioner > &, const size_type size, const Number a, const ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Default > &v_data, ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Default > &data)
static Number mean_value(const std::shared_ptr<::parallel::internal::TBBPartitioner > &, const size_type, const ::MemorySpace::MemorySpaceData< Number, MemorySpace > &)
static void equ_au(const std::shared_ptr<::parallel::internal::TBBPartitioner > &, const size_type, const Number, const ::MemorySpace::MemorySpaceData< Number, MemorySpace > &, ::MemorySpace::MemorySpaceData< Number, MemorySpace > &)
static void add_avpbw(const std::shared_ptr<::parallel::internal::TBBPartitioner > &, const size_type, const Number, const Number, const ::MemorySpace::MemorySpaceData< Number, MemorySpace > &, const ::MemorySpace::MemorySpaceData< Number, MemorySpace > &, ::MemorySpace::MemorySpaceData< Number, MemorySpace > &)
static void sadd_xv(const std::shared_ptr<::parallel::internal::TBBPartitioner > &, const size_type, const Number, const ::MemorySpace::MemorySpaceData< Number, MemorySpace > &, ::MemorySpace::MemorySpaceData< Number, MemorySpace > &)
static void add_factor(const std::shared_ptr<::parallel::internal::TBBPartitioner > &, const size_type, Number, ::MemorySpace::MemorySpaceData< Number, MemorySpace > &)
static void norm_2(const std::shared_ptr<::parallel::internal::TBBPartitioner > &, const size_type, real_type &, const ::MemorySpace::MemorySpaceData< Number, MemorySpace > &, ::MemorySpace::MemorySpaceData< Number, MemorySpace > &)
static void add_vector(const std::shared_ptr<::parallel::internal::TBBPartitioner > &, const size_type, const ::MemorySpace::MemorySpaceData< Number, MemorySpace > &, ::MemorySpace::MemorySpaceData< Number, MemorySpace > &)
static void scale(const std::shared_ptr<::parallel::internal::TBBPartitioner > &, const size_type, const ::MemorySpace::MemorySpaceData< Number, MemorySpace > &, ::MemorySpace::MemorySpaceData< Number, MemorySpace > &)
static void subtract_vector(const std::shared_ptr<::parallel::internal::TBBPartitioner > &, const size_type, const ::MemorySpace::MemorySpaceData< Number, MemorySpace > &, ::MemorySpace::MemorySpaceData< Number, MemorySpace > &)
static void sadd_xavbw(const std::shared_ptr<::parallel::internal::TBBPartitioner > &, const size_type, const Number, const Number, const Number, const ::MemorySpace::MemorySpaceData< Number, MemorySpace > &, const ::MemorySpace::MemorySpaceData< Number, MemorySpace > &, ::MemorySpace::MemorySpaceData< Number, MemorySpace > &)
static Number dot(const std::shared_ptr<::parallel::internal::TBBPartitioner > &, const size_type, const ::MemorySpace::MemorySpaceData< Number2, MemorySpace > &, ::MemorySpace::MemorySpaceData< Number, MemorySpace > &)
static Number add_and_dot(const std::shared_ptr<::parallel::internal::TBBPartitioner > &, const size_type, const Number, const ::MemorySpace::MemorySpaceData< Number, MemorySpace > &, const ::MemorySpace::MemorySpaceData< Number, MemorySpace > &, ::MemorySpace::MemorySpaceData< Number, MemorySpace > &)
static void sadd_xav(const std::shared_ptr<::parallel::internal::TBBPartitioner > &, const size_type, const Number, const Number, const ::MemorySpace::MemorySpaceData< Number, MemorySpace > &, ::MemorySpace::MemorySpaceData< Number, MemorySpace > &)
static void import_elements(const std::shared_ptr<::parallel::internal::TBBPartitioner > &, const size_type, VectorOperation::values, const ::MemorySpace::MemorySpaceData< Number, MemorySpace2 > &, ::MemorySpace::MemorySpaceData< Number, MemorySpace > &)
static void copy(const std::shared_ptr<::parallel::internal::TBBPartitioner > &, const size_type, const ::MemorySpace::MemorySpaceData< Number2, MemorySpace > &, ::MemorySpace::MemorySpaceData< Number, MemorySpace > &)
static void norm_1(const std::shared_ptr<::parallel::internal::TBBPartitioner > &, const size_type, real_type &, Number *, Number *)
static void add_av(const std::shared_ptr<::parallel::internal::TBBPartitioner > &, const size_type, const Number, const ::MemorySpace::MemorySpaceData< Number, MemorySpace > &, ::MemorySpace::MemorySpaceData< Number, MemorySpace > &)
static void multiply_factor(const std::shared_ptr<::parallel::internal::TBBPartitioner > &, const size_type, const Number, ::MemorySpace::MemorySpaceData< Number, MemorySpace > &)
static void norm_p(const std::shared_ptr<::parallel::internal::TBBPartitioner > &, const size_type, real_type &, real_type, ::MemorySpace::MemorySpaceData< Number, MemorySpace > &)
static void equ_aubv(const std::shared_ptr<::parallel::internal::TBBPartitioner > &, const size_type, const Number, const Number, const ::MemorySpace::MemorySpaceData< Number, MemorySpace > &, const ::MemorySpace::MemorySpaceData< Number, MemorySpace > &, ::MemorySpace::MemorySpaceData< Number, MemorySpace > &)
static void set(const std::shared_ptr<::parallel::internal::TBBPartitioner > &, const size_type, const Number, ::MemorySpace::MemorySpaceData< Number, MemorySpace > &)
static real_type abs(const number &x)
Definition numbers.h:593
static constexpr real_type abs_square(const number &x)
Definition numbers.h:584