Reference documentation for deal.II version GIT d2cc530c04 2023-03-22 15:10:02+00:00
\(\newcommand{\dealvcentcolon}{\mathrel{\mathop{:}}}\) \(\newcommand{\dealcoloneq}{\dealvcentcolon\mathrel{\mkern-1.2mu}=}\) \(\newcommand{\jump}[1]{\left[\!\left[ #1 \right]\!\right]}\) \(\newcommand{\average}[1]{\left\{\!\left\{ #1 \right\}\!\right\}}\)
vector_operations_internal.h
Go to the documentation of this file.
1 // ---------------------------------------------------------------------
2 //
3 // Copyright (C) 2016 - 2020 by the deal.II authors
4 //
5 // This file is part of the deal.II library.
6 //
7 // The deal.II library is free software; you can use it, redistribute
8 // it, and/or modify it under the terms of the GNU Lesser General
9 // Public License as published by the Free Software Foundation; either
10 // version 2.1 of the License, or (at your option) any later version.
11 // The full text of the license can be found in the file LICENSE.md at
12 // the top level directory of deal.II.
13 //
14 // ---------------------------------------------------------------------
15 
16 
17 #ifndef dealii_vector_operations_internal_h
18 #define dealii_vector_operations_internal_h
19 
20 #include <deal.II/base/config.h>
21 
25 #include <deal.II/base/parallel.h>
26 #include <deal.II/base/types.h>
28 
30 #include <deal.II/lac/cuda_kernels.templates.h>
32 
33 #include <cstdio>
34 #include <cstring>
35 
37 
38 namespace internal
39 {
40  namespace VectorOperations
41  {
43 
44  template <typename T>
45  bool
46  is_non_negative(const T &t)
47  {
48  return t >= 0;
49  }
50 
51 
52  template <typename T>
53  bool
54  is_non_negative(const std::complex<T> &)
55  {
56  Assert(false, ExcMessage("Complex numbers do not have an ordering."));
57 
58  return false;
59  }
60 
61 
62  // call std::copy, except for in
63  // the case where we want to copy
64  // from std::complex to a
65  // non-complex type
66  template <typename T, typename U>
67  void
68  copy(const T *begin, const T *end, U *dest)
69  {
70  std::copy(begin, end, dest);
71  }
72 
73  template <typename T, typename U>
74  void
75  copy(const std::complex<T> *begin,
76  const std::complex<T> *end,
77  std::complex<U> * dest)
78  {
79  std::copy(begin, end, dest);
80  }
81 
82  template <typename T, typename U>
83  void
84  copy(const std::complex<T> *, const std::complex<T> *, U *)
85  {
86  Assert(false,
87  ExcMessage("Can't convert a vector of complex numbers "
88  "into a vector of reals/doubles"));
89  }
90 
91 
92 
93 #ifdef DEAL_II_WITH_TBB
102  template <typename Functor>
104  {
106  const size_type start,
107  const size_type end)
108  : functor(functor)
109  , start(start)
110  , end(end)
111  {
112  const size_type vec_size = end - start;
113  // set chunk size for sub-tasks
114  const unsigned int gs =
116  n_chunks =
117  std::min(static_cast<size_type>(4 * MultithreadInfo::n_threads()),
118  vec_size / gs);
119  chunk_size = vec_size / n_chunks;
120 
121  // round to next multiple of 512 (or minimum grain size if that happens
122  // to be smaller). this is advantageous because our accumulation
123  // algorithms favor lengths of a power of 2 due to pairwise summation ->
124  // at most one 'oddly' sized chunk
125  if (chunk_size > 512)
126  chunk_size = ((chunk_size + 511) / 512) * 512;
127  n_chunks = (vec_size + chunk_size - 1) / chunk_size;
128  AssertIndexRange((n_chunks - 1) * chunk_size, vec_size);
129  AssertIndexRange(vec_size, n_chunks * chunk_size + 1);
130  }
131 
132  void
133  operator()(const tbb::blocked_range<size_type> &range) const
134  {
135  const size_type r_begin = start + range.begin() * chunk_size;
136  const size_type r_end = std::min(start + range.end() * chunk_size, end);
137  functor(r_begin, r_end);
138  }
139 
140  Functor & functor;
142  const size_type end;
143  unsigned int n_chunks;
145  };
146 #endif
147 
148  template <typename Functor>
149  void
151  Functor & functor,
152  const size_type start,
153  const size_type end,
154  const std::shared_ptr<::parallel::internal::TBBPartitioner>
155  &partitioner)
156  {
157 #ifdef DEAL_II_WITH_TBB
158  const size_type vec_size = end - start;
159  // only go to the parallel function in case there are at least 4 parallel
160  // items, otherwise the overhead is too large
161  if (vec_size >=
164  {
165  Assert(partitioner.get() != nullptr,
167  "Unexpected initialization of Vector that does "
168  "not set the TBB partitioner to a usable state."));
169  std::shared_ptr<tbb::affinity_partitioner> tbb_partitioner =
170  partitioner->acquire_one_partitioner();
171 
172  TBBForFunctor<Functor> generic_functor(functor, start, end);
173  // We use a minimum grain size of 1 here since the grains at this
174  // stage of dividing the work refer to the number of vector chunks
175  // that are processed by (possibly different) threads in the
176  // parallelized for loop (i.e., they do not refer to individual
177  // vector entries). The number of chunks here is calculated inside
178  // TBBForFunctor. See also GitHub issue #2496 for further discussion
179  // of this strategy.
181  static_cast<size_type>(0),
182  static_cast<size_type>(generic_functor.n_chunks),
183  generic_functor,
184  1,
185  tbb_partitioner);
186  partitioner->release_one_partitioner(tbb_partitioner);
187  }
188  else if (vec_size > 0)
189  functor(start, end);
190 #else
191  functor(start, end);
192  (void)partitioner;
193 #endif
194  }
195 
196 
197  // Define the functors necessary to use SIMD with TBB. we also include the
198  // simple copy and set operations
199 
200  template <typename Number>
201  struct Vector_set
202  {
203  Vector_set(const Number value, Number *const dst)
204  : value(value)
205  , dst(dst)
206  {
207  Assert(dst != nullptr, ExcInternalError());
208  }
209 
210  void
211  operator()(const size_type begin, const size_type end) const
212  {
214 
215  if (value == Number())
216  {
217  if DEAL_II_CONSTEXPR_IN_CONDITIONAL (std::is_trivial<Number>::value)
218  {
219  std::memset(dst + begin, 0, sizeof(Number) * (end - begin));
220  return;
221  }
222  }
223  std::fill(dst + begin, dst + end, value);
224  }
225 
226  const Number value;
227  Number *const dst;
228  };
229 
230  template <typename Number, typename OtherNumber>
231  struct Vector_copy
232  {
233  Vector_copy(const OtherNumber *const src, Number *const dst)
234  : src(src)
235  , dst(dst)
236  {
237  Assert(src != nullptr, ExcInternalError());
238  Assert(dst != nullptr, ExcInternalError());
239  }
240 
241  void
242  operator()(const size_type begin, const size_type end) const
243  {
245 
246  if DEAL_II_CONSTEXPR_IN_CONDITIONAL (std::is_trivially_copyable<
247  Number>() &&
248  std::is_same<Number,
249  OtherNumber>::value)
250  std::memcpy(dst + begin, src + begin, (end - begin) * sizeof(Number));
251  else
252  {
254  for (size_type i = begin; i < end; ++i)
255  dst[i] = src[i];
256  }
257  }
258 
259  const OtherNumber *const src;
260  Number *const dst;
261  };
262 
263  template <typename Number>
265  {
266  Vectorization_multiply_factor(Number *const val, const Number factor)
267  : val(val)
268  , stored_factor(factor)
269  {}
270 
271  void
272  operator()(const size_type begin, const size_type end) const
273  {
274  // create a local copy of the variable to help the compiler with the
275  // aliasing analysis
276  const Number factor = stored_factor;
277 
279  {
281  for (size_type i = begin; i < end; ++i)
282  val[i] *= factor;
283  }
284  else
285  {
286  for (size_type i = begin; i < end; ++i)
287  val[i] *= factor;
288  }
289  }
290 
291  Number *const val;
292  const Number stored_factor;
293  };
294 
295  template <typename Number>
297  {
298  Vectorization_add_av(Number *const val,
299  const Number *const v_val,
300  const Number factor)
301  : val(val)
302  , v_val(v_val)
303  , stored_factor(factor)
304  {}
305 
306  void
307  operator()(const size_type begin, const size_type end) const
308  {
309  // create a local copy of the variable to help the compiler with the
310  // aliasing analysis
311  const Number factor = stored_factor;
313  {
315  for (size_type i = begin; i < end; ++i)
316  val[i] += factor * v_val[i];
317  }
318  else
319  {
320  for (size_type i = begin; i < end; ++i)
321  val[i] += factor * v_val[i];
322  }
323  }
324 
325  Number *const val;
326  const Number *const v_val;
327  const Number stored_factor;
328  };
329 
330  template <typename Number>
332  {
334  const Number *const v_val,
335  const Number a,
336  const Number x)
337  : val(val)
338  , v_val(v_val)
339  , stored_a(a)
340  , stored_x(x)
341  {}
342 
343  void
344  operator()(const size_type begin, const size_type end) const
345  {
346  // create a local copy of the variable to help the compiler with the
347  // aliasing analysis
348  const Number x = stored_x, a = stored_a;
349 
351  {
353  for (size_type i = begin; i < end; ++i)
354  val[i] = x * val[i] + a * v_val[i];
355  }
356  else
357  {
358  for (size_type i = begin; i < end; ++i)
359  val[i] = x * val[i] + a * v_val[i];
360  }
361  }
362 
363  Number *const val;
364  const Number *const v_val;
365  const Number stored_a;
366  const Number stored_x;
367  };
368 
369  template <typename Number>
371  {
372  Vectorization_subtract_v(Number *val, const Number *const v_val)
373  : val(val)
374  , v_val(v_val)
375  {}
376 
377  void
378  operator()(const size_type begin, const size_type end) const
379  {
381  {
383  for (size_type i = begin; i < end; ++i)
384  val[i] -= v_val[i];
385  }
386  else
387  {
388  for (size_type i = begin; i < end; ++i)
389  val[i] -= v_val[i];
390  }
391  }
392 
393  Number *const val;
394  const Number *const v_val;
395  };
396 
397  template <typename Number>
399  {
400  Vectorization_add_factor(Number *const val, const Number factor)
401  : val(val)
402  , stored_factor(factor)
403  {}
404 
405  void
406  operator()(const size_type begin, const size_type end) const
407  {
408  const Number factor = stored_factor;
409 
411  {
413  for (size_type i = begin; i < end; ++i)
414  val[i] += factor;
415  }
416  else
417  {
418  for (size_type i = begin; i < end; ++i)
419  val[i] += factor;
420  }
421  }
422 
423  Number *const val;
424  const Number stored_factor;
425  };
426 
427  template <typename Number>
429  {
430  Vectorization_add_v(Number *const val, const Number *const v_val)
431  : val(val)
432  , v_val(v_val)
433  {}
434 
435  void
436  operator()(const size_type begin, const size_type end) const
437  {
439  {
441  for (size_type i = begin; i < end; ++i)
442  val[i] += v_val[i];
443  }
444  else
445  {
446  for (size_type i = begin; i < end; ++i)
447  val[i] += v_val[i];
448  }
449  }
450 
451  Number *const val;
452  const Number *const v_val;
453  };
454 
455  template <typename Number>
457  {
459  const Number *const v_val,
460  const Number *const w_val,
461  const Number a,
462  const Number b)
463  : val(val)
464  , v_val(v_val)
465  , w_val(w_val)
466  , stored_a(a)
467  , stored_b(b)
468  {}
469 
470  void
471  operator()(const size_type begin, const size_type end) const
472  {
473  const Number a = stored_a, b = stored_b;
474 
476  {
478  for (size_type i = begin; i < end; ++i)
479  val[i] = val[i] + a * v_val[i] + b * w_val[i];
480  }
481  else
482  {
483  for (size_type i = begin; i < end; ++i)
484  val[i] = val[i] + a * v_val[i] + b * w_val[i];
485  }
486  }
487 
488  Number *const val;
489  const Number *const v_val;
490  const Number *const w_val;
491  const Number stored_a;
492  const Number stored_b;
493  };
494 
495  template <typename Number>
497  {
498  Vectorization_sadd_xv(Number *const val,
499  const Number *const v_val,
500  const Number x)
501  : val(val)
502  , v_val(v_val)
503  , stored_x(x)
504  {}
505 
506  void
507  operator()(const size_type begin, const size_type end) const
508  {
509  const Number x = stored_x;
510 
512  {
514  for (size_type i = begin; i < end; ++i)
515  val[i] = x * val[i] + v_val[i];
516  }
517  else
518  {
519  for (size_type i = begin; i < end; ++i)
520  val[i] = x * val[i] + v_val[i];
521  }
522  }
523 
524  Number *const val;
525  const Number *const v_val;
526  const Number stored_x;
527  };
528 
529  template <typename Number>
531  {
533  const Number *v_val,
534  const Number *w_val,
535  Number x,
536  Number a,
537  Number b)
538  : val(val)
539  , v_val(v_val)
540  , w_val(w_val)
541  , stored_x(x)
542  , stored_a(a)
543  , stored_b(b)
544  {}
545 
546  void
547  operator()(const size_type begin, const size_type end) const
548  {
549  const Number x = stored_x, a = stored_a, b = stored_b;
550 
552  {
554  for (size_type i = begin; i < end; ++i)
555  val[i] = x * val[i] + a * v_val[i] + b * w_val[i];
556  }
557  else
558  {
559  for (size_type i = begin; i < end; ++i)
560  val[i] = x * val[i] + a * v_val[i] + b * w_val[i];
561  }
562  }
563 
564  Number *const val;
565  const Number *const v_val;
566  const Number *const w_val;
567  const Number stored_x;
568  const Number stored_a;
569  const Number stored_b;
570  };
571 
572  template <typename Number>
574  {
575  Vectorization_scale(Number *const val, const Number *const v_val)
576  : val(val)
577  , v_val(v_val)
578  {}
579 
580  void
581  operator()(const size_type begin, const size_type end) const
582  {
584  {
586  for (size_type i = begin; i < end; ++i)
587  val[i] *= v_val[i];
588  }
589  else
590  {
591  for (size_type i = begin; i < end; ++i)
592  val[i] *= v_val[i];
593  }
594  }
595 
596  Number *const val;
597  const Number *const v_val;
598  };
599 
600  template <typename Number>
602  {
603  Vectorization_equ_au(Number *const val,
604  const Number *const u_val,
605  const Number a)
606  : val(val)
607  , u_val(u_val)
608  , stored_a(a)
609  {}
610 
611  void
612  operator()(const size_type begin, const size_type end) const
613  {
614  const Number a = stored_a;
615 
617  {
619  for (size_type i = begin; i < end; ++i)
620  val[i] = a * u_val[i];
621  }
622  else
623  {
624  for (size_type i = begin; i < end; ++i)
625  val[i] = a * u_val[i];
626  }
627  }
628 
629  Number *const val;
630  const Number *const u_val;
631  const Number stored_a;
632  };
633 
634  template <typename Number>
636  {
638  const Number *const u_val,
639  const Number *const v_val,
640  const Number a,
641  const Number b)
642  : val(val)
643  , u_val(u_val)
644  , v_val(v_val)
645  , stored_a(a)
646  , stored_b(b)
647  {}
648 
649  void
650  operator()(const size_type begin, const size_type end) const
651  {
652  const Number a = stored_a, b = stored_b;
653 
655  {
657  for (size_type i = begin; i < end; ++i)
658  val[i] = a * u_val[i] + b * v_val[i];
659  }
660  else
661  {
662  for (size_type i = begin; i < end; ++i)
663  val[i] = a * u_val[i] + b * v_val[i];
664  }
665  }
666 
667  Number *const val;
668  const Number *const u_val;
669  const Number *const v_val;
670  const Number stored_a;
671  const Number stored_b;
672  };
673 
674  template <typename Number>
676  {
678  const Number *u_val,
679  const Number *v_val,
680  const Number *w_val,
681  const Number a,
682  const Number b,
683  const Number c)
684  : val(val)
685  , u_val(u_val)
686  , v_val(v_val)
687  , w_val(w_val)
688  , stored_a(a)
689  , stored_b(b)
690  , stored_c(c)
691  {}
692 
693  void
694  operator()(const size_type begin, const size_type end) const
695  {
696  const Number a = stored_a, b = stored_b, c = stored_c;
697 
699  {
701  for (size_type i = begin; i < end; ++i)
702  val[i] = a * u_val[i] + b * v_val[i] + c * w_val[i];
703  }
704  else
705  {
706  for (size_type i = begin; i < end; ++i)
707  val[i] = a * u_val[i] + b * v_val[i] + c * w_val[i];
708  }
709  }
710 
711  Number *const val;
712  const Number *const u_val;
713  const Number *const v_val;
714  const Number *const w_val;
715  const Number stored_a;
716  const Number stored_b;
717  const Number stored_c;
718  };
719 
720  template <typename Number>
722  {
723  Vectorization_ratio(Number *val, const Number *a_val, const Number *b_val)
724  : val(val)
725  , a_val(a_val)
726  , b_val(b_val)
727  {}
728 
729  void
730  operator()(const size_type begin, const size_type end) const
731  {
733  {
735  for (size_type i = begin; i < end; ++i)
736  val[i] = a_val[i] / b_val[i];
737  }
738  else
739  {
740  for (size_type i = begin; i < end; ++i)
741  val[i] = a_val[i] / b_val[i];
742  }
743  }
744 
745  Number *const val;
746  const Number *const a_val;
747  const Number *const b_val;
748  };
749 
750 
751 
752  // All sums over all the vector entries (l2-norm, inner product, etc.) are
753  // performed with the same code, using a templated operation defined
754  // here. There are always two versions defined, a standard one that covers
755  // most cases and a vectorized one which is only for equal types and float
756  // and double.
757  template <typename Number, typename Number2>
758  struct Dot
759  {
760  static constexpr bool vectorizes = std::is_same<Number, Number2>::value &&
762 
763  Dot(const Number *const X, const Number2 *const Y)
764  : X(X)
765  , Y(Y)
766  {}
767 
768  Number
769  operator()(const size_type i) const
770  {
771  return X[i] * Number(numbers::NumberTraits<Number2>::conjugate(Y[i]));
772  }
773 
775  do_vectorized(const size_type i) const
776  {
778  x.load(X + i);
779  y.load(Y + i);
780 
781  // the following operation in VectorizedArray does an element-wise
782  // scalar product without taking into account complex values and
783  // the need to take the complex-conjugate of one argument. this
784  // may be a bug, but because all VectorizedArray classes only
785  // work on real scalars, it doesn't really matter very much.
786  // in any case, assert that we really don't get here for
787  // complex-valued objects
788  static_assert(numbers::NumberTraits<Number>::is_complex == false,
789  "This operation is not correctly implemented for "
790  "complex-valued objects.");
791  return x * y;
792  }
793 
794  const Number *const X;
795  const Number2 *const Y;
796  };
797 
798  template <typename Number, typename RealType>
799  struct Norm2
800  {
801  static const bool vectorizes = VectorizedArray<Number>::size() > 1;
802 
803  Norm2(const Number *const X)
804  : X(X)
805  {}
806 
807  RealType
808  operator()(const size_type i) const
809  {
811  }
812 
814  do_vectorized(const size_type i) const
815  {
817  x.load(X + i);
818  return x * x;
819  }
820 
821  const Number *const X;
822  };
823 
824  template <typename Number, typename RealType>
825  struct Norm1
826  {
827  static const bool vectorizes = VectorizedArray<Number>::size() > 1;
828 
829  Norm1(const Number *X)
830  : X(X)
831  {}
832 
833  RealType
834  operator()(const size_type i) const
835  {
837  }
838 
840  do_vectorized(const size_type i) const
841  {
843  x.load(X + i);
844  return std::abs(x);
845  }
846 
847  const Number *X;
848  };
849 
850  template <typename Number, typename RealType>
851  struct NormP
852  {
853  static const bool vectorizes = VectorizedArray<Number>::size() > 1;
854 
855  NormP(const Number *X, RealType p)
856  : X(X)
857  , p(p)
858  {}
859 
860  RealType
861  operator()(const size_type i) const
862  {
863  return std::pow(numbers::NumberTraits<Number>::abs(X[i]), p);
864  }
865 
867  do_vectorized(const size_type i) const
868  {
870  x.load(X + i);
871  return std::pow(std::abs(x), p);
872  }
873 
874  const Number * X;
875  const RealType p;
876  };
877 
878  template <typename Number>
879  struct MeanValue
880  {
881  static const bool vectorizes = VectorizedArray<Number>::size() > 1;
882 
883  MeanValue(const Number *X)
884  : X(X)
885  {}
886 
887  Number
888  operator()(const size_type i) const
889  {
890  return X[i];
891  }
892 
894  do_vectorized(const size_type i) const
895  {
897  x.load(X + i);
898  return x;
899  }
900 
901  const Number *X;
902  };
903 
904  template <typename Number>
905  struct AddAndDot
906  {
907  static const bool vectorizes = VectorizedArray<Number>::size() > 1;
908 
909  AddAndDot(Number *const X,
910  const Number *const V,
911  const Number *const W,
912  const Number a)
913  : X(X)
914  , V(V)
915  , W(W)
916  , a(a)
917  {}
918 
919  Number
920  operator()(const size_type i) const
921  {
922  X[i] += a * V[i];
923  return X[i] * Number(numbers::NumberTraits<Number>::conjugate(W[i]));
924  }
925 
927  do_vectorized(const size_type i) const
928  {
930  x.load(X + i);
931  v.load(V + i);
932  x += a * v;
933  x.store(X + i);
934  // may only load from W after storing in X because the pointers might
935  // point to the same memory
936  w.load(W + i);
937 
938  // the following operation in VectorizedArray does an element-wise
939  // scalar product without taking into account complex values and
940  // the need to take the complex-conjugate of one argument. this
941  // may be a bug, but because all VectorizedArray classes only
942  // work on real scalars, it doesn't really matter very much.
943  // in any case, assert that we really don't get here for
944  // complex-valued objects
945  static_assert(numbers::NumberTraits<Number>::is_complex == false,
946  "This operation is not correctly implemented for "
947  "complex-valued objects.");
948  return x * w;
949  }
950 
951  Number *const X;
952  const Number *const V;
953  const Number *const W;
954  const Number a;
955  };
956 
957 
958 
959  // this is the main working loop for all vector sums using the templated
960  // operation above. it accumulates the sums using a block-wise summation
961  // algorithm with post-update. this blocked algorithm has been proposed in
962  // a similar form by Castaldo, Whaley and Chronopoulos (SIAM
963  // J. Sci. Comput. 31, 1156-1174, 2008) and we use the smallest possible
964  // block size, 2. Sometimes it is referred to as pairwise summation. The
965  // worst case error made by this algorithm is on the order O(eps *
966  // log2(vec_size)), whereas a naive summation is O(eps * vec_size). Even
967  // though the Kahan summation is even more accurate with an error O(eps)
968  // by carrying along remainders not captured by the main sum, that involves
969  // additional costs which are not worthwhile. See the Wikipedia article on
970  // the Kahan summation algorithm.
971 
972  // The algorithm implemented here has the additional benefit that it is
973  // easily parallelized without changing the order of how the elements are
974  // added (floating point addition is not associative). For the same vector
975  // size and minimum_parallel_grainsize, the blocks are always the
976  // same and added pairwise.
977 
978  // The depth of recursion is controlled by the 'magic' parameter
979  // vector_accumulation_recursion_threshold: If the length is below
980  // vector_accumulation_recursion_threshold * 32 (32 is the part of code we
981  // unroll), a straight loop instead of recursion will be used. At the
982  // innermost level, eight values are added consecutively in order to better
983  // balance multiplications and additions.
984 
985  // Loops are unrolled as follows: the range [first,last) is broken into
986  // @p n_chunks each of size 32 plus the @p remainder.
987  // accumulate_regular() does the work on 32*n_chunks elements employing SIMD
988  // if possible and stores the result of the operation for each chunk in @p outer_results.
989 
990  // The code returns the result as the last argument in order to make
991  // spawning tasks simpler and use automatic template deduction.
992 
993 
999  const unsigned int vector_accumulation_recursion_threshold = 128;
1000 
1001  template <typename Operation, typename ResultType>
1002  void
1003  accumulate_recursive(const Operation &op,
1004  const size_type first,
1005  const size_type last,
1006  ResultType & result)
1007  {
1008  if (first == last)
1009  {
1010  result = ResultType();
1011  return;
1012  }
1013 
1014  const size_type vec_size = last - first;
1015  if (vec_size <= vector_accumulation_recursion_threshold * 32)
1016  {
1017  // The vector is short enough so we perform the summation. We store
1018  // the number of chunks (each 32 indices) for the given vector
1019  // length; all results are stored in outer_results[0,n_chunks). We
1020  // keep twice the number around to be able to do the pairwise
1021  // summation with a single for loop (see the loop over j below)
1022  ResultType outer_results[vector_accumulation_recursion_threshold * 2];
1023 
1024  // Select between the regular version and vectorized version based
1025  // on the number types we are given. To choose the vectorized
1026  // version often enough, we need to have all tasks but the last one
1027  // to be divisible by the vectorization length
1028  size_type n_chunks = do_accumulate(
1029  op,
1030  vec_size,
1031  first,
1032  outer_results,
1033  std::integral_constant<bool, Operation::vectorizes>());
1034 
1035  AssertIndexRange(n_chunks,
1037 
1038  // now sum the results from the chunks stored in
1039  // outer_results[0,n_chunks) recursively
1040  unsigned int j = 0;
1041  constexpr unsigned int n_lanes = VectorizedArray<ResultType>::size();
1042  for (; j + 2 * n_lanes - 1 < n_chunks;
1043  j += 2 * n_lanes, n_chunks += n_lanes)
1044  {
1046  a.load(outer_results + j);
1047  b.load(outer_results + j + n_lanes);
1048  a += b;
1049  a.store(outer_results + n_chunks);
1050  }
1051 
1052  // In the vectorized case, we know the loop bounds and can do things
1053  // more efficiently
1054  if (Operation::vectorizes)
1055  {
1056  AssertDimension(j + n_lanes, n_chunks);
1057  AssertIndexRange(n_chunks,
1059  ResultType *result_ptr = outer_results + j;
1060  if (n_lanes >= 16)
1061  for (unsigned int i = 0; i < 8; ++i)
1062  result_ptr[i] = result_ptr[i] + result_ptr[i + 8];
1063  if (n_lanes >= 8)
1064  for (unsigned int i = 0; i < 4; ++i)
1065  result_ptr[i] = result_ptr[i] + result_ptr[i + 4];
1066  if (n_lanes >= 4)
1067  for (unsigned int i = 0; i < 2; ++i)
1068  result_ptr[i] = result_ptr[i] + result_ptr[i + 2];
1069  result = result_ptr[0] + result_ptr[1];
1070  }
1071  else
1072  {
1073  // Without vectorization, we do not know the exact bounds, so we
1074  // need to continue the variable-length pairwise summation loop
1075  // from above
1076  for (; j + 1 < n_chunks; j += 2, ++n_chunks)
1077  outer_results[n_chunks] =
1078  outer_results[j] + outer_results[j + 1];
1079 
1080  AssertIndexRange(n_chunks,
1082  Assert(n_chunks > 0, ExcInternalError());
1083  result = outer_results[n_chunks - 1];
1084  }
1085  }
1086  else
1087  {
1088  // split vector into four pieces and work on the pieces
1089  // recursively. Make pieces (except last) divisible by one fourth the
1090  // recursion threshold.
1091  const size_type new_size =
1092  (vec_size / (vector_accumulation_recursion_threshold * 32)) *
1094  Assert(first + 3 * new_size < last, ExcInternalError());
1095  ResultType r0, r1, r2, r3;
1096  accumulate_recursive(op, first, first + new_size, r0);
1097  accumulate_recursive(op, first + new_size, first + 2 * new_size, r1);
1099  first + 2 * new_size,
1100  first + 3 * new_size,
1101  r2);
1102  accumulate_recursive(op, first + 3 * new_size, last, r3);
1103  result = (r0 + r1) + (r2 + r3);
1104  }
1105  }
1106 
1107 
1108  // this is the inner working routine for the accumulation loops below. We
1109  // pulled this part out of the regular accumulate routine because we might
1110  // do this thing vectorized (see specialized function below; this is the
1111  // un-vectorized version). As opposed to the vector add functions above,
1112  // we here pass the functor 'op' by value, because we cannot create a copy
1113  // of the scalar inline, and instead make sure that the numbers get local
1114  // (and thus definitely not aliased) for the compiler
1115  template <typename Operation, typename ResultType>
1116  size_type
1117  do_accumulate(const Operation op,
1118  const size_type vec_size,
1119  const size_type start_index,
1120  ResultType * outer_results,
1121  std::integral_constant<bool, false>)
1122  {
1123  // Create local copy to indicate no aliasing to the compiler
1124  size_type index = start_index;
1125 
1126  // choose each chunk to have a width of 32, thereby the index
1127  // is incremented by 4*8 for each @p i.
1128  size_type n_chunks = vec_size / 32;
1129  for (size_type i = 0; i < n_chunks; ++i)
1130  {
1131  ResultType r = {};
1132  for (unsigned int k = 0; k < 2; ++k)
1133  {
1134  ResultType r0 = op(index);
1135  ResultType r1 = op(index + 1);
1136  ResultType r2 = op(index + 2);
1137  ResultType r3 = op(index + 3);
1138  index += 4;
1139  for (size_type j = 1; j < 4; ++j, index += 4)
1140  {
1141  r0 += op(index);
1142  r1 += op(index + 1);
1143  r2 += op(index + 2);
1144  r3 += op(index + 3);
1145  }
1146  r += (r0 + r1) + (r2 + r3);
1147  }
1148  outer_results[i] = r;
1149  }
1150 
1151  if (n_chunks * 32 < vec_size)
1152  {
1153  const size_type remainder = vec_size - n_chunks * 32;
1154  const size_type inner_chunks = remainder / 8;
1155  const size_type remainder_inner = remainder % 8;
1156  ResultType r0 = ResultType(), r1 = ResultType(), r2 = ResultType();
1157  switch (inner_chunks)
1158  {
1159  case 3:
1160  r2 = op(index++);
1161  for (size_type j = 1; j < 8; ++j)
1162  r2 += op(index++);
1164  case 2:
1165  r1 = op(index++);
1166  for (size_type j = 1; j < 8; ++j)
1167  r1 += op(index++);
1168  r1 += r2;
1170  case 1:
1171  r2 = op(index++);
1172  for (size_type j = 1; j < 8; ++j)
1173  r2 += op(index++);
1175  default:
1176  for (size_type j = 0; j < remainder_inner; ++j)
1177  r0 += op(index++);
1178  outer_results[n_chunks++] = (r0 + r2) + r1;
1179  break;
1180  }
1181  }
1182 
1183  // make sure we worked through all indices
1184  AssertDimension(index, start_index + vec_size);
1185 
1186  return n_chunks;
1187  }
1188 
1189 
1190 
1191  // this is the inner working routine for the accumulation loops
1192  // below. This is the specialized case where we can vectorize. We request
1193  // the 'do_vectorized' routine of the operation instead of the regular one
1194  // which does several operations at once. As above, pass in the functor by
1195  // value to create a local copy of the scalar factors in the function (if
1196  // there are any).
1197  template <typename Operation, typename Number>
1198  size_type
1199  do_accumulate(const Operation op,
1200  const size_type vec_size,
1201  const size_type start_index,
1202  Number * outer_results,
1203  std::integral_constant<bool, true>)
1204  {
1205  // Create local copy to indicate no aliasing to the compiler
1206  size_type index = start_index;
1207 
1208  // we start from @p index and workout @p n_chunks each of size 32.
1209  // in order employ SIMD and work on @p nvecs at a time, we split this
1210  // loop yet again:
1211  // First we work on (n_chunks/nvecs) chunks, where each chunk processes
1212  // nvecs*(4*8) elements.
1213 
1214  constexpr size_type n_lanes = VectorizedArray<Number>::size();
1215  const size_type regular_chunks = vec_size / (32 * n_lanes);
1216  for (size_type i = 0; i < regular_chunks; ++i)
1217  {
1218  VectorizedArray<Number> r = {};
1219  for (unsigned int k = 0; k < 2; ++k)
1220  {
1221  VectorizedArray<Number> r0 = op.do_vectorized(index);
1222  VectorizedArray<Number> r1 = op.do_vectorized(index + n_lanes);
1224  op.do_vectorized(index + 2 * n_lanes);
1226  op.do_vectorized(index + 3 * n_lanes);
1227  index += n_lanes * 4;
1228  for (size_type j = 1; j < 4; ++j, index += n_lanes * 4)
1229  {
1230  r0 += op.do_vectorized(index);
1231  r1 += op.do_vectorized(index + n_lanes);
1232  r2 += op.do_vectorized(index + 2 * n_lanes);
1233  r3 += op.do_vectorized(index + 3 * n_lanes);
1234  }
1235  r += (r0 + r1) + (r2 + r3);
1236  }
1237  r.store(&outer_results[i * n_lanes]);
1238  }
1239 
1240  // If we are treating a case where the vector length is not divisible by
1241  // the vectorization length, need a cleanup loop
1242  // The remaining chunks are processed one by one starting from
1243  // regular_chunks * n_lanes; We do as much as possible with 2 SIMD
1244  // operations within each chunk. Here we assume that n_lanes < 32/2 = 16
1245  // as well as 16 % n_lanes == 0.
1246  static_assert(n_lanes <= 16 && 16 % n_lanes == 0,
1247  "VectorizedArray::size() must be 1, 2, 4, 8, or 16");
1248  size_type n_chunks = regular_chunks * n_lanes;
1249  const size_type start_irregular = regular_chunks * n_lanes * 32;
1250  if (start_irregular < vec_size)
1251  {
1253  r1 = VectorizedArray<Number>();
1254  const size_type remainder = vec_size - start_irregular;
1255  const size_type loop_length = remainder / (2 * n_lanes);
1256  for (size_type j = 0; j < loop_length; ++j, index += 2 * n_lanes)
1257  {
1258  r0 += op.do_vectorized(index);
1259  r1 += op.do_vectorized(index + n_lanes);
1260  }
1261  Number scalar_part = Number();
1262  size_type last = remainder % (2 * n_lanes);
1263  if (last > 0)
1264  {
1265  if (last >= n_lanes)
1266  {
1267  r0 += op.do_vectorized(index);
1268  index += n_lanes;
1269  last -= n_lanes;
1270  }
1271  for (unsigned int i = 0; i < last; ++i)
1272  scalar_part += op(index++);
1273  }
1274 
1275  r0 += r1;
1276  r0.store(&outer_results[n_chunks]);
1277  outer_results[n_chunks] += scalar_part;
1278 
1279  // update n_chunks to denote range of entries to sum up in
1280  // outer_results[].
1281  n_chunks += n_lanes;
1282  }
1283 
1284  // make sure we worked through all indices
1285  AssertDimension(index, start_index + vec_size);
1286 
1287  return n_chunks;
1288  }
1289 
1290 
1291 
1292 #ifdef DEAL_II_WITH_TBB
1321  template <typename Operation, typename ResultType>
1323  {
1324  static const unsigned int threshold_array_allocate = 512;
1325 
1326  TBBReduceFunctor(const Operation &op,
1327  const size_type start,
1328  const size_type end)
1329  : op(op)
1330  , start(start)
1331  , end(end)
1332  {
1333  const size_type vec_size = end - start;
1334  // set chunk size for sub-tasks
1335  const unsigned int gs =
1337  n_chunks =
1338  std::min(static_cast<size_type>(4 * MultithreadInfo::n_threads()),
1339  vec_size / gs);
1340  chunk_size = vec_size / n_chunks;
1341 
1342  // round to next multiple of 512 (or leave it at the minimum grain size
1343  // if that happens to be smaller). this is advantageous because our
1344  // algorithm favors lengths of a power of 2 due to pairwise summation ->
1345  // at most one 'oddly' sized chunk
1346  if (chunk_size > 512)
1347  chunk_size = ((chunk_size + 511) / 512) * 512;
1348  n_chunks = (vec_size + chunk_size - 1) / chunk_size;
1349  AssertIndexRange((n_chunks - 1) * chunk_size, vec_size);
1350  AssertIndexRange(vec_size, n_chunks * chunk_size + 1);
1351 
1353  {
1354  // make sure we allocate an even number of elements,
1355  // access to the new last element is needed in do_sum()
1356  large_array.resize(2 * ((n_chunks + 1) / 2));
1357  array_ptr = large_array.data();
1358  }
1359  else
1360  array_ptr = &small_array[0];
1361  }
1362 
1367  void
1368  operator()(const tbb::blocked_range<size_type> &range) const
1369  {
1370  for (size_type i = range.begin(); i < range.end(); ++i)
1372  start + i * chunk_size,
1373  std::min(start + (i + 1) * chunk_size, end),
1374  array_ptr[i]);
1375  }
1376 
1377  ResultType
1378  do_sum() const
1379  {
1380  while (n_chunks > 1)
1381  {
1382  if (n_chunks % 2 == 1)
1383  array_ptr[n_chunks++] = ResultType();
1384  for (size_type i = 0; i < n_chunks; i += 2)
1385  array_ptr[i / 2] = array_ptr[i] + array_ptr[i + 1];
1386  n_chunks /= 2;
1387  }
1388  return array_ptr[0];
1389  }
1390 
1391  const Operation &op;
1394 
1395  mutable unsigned int n_chunks;
1396  unsigned int chunk_size;
1398  std::vector<ResultType> large_array;
1399  // this variable either points to small_array or large_array depending on
1400  // the number of threads we want to feed
1401  mutable ResultType *array_ptr;
1402  };
1403 #endif
1404 
1405 
1406 
1411  template <typename Operation, typename ResultType>
1412 #ifndef DEBUG
1414 #endif
1415  inline void
1417  const Operation &op,
1418  const size_type start,
1419  const size_type end,
1420  ResultType & result,
1421  const std::shared_ptr<::parallel::internal::TBBPartitioner>
1422  &partitioner)
1423  {
1424 #ifdef DEAL_II_WITH_TBB
1425  const size_type vec_size = end - start;
1426  // only go to the parallel function in case there are at least 4 parallel
1427  // items, otherwise the overhead is too large
1428  if (vec_size >=
1431  {
1432  Assert(partitioner.get() != nullptr,
1434  "Unexpected initialization of Vector that does "
1435  "not set the TBB partitioner to a usable state."));
1436  std::shared_ptr<tbb::affinity_partitioner> tbb_partitioner =
1437  partitioner->acquire_one_partitioner();
1438 
1439  TBBReduceFunctor<Operation, ResultType> generic_functor(op,
1440  start,
1441  end);
1442  // We use a minimum grain size of 1 here since the grains at this
1443  // stage of dividing the work refer to the number of vector chunks
1444  // that are processed by (possibly different) threads in the
1445  // parallelized for loop (i.e., they do not refer to individual
1446  // vector entries). The number of chunks here is calculated inside
1447  // TBBForFunctor. See also GitHub issue #2496 for further discussion
1448  // of this strategy.
1450  static_cast<size_type>(0),
1451  static_cast<size_type>(generic_functor.n_chunks),
1452  generic_functor,
1453  1,
1454  tbb_partitioner);
1455  partitioner->release_one_partitioner(tbb_partitioner);
1456  result = generic_functor.do_sum();
1457  }
1458  else
1459  accumulate_recursive(op, start, end, result);
1460 #else
1461  accumulate_recursive(op, start, end, result);
1462  (void)partitioner;
1463 #endif
1464  }
1465 
1466 
1467  template <typename Number, typename Number2, typename MemorySpace>
1468  struct functions
1469  {
1470  static void
1472  const std::shared_ptr<::parallel::internal::TBBPartitioner> &
1473  /*thread_loop_partitioner*/,
1474  const size_type /*size*/,
1475  const ::MemorySpace::MemorySpaceData<Number2, MemorySpace>
1476  & /*v_data*/,
1478  {
1479  static_assert(
1480  std::is_same<MemorySpace, ::MemorySpace::Default>::value &&
1481  std::is_same<Number, Number2>::value,
1482  "For the Default MemorySpace Number and Number2 should be the same type");
1483  }
1484 
1485  static void
1487  const std::shared_ptr<::parallel::internal::TBBPartitioner> &
1488  /*thread_loop_partitioner*/,
1489  const size_type /*size*/,
1490  const Number /*s*/,
1492  {}
1493 
1494  static void
1496  const std::shared_ptr<::parallel::internal::TBBPartitioner> &
1497  /*thread_loop_partitioner*/,
1498  const size_type /*size*/,
1499  const ::MemorySpace::MemorySpaceData<Number, MemorySpace>
1500  & /*v_data*/,
1502  {}
1503 
1504  static void
1506  const std::shared_ptr<::parallel::internal::TBBPartitioner> &
1507  /*thread_loop_partitioner*/,
1508  const size_type /*size*/,
1509  const ::MemorySpace::MemorySpaceData<Number, MemorySpace>
1510  & /*v_data*/,
1512  {}
1513 
1514  static void
1516  const std::shared_ptr<::parallel::internal::TBBPartitioner> &
1517  /*thread_loop_partitioner*/,
1518  const size_type /*size*/,
1519  Number /*a*/,
1521  {}
1522 
1523  static void
1525  const std::shared_ptr<::parallel::internal::TBBPartitioner> &
1526  /*thread_loop_partitioner*/,
1527  const size_type /*size*/,
1528  const Number /*a*/,
1529  const ::MemorySpace::MemorySpaceData<Number, MemorySpace>
1530  & /*v_data*/,
1532  {}
1533 
1534  static void
1536  const std::shared_ptr<::parallel::internal::TBBPartitioner> &
1537  /*thread_loop_partitioner*/,
1538  const size_type /*size*/,
1539  const Number /*a*/,
1540  const Number /*b*/,
1541  const ::MemorySpace::MemorySpaceData<Number, MemorySpace>
1542  & /*v_data*/,
1543  const ::MemorySpace::MemorySpaceData<Number, MemorySpace>
1544  & /*w_data*/,
1546  {}
1547 
1548  static void
1550  const std::shared_ptr<::parallel::internal::TBBPartitioner> &
1551  /*thread_loop_partitioner*/,
1552  const size_type /*size*/,
1553  const Number /*x*/,
1554  const ::MemorySpace::MemorySpaceData<Number, MemorySpace>
1555  & /*v_data*/,
1557  {}
1558 
1559  static void
1561  const std::shared_ptr<::parallel::internal::TBBPartitioner> &
1562  /*thread_loop_partitioner*/,
1563  const size_type /*size*/,
1564  const Number /*x*/,
1565  const Number /*a*/,
1566  const ::MemorySpace::MemorySpaceData<Number, MemorySpace>
1567  & /*v_data*/,
1569  {}
1570 
1571  static void
1573  const std::shared_ptr<::parallel::internal::TBBPartitioner> &
1574  /*thread_loop_partitioner*/,
1575  const size_type /*size*/,
1576  const Number /*x*/,
1577  const Number /*a*/,
1578  const Number /*b*/,
1579  const ::MemorySpace::MemorySpaceData<Number, MemorySpace>
1580  & /*v_data*/,
1581  const ::MemorySpace::MemorySpaceData<Number, MemorySpace>
1582  & /*w_data*/,
1584  {}
1585 
1586  static void
1588  const std::shared_ptr<::parallel::internal::TBBPartitioner> &
1589  /*thread_loop_partitioner*/,
1590  const size_type /*size*/,
1591  const Number /*factor*/,
1593  {}
1594 
1595  static void
1597  const std::shared_ptr<::parallel::internal::TBBPartitioner> &
1598  /*thread_loop_partitioner*/,
1599  const size_type /*size*/,
1600  const ::MemorySpace::MemorySpaceData<Number, MemorySpace>
1601  & /*v_data*/,
1603  {}
1604 
1605  static void
1607  const std::shared_ptr<::parallel::internal::TBBPartitioner> &
1608  /*thread_loop_partitioner*/,
1609  const size_type /*size*/,
1610  const Number /*a*/,
1611  const ::MemorySpace::MemorySpaceData<Number, MemorySpace>
1612  & /*v_data*/,
1614  {}
1615 
1616  static void
1618  const std::shared_ptr<::parallel::internal::TBBPartitioner> &
1619  /*thread_loop_partitioner*/,
1620  const size_type /*size*/,
1621  const Number /*a*/,
1622  const Number /*b*/,
1623  const ::MemorySpace::MemorySpaceData<Number, MemorySpace>
1624  & /*v_data*/,
1625  const ::MemorySpace::MemorySpaceData<Number, MemorySpace>
1626  & /*w_data*/,
1628  {}
1629 
1630  static Number
1632  const std::shared_ptr<::parallel::internal::TBBPartitioner> &
1633  /*thread_loop_partitioner*/,
1634  const size_type /*size*/,
1635  const ::MemorySpace::MemorySpaceData<Number2, MemorySpace>
1636  & /*v_data*/,
1638  {
1639  return Number();
1640  }
1641 
1642  template <typename real_type>
1643  static void
1645  const std::shared_ptr<::parallel::internal::TBBPartitioner> &
1646  /*thread_loop_partitioner*/,
1647  const size_type /*size*/,
1648  real_type & /*sum*/,
1649  const ::MemorySpace::MemorySpaceData<Number, MemorySpace>
1650  & /*v_data*/,
1652  {}
1653 
1654  static Number
1656  const std::shared_ptr<::parallel::internal::TBBPartitioner> &
1657  /*thread_loop_partitioner*/,
1658  const size_type /*size*/,
1659  const ::MemorySpace::MemorySpaceData<Number, MemorySpace>
1660  & /*data*/)
1661  {
1662  return Number();
1663  }
1664 
1665  template <typename real_type>
1666  static void
1668  const std::shared_ptr<::parallel::internal::TBBPartitioner> &
1669  /*thread_loop_partitioner*/,
1670  const size_type /*size*/,
1671  real_type & /*sum*/,
1672  Number * /*values*/,
1673  Number * /*values*/)
1674  {}
1675 
1676  template <typename real_type>
1677  static void
1679  const std::shared_ptr<::parallel::internal::TBBPartitioner> &
1680  /*thread_loop_partitioner*/,
1681  const size_type /*size*/,
1682  real_type & /*sum*/,
1683  real_type /*p*/,
1685  {}
1686 
1687  static Number
1689  const std::shared_ptr<::parallel::internal::TBBPartitioner> &
1690  /*thread_loop_partitioner*/,
1691  const size_type /*size*/,
1692  const Number /*a*/,
1693  const ::MemorySpace::MemorySpaceData<Number, MemorySpace>
1694  & /*v_data*/,
1695  const ::MemorySpace::MemorySpaceData<Number, MemorySpace>
1696  & /*w_data*/,
1698  {
1699  return Number();
1700  }
1701 
1702  template <typename MemorySpace2>
1703  static void
1705  const std::shared_ptr<::parallel::internal::TBBPartitioner> &
1706  /*thread_loop_partitioner*/,
1707  const size_type /*size*/,
1708  VectorOperation::values /*operation*/,
1709  const ::MemorySpace::MemorySpaceData<Number, MemorySpace2>
1710  & /*v_data*/,
1712  {}
1713  };
1714 
1715 
1716 
1717  template <typename Number, typename Number2>
1718  struct functions<Number, Number2, ::MemorySpace::Host>
1719  {
1720  static void
1721  copy(const std::shared_ptr<::parallel::internal::TBBPartitioner>
1722  & thread_loop_partitioner,
1723  const size_type size,
1724  const ::MemorySpace::
1725  MemorySpaceData<Number2, ::MemorySpace::Host> &v_data,
1728  &data)
1729  {
1730  Vector_copy<Number, Number2> copier(v_data.values.data(),
1731  data.values.data());
1732  parallel_for(copier, 0, size, thread_loop_partitioner);
1733  }
1734 
1735  static void
1736  set(const std::shared_ptr<::parallel::internal::TBBPartitioner>
1737  & thread_loop_partitioner,
1738  const size_type size,
1739  const Number s,
1742  &data)
1743  {
1744  Vector_set<Number> setter(s, data.values.data());
1745  parallel_for(setter, 0, size, thread_loop_partitioner);
1746  }
1747 
1748  static void
1750  const std::shared_ptr<::parallel::internal::TBBPartitioner>
1751  & thread_loop_partitioner,
1752  const size_type size,
1753  const ::MemorySpace::
1754  MemorySpaceData<Number, ::MemorySpace::Host> &v_data,
1757  &data)
1758  {
1759  Vectorization_add_v<Number> vector_add(data.values.data(),
1760  v_data.values.data());
1761  parallel_for(vector_add, 0, size, thread_loop_partitioner);
1762  }
1763 
1764  static void
1766  const std::shared_ptr<::parallel::internal::TBBPartitioner>
1767  & thread_loop_partitioner,
1768  const size_type size,
1769  const ::MemorySpace::
1770  MemorySpaceData<Number, ::MemorySpace::Host> &v_data,
1773  &data)
1774  {
1775  Vectorization_subtract_v<Number> vector_subtract(data.values.data(),
1776  v_data.values.data());
1777  parallel_for(vector_subtract, 0, size, thread_loop_partitioner);
1778  }
1779 
1780  static void
1782  const std::shared_ptr<::parallel::internal::TBBPartitioner>
1783  & thread_loop_partitioner,
1784  const size_type size,
1785  Number a,
1788  &data)
1789  {
1790  Vectorization_add_factor<Number> vector_add(data.values.data(), a);
1791  parallel_for(vector_add, 0, size, thread_loop_partitioner);
1792  }
1793 
1794  static void
1795  add_av(const std::shared_ptr<::parallel::internal::TBBPartitioner>
1796  & thread_loop_partitioner,
1797  const size_type size,
1798  const Number a,
1799  const ::MemorySpace::
1800  MemorySpaceData<Number, ::MemorySpace::Host> &v_data,
1803  &data)
1804  {
1805  Vectorization_add_av<Number> vector_add(data.values.data(),
1806  v_data.values.data(),
1807  a);
1808  parallel_for(vector_add, 0, size, thread_loop_partitioner);
1809  }
1810 
1811  static void
1813  const std::shared_ptr<::parallel::internal::TBBPartitioner>
1814  & thread_loop_partitioner,
1815  const size_type size,
1816  const Number a,
1817  const Number b,
1818  const ::MemorySpace::
1819  MemorySpaceData<Number, ::MemorySpace::Host> &v_data,
1820  const ::MemorySpace::
1821  MemorySpaceData<Number, ::MemorySpace::Host> &w_data,
1824  &data)
1825  {
1827  data.values.data(), v_data.values.data(), w_data.values.data(), a, b);
1828  parallel_for(vector_add, 0, size, thread_loop_partitioner);
1829  }
1830 
1831  static void
1833  const std::shared_ptr<::parallel::internal::TBBPartitioner>
1834  & thread_loop_partitioner,
1835  const size_type size,
1836  const Number x,
1837  const ::MemorySpace::
1838  MemorySpaceData<Number, ::MemorySpace::Host> &v_data,
1841  &data)
1842  {
1843  Vectorization_sadd_xv<Number> vector_sadd(data.values.data(),
1844  v_data.values.data(),
1845  x);
1846  parallel_for(vector_sadd, 0, size, thread_loop_partitioner);
1847  }
1848 
1849  static void
1851  const std::shared_ptr<::parallel::internal::TBBPartitioner>
1852  & thread_loop_partitioner,
1853  const size_type size,
1854  const Number x,
1855  const Number a,
1856  const ::MemorySpace::
1857  MemorySpaceData<Number, ::MemorySpace::Host> &v_data,
1860  &data)
1861  {
1862  Vectorization_sadd_xav<Number> vector_sadd(data.values.data(),
1863  v_data.values.data(),
1864  a,
1865  x);
1866  parallel_for(vector_sadd, 0, size, thread_loop_partitioner);
1867  }
1868 
1869  static void
1871  const std::shared_ptr<::parallel::internal::TBBPartitioner>
1872  & thread_loop_partitioner,
1873  const size_type size,
1874  const Number x,
1875  const Number a,
1876  const Number b,
1877  const ::MemorySpace::
1878  MemorySpaceData<Number, ::MemorySpace::Host> &v_data,
1879  const ::MemorySpace::
1880  MemorySpaceData<Number, ::MemorySpace::Host> &w_data,
1883  &data)
1884  {
1885  Vectorization_sadd_xavbw<Number> vector_sadd(data.values.data(),
1886  v_data.values.data(),
1887  w_data.values.data(),
1888  x,
1889  a,
1890  b);
1891  parallel_for(vector_sadd, 0, size, thread_loop_partitioner);
1892  }
1893 
1894  static void
1896  const std::shared_ptr<::parallel::internal::TBBPartitioner>
1897  & thread_loop_partitioner,
1898  const size_type size,
1899  const Number factor,
1902  &data)
1903  {
1904  Vectorization_multiply_factor<Number> vector_multiply(
1905  data.values.data(), factor);
1906  parallel_for(vector_multiply, 0, size, thread_loop_partitioner);
1907  }
1908 
1909  static void
1910  scale(const std::shared_ptr<::parallel::internal::TBBPartitioner>
1911  & thread_loop_partitioner,
1912  const size_type size,
1913  const ::MemorySpace::
1914  MemorySpaceData<Number, ::MemorySpace::Host> &v_data,
1917  &data)
1918  {
1919  Vectorization_scale<Number> vector_scale(data.values.data(),
1920  v_data.values.data());
1921  parallel_for(vector_scale, 0, size, thread_loop_partitioner);
1922  }
1923 
1924  static void
1925  equ_au(const std::shared_ptr<::parallel::internal::TBBPartitioner>
1926  & thread_loop_partitioner,
1927  const size_type size,
1928  const Number a,
1929  const ::MemorySpace::
1930  MemorySpaceData<Number, ::MemorySpace::Host> &v_data,
1933  &data)
1934  {
1935  Vectorization_equ_au<Number> vector_equ(data.values.data(),
1936  v_data.values.data(),
1937  a);
1938  parallel_for(vector_equ, 0, size, thread_loop_partitioner);
1939  }
1940 
1941  static void
1943  const std::shared_ptr<::parallel::internal::TBBPartitioner>
1944  & thread_loop_partitioner,
1945  const size_type size,
1946  const Number a,
1947  const Number b,
1948  const ::MemorySpace::
1949  MemorySpaceData<Number, ::MemorySpace::Host> &v_data,
1950  const ::MemorySpace::
1951  MemorySpaceData<Number, ::MemorySpace::Host> &w_data,
1954  &data)
1955  {
1956  Vectorization_equ_aubv<Number> vector_equ(
1957  data.values.data(), v_data.values.data(), w_data.values.data(), a, b);
1958  parallel_for(vector_equ, 0, size, thread_loop_partitioner);
1959  }
1960 
1961  static Number
1962  dot(const std::shared_ptr<::parallel::internal::TBBPartitioner>
1963  & thread_loop_partitioner,
1964  const size_type size,
1965  const ::MemorySpace::
1966  MemorySpaceData<Number2, ::MemorySpace::Host> &v_data,
1969  &data)
1970  {
1971  Number sum;
1973  data.values.data(), v_data.values.data());
1975  dot, 0, size, sum, thread_loop_partitioner);
1977 
1978  return sum;
1979  }
1980 
1981  template <typename real_type>
1982  static void
1983  norm_2(const std::shared_ptr<::parallel::internal::TBBPartitioner>
1984  & thread_loop_partitioner,
1985  const size_type size,
1986  real_type & sum,
1989  &data)
1990  {
1991  Norm2<Number, real_type> norm2(data.values.data());
1992  parallel_reduce(norm2, 0, size, sum, thread_loop_partitioner);
1993  }
1994 
1995  static Number
1997  const std::shared_ptr<::parallel::internal::TBBPartitioner>
1998  & thread_loop_partitioner,
1999  const size_type size,
2000  const ::MemorySpace::
2001  MemorySpaceData<Number, ::MemorySpace::Host> &data)
2002  {
2003  Number sum;
2004  MeanValue<Number> mean(data.values.data());
2005  parallel_reduce(mean, 0, size, sum, thread_loop_partitioner);
2006 
2007  return sum;
2008  }
2009 
2010  template <typename real_type>
2011  static void
2012  norm_1(const std::shared_ptr<::parallel::internal::TBBPartitioner>
2013  & thread_loop_partitioner,
2014  const size_type size,
2015  real_type & sum,
2018  &data)
2019  {
2020  Norm1<Number, real_type> norm1(data.values.data());
2021  parallel_reduce(norm1, 0, size, sum, thread_loop_partitioner);
2022  }
2023 
2024  template <typename real_type>
2025  static void
2026  norm_p(const std::shared_ptr<::parallel::internal::TBBPartitioner>
2027  & thread_loop_partitioner,
2028  const size_type size,
2029  real_type & sum,
2030  const real_type p,
2033  &data)
2034  {
2035  NormP<Number, real_type> normp(data.values.data(), p);
2036  parallel_reduce(normp, 0, size, sum, thread_loop_partitioner);
2037  }
2038 
2039  static Number
2041  const std::shared_ptr<::parallel::internal::TBBPartitioner>
2042  & thread_loop_partitioner,
2043  const size_type size,
2044  const Number a,
2045  const ::MemorySpace::
2046  MemorySpaceData<Number, ::MemorySpace::Host> &v_data,
2047  const ::MemorySpace::
2048  MemorySpaceData<Number, ::MemorySpace::Host> &w_data,
2051  &data)
2052  {
2053  Number sum;
2054  AddAndDot<Number> adder(data.values.data(),
2055  v_data.values.data(),
2056  w_data.values.data(),
2057  a);
2058  parallel_reduce(adder, 0, size, sum, thread_loop_partitioner);
2059 
2060  return sum;
2061  }
2062 
2063  template <typename MemorySpace2>
2064  static void
2066  const std::shared_ptr<::parallel::internal::TBBPartitioner>
2067  & thread_loop_partitioner,
2068  const size_type size,
2069  VectorOperation::values operation,
2070  const ::MemorySpace::MemorySpaceData<Number, MemorySpace2>
2071  &v_data,
2074  &data,
2075  std::enable_if_t<
2076  std::is_same<MemorySpace2, ::MemorySpace::Host>::value,
2077  int> = 0)
2078  {
2079  if (operation == VectorOperation::insert)
2080  {
2081  copy(thread_loop_partitioner, size, v_data, data);
2082  }
2083  else if (operation == VectorOperation::add)
2084  {
2085  add_vector(thread_loop_partitioner, size, v_data, data);
2086  }
2087  else
2088  {
2089  AssertThrow(false, ExcNotImplemented());
2090  }
2091  }
2092 
2093  template <typename MemorySpace2>
2094  static void
2096  const std::shared_ptr<::parallel::internal::TBBPartitioner>
2097  & /*thread_loop_partitioner*/,
2098  const size_type size,
2099  VectorOperation::values operation,
2100  const ::MemorySpace::MemorySpaceData<Number, MemorySpace2>
2101  &v_data,
2104  &data,
2105  std::enable_if_t<
2106  std::is_same<MemorySpace2, ::MemorySpace::Default>::value,
2107  int> = 0)
2108  {
2109  if (operation == VectorOperation::insert)
2110  {
2111  Kokkos::deep_copy(
2112  Kokkos::subview(data.values,
2113  Kokkos::pair<size_type, size_type>(0, size)),
2114  Kokkos::subview(v_data.values,
2115  Kokkos::pair<size_type, size_type>(0, size)));
2116  }
2117  else
2118  {
2119  AssertThrow(false, ExcNotImplemented());
2120  }
2121  }
2122  };
2123 
2124 
2125 
2126  template <typename Number>
2127  struct functions<Number, Number, ::MemorySpace::Default>
2128  {
2129  static void
2131  const std::shared_ptr<::parallel::internal::TBBPartitioner> &,
2132  const size_type size,
2133  const ::MemorySpace::
2134  MemorySpaceData<Number, ::MemorySpace::Default> &v_data,
2137  &data)
2138  {
2139  Kokkos::deep_copy(
2140  Kokkos::subview(data.values,
2141  Kokkos::pair<size_type, size_type>(0, size)),
2142  Kokkos::subview(v_data.values,
2143  Kokkos::pair<size_type, size_type>(0, size)));
2144  }
2145 
2146  static void
2147  set(const std::shared_ptr<::parallel::internal::TBBPartitioner> &,
2148  const size_type size,
2149  const Number s,
2152  &data)
2153  {
2154  Kokkos::deep_copy(
2155  Kokkos::subview(data.values,
2156  Kokkos::pair<size_type, size_type>(0, size)),
2157  s);
2158  }
2159 
2160  static void
2162  const std::shared_ptr<::parallel::internal::TBBPartitioner> &,
2163  const size_type size,
2164  const ::MemorySpace::
2165  MemorySpaceData<Number, ::MemorySpace::Default> &v_data,
2168  &data)
2169  {
2170  auto exec = typename ::MemorySpace::Default::kokkos_space::
2171  execution_space{};
2173  "add_vector",
2174  Kokkos::RangePolicy<
2175  ::MemorySpace::Default::kokkos_space::execution_space>(
2176  exec, 0, size),
2177  KOKKOS_LAMBDA(int i) { data.values(i) += v_data.values(i); });
2178  exec.fence();
2179  }
2180 
2181  static void
2183  const std::shared_ptr<::parallel::internal::TBBPartitioner> &,
2184  const size_type size,
2185  const ::MemorySpace::
2186  MemorySpaceData<Number, ::MemorySpace::Default> &v_data,
2189  &data)
2190  {
2191  auto exec = typename ::MemorySpace::Default::kokkos_space::
2192  execution_space{};
2194  "subtract_vector",
2195  Kokkos::RangePolicy<
2196  ::MemorySpace::Default::kokkos_space::execution_space>(
2197  exec, 0, size),
2198  KOKKOS_LAMBDA(size_type i) { data.values(i) -= v_data.values(i); });
2199  exec.fence();
2200  }
2201 
2202  static void
2204  const std::shared_ptr<::parallel::internal::TBBPartitioner> &,
2205  const size_type size,
2206  Number a,
2209  &data)
2210  {
2211  auto exec = typename ::MemorySpace::Default::kokkos_space::
2212  execution_space{};
2214  "add_factor",
2215  Kokkos::RangePolicy<
2216  ::MemorySpace::Default::kokkos_space::execution_space>(
2217  exec, 0, size),
2218  KOKKOS_LAMBDA(size_type i) { data.values(i) += a; });
2219  exec.fence();
2220  }
2221 
2222  static void
2224  const std::shared_ptr<::parallel::internal::TBBPartitioner> &,
2225  const size_type size,
2226  const Number a,
2227  const ::MemorySpace::
2228  MemorySpaceData<Number, ::MemorySpace::Default> &v_data,
2231  &data)
2232  {
2233  auto exec = typename ::MemorySpace::Default::kokkos_space::
2234  execution_space{};
2236  "add_av",
2237  Kokkos::RangePolicy<
2238  ::MemorySpace::Default::kokkos_space::execution_space>(
2239  exec, 0, size),
2240  KOKKOS_LAMBDA(size_type i) {
2241  data.values(i) += a * v_data.values(i);
2242  });
2243  exec.fence();
2244  }
2245 
2246  static void
2248  const std::shared_ptr<::parallel::internal::TBBPartitioner> &,
2249  const size_type size,
2250  const Number a,
2251  const Number b,
2252  const ::MemorySpace::
2253  MemorySpaceData<Number, ::MemorySpace::Default> &v_data,
2254  const ::MemorySpace::
2255  MemorySpaceData<Number, ::MemorySpace::Default> &w_data,
2258  &data)
2259  {
2260  auto exec = typename ::MemorySpace::Default::kokkos_space::
2261  execution_space{};
2263  "add_avpbw",
2264  Kokkos::RangePolicy<
2265  ::MemorySpace::Default::kokkos_space::execution_space>(
2266  exec, 0, size),
2267  KOKKOS_LAMBDA(size_type i) {
2268  data.values(i) += a * v_data.values(i) + b * w_data.values(i);
2269  });
2270  exec.fence();
2271  }
2272 
2273  static void
2275  const std::shared_ptr<::parallel::internal::TBBPartitioner> &,
2276  const size_type size,
2277  const Number x,
2278  const ::MemorySpace::
2279  MemorySpaceData<Number, ::MemorySpace::Default> &v_data,
2282  &data)
2283  {
2284  auto exec = typename ::MemorySpace::Default::kokkos_space::
2285  execution_space{};
2287  "sadd_xv",
2288  Kokkos::RangePolicy<
2289  ::MemorySpace::Default::kokkos_space::execution_space>(
2290  exec, 0, size),
2291  KOKKOS_LAMBDA(size_type i) {
2292  data.values(i) = x * data.values(i) + v_data.values(i);
2293  });
2294  exec.fence();
2295  }
2296 
2297  static void
2299  const std::shared_ptr<::parallel::internal::TBBPartitioner> &,
2300  const size_type size,
2301  const Number x,
2302  const Number a,
2303  const ::MemorySpace::
2304  MemorySpaceData<Number, ::MemorySpace::Default> &v_data,
2307  &data)
2308  {
2309  auto exec = typename ::MemorySpace::Default::kokkos_space::
2310  execution_space{};
2312  "sadd_xav",
2313  Kokkos::RangePolicy<
2314  ::MemorySpace::Default::kokkos_space::execution_space>(
2315  exec, 0, size),
2316  KOKKOS_LAMBDA(size_type i) {
2317  data.values(i) = x * data.values(i) + a * v_data.values(i);
2318  });
2319  exec.fence();
2320  }
2321 
2322  static void
2324  const std::shared_ptr<::parallel::internal::TBBPartitioner> &,
2325  const size_type size,
2326  const Number x,
2327  const Number a,
2328  const Number b,
2329  const ::MemorySpace::
2330  MemorySpaceData<Number, ::MemorySpace::Default> &v_data,
2331  const ::MemorySpace::
2332  MemorySpaceData<Number, ::MemorySpace::Default> &w_data,
2335  &data)
2336  {
2337  auto exec = typename ::MemorySpace::Default::kokkos_space::
2338  execution_space{};
2340  "sadd_xavbw",
2341  Kokkos::RangePolicy<
2342  ::MemorySpace::Default::kokkos_space::execution_space>(
2343  exec, 0, size),
2344  KOKKOS_LAMBDA(size_type i) {
2345  data.values(i) =
2346  x * data.values(i) + a * v_data.values(i) + b * w_data.values(i);
2347  });
2348  exec.fence();
2349  }
2350 
2351  static void
2353  const std::shared_ptr<::parallel::internal::TBBPartitioner> &,
2354  const size_type size,
2355  const Number factor,
2358  &data)
2359  {
2360  auto exec = typename ::MemorySpace::Default::kokkos_space::
2361  execution_space{};
2363  "multiply_factor",
2364  Kokkos::RangePolicy<
2365  ::MemorySpace::Default::kokkos_space::execution_space>(
2366  exec, 0, size),
2367  KOKKOS_LAMBDA(size_type i) { data.values(i) *= factor; });
2368  exec.fence();
2369  }
2370 
2371  static void
2373  const std::shared_ptr<::parallel::internal::TBBPartitioner> &,
2374  const size_type size,
2375  const ::MemorySpace::
2376  MemorySpaceData<Number, ::MemorySpace::Default> &v_data,
2379  &data)
2380  {
2381  auto exec = typename ::MemorySpace::Default::kokkos_space::
2382  execution_space{};
2384  "scale",
2385  Kokkos::RangePolicy<
2386  ::MemorySpace::Default::kokkos_space::execution_space>(
2387  exec, 0, size),
2388  KOKKOS_LAMBDA(size_type i) { data.values(i) *= v_data.values(i); });
2389  exec.fence();
2390  }
2391 
2392  static void
2394  const std::shared_ptr<::parallel::internal::TBBPartitioner> &,
2395  const size_type size,
2396  const Number a,
2397  const ::MemorySpace::
2398  MemorySpaceData<Number, ::MemorySpace::Default> &v_data,
2401  &data)
2402  {
2403  auto exec = typename ::MemorySpace::Default::kokkos_space::
2404  execution_space{};
2406  "equ_au",
2407  Kokkos::RangePolicy<
2408  ::MemorySpace::Default::kokkos_space::execution_space>(
2409  exec, 0, size),
2410  KOKKOS_LAMBDA(size_type i) {
2411  data.values(i) = a * v_data.values(i);
2412  });
2413  exec.fence();
2414  }
2415 
2416  static void
2418  const std::shared_ptr<::parallel::internal::TBBPartitioner> &,
2419  const size_type size,
2420  const Number a,
2421  const Number b,
2422  const ::MemorySpace::
2423  MemorySpaceData<Number, ::MemorySpace::Default> &v_data,
2424  const ::MemorySpace::
2425  MemorySpaceData<Number, ::MemorySpace::Default> &w_data,
2428  &data)
2429  {
2430  auto exec = typename ::MemorySpace::Default::kokkos_space::
2431  execution_space{};
2433  "equ_aubv",
2434  Kokkos::RangePolicy<
2435  ::MemorySpace::Default::kokkos_space::execution_space>(
2436  exec, 0, size),
2437  KOKKOS_LAMBDA(size_type i) {
2438  data.values(i) = a * v_data.values(i) + b * w_data.values(i);
2439  });
2440  exec.fence();
2441  }
2442 
2443  static Number
2444  dot(const std::shared_ptr<::parallel::internal::TBBPartitioner> &,
2445  const size_type size,
2446  const ::MemorySpace::
2447  MemorySpaceData<Number, ::MemorySpace::Default> &v_data,
2450  &data)
2451  {
2452  Number result;
2453 
2454  auto exec = typename ::MemorySpace::Default::kokkos_space::
2455  execution_space{};
2457  "dot",
2458  Kokkos::RangePolicy<
2459  ::MemorySpace::Default::kokkos_space::execution_space>(
2460  exec, 0, size),
2461  KOKKOS_LAMBDA(size_type i, Number & update) {
2462  update += data.values(i) * v_data.values(i);
2463  },
2464  result);
2465 
2466  AssertIsFinite(result);
2467  return result;
2468  }
2469 
2470  template <typename real_type>
2471  static void
2472  norm_2(const std::shared_ptr<::parallel::internal::TBBPartitioner>
2473  & thread_loop_partitioner,
2474  const size_type size,
2475  real_type & sum,
2476  ::MemorySpace::
2477  MemorySpaceData<Number, ::MemorySpace::Default> &data)
2478  {
2479  sum = dot(thread_loop_partitioner, size, data, data);
2480  }
2481 
2482  static Number
2484  const std::shared_ptr<::parallel::internal::TBBPartitioner> &,
2485  const size_type size,
2486  const ::MemorySpace::
2487  MemorySpaceData<Number, ::MemorySpace::Default> &data)
2488  {
2489  Number result;
2490 
2491  auto exec = typename ::MemorySpace::Default::kokkos_space::
2492  execution_space{};
2494  "mean_value",
2495  Kokkos::RangePolicy<
2496  ::MemorySpace::Default::kokkos_space::execution_space>(
2497  exec, 0, size),
2498  KOKKOS_LAMBDA(size_type i, Number & update) {
2499  update += data.values(i);
2500  },
2501  result);
2502 
2503  AssertIsFinite(result);
2504  return result;
2505  }
2506 
2507  template <typename real_type>
2508  static void
2510  const std::shared_ptr<::parallel::internal::TBBPartitioner> &,
2511  const size_type size,
2512  real_type & sum,
2515  &data)
2516  {
2517  auto exec = typename ::MemorySpace::Default::kokkos_space::
2518  execution_space{};
2520  "norm_1",
2521  Kokkos::RangePolicy<
2522  ::MemorySpace::Default::kokkos_space::execution_space>(
2523  exec, 0, size),
2524  KOKKOS_LAMBDA(size_type i, Number & update) {
2525 #if KOKKOS_VERSION < 30400
2526  update += std::abs(data.values(i));
2527 #elif KOKKOS_VERSION < 30700
2528  update += Kokkos::Experimental::fabs(data.values(i));
2529 #else
2530  update += Kokkos::abs(data.values(i));
2531 #endif
2532  },
2533  sum);
2534  }
2535 
2536  template <typename real_type>
2537  static void
2539  const std::shared_ptr<::parallel::internal::TBBPartitioner> &,
2540  const size_type size,
2541  real_type & sum,
2542  real_type exp,
2545  &data)
2546  {
2547  auto exec = typename ::MemorySpace::Default::kokkos_space::
2548  execution_space{};
2550  "norm_p",
2551  Kokkos::RangePolicy<
2552  ::MemorySpace::Default::kokkos_space::execution_space>(
2553  exec, 0, size),
2554  KOKKOS_LAMBDA(size_type i, Number & update) {
2555 #if KOKKOS_VERSION < 30400
2556  update += std::pow(fabs(data.values(i)), exp);
2557 #elif KOKKOS_VERSION < 30700
2558  update += Kokkos::Experimental::pow(
2559  Kokkos::Experimental::fabs(data.values(i)), exp);
2560 #else
2561  update += Kokkos::pow(Kokkos::abs(data.values(i)), exp);
2562 #endif
2563  },
2564  sum);
2565  }
2566 
2567  static Number
2569  const std::shared_ptr<::parallel::internal::TBBPartitioner> &,
2570  const size_type size,
2571  const Number a,
2572  const ::MemorySpace::
2573  MemorySpaceData<Number, ::MemorySpace::Default> &v_data,
2574  const ::MemorySpace::
2575  MemorySpaceData<Number, ::MemorySpace::Default> &w_data,
2578  &data)
2579  {
2580  Number res;
2581 
2582  auto exec = typename ::MemorySpace::Default::kokkos_space::
2583  execution_space{};
2585  "add_and_dot",
2586  Kokkos::RangePolicy<
2587  ::MemorySpace::Default::kokkos_space::execution_space>(
2588  exec, 0, size),
2589  KOKKOS_LAMBDA(size_type i, Number & update) {
2590  data.values(i) += a * v_data.values(i);
2591  update +=
2592  data.values(i) * Number(numbers::NumberTraits<Number>::conjugate(
2593  w_data.values(i)));
2594  },
2595  res);
2596 
2597  return res;
2598  }
2599 
2600  template <typename MemorySpace2>
2601  static void
2603  const std::shared_ptr<::parallel::internal::TBBPartitioner>
2604  & thread_loop_partitioner,
2605  const size_type size,
2606  VectorOperation::values operation,
2607  const ::MemorySpace::MemorySpaceData<Number, MemorySpace2>
2608  &v_data,
2611  &data,
2612  std::enable_if_t<
2613  std::is_same<MemorySpace2, ::MemorySpace::Default>::value,
2614  int> = 0)
2615  {
2616  if (operation == VectorOperation::insert)
2617  {
2618  copy(thread_loop_partitioner, size, v_data, data);
2619  }
2620  else if (operation == VectorOperation::add)
2621  {
2622  add_vector(thread_loop_partitioner, size, v_data, data);
2623  }
2624  else
2625  {
2626  AssertThrow(false, ExcNotImplemented());
2627  }
2628  }
2629 
2630  template <typename MemorySpace2>
2631  static void
2633  const std::shared_ptr<::parallel::internal::TBBPartitioner>
2634  & /*thread_loop_partitioner*/,
2635  const size_type size,
2636  VectorOperation::values operation,
2637  const ::MemorySpace::MemorySpaceData<Number, MemorySpace2>
2638  &v_data,
2641  &data,
2642  std::enable_if_t<
2643  std::is_same<MemorySpace2, ::MemorySpace::Host>::value,
2644  int> = 0)
2645  {
2646  if (operation == VectorOperation::insert)
2647  {
2648  Kokkos::deep_copy(
2649  Kokkos::subview(data.values,
2650  Kokkos::pair<size_type, size_type>(0, size)),
2651  Kokkos::subview(v_data.values,
2652  Kokkos::pair<size_type, size_type>(0, size)));
2653  }
2654  else
2655  {
2656  AssertThrow(false, ExcNotImplemented());
2657  }
2658  }
2659  };
2660  } // namespace VectorOperations
2661 } // namespace internal
2662 
2664 
2665 #endif
static unsigned int n_threads()
void store(OtherNumber *ptr) const
void load(const OtherNumber *ptr)
#define DEAL_II_CONSTEXPR_IN_CONDITIONAL
Definition: config.h:576
#define DEAL_II_ALWAYS_INLINE
Definition: config.h:108
#define DEAL_II_OPENMP_SIMD_PRAGMA
Definition: config.h:140
#define DEAL_II_NAMESPACE_OPEN
Definition: config.h:474
#define DEAL_II_NAMESPACE_CLOSE
Definition: config.h:475
#define DEAL_II_FALLTHROUGH
Definition: config.h:186
Point< 2 > first
Definition: grid_out.cc:4615
static ::ExceptionBase & ExcInternalError()
#define Assert(cond, exc)
Definition: exceptions.h:1586
static ::ExceptionBase & ExcNotImplemented()
#define AssertIsFinite(number)
Definition: exceptions.h:1854
#define AssertDimension(dim1, dim2)
Definition: exceptions.h:1759
#define AssertIndexRange(index, range)
Definition: exceptions.h:1827
static ::ExceptionBase & ExcMessage(std::string arg1)
#define AssertThrow(cond, exc)
Definition: exceptions.h:1675
Expression fabs(const Expression &x)
static const char U
static const char T
Tensor< 2, dim, Number > w(const Tensor< 2, dim, Number > &F, const Tensor< 2, dim, Number > &dF_dt)
SymmetricTensor< 2, dim, Number > b(const Tensor< 2, dim, Number > &F)
VectorType::value_type * begin(VectorType &V)
VectorType::value_type * end(VectorType &V)
T sum(const T &t, const MPI_Comm &mpi_communicator)
unsigned int minimum_parallel_grain_size
Definition: parallel.cc:34
void accumulate_recursive(const Operation &op, const size_type first, const size_type last, ResultType &result)
void parallel_reduce(const Operation &op, const size_type start, const size_type end, ResultType &result, const std::shared_ptr<::parallel::internal::TBBPartitioner > &partitioner)
void copy(const T *begin, const T *end, U *dest)
void parallel_for(Functor &functor, const size_type start, const size_type end, const std::shared_ptr<::parallel::internal::TBBPartitioner > &partitioner)
void copy(const std::complex< T > *, const std::complex< T > *, U *)
const unsigned int vector_accumulation_recursion_threshold
size_type do_accumulate(const Operation op, const size_type vec_size, const size_type start_index, ResultType *outer_results, std::integral_constant< bool, false >)
unsigned int global_dof_index
Definition: types.h:82
AddAndDot(Number *const X, const Number *const V, const Number *const W, const Number a)
VectorizedArray< Number > do_vectorized(const size_type i) const
Number operator()(const size_type i) const
Dot(const Number *const X, const Number2 *const Y)
Number operator()(const size_type i) const
VectorizedArray< Number > do_vectorized(const size_type i) const
Number operator()(const size_type i) const
VectorizedArray< Number > do_vectorized(const size_type i) const
RealType operator()(const size_type i) const
VectorizedArray< Number > do_vectorized(const size_type i) const
RealType operator()(const size_type i) const
VectorizedArray< Number > do_vectorized(const size_type i) const
RealType operator()(const size_type i) const
VectorizedArray< Number > do_vectorized(const size_type i) const
void operator()(const tbb::blocked_range< size_type > &range) const
TBBForFunctor(Functor &functor, const size_type start, const size_type end)
TBBReduceFunctor(const Operation &op, const size_type start, const size_type end)
ResultType small_array[threshold_array_allocate]
void operator()(const tbb::blocked_range< size_type > &range) const
Vector_copy(const OtherNumber *const src, Number *const dst)
void operator()(const size_type begin, const size_type end) const
Vector_set(const Number value, Number *const dst)
void operator()(const size_type begin, const size_type end) const
Vectorization_add_av(Number *const val, const Number *const v_val, const Number factor)
void operator()(const size_type begin, const size_type end) const
Vectorization_add_avpbw(Number *const val, const Number *const v_val, const Number *const w_val, const Number a, const Number b)
void operator()(const size_type begin, const size_type end) const
void operator()(const size_type begin, const size_type end) const
Vectorization_add_factor(Number *const val, const Number factor)
void operator()(const size_type begin, const size_type end) const
Vectorization_add_v(Number *const val, const Number *const v_val)
Vectorization_equ_au(Number *const val, const Number *const u_val, const Number a)
void operator()(const size_type begin, const size_type end) const
Vectorization_equ_aubv(Number *const val, const Number *const u_val, const Number *const v_val, const Number a, const Number b)
void operator()(const size_type begin, const size_type end) const
Vectorization_equ_aubvcw(Number *val, const Number *u_val, const Number *v_val, const Number *w_val, const Number a, const Number b, const Number c)
void operator()(const size_type begin, const size_type end) const
Vectorization_multiply_factor(Number *const val, const Number factor)
void operator()(const size_type begin, const size_type end) const
void operator()(const size_type begin, const size_type end) const
Vectorization_ratio(Number *val, const Number *a_val, const Number *b_val)
Vectorization_sadd_xav(Number *val, const Number *const v_val, const Number a, const Number x)
void operator()(const size_type begin, const size_type end) const
void operator()(const size_type begin, const size_type end) const
Vectorization_sadd_xavbw(Number *val, const Number *v_val, const Number *w_val, Number x, Number a, Number b)
Vectorization_sadd_xv(Number *const val, const Number *const v_val, const Number x)
void operator()(const size_type begin, const size_type end) const
void operator()(const size_type begin, const size_type end) const
Vectorization_scale(Number *const val, const Number *const v_val)
Vectorization_subtract_v(Number *val, const Number *const v_val)
void operator()(const size_type begin, const size_type end) const
static void set(const std::shared_ptr<::parallel::internal::TBBPartitioner > &thread_loop_partitioner, const size_type size, const Number s, ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Host > &data)
static void norm_2(const std::shared_ptr<::parallel::internal::TBBPartitioner > &thread_loop_partitioner, const size_type size, real_type &sum, ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Host > &data)
static void norm_p(const std::shared_ptr<::parallel::internal::TBBPartitioner > &thread_loop_partitioner, const size_type size, real_type &sum, const real_type p, ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Host > &data)
static void import_elements(const std::shared_ptr<::parallel::internal::TBBPartitioner > &thread_loop_partitioner, const size_type size, VectorOperation::values operation, const ::MemorySpace::MemorySpaceData< Number, MemorySpace2 > &v_data, ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Host > &data, std::enable_if_t< std::is_same< MemorySpace2, ::MemorySpace::Host >::value, int >=0)
static void add_avpbw(const std::shared_ptr<::parallel::internal::TBBPartitioner > &thread_loop_partitioner, const size_type size, const Number a, const Number b, const ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Host > &v_data, const ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Host > &w_data, ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Host > &data)
static void sadd_xav(const std::shared_ptr<::parallel::internal::TBBPartitioner > &thread_loop_partitioner, const size_type size, const Number x, const Number a, const ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Host > &v_data, ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Host > &data)
static void import_elements(const std::shared_ptr<::parallel::internal::TBBPartitioner > &, const size_type size, VectorOperation::values operation, const ::MemorySpace::MemorySpaceData< Number, MemorySpace2 > &v_data, ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Host > &data, std::enable_if_t< std::is_same< MemorySpace2, ::MemorySpace::Default >::value, int >=0)
static void equ_au(const std::shared_ptr<::parallel::internal::TBBPartitioner > &thread_loop_partitioner, const size_type size, const Number a, const ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Host > &v_data, ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Host > &data)
static void scale(const std::shared_ptr<::parallel::internal::TBBPartitioner > &thread_loop_partitioner, const size_type size, const ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Host > &v_data, ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Host > &data)
static Number mean_value(const std::shared_ptr<::parallel::internal::TBBPartitioner > &thread_loop_partitioner, const size_type size, const ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Host > &data)
static void add_factor(const std::shared_ptr<::parallel::internal::TBBPartitioner > &thread_loop_partitioner, const size_type size, Number a, ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Host > &data)
static void sadd_xv(const std::shared_ptr<::parallel::internal::TBBPartitioner > &thread_loop_partitioner, const size_type size, const Number x, const ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Host > &v_data, ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Host > &data)
static void norm_1(const std::shared_ptr<::parallel::internal::TBBPartitioner > &thread_loop_partitioner, const size_type size, real_type &sum, ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Host > &data)
static void copy(const std::shared_ptr<::parallel::internal::TBBPartitioner > &thread_loop_partitioner, const size_type size, const ::MemorySpace::MemorySpaceData< Number2, ::MemorySpace::Host > &v_data, ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Host > &data)
static void add_av(const std::shared_ptr<::parallel::internal::TBBPartitioner > &thread_loop_partitioner, const size_type size, const Number a, const ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Host > &v_data, ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Host > &data)
static void add_vector(const std::shared_ptr<::parallel::internal::TBBPartitioner > &thread_loop_partitioner, const size_type size, const ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Host > &v_data, ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Host > &data)
static void sadd_xavbw(const std::shared_ptr<::parallel::internal::TBBPartitioner > &thread_loop_partitioner, const size_type size, const Number x, const Number a, const Number b, const ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Host > &v_data, const ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Host > &w_data, ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Host > &data)
static void subtract_vector(const std::shared_ptr<::parallel::internal::TBBPartitioner > &thread_loop_partitioner, const size_type size, const ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Host > &v_data, ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Host > &data)
static Number add_and_dot(const std::shared_ptr<::parallel::internal::TBBPartitioner > &thread_loop_partitioner, const size_type size, const Number a, const ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Host > &v_data, const ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Host > &w_data, ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Host > &data)
static Number dot(const std::shared_ptr<::parallel::internal::TBBPartitioner > &thread_loop_partitioner, const size_type size, const ::MemorySpace::MemorySpaceData< Number2, ::MemorySpace::Host > &v_data, ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Host > &data)
static void equ_aubv(const std::shared_ptr<::parallel::internal::TBBPartitioner > &thread_loop_partitioner, const size_type size, const Number a, const Number b, const ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Host > &v_data, const ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Host > &w_data, ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Host > &data)
static void multiply_factor(const std::shared_ptr<::parallel::internal::TBBPartitioner > &thread_loop_partitioner, const size_type size, const Number factor, ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Host > &data)
static void subtract_vector(const std::shared_ptr<::parallel::internal::TBBPartitioner > &, const size_type size, const ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Default > &v_data, ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Default > &data)
static void sadd_xav(const std::shared_ptr<::parallel::internal::TBBPartitioner > &, const size_type size, const Number x, const Number a, const ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Default > &v_data, ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Default > &data)
static Number dot(const std::shared_ptr<::parallel::internal::TBBPartitioner > &, const size_type size, const ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Default > &v_data, ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Default > &data)
static void add_av(const std::shared_ptr<::parallel::internal::TBBPartitioner > &, const size_type size, const Number a, const ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Default > &v_data, ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Default > &data)
static void copy(const std::shared_ptr<::parallel::internal::TBBPartitioner > &, const size_type size, const ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Default > &v_data, ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Default > &data)
static Number add_and_dot(const std::shared_ptr<::parallel::internal::TBBPartitioner > &, const size_type size, const Number a, const ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Default > &v_data, const ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Default > &w_data, ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Default > &data)
static void sadd_xavbw(const std::shared_ptr<::parallel::internal::TBBPartitioner > &, const size_type size, const Number x, const Number a, const Number b, const ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Default > &v_data, const ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Default > &w_data, ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Default > &data)
static Number mean_value(const std::shared_ptr<::parallel::internal::TBBPartitioner > &, const size_type size, const ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Default > &data)
static void norm_p(const std::shared_ptr<::parallel::internal::TBBPartitioner > &, const size_type size, real_type &sum, real_type exp, ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Default > &data)
static void add_factor(const std::shared_ptr<::parallel::internal::TBBPartitioner > &, const size_type size, Number a, ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Default > &data)
static void add_vector(const std::shared_ptr<::parallel::internal::TBBPartitioner > &, const size_type size, const ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Default > &v_data, ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Default > &data)
static void import_elements(const std::shared_ptr<::parallel::internal::TBBPartitioner > &thread_loop_partitioner, const size_type size, VectorOperation::values operation, const ::MemorySpace::MemorySpaceData< Number, MemorySpace2 > &v_data, ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Default > &data, std::enable_if_t< std::is_same< MemorySpace2, ::MemorySpace::Default >::value, int >=0)
static void add_avpbw(const std::shared_ptr<::parallel::internal::TBBPartitioner > &, const size_type size, const Number a, const Number b, const ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Default > &v_data, const ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Default > &w_data, ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Default > &data)
static void set(const std::shared_ptr<::parallel::internal::TBBPartitioner > &, const size_type size, const Number s, ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Default > &data)
static void norm_2(const std::shared_ptr<::parallel::internal::TBBPartitioner > &thread_loop_partitioner, const size_type size, real_type &sum, ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Default > &data)
static void import_elements(const std::shared_ptr<::parallel::internal::TBBPartitioner > &, const size_type size, VectorOperation::values operation, const ::MemorySpace::MemorySpaceData< Number, MemorySpace2 > &v_data, ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Default > &data, std::enable_if_t< std::is_same< MemorySpace2, ::MemorySpace::Host >::value, int >=0)
static void sadd_xv(const std::shared_ptr<::parallel::internal::TBBPartitioner > &, const size_type size, const Number x, const ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Default > &v_data, ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Default > &data)
static void multiply_factor(const std::shared_ptr<::parallel::internal::TBBPartitioner > &, const size_type size, const Number factor, ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Default > &data)
static void scale(const std::shared_ptr<::parallel::internal::TBBPartitioner > &, const size_type size, const ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Default > &v_data, ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Default > &data)
static void equ_aubv(const std::shared_ptr<::parallel::internal::TBBPartitioner > &, const size_type size, const Number a, const Number b, const ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Default > &v_data, const ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Default > &w_data, ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Default > &data)
static void norm_1(const std::shared_ptr<::parallel::internal::TBBPartitioner > &, const size_type size, real_type &sum, ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Default > &data)
static void equ_au(const std::shared_ptr<::parallel::internal::TBBPartitioner > &, const size_type size, const Number a, const ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Default > &v_data, ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Default > &data)
static Number mean_value(const std::shared_ptr<::parallel::internal::TBBPartitioner > &, const size_type, const ::MemorySpace::MemorySpaceData< Number, MemorySpace > &)
static void equ_au(const std::shared_ptr<::parallel::internal::TBBPartitioner > &, const size_type, const Number, const ::MemorySpace::MemorySpaceData< Number, MemorySpace > &, ::MemorySpace::MemorySpaceData< Number, MemorySpace > &)
static void add_avpbw(const std::shared_ptr<::parallel::internal::TBBPartitioner > &, const size_type, const Number, const Number, const ::MemorySpace::MemorySpaceData< Number, MemorySpace > &, const ::MemorySpace::MemorySpaceData< Number, MemorySpace > &, ::MemorySpace::MemorySpaceData< Number, MemorySpace > &)
static void sadd_xv(const std::shared_ptr<::parallel::internal::TBBPartitioner > &, const size_type, const Number, const ::MemorySpace::MemorySpaceData< Number, MemorySpace > &, ::MemorySpace::MemorySpaceData< Number, MemorySpace > &)
static void add_factor(const std::shared_ptr<::parallel::internal::TBBPartitioner > &, const size_type, Number, ::MemorySpace::MemorySpaceData< Number, MemorySpace > &)
static void norm_2(const std::shared_ptr<::parallel::internal::TBBPartitioner > &, const size_type, real_type &, const ::MemorySpace::MemorySpaceData< Number, MemorySpace > &, ::MemorySpace::MemorySpaceData< Number, MemorySpace > &)
static void add_vector(const std::shared_ptr<::parallel::internal::TBBPartitioner > &, const size_type, const ::MemorySpace::MemorySpaceData< Number, MemorySpace > &, ::MemorySpace::MemorySpaceData< Number, MemorySpace > &)
static void scale(const std::shared_ptr<::parallel::internal::TBBPartitioner > &, const size_type, const ::MemorySpace::MemorySpaceData< Number, MemorySpace > &, ::MemorySpace::MemorySpaceData< Number, MemorySpace > &)
static void subtract_vector(const std::shared_ptr<::parallel::internal::TBBPartitioner > &, const size_type, const ::MemorySpace::MemorySpaceData< Number, MemorySpace > &, ::MemorySpace::MemorySpaceData< Number, MemorySpace > &)
static void sadd_xavbw(const std::shared_ptr<::parallel::internal::TBBPartitioner > &, const size_type, const Number, const Number, const Number, const ::MemorySpace::MemorySpaceData< Number, MemorySpace > &, const ::MemorySpace::MemorySpaceData< Number, MemorySpace > &, ::MemorySpace::MemorySpaceData< Number, MemorySpace > &)
static Number dot(const std::shared_ptr<::parallel::internal::TBBPartitioner > &, const size_type, const ::MemorySpace::MemorySpaceData< Number2, MemorySpace > &, ::MemorySpace::MemorySpaceData< Number, MemorySpace > &)
static Number add_and_dot(const std::shared_ptr<::parallel::internal::TBBPartitioner > &, const size_type, const Number, const ::MemorySpace::MemorySpaceData< Number, MemorySpace > &, const ::MemorySpace::MemorySpaceData< Number, MemorySpace > &, ::MemorySpace::MemorySpaceData< Number, MemorySpace > &)
static void sadd_xav(const std::shared_ptr<::parallel::internal::TBBPartitioner > &, const size_type, const Number, const Number, const ::MemorySpace::MemorySpaceData< Number, MemorySpace > &, ::MemorySpace::MemorySpaceData< Number, MemorySpace > &)
static void import_elements(const std::shared_ptr<::parallel::internal::TBBPartitioner > &, const size_type, VectorOperation::values, const ::MemorySpace::MemorySpaceData< Number, MemorySpace2 > &, ::MemorySpace::MemorySpaceData< Number, MemorySpace > &)
static void copy(const std::shared_ptr<::parallel::internal::TBBPartitioner > &, const size_type, const ::MemorySpace::MemorySpaceData< Number2, MemorySpace > &, ::MemorySpace::MemorySpaceData< Number, MemorySpace > &)
static void norm_1(const std::shared_ptr<::parallel::internal::TBBPartitioner > &, const size_type, real_type &, Number *, Number *)
static void add_av(const std::shared_ptr<::parallel::internal::TBBPartitioner > &, const size_type, const Number, const ::MemorySpace::MemorySpaceData< Number, MemorySpace > &, ::MemorySpace::MemorySpaceData< Number, MemorySpace > &)
static void multiply_factor(const std::shared_ptr<::parallel::internal::TBBPartitioner > &, const size_type, const Number, ::MemorySpace::MemorySpaceData< Number, MemorySpace > &)
static void norm_p(const std::shared_ptr<::parallel::internal::TBBPartitioner > &, const size_type, real_type &, real_type, ::MemorySpace::MemorySpaceData< Number, MemorySpace > &)
static void equ_aubv(const std::shared_ptr<::parallel::internal::TBBPartitioner > &, const size_type, const Number, const Number, const ::MemorySpace::MemorySpaceData< Number, MemorySpace > &, const ::MemorySpace::MemorySpaceData< Number, MemorySpace > &, ::MemorySpace::MemorySpaceData< Number, MemorySpace > &)
static void set(const std::shared_ptr<::parallel::internal::TBBPartitioner > &, const size_type, const Number, ::MemorySpace::MemorySpaceData< Number, MemorySpace > &)
static real_type abs(const number &x)
Definition: numbers.h:593
static constexpr real_type abs_square(const number &x)
Definition: numbers.h:584