deal.II version GIT relicensing-2901-g19332422bd 2025-03-23 19:50:00+00:00
\(\newcommand{\dealvcentcolon}{\mathrel{\mathop{:}}}\) \(\newcommand{\dealcoloneq}{\dealvcentcolon\mathrel{\mkern-1.2mu}=}\) \(\newcommand{\jump}[1]{\left[\!\left[ #1 \right]\!\right]}\) \(\newcommand{\average}[1]{\left\{\!\left\{ #1 \right\}\!\right\}}\)
Loading...
Searching...
No Matches
portable_matrix_free.h
Go to the documentation of this file.
1// ------------------------------------------------------------------------
2//
3// SPDX-License-Identifier: LGPL-2.1-or-later
4// Copyright (C) 2017 - 2024 by the deal.II authors
5//
6// This file is part of the deal.II library.
7//
8// Part of the source code is dual licensed under Apache-2.0 WITH
9// LLVM-exception OR LGPL-2.1-or-later. Detailed license information
10// governing the source code and code contributions can be found in
11// LICENSE.md and CONTRIBUTING.md at the top level directory of deal.II.
12//
13// ------------------------------------------------------------------------
14
15
16#ifndef dealii_portable_matrix_free_h
17#define dealii_portable_matrix_free_h
18
19#include <deal.II/base/config.h>
20
25#include <deal.II/base/tensor.h>
27
29
31#include <deal.II/fe/mapping.h>
32
34
37
39
40#include <Kokkos_Core.hpp>
41
42
43
45
46// Forward declaration
47namespace internal
48{
49 namespace MatrixFreeFunctions
50 {
51 enum class ConstraintKinds : std::uint16_t;
52 }
53} // namespace internal
54
55namespace Portable
56{
57 // forward declaration
58#ifndef DOXYGEN
59 namespace internal
60 {
61 template <int dim, typename Number>
62 class ReinitHelper;
63 }
64 template <int dim, typename Number>
65 struct SharedData;
66#endif
67
92 template <int dim, typename Number = double>
94 {
95 public:
98 using CellFilter =
100
105 {
112 const bool use_coloring = false,
113 const bool overlap_communication_computation = false)
117 {
118#ifndef DEAL_II_MPI_WITH_DEVICE_SUPPORT
122 "Overlapping communication and computation requires device-aware MPI."));
123#endif
128 "Overlapping communication and coloring are incompatible options. Only one of them can be enabled."));
129 }
140
147
153 };
154
161 {
165 Kokkos::View<point_type **, MemorySpace::Default::kokkos_space> q_points;
166
171 Kokkos::View<types::global_dof_index **,
174
178 Kokkos::View<Number **[dim][dim], MemorySpace::Default::kokkos_space>
180
184 Kokkos::View<Number **, MemorySpace::Default::kokkos_space> JxW;
185
192
196 Kokkos::View<Number *, MemorySpace::Default::kokkos_space> shape_values;
197
201 Kokkos::View<Number *, MemorySpace::Default::kokkos_space>
203
207 Kokkos::View<Number *, MemorySpace::Default::kokkos_space>
209
213 Kokkos::View<Number *, MemorySpace::Default::kokkos_space>
215
219 unsigned int n_cells;
220
224 unsigned int n_components;
225
229 unsigned int padding_length;
230
234 unsigned int row_start;
235
241
248
252 unsigned int scratch_pad_size;
253 };
254
255
263 struct Data
264 {
265 using TeamHandle = Kokkos::TeamPolicy<
266 MemorySpace::Default::kokkos_space::execution_space>::member_type;
267
272
273 const int cell_index;
276
281 DEAL_II_HOST_DEVICE unsigned int
282 local_q_point_id(const unsigned int cell,
283 const unsigned int n_q_points,
284 const unsigned int q_point) const
285 {
287 cell) *
288 n_q_points +
289 q_point;
290 }
291
292
298 get_quadrature_point(const unsigned int cell,
299 const unsigned int q_point) const
300 {
301 return precomputed_data->q_points(cell, q_point);
302 }
303 };
304
309
313 unsigned int
315
326 template <typename IteratorFiltersType>
327 void
328 reinit(const Mapping<dim> &mapping,
330 const AffineConstraints<Number> &constraints,
331 const Quadrature<1> &quad,
332 const IteratorFiltersType &iterator_filter,
333 const AdditionalData &additional_data = AdditionalData());
334
338 void
339 reinit(const Mapping<dim> &mapping,
341 const AffineConstraints<Number> &constraints,
342 const Quadrature<1> &quad,
343 const AdditionalData &additional_data = AdditionalData());
344
348 void
350 const AffineConstraints<Number> &constraints,
351 const Quadrature<1> &quad,
352 const AdditionalData &additional_data = AdditionalData());
353
358 get_data(unsigned int color) const;
359
360 // clang-format off
375 // clang-format on
376 template <typename Functor, typename VectorType>
377 void
378 cell_loop(const Functor &func,
379 const VectorType &src,
380 VectorType &dst) const;
381
395 template <typename Functor>
396 void
397 evaluate_coefficients(Functor func) const;
398
403 template <typename VectorType>
404 void
405 copy_constrained_values(const VectorType &src, VectorType &dst) const;
406
412 template <typename VectorType>
413 void
414 set_constrained_values(const Number val, VectorType &dst) const;
415
421 template <typename MemorySpaceType>
422 void
425
429 const std::vector<std::vector<CellFilter>> &
431
442 const std::shared_ptr<const Utilities::MPI::Partitioner> &
444
448 const DoFHandler<dim> &
450
454 std::size_t
456
457 private:
461 template <typename IteratorFiltersType>
462 void
465 const AffineConstraints<Number> &constraints,
466 const Quadrature<1> &quad,
467 const IteratorFiltersType &iterator_filter,
468 const std::shared_ptr<const MPI_Comm> &comm,
469 const AdditionalData additional_data);
470
475 template <typename Functor, typename VectorType>
476 void
477 serial_cell_loop(const Functor &func,
478 const VectorType &src,
479 VectorType &dst) const;
480
485 template <typename Functor>
486 void
488 const Functor &func,
490 &src,
492 const;
493
497 int my_id;
498
505
511
516
523
527 unsigned int scratch_pad_size;
528
532 unsigned int fe_degree;
533
537 unsigned int n_components;
538
543
547 unsigned int dofs_per_cell;
548
552 unsigned int n_constrained_dofs;
553
557 unsigned int q_points_per_cell;
558
562 unsigned int n_colors;
563
567 std::vector<unsigned int> n_cells;
568
573 std::vector<Kokkos::View<point_type **, MemorySpace::Default::kokkos_space>>
575
580 std::vector<Kokkos::View<types::global_dof_index **,
583
588 std::vector<
589 Kokkos::View<Number **[dim][dim], MemorySpace::Default::kokkos_space>>
591
596 std::vector<Kokkos::View<Number **, MemorySpace::Default::kokkos_space>>
598
602 Kokkos::View<types::global_dof_index *, MemorySpace::Default::kokkos_space>
604
608 std::vector<
612
616 Kokkos::View<Number *, MemorySpace::Default::kokkos_space> shape_values;
617
621 Kokkos::View<Number *, MemorySpace::Default::kokkos_space> shape_gradients;
622
626 Kokkos::View<Number *, MemorySpace::Default::kokkos_space>
628
632 Kokkos::View<Number *, MemorySpace::Default::kokkos_space>
634
639 std::shared_ptr<const Utilities::MPI::Partitioner> partitioner;
640
641
646 unsigned int padding_length;
647
651 std::vector<unsigned int> row_start;
652
657
661 std::vector<std::vector<CellFilter>> graph;
662
663 friend class internal::ReinitHelper<dim, Number>;
664 };
665
666
667
668 template <int dim, typename Number>
670 {
671 using SharedViewValues = Kokkos::View<
672 Number **,
673 MemorySpace::Default::kokkos_space::execution_space::scratch_memory_space,
674 Kokkos::MemoryTraits<Kokkos::Unmanaged>>;
675 using SharedViewGradients = Kokkos::View<
676 Number ***,
677 MemorySpace::Default::kokkos_space::execution_space::scratch_memory_space,
678 Kokkos::MemoryTraits<Kokkos::Unmanaged>>;
679 using SharedViewScratchPad = Kokkos::View<
680 Number *,
681 MemorySpace::Default::kokkos_space::execution_space::scratch_memory_space,
682 Kokkos::MemoryTraits<Kokkos::Unmanaged>>;
683
692
697
702
707 };
708
709
710
715 template <int dim, typename Number>
716 struct DataHost
717 {
721 typename Kokkos::View<Point<dim, Number> **,
724
729 typename Kokkos::View<types::global_dof_index **,
732
736 typename Kokkos::View<Number **[dim][dim],
739
743 typename Kokkos::View<Number **,
745
749 unsigned int n_cells;
750
754 unsigned int padding_length;
755
759 unsigned int row_start;
760
764 typename Kokkos::View<
767
773
774
775
779 unsigned int
780 local_q_point_id(const unsigned int cell,
781 const unsigned int n_q_points,
782 const unsigned int q_point) const
783 {
784 return (row_start / padding_length + cell) * n_q_points + q_point;
785 }
786
787
788
793 get_quadrature_point(const unsigned int cell,
794 const unsigned int q_point) const
795 {
796 return q_points(cell, q_point);
797 }
798 };
799
800
801
808 template <int dim, typename Number>
809 DataHost<dim, Number>
811 const typename ::Portable::MatrixFree<dim, Number>::PrecomputedData
812 &data,
813 const UpdateFlags &update_flags)
814 {
815 DataHost<dim, Number> data_host;
816
817 data_host.n_cells = data.n_cells;
818 data_host.padding_length = data.padding_length;
819 data_host.row_start = data.row_start;
820 data_host.use_coloring = data.use_coloring;
821
822 if (update_flags & update_quadrature_points)
823 {
824 data_host.q_points = Kokkos::create_mirror(data.q_points);
825 Kokkos::deep_copy(data_host.q_points, data.q_points);
826 }
827
828 data_host.local_to_global = Kokkos::create_mirror(data.local_to_global);
829 Kokkos::deep_copy(data_host.local_to_global, data.local_to_global);
830
831 if (update_flags & update_gradients)
832 {
833 data_host.inv_jacobian = Kokkos::create_mirror(data.inv_jacobian);
834 Kokkos::deep_copy(data_host.inv_jacobian, data.inv_jacobian);
835 }
836
837 if (update_flags & update_JxW_values)
838 {
839 data_host.JxW = Kokkos::create_mirror(data.JxW);
840 Kokkos::deep_copy(data_host.JxW, data.JxW);
841 }
842
843 data_host.constraint_mask = Kokkos::create_mirror(data.constraint_mask);
844 Kokkos::deep_copy(data_host.constraint_mask, data.constraint_mask);
845
846 return data_host;
847 }
848
849
850 /*----------------------- Inline functions ---------------------------------*/
851
852#ifndef DOXYGEN
853
854 template <int dim, typename Number>
855 inline const std::vector<std::vector<
858 {
859 return graph;
860 }
861
862
863
864 template <int dim, typename Number>
865 inline const std::shared_ptr<const Utilities::MPI::Partitioner> &
867 {
868 return partitioner;
869 }
870
871
872
873 template <int dim, typename Number>
874 inline const DoFHandler<dim> &
876 {
877 Assert(dof_handler != nullptr, ExcNotInitialized());
878
879 return *dof_handler;
880 }
881
882#endif
883
884} // namespace Portable
885
887
888#endif
Abstract base class for mapping classes.
Definition mapping.h:320
const std::vector< std::vector< CellFilter > > & get_colored_graph() const
std::vector< std::vector< CellFilter > > graph
void reinit(const DoFHandler< dim > &dof_handler, const AffineConstraints< Number > &constraints, const Quadrature< 1 > &quad, const AdditionalData &additional_data=AdditionalData())
void internal_reinit(const Mapping< dim > &mapping, const DoFHandler< dim > &dof_handler, const AffineConstraints< Number > &constraints, const Quadrature< 1 > &quad, const IteratorFiltersType &iterator_filter, const std::shared_ptr< const MPI_Comm > &comm, const AdditionalData additional_data)
std::vector< Kokkos::View<::internal::MatrixFreeFunctions::ConstraintKinds *, MemorySpace::Default::kokkos_space > > constraint_mask
unsigned int get_padding_length() const
Kokkos::View< Number *, MemorySpace::Default::kokkos_space > constraint_weights
const DoFHandler< dim > * dof_handler
Kokkos::View< Number *, MemorySpace::Default::kokkos_space > shape_gradients
void initialize_dof_vector(LinearAlgebra::distributed::Vector< Number, MemorySpaceType > &vec) const
void reinit(const Mapping< dim > &mapping, const DoFHandler< dim > &dof_handler, const AffineConstraints< Number > &constraints, const Quadrature< 1 > &quad, const IteratorFiltersType &iterator_filter, const AdditionalData &additional_data=AdditionalData())
std::vector< Kokkos::View< Number **[dim][dim], MemorySpace::Default::kokkos_space > > inv_jacobian
PrecomputedData get_data(unsigned int color) const
Kokkos::View< types::global_dof_index *, MemorySpace::Default::kokkos_space > constrained_dofs
std::vector< Kokkos::View< point_type **, MemorySpace::Default::kokkos_space > > q_points
types::global_dof_index n_dofs
void distributed_cell_loop(const Functor &func, const LinearAlgebra::distributed::Vector< Number, MemorySpace::Default > &src, LinearAlgebra::distributed::Vector< Number, MemorySpace::Default > &dst) const
std::vector< unsigned int > row_start
Kokkos::View< Number *, MemorySpace::Default::kokkos_space > co_shape_gradients
std::vector< Kokkos::View< types::global_dof_index **, MemorySpace::Default::kokkos_space > > local_to_global
Kokkos::View< Number *, MemorySpace::Default::kokkos_space > shape_values
std::size_t memory_consumption() const
void set_constrained_values(const Number val, VectorType &dst) const
void copy_constrained_values(const VectorType &src, VectorType &dst) const
std::vector< Kokkos::View< Number **, MemorySpace::Default::kokkos_space > > JxW
const DoFHandler< dim > & get_dof_handler() const
std::shared_ptr< const Utilities::MPI::Partitioner > partitioner
void serial_cell_loop(const Functor &func, const VectorType &src, VectorType &dst) const
void evaluate_coefficients(Functor func) const
DataHost< dim, Number > copy_mf_data_to_host(const typename::Portable::MatrixFree< dim, Number >::PrecomputedData &data, const UpdateFlags &update_flags)
::internal::MatrixFreeFunctions::ElementType element_type
void reinit(const Mapping< dim > &mapping, const DoFHandler< dim > &dof_handler, const AffineConstraints< Number > &constraints, const Quadrature< 1 > &quad, const AdditionalData &additional_data=AdditionalData())
const std::shared_ptr< const Utilities::MPI::Partitioner > & get_vector_partitioner() const
std::vector< unsigned int > n_cells
void cell_loop(const Functor &func, const VectorType &src, VectorType &dst) const
#define DEAL_II_NAMESPACE_OPEN
Definition config.h:40
#define DEAL_II_HOST_DEVICE
Definition config.h:169
#define DEAL_II_NAMESPACE_CLOSE
Definition config.h:41
#define Assert(cond, exc)
static ::ExceptionBase & ExcNotInitialized()
static ::ExceptionBase & ExcMessage(std::string arg1)
#define AssertThrow(cond, exc)
UpdateFlags
@ update_JxW_values
Transformed quadrature weights.
@ update_gradients
Shape function gradients.
@ update_quadrature_points
Transformed quadrature points.
std::vector< index_type > data
Definition mpi.cc:746
const MPI_Comm comm
Definition mpi.cc:924
::Kokkos::DefaultExecutionSpace::memory_space kokkos_space
Kokkos::View< types::global_dof_index **, MemorySpace::Default::kokkos_space >::HostMirror local_to_global
Kokkos::View< Number **, MemorySpace::Default::kokkos_space >::HostMirror JxW
Point< dim, Number > get_quadrature_point(const unsigned int cell, const unsigned int q_point) const
Kokkos::View<::internal::MatrixFreeFunctions::ConstraintKinds *, MemorySpace::Default::kokkos_space >::HostMirror constraint_mask
Kokkos::View< Number **[dim][dim], MemorySpace::Default::kokkos_space >::HostMirror inv_jacobian
Kokkos::View< Point< dim, Number > **, MemorySpace::Default::kokkos_space >::HostMirror q_points
unsigned int local_q_point_id(const unsigned int cell, const unsigned int n_q_points, const unsigned int q_point) const
AdditionalData(const UpdateFlags mapping_update_flags=update_gradients|update_JxW_values|update_quadrature_points, const bool use_coloring=false, const bool overlap_communication_computation=false)
unsigned int local_q_point_id(const unsigned int cell, const unsigned int n_q_points, const unsigned int q_point) const
SharedData< dim, Number > * shared_data
Kokkos::TeamPolicy< MemorySpace::Default::kokkos_space::execution_space >::member_type TeamHandle
const PrecomputedData * precomputed_data
Portable::MatrixFree< dim, Number >::point_type & get_quadrature_point(const unsigned int cell, const unsigned int q_point) const
Kokkos::View< Number *, MemorySpace::Default::kokkos_space > shape_gradients
Kokkos::View< point_type **, MemorySpace::Default::kokkos_space > q_points
Kokkos::View< Number *, MemorySpace::Default::kokkos_space > shape_values
Kokkos::View< Number *, MemorySpace::Default::kokkos_space > constraint_weights
Kokkos::View< Number *, MemorySpace::Default::kokkos_space > co_shape_gradients
Kokkos::View< Number **[dim][dim], MemorySpace::Default::kokkos_space > inv_jacobian
::internal::MatrixFreeFunctions::ElementType element_type
Kokkos::View< Number **, MemorySpace::Default::kokkos_space > JxW
Kokkos::View<::internal::MatrixFreeFunctions::ConstraintKinds *, MemorySpace::Default::kokkos_space > constraint_mask
Kokkos::View< types::global_dof_index **, MemorySpace::Default::kokkos_space > local_to_global
Kokkos::View< Number ***, MemorySpace::Default::kokkos_space::execution_space::scratch_memory_space, Kokkos::MemoryTraits< Kokkos::Unmanaged > > SharedViewGradients
SharedViewScratchPad scratch_pad
Kokkos::View< Number *, MemorySpace::Default::kokkos_space::execution_space::scratch_memory_space, Kokkos::MemoryTraits< Kokkos::Unmanaged > > SharedViewScratchPad
SharedData(const SharedViewValues &values, const SharedViewGradients &gradients, const SharedViewScratchPad &scratch_pad)
Kokkos::View< Number **, MemorySpace::Default::kokkos_space::execution_space::scratch_memory_space, Kokkos::MemoryTraits< Kokkos::Unmanaged > > SharedViewValues
SharedViewGradients gradients