deal.II version GIT relicensing-2233-g9bd70d8389 2024-12-08 16:10:00+00:00
\(\newcommand{\dealvcentcolon}{\mathrel{\mathop{:}}}\) \(\newcommand{\dealcoloneq}{\dealvcentcolon\mathrel{\mkern-1.2mu}=}\) \(\newcommand{\jump}[1]{\left[\!\left[ #1 \right]\!\right]}\) \(\newcommand{\average}[1]{\left\{\!\left\{ #1 \right\}\!\right\}}\)
Loading...
Searching...
No Matches
portable_matrix_free.h
Go to the documentation of this file.
1// ------------------------------------------------------------------------
2//
3// SPDX-License-Identifier: LGPL-2.1-or-later
4// Copyright (C) 2017 - 2024 by the deal.II authors
5//
6// This file is part of the deal.II library.
7//
8// Part of the source code is dual licensed under Apache-2.0 WITH
9// LLVM-exception OR LGPL-2.1-or-later. Detailed license information
10// governing the source code and code contributions can be found in
11// LICENSE.md and CONTRIBUTING.md at the top level directory of deal.II.
12//
13// ------------------------------------------------------------------------
14
15
16#ifndef dealii_portable_matrix_free_h
17#define dealii_portable_matrix_free_h
18
19#include <deal.II/base/config.h>
20
25#include <deal.II/base/tensor.h>
27
29
31#include <deal.II/fe/mapping.h>
32
34
37
38#include <Kokkos_Core.hpp>
39
40
41
43
44// Forward declaration
45namespace internal
46{
47 namespace MatrixFreeFunctions
48 {
49 enum class ConstraintKinds : std::uint16_t;
50 }
51} // namespace internal
52
53namespace Portable
54{
55 // forward declaration
56#ifndef DOXYGEN
57 namespace internal
58 {
59 template <int dim, typename Number>
60 class ReinitHelper;
61 }
62#endif
63
88 template <int dim, typename Number = double>
90 {
91 public:
94 using CellFilter =
96
101 {
108 const bool use_coloring = false,
109 const bool overlap_communication_computation = false)
113 {
114#ifndef DEAL_II_MPI_WITH_DEVICE_SUPPORT
118 "Overlapping communication and computation requires device-aware MPI."));
119#endif
124 "Overlapping communication and coloring are incompatible options. Only one of them can be enabled."));
125 }
136
143
149 };
150
155 struct Data
156 {
160 Kokkos::View<point_type **, MemorySpace::Default::kokkos_space> q_points;
161
166 Kokkos::View<types::global_dof_index **,
169
173 Kokkos::View<Number **[dim][dim], MemorySpace::Default::kokkos_space>
175
179 Kokkos::View<Number **, MemorySpace::Default::kokkos_space> JxW;
180
187
191 Kokkos::View<Number *, MemorySpace::Default::kokkos_space> shape_values;
192
196 Kokkos::View<Number *, MemorySpace::Default::kokkos_space>
198
202 Kokkos::View<Number *, MemorySpace::Default::kokkos_space>
204
208 Kokkos::View<Number *, MemorySpace::Default::kokkos_space>
210
214 unsigned int n_cells;
215
219 unsigned int n_components;
220
224 unsigned int padding_length;
225
229 unsigned int row_start;
230
236
241 DEAL_II_HOST_DEVICE unsigned int
242 local_q_point_id(const unsigned int cell,
243 const unsigned int n_q_points,
244 const unsigned int q_point) const
245 {
246 return (row_start / padding_length + cell) * n_q_points + q_point;
247 }
248
249
255 get_quadrature_point(const unsigned int cell,
256 const unsigned int q_point) const
257 {
258 return q_points(cell, q_point);
259 }
260 };
261
266
270 unsigned int
272
283 template <typename IteratorFiltersType>
284 void
285 reinit(const Mapping<dim> &mapping,
287 const AffineConstraints<Number> &constraints,
288 const Quadrature<1> &quad,
289 const IteratorFiltersType &iterator_filter,
290 const AdditionalData &additional_data = AdditionalData());
291
295 void
296 reinit(const Mapping<dim> &mapping,
298 const AffineConstraints<Number> &constraints,
299 const Quadrature<1> &quad,
300 const AdditionalData &additional_data = AdditionalData());
301
305 void
307 const AffineConstraints<Number> &constraints,
308 const Quadrature<1> &quad,
309 const AdditionalData &additional_data = AdditionalData());
310
314 Data
315 get_data(unsigned int color) const;
316
317 // clang-format off
335 // clang-format on
336 template <typename Functor, typename VectorType>
337 void
338 cell_loop(const Functor &func,
339 const VectorType &src,
340 VectorType &dst) const;
341
357 template <typename Functor>
358 void
359 evaluate_coefficients(Functor func) const;
360
365 template <typename VectorType>
366 void
367 copy_constrained_values(const VectorType &src, VectorType &dst) const;
368
374 template <typename VectorType>
375 void
376 set_constrained_values(const Number val, VectorType &dst) const;
377
383 template <typename MemorySpaceType>
384 void
387
391 const std::vector<std::vector<CellFilter>> &
393
404 const std::shared_ptr<const Utilities::MPI::Partitioner> &
406
410 const DoFHandler<dim> &
412
416 std::size_t
418
419 private:
423 template <typename IteratorFiltersType>
424 void
427 const AffineConstraints<Number> &constraints,
428 const Quadrature<1> &quad,
429 const IteratorFiltersType &iterator_filter,
430 const std::shared_ptr<const MPI_Comm> &comm,
431 const AdditionalData additional_data);
432
437 template <typename Functor, typename VectorType>
438 void
439 serial_cell_loop(const Functor &func,
440 const VectorType &src,
441 VectorType &dst) const;
442
447 template <typename Functor>
448 void
450 const Functor &func,
452 &src,
454 const;
455
459 int my_id;
460
467
473
478
482 unsigned int fe_degree;
483
487 unsigned int n_components;
488
493
497 unsigned int dofs_per_cell;
498
502 unsigned int n_constrained_dofs;
503
507 unsigned int q_points_per_cell;
508
512 unsigned int n_colors;
513
517 std::vector<unsigned int> n_cells;
518
523 std::vector<Kokkos::View<point_type **, MemorySpace::Default::kokkos_space>>
525
530 std::vector<Kokkos::View<types::global_dof_index **,
533
538 std::vector<
539 Kokkos::View<Number **[dim][dim], MemorySpace::Default::kokkos_space>>
541
546 std::vector<Kokkos::View<Number **, MemorySpace::Default::kokkos_space>>
548
552 Kokkos::View<types::global_dof_index *, MemorySpace::Default::kokkos_space>
554
558 std::vector<
562
566 Kokkos::View<Number *, MemorySpace::Default::kokkos_space> shape_values;
567
571 Kokkos::View<Number *, MemorySpace::Default::kokkos_space> shape_gradients;
572
576 Kokkos::View<Number *, MemorySpace::Default::kokkos_space>
578
582 Kokkos::View<Number *, MemorySpace::Default::kokkos_space>
584
589 std::shared_ptr<const Utilities::MPI::Partitioner> partitioner;
590
591
596 unsigned int padding_length;
597
601 std::vector<unsigned int> row_start;
602
607
611 std::vector<std::vector<CellFilter>> graph;
612
613 friend class internal::ReinitHelper<dim, Number>;
614 };
615
616
617
618 template <int dim, typename Number>
620 {
621 using TeamHandle = Kokkos::TeamPolicy<
622 MemorySpace::Default::kokkos_space::execution_space>::member_type;
623
624 using SharedViewValues = Kokkos::View<
625 Number **,
626 MemorySpace::Default::kokkos_space::execution_space::scratch_memory_space,
627 Kokkos::MemoryTraits<Kokkos::Unmanaged>>;
628 using SharedViewGradients = Kokkos::View<
629 Number ***,
630 MemorySpace::Default::kokkos_space::execution_space::scratch_memory_space,
631 Kokkos::MemoryTraits<Kokkos::Unmanaged>>;
632
641
646
651
656 };
657
658
659
664 template <int dim, typename Number>
665 struct DataHost
666 {
670 typename Kokkos::View<Point<dim, Number> **,
673
678 typename Kokkos::View<types::global_dof_index **,
681
685 typename Kokkos::View<Number **[dim][dim],
688
692 typename Kokkos::View<Number **,
694
698 unsigned int n_cells;
699
703 unsigned int padding_length;
704
708 unsigned int row_start;
709
713 typename Kokkos::View<
716
722
723
724
728 unsigned int
729 local_q_point_id(const unsigned int cell,
730 const unsigned int n_q_points,
731 const unsigned int q_point) const
732 {
733 return (row_start / padding_length + cell) * n_q_points + q_point;
734 }
735
736
737
742 get_quadrature_point(const unsigned int cell,
743 const unsigned int q_point) const
744 {
745 return q_points(cell, q_point);
746 }
747 };
748
749
750
757 template <int dim, typename Number>
758 DataHost<dim, Number>
760 const typename ::Portable::MatrixFree<dim, Number>::Data &data,
761 const UpdateFlags &update_flags)
762 {
763 DataHost<dim, Number> data_host;
764
765 data_host.n_cells = data.n_cells;
766 data_host.padding_length = data.padding_length;
767 data_host.row_start = data.row_start;
768 data_host.use_coloring = data.use_coloring;
769
770 if (update_flags & update_quadrature_points)
771 {
772 data_host.q_points = Kokkos::create_mirror(data.q_points);
773 Kokkos::deep_copy(data_host.q_points, data.q_points);
774 }
775
776 data_host.local_to_global = Kokkos::create_mirror(data.local_to_global);
777 Kokkos::deep_copy(data_host.local_to_global, data.local_to_global);
778
779 if (update_flags & update_gradients)
780 {
781 data_host.inv_jacobian = Kokkos::create_mirror(data.inv_jacobian);
782 Kokkos::deep_copy(data_host.inv_jacobian, data.inv_jacobian);
783 }
784
785 if (update_flags & update_JxW_values)
786 {
787 data_host.JxW = Kokkos::create_mirror(data.JxW);
788 Kokkos::deep_copy(data_host.JxW, data.JxW);
789 }
790
791 data_host.constraint_mask = Kokkos::create_mirror(data.constraint_mask);
792 Kokkos::deep_copy(data_host.constraint_mask, data.constraint_mask);
793
794 return data_host;
795 }
796
797
798 /*----------------------- Inline functions ---------------------------------*/
799
800#ifndef DOXYGEN
801
802 template <int dim, typename Number>
803 inline const std::vector<std::vector<
806 {
807 return graph;
808 }
809
810
811
812 template <int dim, typename Number>
813 inline const std::shared_ptr<const Utilities::MPI::Partitioner> &
815 {
816 return partitioner;
817 }
818
819
820
821 template <int dim, typename Number>
822 inline const DoFHandler<dim> &
824 {
825 Assert(dof_handler != nullptr, ExcNotInitialized());
826
827 return *dof_handler;
828 }
829
830#endif
831
832} // namespace Portable
833
835
836#endif
Abstract base class for mapping classes.
Definition mapping.h:318
const std::vector< std::vector< CellFilter > > & get_colored_graph() const
std::vector< std::vector< CellFilter > > graph
void reinit(const DoFHandler< dim > &dof_handler, const AffineConstraints< Number > &constraints, const Quadrature< 1 > &quad, const AdditionalData &additional_data=AdditionalData())
void internal_reinit(const Mapping< dim > &mapping, const DoFHandler< dim > &dof_handler, const AffineConstraints< Number > &constraints, const Quadrature< 1 > &quad, const IteratorFiltersType &iterator_filter, const std::shared_ptr< const MPI_Comm > &comm, const AdditionalData additional_data)
std::vector< Kokkos::View<::internal::MatrixFreeFunctions::ConstraintKinds *, MemorySpace::Default::kokkos_space > > constraint_mask
unsigned int get_padding_length() const
Kokkos::View< Number *, MemorySpace::Default::kokkos_space > constraint_weights
const DoFHandler< dim > * dof_handler
Kokkos::View< Number *, MemorySpace::Default::kokkos_space > shape_gradients
void initialize_dof_vector(LinearAlgebra::distributed::Vector< Number, MemorySpaceType > &vec) const
void reinit(const Mapping< dim > &mapping, const DoFHandler< dim > &dof_handler, const AffineConstraints< Number > &constraints, const Quadrature< 1 > &quad, const IteratorFiltersType &iterator_filter, const AdditionalData &additional_data=AdditionalData())
std::vector< Kokkos::View< Number **[dim][dim], MemorySpace::Default::kokkos_space > > inv_jacobian
Kokkos::View< types::global_dof_index *, MemorySpace::Default::kokkos_space > constrained_dofs
std::vector< Kokkos::View< point_type **, MemorySpace::Default::kokkos_space > > q_points
types::global_dof_index n_dofs
void distributed_cell_loop(const Functor &func, const LinearAlgebra::distributed::Vector< Number, MemorySpace::Default > &src, LinearAlgebra::distributed::Vector< Number, MemorySpace::Default > &dst) const
std::vector< unsigned int > row_start
Kokkos::View< Number *, MemorySpace::Default::kokkos_space > co_shape_gradients
std::vector< Kokkos::View< types::global_dof_index **, MemorySpace::Default::kokkos_space > > local_to_global
Kokkos::View< Number *, MemorySpace::Default::kokkos_space > shape_values
std::size_t memory_consumption() const
void set_constrained_values(const Number val, VectorType &dst) const
void copy_constrained_values(const VectorType &src, VectorType &dst) const
std::vector< Kokkos::View< Number **, MemorySpace::Default::kokkos_space > > JxW
Data get_data(unsigned int color) const
const DoFHandler< dim > & get_dof_handler() const
std::shared_ptr< const Utilities::MPI::Partitioner > partitioner
void serial_cell_loop(const Functor &func, const VectorType &src, VectorType &dst) const
void evaluate_coefficients(Functor func) const
void reinit(const Mapping< dim > &mapping, const DoFHandler< dim > &dof_handler, const AffineConstraints< Number > &constraints, const Quadrature< 1 > &quad, const AdditionalData &additional_data=AdditionalData())
const std::shared_ptr< const Utilities::MPI::Partitioner > & get_vector_partitioner() const
DataHost< dim, Number > copy_mf_data_to_host(const typename::Portable::MatrixFree< dim, Number >::Data &data, const UpdateFlags &update_flags)
std::vector< unsigned int > n_cells
void cell_loop(const Functor &func, const VectorType &src, VectorType &dst) const
#define DEAL_II_NAMESPACE_OPEN
Definition config.h:498
#define DEAL_II_NAMESPACE_CLOSE
Definition config.h:499
#define Assert(cond, exc)
static ::ExceptionBase & ExcNotInitialized()
static ::ExceptionBase & ExcMessage(std::string arg1)
#define AssertThrow(cond, exc)
UpdateFlags
@ update_JxW_values
Transformed quadrature weights.
@ update_gradients
Shape function gradients.
@ update_quadrature_points
Transformed quadrature points.
std::vector< index_type > data
Definition mpi.cc:735
const MPI_Comm comm
Definition mpi.cc:913
#define DEAL_II_HOST_DEVICE
Definition numbers.h:30
::Kokkos::DefaultExecutionSpace::memory_space kokkos_space
Kokkos::View< types::global_dof_index **, MemorySpace::Default::kokkos_space >::HostMirror local_to_global
Kokkos::View< Number **, MemorySpace::Default::kokkos_space >::HostMirror JxW
Point< dim, Number > get_quadrature_point(const unsigned int cell, const unsigned int q_point) const
Kokkos::View<::internal::MatrixFreeFunctions::ConstraintKinds *, MemorySpace::Default::kokkos_space >::HostMirror constraint_mask
Kokkos::View< Number **[dim][dim], MemorySpace::Default::kokkos_space >::HostMirror inv_jacobian
Kokkos::View< Point< dim, Number > **, MemorySpace::Default::kokkos_space >::HostMirror q_points
unsigned int local_q_point_id(const unsigned int cell, const unsigned int n_q_points, const unsigned int q_point) const
AdditionalData(const UpdateFlags mapping_update_flags=update_gradients|update_JxW_values|update_quadrature_points, const bool use_coloring=false, const bool overlap_communication_computation=false)
Kokkos::View< Number **, MemorySpace::Default::kokkos_space > JxW
Kokkos::View< Number *, MemorySpace::Default::kokkos_space > constraint_weights
unsigned int local_q_point_id(const unsigned int cell, const unsigned int n_q_points, const unsigned int q_point) const
Kokkos::View<::internal::MatrixFreeFunctions::ConstraintKinds *, MemorySpace::Default::kokkos_space > constraint_mask
Kokkos::View< Number *, MemorySpace::Default::kokkos_space > shape_values
Kokkos::View< Number **[dim][dim], MemorySpace::Default::kokkos_space > inv_jacobian
Kokkos::View< point_type **, MemorySpace::Default::kokkos_space > q_points
Kokkos::View< types::global_dof_index **, MemorySpace::Default::kokkos_space > local_to_global
Kokkos::View< Number *, MemorySpace::Default::kokkos_space > shape_gradients
Kokkos::View< Number *, MemorySpace::Default::kokkos_space > co_shape_gradients
Portable::MatrixFree< dim, Number >::point_type & get_quadrature_point(const unsigned int cell, const unsigned int q_point) const
Kokkos::View< Number ***, MemorySpace::Default::kokkos_space::execution_space::scratch_memory_space, Kokkos::MemoryTraits< Kokkos::Unmanaged > > SharedViewGradients
SharedData(const TeamHandle &team_member, const SharedViewValues &values, const SharedViewGradients &gradients)
Kokkos::TeamPolicy< MemorySpace::Default::kokkos_space::execution_space >::member_type TeamHandle
Kokkos::View< Number **, MemorySpace::Default::kokkos_space::execution_space::scratch_memory_space, Kokkos::MemoryTraits< Kokkos::Unmanaged > > SharedViewValues
SharedViewGradients gradients