18 #ifdef DEAL_II_WITH_PETSC
26 # define AssertPETSc(code) \
29 PetscErrorCode ierr = (code); \
30 AssertThrow(ierr == 0, ExcPETScError(ierr)); \
52 const MPI_Comm communicator)
57 AssertPETSc(PetscLayoutCreate(communicator, &layout));
58 AssertPETSc(PetscLayoutSetLocalSize(layout, local_size));
77 PetscSFSetGraphLayout(
sf, layout, n,
nullptr, PETSC_OWN_POINTER, idxs));
90 const MPI_Comm communicator)
92 std::vector<types::global_dof_index> in_deal;
94 std::vector<PetscInt> in_petsc(in_deal.begin(), in_deal.end());
96 std::vector<types::global_dof_index> out_deal;
98 std::vector<PetscInt> out_petsc(out_deal.begin(), out_deal.end());
100 std::vector<PetscInt> dummy;
102 this->
do_reinit(in_petsc, dummy, out_petsc, dummy, communicator);
109 const std::vector<types::global_dof_index> &indices_has,
110 const std::vector<types::global_dof_index> &indices_want,
111 const MPI_Comm communicator)
114 std::vector<PetscInt> indices_has_clean, indices_has_loc;
115 std::vector<PetscInt> indices_want_clean, indices_want_loc;
116 indices_want_clean.reserve(indices_want.size());
117 indices_want_loc.reserve(indices_want.size());
118 indices_has_clean.reserve(indices_has.size());
119 indices_has_loc.reserve(indices_has.size());
122 bool has_invalid =
false;
123 for (
const auto i : indices_has)
127 indices_has_clean.push_back(
static_cast<PetscInt
>(i));
128 indices_has_loc.push_back(loc);
135 indices_has_loc.clear();
139 for (
const auto i : indices_want)
143 indices_want_clean.push_back(
static_cast<PetscInt
>(i));
144 indices_want_loc.push_back(loc);
151 indices_want_loc.clear();
164 const std::vector<PetscInt> &inloc,
165 const std::vector<PetscInt> &outidx,
166 const std::vector<PetscInt> &outloc,
167 const MPI_Comm communicator)
185 PetscInt n =
static_cast<PetscInt
>(inidx.size());
186 PetscInt lN = n > 0 ? *std::max_element(inidx.begin(), inidx.end()) : -1;
189 Utilities::MPI::internal::all_reduce<PetscInt>(
195 PetscSFNode *remotes;
199 AssertPETSc(PetscLayoutCreate(communicator, &layout));
204 const PetscInt *ranges;
205 AssertPETSc(PetscLayoutGetRanges(layout, &ranges));
208 PetscMPIInt owner = 0;
209 for (
const auto idx : inidx)
212 if (idx < ranges[owner] || ranges[owner + 1] <= idx)
214 AssertPETSc(PetscLayoutFindOwner(layout, idx, &owner));
216 remotes[cnt].rank = owner;
217 remotes[cnt].index = idx - ranges[owner];
225 const_cast<PetscInt *
>(
226 inloc.size() > 0 ? inloc.data() :
nullptr),
236 n =
static_cast<PetscInt
>(outidx.size());
242 const_cast<PetscInt *
>(outloc.size() > 0 ? outloc.data() :
nullptr),
244 const_cast<PetscInt *
>(n > 0 ? outidx.data() :
nullptr)));
269 return PetscObjectComm(
reinterpret_cast<PetscObject
>(
sf));
274 template <
typename Number>
280 auto datatype = Utilities::MPI::mpi_type_id_for_type<Number>;
282 # if DEAL_II_PETSC_VERSION_LT(3, 15, 0)
286 PetscSFBcastBegin(
sf, datatype, src.
data(), dst.
data(), MPI_REPLACE));
292 template <
typename Number>
298 auto datatype = Utilities::MPI::mpi_type_id_for_type<Number>;
300 # if DEAL_II_PETSC_VERSION_LT(3, 15, 0)
304 PetscSFBcastEnd(
sf, datatype, src.
data(), dst.
data(), MPI_REPLACE));
310 template <
typename Number>
322 template <
typename Number>
330 auto datatype = Utilities::MPI::mpi_type_id_for_type<Number>;
333 PetscSFReduceBegin(
sf, datatype, src.
data(), dst.
data(), mpiop));
338 template <
typename Number>
346 auto datatype = Utilities::MPI::mpi_type_id_for_type<Number>;
353 template <
typename Number>
371 , ghost_indices_data()
381 const MPI_Comm communicator)
397 const IndexSet &larger_ghost_indices,
398 const MPI_Comm communicator)
400 std::vector<types::global_dof_index> local_indices;
407 std::vector<types::global_dof_index> expanded_ghost_indices(
412 ExcMessage(
"The given larger ghost index set must contain "
413 "all indices in the actual index set."));
415 expanded_ghost_indices[tmp_index] =
index;
430 template <
typename Number>
445 template <
typename Number>
461 template <
typename Number>
470 template <
typename Number>
487 template <
typename Number>
504 template <
typename Number>
517 # include "petsc_communication_pattern.inst"
value_type * data() const noexcept
IS make_petsc_is(const MPI_Comm communicator=MPI_COMM_WORLD) const
void fill_index_vector(std::vector< size_type > &indices) const
size_type index_within_set(const size_type global_index) const
size_type n_elements() const
bool is_element(const size_type index) const
void subtract_set(const IndexSet &other)
void add_range(const size_type begin, const size_type end)
void add_indices(const ForwardIterator &begin, const ForwardIterator &end)
MPI_Comm get_mpi_communicator() const override
void import_from_ghosted_array(const VectorOperation::values op, const ArrayView< const Number > &ghost_array, const ArrayView< Number > &locally_owned_array) const
virtual ~CommunicationPattern() override
void export_to_ghosted_array_start(const ArrayView< const Number > &locally_owned_array, const ArrayView< Number > &ghost_array) const
void export_to_ghosted_array(const ArrayView< const Number > &locally_owned_array, const ArrayView< Number > &ghost_array) const
void export_to_ghosted_array_finish(const ArrayView< const Number > &locally_owned_array, const ArrayView< Number > &ghost_array) const
void import_from_ghosted_array_finish(const VectorOperation::values op, const ArrayView< const Number > &ghost_array, const ArrayView< Number > &locally_owned_array) const
void do_reinit(const std::vector< PetscInt > &inidx, const std::vector< PetscInt > &inloc, const std::vector< PetscInt > &outidx, const std::vector< PetscInt > &outloc, const MPI_Comm communicator)
virtual void reinit(const IndexSet &locally_owned_indices, const IndexSet &ghost_indices, const MPI_Comm communicator) override
void import_from_ghosted_array_start(const VectorOperation::values op, const ArrayView< const Number > &ghost_array, const ArrayView< Number > &locally_owned_array) const
types::global_dof_index n_ghost_indices_data
virtual void reinit(const IndexSet &locally_owned_indices, const IndexSet &ghost_indices, const MPI_Comm communicator) override
CommunicationPattern ghost
IndexSet ghost_indices_data
void import_from_ghosted_array(const VectorOperation::values op, const ArrayView< const Number > &ghost_array, const ArrayView< Number > &locally_owned_array) const
void export_to_ghosted_array_finish(const ArrayView< const Number > &locally_owned_array, const ArrayView< Number > &ghost_array) const
MPI_Comm get_mpi_communicator() const override
const IndexSet & ghost_indices() const
void export_to_ghosted_array(const ArrayView< const Number > &locally_owned_array, const ArrayView< Number > &ghost_array) const
void export_to_ghosted_array_start(const ArrayView< const Number > &locally_owned_array, const ArrayView< Number > &ghost_array) const
void import_from_ghosted_array_start(const VectorOperation::values op, const ArrayView< const Number > &ghost_array, const ArrayView< Number > &locally_owned_array) const
void import_from_ghosted_array_finish(const VectorOperation::values op, const ArrayView< const Number > &ghost_array, const ArrayView< Number > &locally_owned_array) const
CommunicationPattern larger_ghost
types::global_dof_index n_ghost_indices_larger
#define DEAL_II_NAMESPACE_OPEN
#define DEAL_II_NAMESPACE_CLOSE
#define Assert(cond, exc)
static ::ExceptionBase & ExcMessage(std::string arg1)
VectorType::value_type * end(VectorType &V)
const types::global_dof_index invalid_dof_index
unsigned int global_dof_index
#define AssertPETSc(code)