deal.II version GIT relicensing-1936-ge55124254b 2024-10-04 21:10:00+00:00
\(\newcommand{\dealvcentcolon}{\mathrel{\mathop{:}}}\) \(\newcommand{\dealcoloneq}{\dealvcentcolon\mathrel{\mkern-1.2mu}=}\) \(\newcommand{\jump}[1]{\left[\!\left[ #1 \right]\!\right]}\) \(\newcommand{\average}[1]{\left\{\!\left\{ #1 \right\}\!\right\}}\)
Loading...
Searching...
No Matches
vector_data_exchange.cc
Go to the documentation of this file.
1// ------------------------------------------------------------------------
2//
3// SPDX-License-Identifier: LGPL-2.1-or-later
4// Copyright (C) 2020 - 2024 by the deal.II authors
5//
6// This file is part of the deal.II library.
7//
8// Part of the source code is dual licensed under Apache-2.0 WITH
9// LLVM-exception OR LGPL-2.1-or-later. Detailed license information
10// governing the source code and code contributions can be found in
11// LICENSE.md and CONTRIBUTING.md at the top level directory of deal.II.
12//
13// ------------------------------------------------------------------------
14
16#include <deal.II/base/mpi.h>
17#include <deal.II/base/mpi.templates.h>
21#include <deal.II/base/timer.h>
22
24
25#include <boost/serialization/utility.hpp>
26
27#include <map>
28#include <vector>
29
30
32
33namespace internal
34{
35 namespace MatrixFreeFunctions
36 {
37 namespace VectorDataExchange
38 {
40 const std::shared_ptr<const Utilities::MPI::Partitioner> &partitioner)
41 : partitioner(partitioner)
42 {}
43
44
45
46 unsigned int
48 {
49 return partitioner->locally_owned_size();
50 }
51
52
53
54 unsigned int
56 {
57 return partitioner->n_ghost_indices();
58 }
59
60
61
62 unsigned int
64 {
65 return partitioner->n_import_indices();
66 }
67
68
69
70 unsigned int
72 {
73 return 0;
74 }
75
76
77
80 {
81 return partitioner->size();
82 }
83
84
85
86 void
88 const unsigned int communication_channel,
89 const ArrayView<const double> &locally_owned_array,
90 const std::vector<ArrayView<const double>> &shared_arrays,
91 const ArrayView<double> &ghost_array,
92 const ArrayView<double> &temporary_storage,
93 std::vector<MPI_Request> &requests) const
94 {
95 (void)shared_arrays;
96#ifndef DEAL_II_WITH_MPI
97 (void)communication_channel;
98 (void)locally_owned_array;
99 (void)ghost_array;
100 (void)temporary_storage;
101 (void)requests;
102#else
103 partitioner->export_to_ghosted_array_start(communication_channel,
104 locally_owned_array,
105 temporary_storage,
106 ghost_array,
107 requests);
108#endif
109 }
110
111
112
113 void
115 const ArrayView<const double> &locally_owned_array,
116 const std::vector<ArrayView<const double>> &shared_arrays,
117 const ArrayView<double> &ghost_array,
118 std::vector<MPI_Request> &requests) const
119 {
120 (void)locally_owned_array;
121 (void)shared_arrays;
122#ifndef DEAL_II_WITH_MPI
123 (void)ghost_array;
124 (void)requests;
125#else
126 partitioner->export_to_ghosted_array_finish(ghost_array, requests);
127#endif
128 }
129
130
131
132 void
134 const VectorOperation::values vector_operation,
135 const unsigned int communication_channel,
136 const ArrayView<const double> &locally_owned_array,
137 const std::vector<ArrayView<const double>> &shared_arrays,
138 const ArrayView<double> &ghost_array,
139 const ArrayView<double> &temporary_storage,
140 std::vector<MPI_Request> &requests) const
141 {
142 (void)locally_owned_array;
143 (void)shared_arrays;
144#ifndef DEAL_II_WITH_MPI
145 (void)vector_operation;
146 (void)communication_channel;
147 (void)ghost_array;
148 (void)temporary_storage;
149 (void)requests;
150#else
151 partitioner->import_from_ghosted_array_start(vector_operation,
152 communication_channel,
153 ghost_array,
154 temporary_storage,
155 requests);
156#endif
157 }
158
159
160
161 void
163 const VectorOperation::values vector_operation,
164 const ArrayView<double> &locally_owned_storage,
165 const std::vector<ArrayView<const double>> &shared_arrays,
166 const ArrayView<double> &ghost_array,
167 const ArrayView<const double> &temporary_storage,
168 std::vector<MPI_Request> &requests) const
169 {
170 (void)shared_arrays;
171#ifndef DEAL_II_WITH_MPI
172 (void)vector_operation;
173 (void)locally_owned_storage;
174 (void)ghost_array;
175 (void)temporary_storage;
176 (void)requests;
177#else
178 partitioner->import_from_ghosted_array_finish(vector_operation,
179 temporary_storage,
180 locally_owned_storage,
181 ghost_array,
182 requests);
183#endif
184 }
185
186
187
188 void
190 const ArrayView<double> &ghost_array) const
191 {
192 reset_ghost_values_impl(ghost_array);
193 }
194
195
196
197 void
199 const unsigned int communication_channel,
200 const ArrayView<const float> &locally_owned_array,
201 const std::vector<ArrayView<const float>> &shared_arrays,
202 const ArrayView<float> &ghost_array,
203 const ArrayView<float> &temporary_storage,
204 std::vector<MPI_Request> &requests) const
205 {
206 (void)shared_arrays;
207#ifndef DEAL_II_WITH_MPI
208 (void)communication_channel;
209 (void)locally_owned_array;
210 (void)ghost_array;
211 (void)temporary_storage;
212 (void)requests;
213#else
214 partitioner->export_to_ghosted_array_start(communication_channel,
215 locally_owned_array,
216 temporary_storage,
217 ghost_array,
218 requests);
219#endif
220 }
221
222
223
224 void
226 const ArrayView<const float> &locally_owned_array,
227 const std::vector<ArrayView<const float>> &shared_arrays,
228 const ArrayView<float> &ghost_array,
229 std::vector<MPI_Request> &requests) const
230 {
231 (void)locally_owned_array;
232 (void)shared_arrays;
233#ifndef DEAL_II_WITH_MPI
234 (void)ghost_array;
235 (void)requests;
236#else
237 partitioner->export_to_ghosted_array_finish(ghost_array, requests);
238#endif
239 }
240
241
242
243 void
245 const VectorOperation::values vector_operation,
246 const unsigned int communication_channel,
247 const ArrayView<const float> &locally_owned_array,
248 const std::vector<ArrayView<const float>> &shared_arrays,
249 const ArrayView<float> &ghost_array,
250 const ArrayView<float> &temporary_storage,
251 std::vector<MPI_Request> &requests) const
252 {
253 (void)locally_owned_array;
254 (void)shared_arrays;
255#ifndef DEAL_II_WITH_MPI
256 (void)vector_operation;
257 (void)communication_channel;
258 (void)ghost_array;
259 (void)temporary_storage;
260 (void)requests;
261#else
262 partitioner->import_from_ghosted_array_start(vector_operation,
263 communication_channel,
264 ghost_array,
265 temporary_storage,
266 requests);
267#endif
268 }
269
270
271
272 void
274 const VectorOperation::values vector_operation,
275 const ArrayView<float> &locally_owned_storage,
276 const std::vector<ArrayView<const float>> &shared_arrays,
277 const ArrayView<float> &ghost_array,
278 const ArrayView<const float> &temporary_storage,
279 std::vector<MPI_Request> &requests) const
280 {
281 (void)shared_arrays;
282#ifndef DEAL_II_WITH_MPI
283 (void)vector_operation;
284 (void)locally_owned_storage;
285 (void)ghost_array;
286 (void)temporary_storage;
287 (void)requests;
288#else
289 partitioner->import_from_ghosted_array_finish(vector_operation,
290 temporary_storage,
291 locally_owned_storage,
292 ghost_array,
293 requests);
294#endif
295 }
296
297
298
299 void
301 const ArrayView<float> &ghost_array) const
302 {
303 reset_ghost_values_impl(ghost_array);
304 }
305
306
307
308 template <typename Number>
309 void
311 const ArrayView<Number> &ghost_array) const
312 {
313 for (const auto &my_ghosts :
314 partitioner->ghost_indices_within_larger_ghost_set())
315 for (unsigned int j = my_ghosts.first; j < my_ghosts.second; ++j)
316 ghost_array[j] = 0.;
317 }
318
319
320
321 namespace internal
322 {
323 std::pair<std::vector<unsigned int>,
324 std::vector<std::pair<unsigned int, unsigned int>>>
326 const std::vector<unsigned int> &sm_export_ptr,
327 const std::vector<unsigned int> &sm_export_indices)
328 {
329 std::vector<unsigned int> recv_ptr = {0};
330 std::vector<unsigned int> recv_indices;
331 std::vector<unsigned int> recv_len;
332
333 for (unsigned int i = 0; i + 1 < sm_export_ptr.size(); ++i)
334 {
335 if (sm_export_ptr[i] != sm_export_ptr[i + 1])
336 {
337 recv_indices.push_back(sm_export_indices[sm_export_ptr[i]]);
338 recv_len.push_back(1);
339
340 for (unsigned int j = sm_export_ptr[i] + 1;
341 j < sm_export_ptr[i + 1];
342 j++)
343 if (recv_indices.back() + recv_len.back() !=
344 sm_export_indices[j])
345 {
346 recv_indices.push_back(sm_export_indices[j]);
347 recv_len.push_back(1);
348 }
349 else
350 recv_len.back()++;
351 }
352 recv_ptr.push_back(recv_indices.size());
353 }
354
355 std::pair<std::vector<unsigned int>,
356 std::vector<std::pair<unsigned int, unsigned int>>>
357 result;
358
359 result.first = recv_ptr;
360
361 for (unsigned int i = 0; i < recv_indices.size(); ++i)
362 result.second.emplace_back(recv_indices[i], recv_len[i]);
363
364 return result;
365 }
366
367 } // namespace internal
368
369
370
372 const std::shared_ptr<const Utilities::MPI::Partitioner> &partitioner,
373 const MPI_Comm communicator_sm)
374 : comm(partitioner->get_mpi_communicator())
375 , comm_sm(communicator_sm)
376 , n_local_elements(partitioner->locally_owned_range().n_elements())
377 , n_ghost_elements(partitioner->ghost_indices().n_elements())
378 , n_global_elements(partitioner->locally_owned_range().size())
379 {
380#ifndef DEAL_II_WITH_MPI
381 Assert(false, ExcNeedsMPI());
382#else
384 return; // nothing to do in serial case
385
386 const auto &is_locally_owned = partitioner->locally_owned_range();
387 const auto &is_locally_ghost = partitioner->ghost_indices();
388 const auto &ghost_indices_within_larger_ghost_set =
389 partitioner->ghost_indices_within_larger_ghost_set();
390
391 // temporal data structures
392 std::vector<unsigned int> n_ghost_indices_in_larger_set_by_remote_rank;
393
394 std::vector<std::array<unsigned int, 3>> ghost_targets_data;
395
396 std::vector<std::array<unsigned int, 3>> import_targets_data;
397
398 std::vector<unsigned int> sm_ghost_ranks;
399
400 std::vector<unsigned int> sm_import_ranks;
401
402 // temporary uncompressed data structures for ghost_indices_subset_data
403 std::vector<unsigned int> ghost_indices_subset_data_ptr = {0};
404 std::vector<unsigned int> ghost_indices_subset_data_indices;
405
406 // ... for import_indices_data
407 std::vector<unsigned int> import_indices_data_ptr = {0};
408 std::vector<unsigned int> import_indices_data_indices;
409
410 // ... for sm_export_data
411 std::vector<unsigned int> sm_export_data_ptr = {0};
412 std::vector<unsigned int> sm_export_data_indices;
413
414 // ... for sm_export_data_this
415 std::vector<unsigned int> sm_export_data_this_ptr = {0};
416 std::vector<unsigned int> sm_export_data_this_indices;
417
418 // ... for sm_import_data
419 std::vector<unsigned int> sm_import_data_ptr = {};
420 std::vector<unsigned int> sm_import_data_indices;
421
422 // ... for sm_import_data_this
423 std::vector<unsigned int> sm_import_data_this_ptr = {0};
424 std::vector<unsigned int> sm_import_data_this_indices;
425
426 // collect ranks of processes of shared-memory domain
427 const auto sm_ranks =
429
430 // determine owners of ghost indices and determine requesters
431 std::vector<unsigned int> owning_ranks_of_ghosts(
432 is_locally_ghost.n_elements());
433
435 process(is_locally_owned,
436 is_locally_ghost,
437 comm,
438 owning_ranks_of_ghosts,
439 /*track_index_requests = */ true);
440
442 std::vector<
443 std::pair<types::global_dof_index, types::global_dof_index>>,
444 std::vector<unsigned int>>
445 consensus_algorithm;
446 consensus_algorithm.run(process, comm);
447
448 // decompress ghost_indices_within_larger_ghost_set for simpler
449 // data access during setup
450 std::vector<unsigned int> shifts_indices;
451 for (const auto &pair : ghost_indices_within_larger_ghost_set)
452 for (unsigned int k = pair.first; k < pair.second; ++k)
453 shifts_indices.push_back(k);
454
455 // process ghost indices
456 {
457 // collect ghost indices according to owning rank
458 std::map<unsigned int, std::vector<types::global_dof_index>>
459 rank_to_local_indices;
460
461 for (unsigned int i = 0; i < owning_ranks_of_ghosts.size(); ++i)
462 rank_to_local_indices[owning_ranks_of_ghosts[i]].push_back(i);
463
464 unsigned int compressed_offset = 0;
465
466 for (const auto &rank_and_local_indices : rank_to_local_indices)
467 {
468 const auto sm_ranks_ptr = std::find(sm_ranks.begin(),
469 sm_ranks.end(),
470 rank_and_local_indices.first);
471
472 if (sm_ranks_ptr == sm_ranks.end()) // remote process
473 {
474 ghost_targets_data.emplace_back(std::array<unsigned int, 3>{{
475 rank_and_local_indices.first, // rank
476 shifts_indices[compressed_offset], // offset
477 static_cast<unsigned int>(
478 rank_and_local_indices.second.size()) // length
479 }});
480
481 for (unsigned int i = 0;
482 i < rank_and_local_indices.second.size();
483 ++i)
484 ghost_indices_subset_data_indices.push_back(
485 shifts_indices[i + compressed_offset]);
486
487 ghost_indices_subset_data_ptr.push_back(
488 ghost_indices_subset_data_indices.size());
489
490 ghost_indices_subset_data.first.push_back(compressed_offset);
491
492 unsigned int i =
494
496 (shifts_indices[ghost_indices_subset_data.first[i] +
497 (ghost_targets_data[i][2] - 1)] -
498 shifts_indices[ghost_indices_subset_data.first[i]]) +
499 1);
500 }
501 else // shared process
502 {
503 sm_ghost_ranks.push_back(
504 std::distance(sm_ranks.begin(), sm_ranks_ptr));
505
506 sm_export_data_ptr.push_back(
507 sm_export_data_ptr.back() +
508 rank_and_local_indices.second.size());
509
510 for (unsigned int i = compressed_offset;
511 i <
512 rank_and_local_indices.second.size() + compressed_offset;
513 ++i)
514 sm_export_data_this_indices.push_back(
515 shifts_indices[i] + is_locally_owned.n_elements());
516
517 sm_export_data_this_ptr.push_back(
518 sm_export_data_this_indices.size());
519 }
520 compressed_offset += rank_and_local_indices.second.size();
521 }
522
523 sm_export_data_indices.resize(sm_export_data_ptr.back());
524 }
525
526 // process requesters
527 {
528 const auto rank_to_global_indices = process.get_requesters();
529
530 for (const auto &rank_and_global_indices : rank_to_global_indices)
531 {
532 const auto sm_ranks_ptr =
533 std::find(sm_ranks.begin(),
534 sm_ranks.end(),
535 rank_and_global_indices.first);
536
537 if (sm_ranks_ptr == sm_ranks.end()) // remote process
538 {
539 import_targets_data.emplace_back(std::array<unsigned int, 3>{{
540 rank_and_global_indices.first, // rank
541 static_cast<unsigned int>(
542 import_indices_data_indices.size()), // offset
543 static_cast<unsigned int>(
544 rank_and_global_indices.second.n_elements()) // length
545 }});
546
547 for (const auto i : rank_and_global_indices.second)
548 import_indices_data_indices.push_back(
549 is_locally_owned.index_within_set(i));
550
551 import_indices_data_ptr.push_back(
552 import_indices_data_indices.size());
553 }
554 else // shared process
555 {
556 sm_import_ranks.push_back(
557 std::distance(sm_ranks.begin(), sm_ranks_ptr));
558
559 for (const auto i : rank_and_global_indices.second)
560 sm_import_data_this_indices.push_back(
561 is_locally_owned.index_within_set(i));
562
563 sm_import_data_this_ptr.push_back(
564 sm_import_data_this_indices.size());
565 }
566 }
567
568 sm_import_data_ptr = sm_import_data_this_ptr;
569 sm_import_data_indices.resize(sm_import_data_this_ptr.back());
570 }
571
572 // send sm_export_data_this to sm-neighbor -> sm_import_data
573 {
574 std::vector<MPI_Request> requests(sm_ghost_ranks.size() +
575 sm_import_ranks.size());
576
577 for (unsigned int i = 0; i < sm_ghost_ranks.size(); ++i)
578 {
579 const int ierr = MPI_Isend(sm_export_data_this_indices.data() +
580 sm_export_data_this_ptr[i],
581 sm_export_data_this_ptr[i + 1] -
582 sm_export_data_this_ptr[i],
583 MPI_UNSIGNED,
585 4,
586 comm_sm,
587 requests.data() + i);
588 AssertThrowMPI(ierr);
589 }
590
591 for (unsigned int i = 0; i < sm_import_ranks.size(); ++i)
592 {
593 const int ierr =
594 MPI_Irecv(sm_import_data_indices.data() + sm_import_data_ptr[i],
595 sm_import_data_ptr[i + 1] - sm_import_data_ptr[i],
596 MPI_UNSIGNED,
598 4,
599 comm_sm,
600 requests.data() + sm_ghost_ranks.size() + i);
601 AssertThrowMPI(ierr);
602 }
603
604 const int ierr =
605 MPI_Waitall(requests.size(), requests.data(), MPI_STATUSES_IGNORE);
606 AssertThrowMPI(ierr);
607 }
608
609 // send sm_import_data_this to sm-neighbor -> sm_export_data_indices
610 {
611 std::vector<MPI_Request> requests(sm_import_ranks.size() +
612 sm_ghost_ranks.size());
613
614 for (unsigned int i = 0; i < sm_import_ranks.size(); ++i)
615 {
616 const int ierr = MPI_Isend(sm_import_data_this_indices.data() +
617 sm_import_data_this_ptr[i],
618 sm_import_data_this_ptr[i + 1] -
619 sm_import_data_this_ptr[i],
620 MPI_UNSIGNED,
622 2,
623 comm_sm,
624 requests.data() + i);
625 AssertThrowMPI(ierr);
626 }
627
628 for (unsigned int i = 0; i < sm_ghost_ranks.size(); ++i)
629 {
630 const int ierr =
631 MPI_Irecv(sm_export_data_indices.data() + sm_export_data_ptr[i],
632 sm_export_data_ptr[i + 1] - sm_export_data_ptr[i],
633 MPI_UNSIGNED,
635 2,
636 comm_sm,
637 requests.data() + sm_import_ranks.size() + i);
638 AssertThrowMPI(ierr);
639 }
640
641 const int ierr =
642 MPI_Waitall(requests.size(), requests.data(), MPI_STATUSES_IGNORE);
643 AssertThrowMPI(ierr);
644 }
645
646 // store data structures and, if needed, compress them
647 this->n_ghost_indices_in_larger_set_by_remote_rank =
649
652 ghost_indices_subset_data_ptr, ghost_indices_subset_data_indices);
653
654 this->ghost_targets_data = ghost_targets_data;
655
656 this->import_targets_data = import_targets_data;
657
658 this->import_indices_data =
659 internal::compress_to_contiguous_ranges(import_indices_data_ptr,
660 import_indices_data_indices);
661
662 this->sm_ghost_ranks = sm_ghost_ranks;
663
664 this->sm_export_data =
666 sm_export_data_indices);
667
668 this->sm_export_data_this =
669 internal::compress_to_contiguous_ranges(sm_export_data_this_ptr,
670 sm_export_data_this_indices);
671
672 this->sm_import_ranks = sm_import_ranks;
673
674 this->sm_import_data =
676 sm_import_data_indices);
677
678 this->sm_import_data_this =
679 internal::compress_to_contiguous_ranges(sm_import_data_this_ptr,
680 sm_import_data_this_indices);
681
682#endif
683 }
684
685
686
687 void
689 const unsigned int communication_channel,
690 const ArrayView<const double> &locally_owned_array,
691 const std::vector<ArrayView<const double>> &shared_arrays,
692 const ArrayView<double> &ghost_array,
693 const ArrayView<double> &temporary_storage,
694 std::vector<MPI_Request> &requests) const
695 {
696 export_to_ghosted_array_start_impl(communication_channel,
697 locally_owned_array,
698 shared_arrays,
699 ghost_array,
700 temporary_storage,
701 requests);
702 }
703
704
705
706 void
708 const ArrayView<const double> &locally_owned_array,
709 const std::vector<ArrayView<const double>> &shared_arrays,
710 const ArrayView<double> &ghost_array,
711 std::vector<MPI_Request> &requests) const
712 {
713 export_to_ghosted_array_finish_impl(locally_owned_array,
714 shared_arrays,
715 ghost_array,
716 requests);
717 }
718
719
720
721 void
723 const VectorOperation::values vector_operation,
724 const unsigned int communication_channel,
725 const ArrayView<const double> &locally_owned_array,
726 const std::vector<ArrayView<const double>> &shared_arrays,
727 const ArrayView<double> &ghost_array,
728 const ArrayView<double> &temporary_storage,
729 std::vector<MPI_Request> &requests) const
730 {
732 communication_channel,
733 locally_owned_array,
734 shared_arrays,
735 ghost_array,
736 temporary_storage,
737 requests);
738 }
739
740
741
742 void
744 const VectorOperation::values vector_operation,
745 const ArrayView<double> &locally_owned_storage,
746 const std::vector<ArrayView<const double>> &shared_arrays,
747 const ArrayView<double> &ghost_array,
748 const ArrayView<const double> &temporary_storage,
749 std::vector<MPI_Request> &requests) const
750 {
752 locally_owned_storage,
753 shared_arrays,
754 ghost_array,
755 temporary_storage,
756 requests);
757 }
758
759
760
761 void
763 const unsigned int communication_channel,
764 const ArrayView<const float> &locally_owned_array,
765 const std::vector<ArrayView<const float>> &shared_arrays,
766 const ArrayView<float> &ghost_array,
767 const ArrayView<float> &temporary_storage,
768 std::vector<MPI_Request> &requests) const
769 {
770 export_to_ghosted_array_start_impl(communication_channel,
771 locally_owned_array,
772 shared_arrays,
773 ghost_array,
774 temporary_storage,
775 requests);
776 }
777
778
779
780 void
782 const ArrayView<const float> &locally_owned_array,
783 const std::vector<ArrayView<const float>> &shared_arrays,
784 const ArrayView<float> &ghost_array,
785 std::vector<MPI_Request> &requests) const
786 {
787 export_to_ghosted_array_finish_impl(locally_owned_array,
788 shared_arrays,
789 ghost_array,
790 requests);
791 }
792
793
794
795 void
797 const VectorOperation::values vector_operation,
798 const unsigned int communication_channel,
799 const ArrayView<const float> &locally_owned_array,
800 const std::vector<ArrayView<const float>> &shared_arrays,
801 const ArrayView<float> &ghost_array,
802 const ArrayView<float> &temporary_storage,
803 std::vector<MPI_Request> &requests) const
804 {
806 communication_channel,
807 locally_owned_array,
808 shared_arrays,
809 ghost_array,
810 temporary_storage,
811 requests);
812 }
813
814
815
816 void
818 const VectorOperation::values vector_operation,
819 const ArrayView<float> &locally_owned_storage,
820 const std::vector<ArrayView<const float>> &shared_arrays,
821 const ArrayView<float> &ghost_array,
822 const ArrayView<const float> &temporary_storage,
823 std::vector<MPI_Request> &requests) const
824 {
826 locally_owned_storage,
827 shared_arrays,
828 ghost_array,
829 temporary_storage,
830 requests);
831 }
832
833
834
835 template <typename Number>
836 void
838 const unsigned int communication_channel,
839 const ArrayView<const Number> &data_this,
840 const std::vector<ArrayView<const Number>> &data_others,
841 const ArrayView<Number> &buffer,
842 const ArrayView<Number> &temporary_storage,
843 std::vector<MPI_Request> &requests) const
844 {
845#ifndef DEAL_II_WITH_MPI
846 Assert(false, ExcNeedsMPI());
847
848 (void)communication_channel;
849 (void)data_this;
850 (void)data_others;
851 (void)buffer;
852 (void)temporary_storage;
853 (void)requests;
854#else
855 (void)data_others;
856
857 requests.resize(sm_import_ranks.size() + sm_ghost_ranks.size() +
859
860 int dummy;
861 // receive a signal that relevant sm neighbors are ready
862 for (unsigned int i = 0; i < sm_ghost_ranks.size(); ++i)
863 {
864 const int ierr =
865 MPI_Irecv(&dummy,
866 0,
867 MPI_INT,
869 communication_channel + 0,
870 comm_sm,
871 requests.data() + sm_import_ranks.size() + i);
872 AssertThrowMPI(ierr);
873 }
874
875 // signal to all relevant sm neighbors that this process is ready
876 for (unsigned int i = 0; i < sm_import_ranks.size(); ++i)
877 {
878 const int ierr = MPI_Isend(&dummy,
879 0,
880 MPI_INT,
882 communication_channel + 0,
883 comm_sm,
884 requests.data() + i);
885 AssertThrowMPI(ierr);
886 }
887
888 // receive data from remote processes
889 for (unsigned int i = 0; i < ghost_targets_data.size(); ++i)
890 {
891 const unsigned int offset =
893 ghost_targets_data[i][2];
894
895 const int ierr = MPI_Irecv(
896 buffer.data() + ghost_targets_data[i][1] + offset,
897 ghost_targets_data[i][2],
898 Utilities::MPI::mpi_type_id_for_type<decltype(*buffer.data())>,
899 ghost_targets_data[i][0],
900 communication_channel + 1,
901 comm,
902 requests.data() + sm_import_ranks.size() + sm_ghost_ranks.size() +
903 i);
904 AssertThrowMPI(ierr);
905 }
906
907 // send data to remote processes
908 for (unsigned int i = 0, k = 0; i < import_targets_data.size(); ++i)
909 {
910 for (unsigned int j = import_indices_data.first[i];
911 j < import_indices_data.first[i + 1];
912 j++)
913 for (unsigned int l = 0; l < import_indices_data.second[j].second;
914 l++, k++)
915 temporary_storage[k] =
916 data_this[import_indices_data.second[j].first + l];
917
918 // send data away
919 const int ierr = MPI_Isend(
920 temporary_storage.data() + import_targets_data[i][1],
922 Utilities::MPI::mpi_type_id_for_type<decltype(*data_this.data())>,
924 communication_channel + 1,
925 comm,
926 requests.data() + sm_import_ranks.size() + sm_ghost_ranks.size() +
927 ghost_targets_data.size() + i);
928 AssertThrowMPI(ierr);
929 }
930#endif
931 }
932
933
934
935 template <typename Number>
936 void
938 const ArrayView<const Number> &data_this,
939 const std::vector<ArrayView<const Number>> &data_others,
940 const ArrayView<Number> &ghost_array,
941 std::vector<MPI_Request> &requests) const
942 {
943 (void)data_this;
944
945#ifndef DEAL_II_WITH_MPI
946 Assert(false, ExcNeedsMPI());
947
948 (void)data_others;
949 (void)ghost_array;
950 (void)requests;
951#else
952
953 AssertDimension(requests.size(),
954 sm_import_ranks.size() + sm_ghost_ranks.size() +
955 ghost_targets_data.size() +
956 import_targets_data.size());
957
958 const auto split =
959 [&](const unsigned int i) -> std::pair<unsigned int, unsigned int> {
961 (sm_ghost_ranks.size() + ghost_targets_data.size()));
962
963 if (i < sm_ghost_ranks.size())
964 return {0, i};
965 else
966 return {1, i - sm_ghost_ranks.size()};
967 };
968
969 for (unsigned int c = 0;
970 c < sm_ghost_ranks.size() + ghost_targets_data.size();
971 c++)
972 {
973 int i;
974 const int ierr =
975 MPI_Waitany(sm_ghost_ranks.size() + ghost_targets_data.size(),
976 requests.data() + sm_import_ranks.size(),
977 &i,
978 MPI_STATUS_IGNORE);
979 AssertThrowMPI(ierr);
980
981 const auto s = split(i);
982 i = s.second;
983
984 if (s.first == 0)
985 {
986 const Number *DEAL_II_RESTRICT data_others_ptr =
987 data_others[sm_ghost_ranks[i]].data();
988 Number *DEAL_II_RESTRICT data_this_ptr = ghost_array.data();
989
990 for (unsigned int lo = sm_export_data.first[i],
991 ko = sm_export_data_this.first[i],
992 li = 0,
993 ki = 0;
994 (lo < sm_export_data.first[i + 1]) &&
995 (ko < sm_export_data_this.first[i + 1]);)
996 {
997 for (; (li < sm_export_data.second[lo].second) &&
998 (ki < sm_export_data_this.second[ko].second);
999 ++li, ++ki)
1000 data_this_ptr[sm_export_data_this.second[ko].first + ki -
1002 data_others_ptr[sm_export_data.second[lo].first + li];
1003
1004 if (li == sm_export_data.second[lo].second)
1005 {
1006 lo++; // increment outer counter
1007 li = 0; // reset inner counter
1008 }
1009
1010 if (ki == sm_export_data_this.second[ko].second)
1011 {
1012 ko++; // increment outer counter
1013 ki = 0; // reset inner counter
1014 }
1015 }
1016 }
1017 else /*if(s.second == 1)*/
1018 {
1019 const unsigned int offset =
1021 ghost_targets_data[i][2];
1022
1023 for (unsigned int c = 0,
1024 ko = ghost_indices_subset_data.first[i],
1025 ki = 0;
1026 c < ghost_targets_data[i][2];
1027 ++c)
1028 {
1030 ghost_indices_subset_data.second.size());
1031
1032 const unsigned int idx_1 =
1033 ghost_indices_subset_data.second[ko].first + ki;
1034 const unsigned int idx_2 =
1035 ghost_targets_data[i][1] + c + offset;
1036
1037 AssertIndexRange(idx_1, ghost_array.size());
1038 AssertIndexRange(idx_2, ghost_array.size());
1039
1040 if (idx_1 == idx_2)
1041 {
1042 // noting to do
1043 }
1044 else if (idx_1 < idx_2)
1045 {
1046 ghost_array[idx_1] = ghost_array[idx_2];
1047 ghost_array[idx_2] = 0.0;
1048 }
1049 else
1050 {
1052 }
1053
1054 ++ki;
1055
1056 if (ki == ghost_indices_subset_data.second[ko].second)
1057 {
1058 ko++; // increment outer counter
1059 ki = 0; // reset inner counter
1060 }
1061 }
1062 }
1063 }
1064
1065 const int ierr =
1066 MPI_Waitall(requests.size(), requests.data(), MPI_STATUSES_IGNORE);
1067 AssertThrowMPI(ierr);
1068
1069#endif
1070 }
1071
1072
1073
1074 template <typename Number>
1075 void
1077 const VectorOperation::values operation,
1078 const unsigned int communication_channel,
1079 const ArrayView<const Number> &data_this,
1080 const std::vector<ArrayView<const Number>> &data_others,
1081 const ArrayView<Number> &buffer,
1082 const ArrayView<Number> &temporary_storage,
1083 std::vector<MPI_Request> &requests) const
1084 {
1085 (void)data_this;
1086
1087#ifndef DEAL_II_WITH_MPI
1088 Assert(false, ExcNeedsMPI());
1089
1090 (void)operation;
1091 (void)communication_channel;
1092 (void)data_others;
1093 (void)buffer;
1094 (void)temporary_storage;
1095 (void)requests;
1096#else
1097 // return;
1098
1099 (void)data_others;
1100 (void)operation;
1101
1103
1104 requests.resize(sm_ghost_ranks.size() + sm_import_ranks.size() +
1105 ghost_targets_data.size() + import_targets_data.size());
1106
1107 int dummy;
1108 for (unsigned int i = 0; i < sm_ghost_ranks.size(); ++i)
1109 {
1110 const int ierr = MPI_Isend(&dummy,
1111 0,
1112 MPI_INT,
1113 sm_ghost_ranks[i],
1114 communication_channel + 1,
1115 comm_sm,
1116 requests.data() + i);
1117 AssertThrowMPI(ierr);
1118 }
1119
1120 for (unsigned int i = 0; i < sm_import_ranks.size(); ++i)
1121 {
1122 const int ierr =
1123 MPI_Irecv(&dummy,
1124 0,
1125 MPI_INT,
1126 sm_import_ranks[i],
1127 communication_channel + 1,
1128 comm_sm,
1129 requests.data() + sm_ghost_ranks.size() + i);
1130 AssertThrowMPI(ierr);
1131 }
1132
1133 for (unsigned int i = 0; i < ghost_targets_data.size(); ++i)
1134 {
1135 for (unsigned int c = 0,
1136 ko = ghost_indices_subset_data.first[i],
1137 ki = 0;
1138 c < ghost_targets_data[i][2];
1139 ++c)
1140 {
1142
1143 const unsigned int idx_1 =
1144 ghost_indices_subset_data.second[ko].first + ki;
1145 const unsigned int idx_2 = ghost_targets_data[i][1] + c;
1146
1147 AssertIndexRange(idx_1, buffer.size());
1148 AssertIndexRange(idx_2, buffer.size());
1149
1150 if (idx_1 == idx_2)
1151 {
1152 // nothing to do
1153 }
1154 else if (idx_2 < idx_1)
1155 {
1156 buffer[idx_2] = buffer[idx_1];
1157 buffer[idx_1] = 0.0;
1158 }
1159 else
1160 {
1162 }
1163
1164 if (++ki == ghost_indices_subset_data.second[ko].second)
1165 {
1166 ko++; // increment outer counter
1167 ki = 0; // reset inner counter
1168 }
1169 }
1170
1171 const int ierr = MPI_Isend(
1172 buffer.data() + ghost_targets_data[i][1],
1173 ghost_targets_data[i][2],
1174 Utilities::MPI::mpi_type_id_for_type<decltype(*buffer.data())>,
1175 ghost_targets_data[i][0],
1176 communication_channel + 0,
1177 comm,
1178 requests.data() + sm_ghost_ranks.size() + sm_import_ranks.size() +
1179 i);
1180 AssertThrowMPI(ierr);
1181 }
1182
1183 for (unsigned int i = 0; i < import_targets_data.size(); ++i)
1184 {
1185 const int ierr =
1186 MPI_Irecv(temporary_storage.data() + import_targets_data[i][1],
1187 import_targets_data[i][2],
1189 decltype(*temporary_storage.data())>,
1190 import_targets_data[i][0],
1191 communication_channel + 0,
1192 comm,
1193 requests.data() + sm_ghost_ranks.size() +
1194 sm_import_ranks.size() + ghost_targets_data.size() +
1195 i);
1196 AssertThrowMPI(ierr);
1197 }
1198#endif
1199 }
1200
1201
1202
1203 template <typename Number>
1204 void
1206 const VectorOperation::values operation,
1207 const ArrayView<Number> &data_this,
1208 const std::vector<ArrayView<const Number>> &data_others,
1209 const ArrayView<Number> &buffer,
1210 const ArrayView<const Number> &temporary_storage,
1211 std::vector<MPI_Request> &requests) const
1212 {
1213#ifndef DEAL_II_WITH_MPI
1214 Assert(false, ExcNeedsMPI());
1215
1216 (void)operation;
1217 (void)data_this;
1218 (void)data_others;
1219 (void)buffer;
1220 (void)temporary_storage;
1221 (void)requests;
1222#else
1223
1224 (void)operation;
1225
1227
1228 AssertDimension(requests.size(),
1229 sm_ghost_ranks.size() + sm_import_ranks.size() +
1230 ghost_targets_data.size() +
1231 import_targets_data.size());
1232
1233 const auto split =
1234 [&](const unsigned int i) -> std::pair<unsigned int, unsigned int> {
1236 (sm_import_ranks.size() + ghost_targets_data.size() +
1237 import_targets_data.size()));
1238
1239 if (i < sm_import_ranks.size())
1240 return {0, i};
1241 else if (i < (sm_import_ranks.size() + ghost_targets_data.size()))
1242 return {2, i - sm_import_ranks.size()};
1243 else
1244 return {1, i - sm_import_ranks.size() - ghost_targets_data.size()};
1245 };
1246
1247 for (unsigned int c = 0;
1248 c < sm_import_ranks.size() + import_targets_data.size() +
1249 ghost_targets_data.size();
1250 c++)
1251 {
1252 int i;
1253 const int ierr =
1254 MPI_Waitany(sm_import_ranks.size() + import_targets_data.size() +
1255 ghost_targets_data.size(),
1256 requests.data() + sm_ghost_ranks.size(),
1257 &i,
1258 MPI_STATUS_IGNORE);
1259 AssertThrowMPI(ierr);
1260
1261 const auto &s = split(i);
1262 i = s.second;
1263
1264 if (s.first == 0)
1265 {
1266 Number *DEAL_II_RESTRICT data_others_ptr =
1267 const_cast<Number *>(data_others[sm_import_ranks[i]].data());
1268 Number *DEAL_II_RESTRICT data_this_ptr = data_this.data();
1269
1270 for (unsigned int lo = sm_import_data_this.first[i],
1271 ko = sm_import_data.first[i],
1272 li = 0,
1273 ki = 0;
1274 (lo < sm_import_data_this.first[i + 1]) &&
1275 (ko < sm_import_data.first[i + 1]);)
1276 {
1277 for (; (li < sm_import_data_this.second[lo].second) &&
1278 (ki < sm_import_data.second[ko].second);
1279 ++li, ++ki)
1280 {
1281 data_this_ptr[sm_import_data_this.second[lo].first +
1282 li] +=
1283 data_others_ptr[sm_import_data.second[ko].first + ki];
1284 data_others_ptr[sm_import_data.second[ko].first + ki] =
1285 0.0;
1286 }
1287
1288 if (li == sm_import_data_this.second[lo].second)
1289 {
1290 lo++; // increment outer counter
1291 li = 0; // reset inner counter
1292 }
1293 if (ki == sm_import_data.second[ko].second)
1294 {
1295 ko++; // increment outer counter
1296 ki = 0; // reset inner counter
1297 }
1298 }
1299 }
1300 else if (s.first == 1)
1301 {
1302 for (unsigned int j = import_indices_data.first[i],
1303 k = import_targets_data[i][1];
1304 j < import_indices_data.first[i + 1];
1305 j++)
1306 for (unsigned int l = 0;
1307 l < import_indices_data.second[j].second;
1308 l++)
1309 data_this[import_indices_data.second[j].first + l] +=
1310 temporary_storage[k++];
1311 }
1312 else /*if (s.first == 2)*/
1313 {
1314 std::memset(buffer.data() + ghost_targets_data[i][1],
1315 0.0,
1316 (ghost_targets_data[i][2]) * sizeof(Number));
1317 }
1318 }
1319
1320 const int ierr =
1321 MPI_Waitall(requests.size(), requests.data(), MPI_STATUSES_IGNORE);
1322 AssertThrowMPI(ierr);
1323#endif
1324 }
1325
1326
1327
1328 unsigned int
1330 {
1331 return n_local_elements;
1332 }
1333
1334
1335
1336 unsigned int
1338 {
1339 return n_ghost_elements;
1340 }
1341
1342
1343
1344 unsigned int
1346 {
1347 if (import_targets_data.empty())
1348 return 0;
1349 return import_targets_data.back()[1] + import_targets_data.back()[2];
1350 }
1351
1352
1353
1354 unsigned int
1356 {
1357 return sm_import_ranks.size() + sm_ghost_ranks.size(); // TODO
1358 }
1359
1360
1361
1364 {
1365 return n_global_elements;
1366 }
1367
1368
1369
1370 MPI_Comm
1372 {
1373 return this->comm_sm;
1374 }
1375
1376
1377
1378 void
1380 {
1381 reset_ghost_values_impl(ghost_array);
1382 }
1383
1384
1385
1386 void
1388 {
1389 reset_ghost_values_impl(ghost_array);
1390 }
1391
1392
1393
1394 template <typename Number>
1395 void
1397 {
1398 // reset ghost values coming from shared-memory neighbors
1399 // TODO: only needed if values are buffered
1400 for (const auto &i : sm_export_data_this.second)
1401 std::memset(ghost_array.data() + (i.first - n_local_elements),
1402 0,
1403 sizeof(Number) * i.second);
1404
1405 // reset ghost values coming from remote neighbors
1406 for (const auto &i : ghost_indices_subset_data.second)
1407 std::memset(ghost_array.data() + i.first,
1408 0,
1409 sizeof(Number) * i.second);
1410 }
1411
1412
1413
1414 } // namespace VectorDataExchange
1415 } // namespace MatrixFreeFunctions
1416} // namespace internal
1417
1418
value_type * data() const noexcept
Definition array_view.h:661
std::size_t size() const
Definition array_view.h:684
virtual std::vector< unsigned int > run(const std::vector< unsigned int > &targets, const std::function< RequestType(const unsigned int)> &create_request, const std::function< AnswerType(const unsigned int, const RequestType &)> &answer_request, const std::function< void(const unsigned int, const AnswerType &)> &process_answer, const MPI_Comm comm) override
void export_to_ghosted_array_start(const unsigned int communication_channel, const ArrayView< const double > &locally_owned_array, const std::vector< ArrayView< const double > > &shared_arrays, const ArrayView< double > &ghost_array, const ArrayView< double > &temporary_storage, std::vector< MPI_Request > &requests) const override
void reset_ghost_values(const ArrayView< double > &ghost_array) const override
std::vector< std::array< unsigned int, 3 > > ghost_targets_data
std::pair< std::vector< unsigned int >, std::vector< std::pair< unsigned int, unsigned int > > > sm_export_data_this
void export_to_ghosted_array_finish(const ArrayView< const double > &locally_owned_array, const std::vector< ArrayView< const double > > &shared_arrays, const ArrayView< double > &ghost_array, std::vector< MPI_Request > &requests) const override
void import_from_ghosted_array_start(const VectorOperation::values vector_operation, const unsigned int communication_channel, const ArrayView< const double > &locally_owned_array, const std::vector< ArrayView< const double > > &shared_arrays, const ArrayView< double > &ghost_array, const ArrayView< double > &temporary_storage, std::vector< MPI_Request > &requests) const override
void export_to_ghosted_array_start_impl(const unsigned int communication_channel, const ArrayView< const Number > &locally_owned_array, const std::vector< ArrayView< const Number > > &shared_arrays, const ArrayView< Number > &ghost_array, const ArrayView< Number > &temporary_storage, std::vector< MPI_Request > &requests) const
void import_from_ghosted_array_finish_impl(const VectorOperation::values vector_operation, const ArrayView< Number > &locally_owned_storage, const std::vector< ArrayView< const Number > > &shared_arrays, const ArrayView< Number > &ghost_array, const ArrayView< const Number > &temporary_storage, std::vector< MPI_Request > &requests) const
std::pair< std::vector< unsigned int >, std::vector< std::pair< unsigned int, unsigned int > > > sm_export_data
std::vector< std::array< unsigned int, 3 > > import_targets_data
std::pair< std::vector< unsigned int >, std::vector< std::pair< unsigned int, unsigned int > > > ghost_indices_subset_data
void import_from_ghosted_array_finish(const VectorOperation::values vector_operation, const ArrayView< double > &locally_owned_storage, const std::vector< ArrayView< const double > > &shared_arrays, const ArrayView< double > &ghost_array, const ArrayView< const double > &temporary_storage, std::vector< MPI_Request > &requests) const override
void export_to_ghosted_array_finish_impl(const ArrayView< const Number > &locally_owned_array, const std::vector< ArrayView< const Number > > &shared_arrays, const ArrayView< Number > &ghost_array, std::vector< MPI_Request > &requests) const
std::pair< std::vector< unsigned int >, std::vector< std::pair< unsigned int, unsigned int > > > sm_import_data
void import_from_ghosted_array_start_impl(const VectorOperation::values vector_operation, const unsigned int communication_channel, const ArrayView< const Number > &locally_owned_array, const std::vector< ArrayView< const Number > > &shared_arrays, const ArrayView< Number > &ghost_array, const ArrayView< Number > &temporary_storage, std::vector< MPI_Request > &requests) const
Full(const std::shared_ptr< const Utilities::MPI::Partitioner > &partitioner, const MPI_Comm communicator_sm)
void reset_ghost_values_impl(const ArrayView< Number > &ghost_array) const
std::pair< std::vector< unsigned int >, std::vector< std::pair< unsigned int, unsigned int > > > sm_import_data_this
virtual types::global_dof_index size() const override
std::pair< std::vector< unsigned int >, std::vector< std::pair< unsigned int, unsigned int > > > import_indices_data
const std::shared_ptr< const Utilities::MPI::Partitioner > partitioner
void export_to_ghosted_array_finish(const ArrayView< const double > &locally_owned_array, const std::vector< ArrayView< const double > > &shared_arrays, const ArrayView< double > &ghost_array, std::vector< MPI_Request > &requests) const override
PartitionerWrapper(const std::shared_ptr< const Utilities::MPI::Partitioner > &partitioner)
void export_to_ghosted_array_start(const unsigned int communication_channel, const ArrayView< const double > &locally_owned_array, const std::vector< ArrayView< const double > > &shared_arrays, const ArrayView< double > &ghost_array, const ArrayView< double > &temporary_storage, std::vector< MPI_Request > &requests) const override
void reset_ghost_values_impl(const ArrayView< Number > &ghost_array) const
void import_from_ghosted_array_finish(const VectorOperation::values vector_operation, const ArrayView< double > &locally_owned_storage, const std::vector< ArrayView< const double > > &shared_arrays, const ArrayView< double > &ghost_array, const ArrayView< const double > &temporary_storage, std::vector< MPI_Request > &requests) const override
void reset_ghost_values(const ArrayView< double > &ghost_array) const override
void import_from_ghosted_array_start(const VectorOperation::values vector_operation, const unsigned int communication_channel, const ArrayView< const double > &locally_owned_array, const std::vector< ArrayView< const double > > &shared_arrays, const ArrayView< double > &ghost_array, const ArrayView< double > &temporary_storage, std::vector< MPI_Request > &requests) const override
#define DEAL_II_NAMESPACE_OPEN
Definition config.h:498
#define DEAL_II_RESTRICT
Definition config.h:110
#define DEAL_II_NAMESPACE_CLOSE
Definition config.h:499
static ::ExceptionBase & ExcNotImplemented()
static ::ExceptionBase & ExcNeedsMPI()
#define Assert(cond, exc)
#define AssertDimension(dim1, dim2)
#define AssertThrowMPI(error_code)
#define AssertIndexRange(index, range)
#define DEAL_II_NOT_IMPLEMENTED()
bool job_supports_mpi()
Definition mpi.cc:681
const MPI_Datatype mpi_type_id_for_type
Definition mpi.h:1610
std::vector< unsigned int > mpi_processes_within_communicator(const MPI_Comm comm_large, const MPI_Comm comm_small)
Definition mpi.cc:123
std::pair< std::vector< unsigned int >, std::vector< std::pair< unsigned int, unsigned int > > > compress_to_contiguous_ranges(const std::vector< unsigned int > &sm_export_ptr, const std::vector< unsigned int > &sm_export_indices)
*braid_SplitCommworld & comm