16 #ifndef dealii_work_stream_h
17 # define dealii_work_stream_h
30 # ifdef DEAL_II_WITH_TBB
32 # ifdef DEAL_II_TBB_WITH_ONEAPI
33 # include <tbb/parallel_pipeline.h>
35 # include <tbb/pipeline.h>
40 # include <functional>
170 # ifdef DEAL_II_WITH_TBB
185 namespace tbb_no_coloring
190 template <
typename Iterator,
typename ScratchData,
typename CopyData>
326 const Iterator &
end,
327 const unsigned int buffer_size,
330 const CopyData & sample_copy_data)
345 item.copy_datas.resize(
chunk_size, sample_copy_data);
346 item.currently_in_use =
false;
372 for (
unsigned int i = 0; i <
item_buffer.size(); ++i)
379 Assert(current_item !=
nullptr,
380 ExcMessage(
"This can't be. There must be a free item!"));
468 template <
typename Worker,
471 typename ScratchData,
478 const ScratchData & sample_scratch_data,
479 const CopyData & sample_copy_data,
480 const unsigned int queue_length,
495 iterator_range_to_item_stream(
begin,
501 auto tbb_item_stream_filter = tbb::make_filter<void, ItemType *>(
502 # ifdef DEAL_II_TBB_WITH_ONEAPI
503 tbb::filter_mode::serial_in_order,
507 [&](tbb::flow_control &fc) -> ItemType * {
508 if (
const auto item = iterator_range_to_item_stream.
get_item())
522 auto tbb_worker_filter = tbb::make_filter<ItemType *, ItemType *>(
523 # ifdef DEAL_II_TBB_WITH_ONEAPI
524 tbb::filter_mode::parallel,
526 tbb::filter::parallel,
529 std::function<
void(
const Iterator &, ScratchData &, CopyData &)>(
532 static_cast<bool>(std::function<
void(
const CopyData &)>(copier))](
533 ItemType *current_item) {
545 ScratchData *scratch_data =
nullptr;
549 for (
auto &p : current_item->scratch_data->get())
550 if (p.currently_in_use ==
false)
552 scratch_data = p.scratch_data.get();
553 p.currently_in_use = true;
559 if (scratch_data ==
nullptr)
562 new ScratchData(*current_item->sample_scratch_data);
563 current_item->scratch_data->get().emplace_back(scratch_data,
572 for (
unsigned int i = 0; i < current_item->n_iterators; ++i)
577 worker(current_item->iterators[i],
579 current_item->copy_datas[i]);
581 catch (
const std::exception &exc)
594 for (
auto &p : current_item->scratch_data->get())
595 if (p.scratch_data.get() == scratch_data)
598 p.currently_in_use =
false;
604 if (copier_exists ==
false)
605 current_item->currently_in_use =
false;
619 auto tbb_copier_filter = tbb::make_filter<ItemType *, void>(
620 # ifdef DEAL_II_TBB_WITH_ONEAPI
621 tbb::filter_mode::serial_in_order,
625 [copier = std::function<
void(
const CopyData &)>(copier)](
626 ItemType *current_item) {
632 for (
unsigned int i = 0; i < current_item->n_iterators; ++i)
636 copier(current_item->copy_datas[i]);
638 catch (
const std::exception &exc)
649 current_item->currently_in_use =
false;
657 tbb::parallel_pipeline(queue_length,
658 tbb_item_stream_filter & tbb_worker_filter &
677 template <
typename Worker,
680 typename ScratchData,
687 const ScratchData & sample_scratch_data,
688 const CopyData & sample_copy_data)
691 ScratchData scratch_data = sample_scratch_data;
692 CopyData copy_data = sample_copy_data;
696 const bool have_worker =
697 (
static_cast<const std::function<
698 void(
const Iterator &, ScratchData &, CopyData &)
> &>(worker)) !=
700 const bool have_copier =
701 (
static_cast<const std::function<
void(
const CopyData &)
> &>(
705 for (Iterator i =
begin; i !=
end; ++i)
708 worker(i, scratch_data, copy_data);
719 template <
typename Worker,
722 typename ScratchData,
725 run(
const std::vector<std::vector<Iterator>> &colored_iterators,
728 const ScratchData & sample_scratch_data,
729 const CopyData & sample_copy_data)
732 ScratchData scratch_data = sample_scratch_data;
733 CopyData copy_data = sample_copy_data;
737 const bool have_worker =
738 (
static_cast<const std::function<
739 void(
const Iterator &, ScratchData &, CopyData &)
> &>(worker)) !=
741 const bool have_copier =
742 (
static_cast<const std::function<
void(
const CopyData &)
> &>(
746 for (
unsigned int color = 0; color < colored_iterators.size(); ++color)
747 if (colored_iterators[color].size() > 0)
748 for (
auto &it : colored_iterators[color])
751 worker(it, scratch_data, copy_data);
761 # ifdef DEAL_II_WITH_TBB
769 namespace tbb_colored
776 template <
typename Iterator,
typename ScratchData,
typename CopyData>
791 std::unique_ptr<CopyData> && q,
813 template <
typename Iterator,
typename ScratchData,
typename CopyData>
821 const std::function<
void(
const Iterator &, ScratchData &, CopyData &)>
823 const std::function<
void(
const CopyData &)> &
copier,
839 typename std::vector<Iterator>::const_iterator> &range)
851 ScratchData *scratch_data =
nullptr;
852 CopyData * copy_data =
nullptr;
858 for (
typename ScratchAndCopyDataList::iterator p =
859 scratch_and_copy_data_list.begin();
860 p != scratch_and_copy_data_list.end();
862 if (p->currently_in_use ==
false)
864 scratch_data = p->scratch_data.get();
865 copy_data = p->copy_data.get();
866 p->currently_in_use =
true;
872 if (scratch_data ==
nullptr)
876 scratch_and_copy_data_list.emplace_back(
881 scratch_and_copy_data_list.back().scratch_data.get();
882 copy_data = scratch_and_copy_data_list.back().copy_data.get();
888 for (
typename std::vector<Iterator>::const_iterator p = range.begin();
895 worker(*p, *scratch_data, *copy_data);
899 catch (
const std::exception &exc)
915 for (
typename ScratchAndCopyDataList::iterator p =
916 scratch_and_copy_data_list.begin();
917 p != scratch_and_copy_data_list.end();
919 if (p->scratch_data.get() == scratch_data)
922 p->currently_in_use =
false;
929 ScratchAndCopyDataObjects<Iterator, ScratchData, CopyData>;
943 const std::function<void(
const Iterator &, ScratchData &, CopyData &)>
950 const std::function<void(
const CopyData &)>
copier;
962 template <
typename Worker,
965 typename ScratchData,
968 run(
const std::vector<std::vector<Iterator>> &colored_iterators,
971 const ScratchData & sample_scratch_data,
972 const CopyData & sample_copy_data,
976 for (
unsigned int color = 0; color < colored_iterators.size(); ++color)
977 if (colored_iterators[color].size() > 0)
980 WorkerAndCopier<Iterator, ScratchData, CopyData>;
988 colored_iterators[color].
begin(),
989 colored_iterators[color].
end(),
990 [&worker_and_copier](
991 const tbb::blocked_range<
992 typename std::vector<Iterator>::const_iterator> &range) {
993 worker_and_copier(range);
1054 template <
typename Worker,
1057 typename ScratchData,
1060 run(
const std::vector<std::vector<Iterator>> &colored_iterators,
1063 const ScratchData & sample_scratch_data,
1064 const CopyData & sample_copy_data,
1118 template <
typename Worker,
1121 typename ScratchData,
1128 const ScratchData & sample_scratch_data,
1129 const CopyData & sample_copy_data,
1134 ExcMessage(
"The queue length must be at least one, and preferably "
1135 "larger than the number of processors on this system."));
1147 # ifdef DEAL_II_WITH_TBB
1148 if (
static_cast<const std::function<
void(
const CopyData &)
> &>(copier))
1155 sample_scratch_data,
1176 std::vector<std::vector<Iterator>> all_iterators(1);
1177 for (Iterator p =
begin; p !=
end; ++p)
1178 all_iterators[0].push_back(p);
1183 sample_scratch_data,
1196 begin,
end, worker, copier, sample_scratch_data, sample_copy_data);
1208 template <
typename Worker,
1210 typename IteratorRangeType,
1211 typename ScratchData,
1213 typename = std::enable_if_t<has_begin_and_end<IteratorRangeType>>>
1215 run(IteratorRangeType iterator_range,
1218 const ScratchData &sample_scratch_data,
1219 const CopyData & sample_copy_data,
1224 run(iterator_range.begin(),
1225 iterator_range.end(),
1228 sample_scratch_data,
1239 template <
typename Worker,
1242 typename ScratchData,
1248 const ScratchData & sample_scratch_data,
1249 const CopyData & sample_copy_data,
1255 iterator_range.
end(),
1258 sample_scratch_data,
1266 template <
typename Worker,
1269 typename ScratchData,
1272 run(
const std::vector<std::vector<Iterator>> &colored_iterators,
1275 const ScratchData & sample_scratch_data,
1276 const CopyData & sample_copy_data,
1277 const unsigned int queue_length,
1281 ExcMessage(
"The queue length must be at least one, and preferably "
1282 "larger than the number of processors on this system."));
1290 # ifdef DEAL_II_WITH_TBB
1294 sample_scratch_data,
1308 sample_scratch_data,
1356 template <
typename MainClass,
1358 typename ScratchData,
1363 MainClass & main_object,
1364 void (MainClass::*worker)(
const Iterator &, ScratchData &, CopyData &),
1365 void (MainClass::*copier)(
const CopyData &),
1366 const ScratchData &sample_scratch_data,
1367 const CopyData & sample_copy_data,
1375 [&main_object, worker](
const Iterator &iterator,
1376 ScratchData & scratch_data,
1377 CopyData & copy_data) {
1378 (main_object.*worker)(iterator, scratch_data, copy_data);
1380 [&main_object, copier](
const CopyData ©_data) {
1381 (main_object.*copier)(copy_data);
1383 sample_scratch_data,
1390 template <
typename MainClass,
1392 typename ScratchData,
1397 MainClass &main_object,
1398 void (MainClass::*worker)(
const Iterator &, ScratchData &, CopyData &),
1399 void (MainClass::*copier)(
const CopyData &),
1400 const ScratchData &sample_scratch_data,
1401 const CopyData & sample_copy_data,
1409 [&main_object, worker](
const Iterator &iterator,
1410 ScratchData & scratch_data,
1411 CopyData & copy_data) {
1412 (main_object.*worker)(iterator, scratch_data, copy_data);
1414 [&main_object, copier](
const CopyData ©_data) {
1415 (main_object.*copier)(copy_data);
1417 sample_scratch_data,
1432 template <
typename MainClass,
1433 typename IteratorRangeType,
1434 typename ScratchData,
1436 typename = std::enable_if_t<has_begin_and_end<IteratorRangeType>>>
1438 run(IteratorRangeType iterator_range,
1439 MainClass & main_object,
1440 void (MainClass::*worker)(
1444 void (MainClass::*copier)(
const CopyData &),
1445 const ScratchData &sample_scratch_data,
1446 const CopyData & sample_copy_data,
1456 sample_scratch_data,
1467 template <
typename MainClass,
1469 typename ScratchData,
1473 MainClass & main_object,
1474 void (MainClass::*worker)(
const Iterator &, ScratchData &, CopyData &),
1475 void (MainClass::*copier)(
const CopyData &),
1476 const ScratchData &sample_scratch_data,
1477 const CopyData & sample_copy_data,
1487 sample_scratch_data,
IteratorOverIterators end() const
IteratorOverIterators begin()
static unsigned int n_threads()
std::list< ScratchAndCopyDataObjects > ScratchAndCopyDataList
WorkerAndCopier(const std::function< void(const Iterator &, ScratchData &, CopyData &)> &worker, const std::function< void(const CopyData &)> &copier, const ScratchData &sample_scratch_data, const CopyData &sample_copy_data)
Threads::ThreadLocalStorage< ScratchAndCopyDataList > data
void operator()(const tbb::blocked_range< typename std::vector< Iterator >::const_iterator > &range)
const std::function< void(const Iterator &, ScratchData &, CopyData &)> worker
typename tbb_colored::ScratchAndCopyDataObjects< Iterator, ScratchData, CopyData > ScratchAndCopyDataObjects
const CopyData & sample_copy_data
const ScratchData & sample_scratch_data
const std::function< void(const CopyData &)> copier
IteratorRangeToItemStream(const Iterator &begin, const Iterator &end, const unsigned int buffer_size, const unsigned int chunk_size, const ScratchData &sample_scratch_data, const CopyData &sample_copy_data)
Threads::ThreadLocalStorage< typename ItemType::ScratchDataList > thread_local_scratch
std::vector< ItemType > item_buffer
const unsigned int chunk_size
std::pair< Iterator, Iterator > remaining_iterator_range
const ScratchData & sample_scratch_data
#define DEAL_II_NAMESPACE_OPEN
#define DEAL_II_DISABLE_EXTRA_DIAGNOSTICS
#define DEAL_II_NAMESPACE_CLOSE
#define DEAL_II_ENABLE_EXTRA_DIAGNOSTICS
static ::ExceptionBase & ExcInternalError()
#define Assert(cond, exc)
static ::ExceptionBase & ExcMessage(std::string arg1)
void handle_unknown_exception()
void handle_std_exception(const std::exception &exc)
VectorType::value_type * begin(VectorType &V)
VectorType::value_type * end(VectorType &V)
std::vector< unsigned int > serial(const std::vector< unsigned int > &targets, const std::function< RequestType(const unsigned int)> &create_request, const std::function< AnswerType(const unsigned int, const RequestType &)> &answer_request, const std::function< void(const unsigned int, const AnswerType &)> &process_answer, const MPI_Comm &comm)
void run(const Iterator &begin, const typename identity< Iterator >::type &end, Worker worker, Copier copier, const ScratchData &sample_scratch_data, const CopyData &sample_copy_data)
void run(const std::vector< std::vector< Iterator >> &colored_iterators, Worker worker, Copier copier, const ScratchData &sample_scratch_data, const CopyData &sample_copy_data, const unsigned int chunk_size)
void run(const Iterator &begin, const typename identity< Iterator >::type &end, Worker worker, Copier copier, const ScratchData &sample_scratch_data, const CopyData &sample_copy_data, const unsigned int queue_length, const unsigned int chunk_size)
void run(const std::vector< std::vector< Iterator >> &colored_iterators, Worker worker, Copier copier, const ScratchData &sample_scratch_data, const CopyData &sample_copy_data, const unsigned int queue_length=2 *MultithreadInfo::n_threads(), const unsigned int chunk_size=8)
void parallel_for(Iterator x_begin, Iterator x_end, const Functor &functor, const unsigned int grainsize)
ScratchAndCopyDataObjects(const ScratchAndCopyDataObjects &)
ScratchAndCopyDataObjects(std::unique_ptr< ScratchData > &&p, std::unique_ptr< CopyData > &&q, const bool in_use)
std::unique_ptr< ScratchData > scratch_data
ScratchAndCopyDataObjects()
std::unique_ptr< CopyData > copy_data
std::unique_ptr< ScratchData > scratch_data
ScratchDataObject(ScratchDataObject &&o) noexcept=default
ScratchDataObject(const ScratchDataObject &)
ScratchDataObject(ScratchData *p, const bool in_use)
const ScratchData * sample_scratch_data
std::list< ScratchDataObject > ScratchDataList
std::vector< Iterator > iterators
std::vector< CopyData > copy_datas
Threads::ThreadLocalStorage< ScratchDataList > * scratch_data