Reference documentation for deal.II version GIT c415589cf0 2022-08-14 18:50:02+00:00
\(\newcommand{\dealvcentcolon}{\mathrel{\mathop{:}}}\) \(\newcommand{\dealcoloneq}{\dealvcentcolon\mathrel{\mkern-1.2mu}=}\) \(\newcommand{\jump}[1]{\left[\!\left[ #1 \right]\!\right]}\) \(\newcommand{\average}[1]{\left\{\!\left\{ #1 \right\}\!\right\}}\)
mpi_large_count.h
Go to the documentation of this file.
1 // ---------------------------------------------------------------------
2 //
3 // Copyright (C) 2022 - 2022 by the deal.II authors
4 //
5 // This file is part of the deal.II library.
6 //
7 // The deal.II library is free software; you can use it, redistribute
8 // it, and/or modify it under the terms of the GNU Lesser General
9 // Public License as published by the Free Software Foundation; either
10 // version 2.1 of the License, or (at your option) any later version.
11 // The full text of the license can be found in the file LICENSE.md at
12 // the top level directory of deal.II.
13 //
14 // ---------------------------------------------------------------------
15 
16 //
17 // The content of this file is a slightly modified version of the
18 // header-only library BigMPICompat - a tiny MPI 4.x compatibility
19 // library released under MIT license at
20 // https://github.com/tjhei/BigMPICompat and relicensed and included
21 // here with permission.
22 //
23 
24 #ifndef BIG_MPI_COMPAT_H
25 #define BIG_MPI_COMPAT_H
26 
27 #include <deal.II/base/config.h>
28 
29 #ifdef DEAL_II_WITH_MPI
30 # include <mpi.h>
31 // required for std::numeric_limits used below.
32 # include <limits>
33 # ifndef MPI_VERSION
34 # error "Your MPI implementation does not define MPI_VERSION!"
35 # endif
36 
37 # if MPI_VERSION < 3
38 # error "BigMPICompat requires at least MPI 3.0"
39 # endif
40 
42 
43 namespace Utilities
44 {
45  namespace MPI
46  {
52  namespace LargeCount
53  {
58  static constexpr MPI_Count mpi_max_int_count =
60 
66  inline int
67  Type_contiguous_c(MPI_Count count,
68  MPI_Datatype oldtype,
69  MPI_Datatype *newtype)
70  {
71 # if MPI_VERSION >= 4
72  return MPI_Type_contiguous_c(count, oldtype, newtype);
73 # else
74  if (count <= LargeCount::mpi_max_int_count)
75  return MPI_Type_contiguous(count, oldtype, newtype);
76  else
77  {
78  int ierr;
79  const MPI_Count max_signed_int = LargeCount::mpi_max_int_count;
80 
81  MPI_Count size_old;
82  ierr = MPI_Type_size_x(oldtype, &size_old);
83 
84  MPI_Count n_chunks = count / max_signed_int;
85  MPI_Count n_remaining_elements = count % max_signed_int;
86 
87  MPI_Datatype chunks;
88  ierr = MPI_Type_vector(
89  n_chunks, max_signed_int, max_signed_int, oldtype, &chunks);
90  if (ierr != MPI_SUCCESS)
91  return ierr;
92 
93  MPI_Datatype remainder;
94  ierr =
95  MPI_Type_contiguous(n_remaining_elements, oldtype, &remainder);
96  if (ierr != MPI_SUCCESS)
97  return ierr;
98 
99  int blocklengths[2] = {1, 1};
100  MPI_Aint displacements[2] = {0,
101  static_cast<MPI_Aint>(n_chunks) *
102  size_old * max_signed_int};
103  MPI_Datatype types[2] = {chunks, remainder};
104  ierr = MPI_Type_create_struct(
105  2, blocklengths, displacements, types, newtype);
106  if (ierr != MPI_SUCCESS)
107  return ierr;
108 
109  ierr = MPI_Type_commit(newtype);
110  if (ierr != MPI_SUCCESS)
111  return ierr;
112 
113  ierr = MPI_Type_free(&chunks);
114  if (ierr != MPI_SUCCESS)
115  return ierr;
116 
117  ierr = MPI_Type_free(&remainder);
118  if (ierr != MPI_SUCCESS)
119  return ierr;
120 
121 # ifndef MPI_COMPAT_SKIP_SIZE_CHECK
122  MPI_Count size_new;
123  ierr = MPI_Type_size_x(*newtype, &size_new);
124  if (ierr != MPI_SUCCESS)
125  return ierr;
126 
127  if (size_old * count != size_new)
128  {
129  // This error can happen when you are using a very old and
130  // buggy MPI implementation. There is nothing we can do
131  // here, unfortunately. Please update your installation.
132  // std::cerr
133  //<< "MPI_Type_contiguous_c() produced an invalid result.
134  // Expected =
135  //"
136  //<< size_old << " * " << count << " = " << size_old * count
137  //<< " but received " << size_new << std::endl;
138  return MPI_ERR_INTERN;
139  }
140 # endif
141 
142  return MPI_SUCCESS;
143  }
144 # endif
145  }
146 
147 
153  inline int
154  Send_c(const void * buf,
155  MPI_Count count,
156  MPI_Datatype datatype,
157  int dest,
158  int tag,
159  MPI_Comm comm)
160  {
161 # if MPI_VERSION >= 4
162  return MPI_Send_c(buf, count, datatype, dest, tag, comm);
163 # else
164  if (count <= LargeCount::mpi_max_int_count)
165  return MPI_Send(buf, count, datatype, dest, tag, comm);
166 
167  MPI_Datatype bigtype;
168  int ierr;
169  ierr = Type_contiguous_c(count, datatype, &bigtype);
170  if (ierr != MPI_SUCCESS)
171  return ierr;
172  ierr = MPI_Type_commit(&bigtype);
173  if (ierr != MPI_SUCCESS)
174  return ierr;
175 
176  ierr = MPI_Send(buf, 1, bigtype, dest, tag, comm);
177  if (ierr != MPI_SUCCESS)
178  return ierr;
179 
180  ierr = MPI_Type_free(&bigtype);
181  if (ierr != MPI_SUCCESS)
182  return ierr;
183  return MPI_SUCCESS;
184 # endif
185  }
186 
192  inline int
193  Recv_c(void * buf,
194  MPI_Count count,
195  MPI_Datatype datatype,
196  int source,
197  int tag,
198  MPI_Comm comm,
199  MPI_Status * status)
200  {
201 # if MPI_VERSION >= 4
202  return MPI_Recv_c(buf, count, datatype, source, tag, comm, status);
203 # else
204  if (count <= LargeCount::mpi_max_int_count)
205  return MPI_Recv(buf, count, datatype, source, tag, comm, status);
206 
207  MPI_Datatype bigtype;
208  int ierr;
209  ierr = Type_contiguous_c(count, datatype, &bigtype);
210  if (ierr != MPI_SUCCESS)
211  return ierr;
212 
213  ierr = MPI_Type_commit(&bigtype);
214  if (ierr != MPI_SUCCESS)
215  return ierr;
216 
217  ierr = MPI_Recv(buf, 1, bigtype, source, tag, comm, status);
218  if (ierr != MPI_SUCCESS)
219  return ierr;
220 
221  ierr = MPI_Type_free(&bigtype);
222  if (ierr != MPI_SUCCESS)
223  return ierr;
224  return MPI_SUCCESS;
225 # endif
226  }
227 
234  inline int
235  Bcast_c(void * buf,
236  MPI_Count count,
237  MPI_Datatype datatype,
238  unsigned int root_mpi_rank,
239  MPI_Comm comm)
240  {
241 # if MPI_VERSION >= 4
242  return MPI_Bcast_c(buf, count, datatype, root_mpi_rank, comm);
243 # else
244  if (count <= LargeCount::mpi_max_int_count)
245  return MPI_Bcast(buf, count, datatype, root_mpi_rank, comm);
246 
247  MPI_Datatype bigtype;
248  int ierr;
249  ierr = Type_contiguous_c(count, datatype, &bigtype);
250  if (ierr != MPI_SUCCESS)
251  return ierr;
252  ierr = MPI_Type_commit(&bigtype);
253  if (ierr != MPI_SUCCESS)
254  return ierr;
255 
256  ierr = MPI_Bcast(buf, 1, bigtype, root_mpi_rank, comm);
257  if (ierr != MPI_SUCCESS)
258  return ierr;
259 
260  ierr = MPI_Type_free(&bigtype);
261  if (ierr != MPI_SUCCESS)
262  return ierr;
263  return MPI_SUCCESS;
264 # endif
265  }
266 
272  inline int
273  File_write_at_c(MPI_File fh,
274  MPI_Offset offset,
275  const void * buf,
276  MPI_Count count,
277  MPI_Datatype datatype,
278  MPI_Status * status)
279  {
280  if (count <= LargeCount::mpi_max_int_count)
281  return MPI_File_write_at(fh, offset, buf, count, datatype, status);
282 
283  MPI_Datatype bigtype;
284  int ierr;
285  ierr = Type_contiguous_c(count, datatype, &bigtype);
286  if (ierr != MPI_SUCCESS)
287  return ierr;
288  ierr = MPI_Type_commit(&bigtype);
289  if (ierr != MPI_SUCCESS)
290  return ierr;
291 
292  ierr = MPI_File_write_at(fh, offset, buf, 1, bigtype, status);
293  if (ierr != MPI_SUCCESS)
294  return ierr;
295 
296  ierr = MPI_Type_free(&bigtype);
297  if (ierr != MPI_SUCCESS)
298  return ierr;
299  return MPI_SUCCESS;
300  }
301 
308  inline int
309  File_write_at_all_c(MPI_File fh,
310  MPI_Offset offset,
311  const void * buf,
312  MPI_Count count,
313  MPI_Datatype datatype,
314  MPI_Status * status)
315  {
316  if (count <= LargeCount::mpi_max_int_count)
317  return MPI_File_write_at_all(
318  fh, offset, buf, count, datatype, status);
319 
320  MPI_Datatype bigtype;
321  int ierr;
322  ierr = Type_contiguous_c(count, datatype, &bigtype);
323  if (ierr != MPI_SUCCESS)
324  return ierr;
325  ierr = MPI_Type_commit(&bigtype);
326  if (ierr != MPI_SUCCESS)
327  return ierr;
328 
329  ierr = MPI_File_write_at_all(fh, offset, buf, 1, bigtype, status);
330  if (ierr != MPI_SUCCESS)
331  return ierr;
332 
333  ierr = MPI_Type_free(&bigtype);
334  if (ierr != MPI_SUCCESS)
335  return ierr;
336  return MPI_SUCCESS;
337  }
338 
344  inline int
345  File_write_ordered_c(MPI_File fh,
346  const void * buf,
347  MPI_Count count,
348  MPI_Datatype datatype,
349  MPI_Status * status)
350  {
351  if (count <= LargeCount::mpi_max_int_count)
352  return MPI_File_write_ordered(fh, buf, count, datatype, status);
353 
354  MPI_Datatype bigtype;
355  int ierr;
356  ierr = Type_contiguous_c(count, datatype, &bigtype);
357  if (ierr != MPI_SUCCESS)
358  return ierr;
359  ierr = MPI_Type_commit(&bigtype);
360  if (ierr != MPI_SUCCESS)
361  return ierr;
362 
363  ierr = MPI_File_write_ordered(fh, buf, 1, bigtype, status);
364  if (ierr != MPI_SUCCESS)
365  return ierr;
366 
367  ierr = MPI_Type_free(&bigtype);
368  if (ierr != MPI_SUCCESS)
369  return ierr;
370  return MPI_SUCCESS;
371  }
372 
379  inline int
380  File_read_at_c(MPI_File fh,
381  MPI_Offset offset,
382  void * buf,
383  MPI_Count count,
384  MPI_Datatype datatype,
385  MPI_Status * status)
386  {
387  if (count <= LargeCount::mpi_max_int_count)
388  return MPI_File_read_at(fh, offset, buf, count, datatype, status);
389 
390  MPI_Datatype bigtype;
391  int ierr;
392  ierr = Type_contiguous_c(count, datatype, &bigtype);
393  if (ierr != MPI_SUCCESS)
394  return ierr;
395  ierr = MPI_Type_commit(&bigtype);
396  if (ierr != MPI_SUCCESS)
397  return ierr;
398 
399  ierr = MPI_File_read_at(fh, offset, buf, 1, bigtype, status);
400  if (ierr != MPI_SUCCESS)
401  return ierr;
402 
403  ierr = MPI_Type_free(&bigtype);
404  if (ierr != MPI_SUCCESS)
405  return ierr;
406  return MPI_SUCCESS;
407  }
408 
415  inline int
416  File_read_at_all_c(MPI_File fh,
417  MPI_Offset offset,
418  void * buf,
419  MPI_Count count,
420  MPI_Datatype datatype,
421  MPI_Status * status)
422  {
423  if (count <= LargeCount::mpi_max_int_count)
424  return MPI_File_read_at_all(fh, offset, buf, count, datatype, status);
425 
426  MPI_Datatype bigtype;
427  int ierr;
428  ierr = Type_contiguous_c(count, datatype, &bigtype);
429  if (ierr != MPI_SUCCESS)
430  return ierr;
431  ierr = MPI_Type_commit(&bigtype);
432  if (ierr != MPI_SUCCESS)
433  return ierr;
434 
435  ierr = MPI_File_read_at_all(fh, offset, buf, 1, bigtype, status);
436  if (ierr != MPI_SUCCESS)
437  return ierr;
438 
439  ierr = MPI_Type_free(&bigtype);
440  if (ierr != MPI_SUCCESS)
441  return ierr;
442  return MPI_SUCCESS;
443  }
444 
445  } // namespace LargeCount
446  } // namespace MPI
447 } // namespace Utilities
448 
450 
451 #endif
452 #endif
#define DEAL_II_NAMESPACE_OPEN
Definition: config.h:442
#define DEAL_II_NAMESPACE_CLOSE
Definition: config.h:443
int File_write_at_c(MPI_File fh, MPI_Offset offset, const void *buf, MPI_Count count, MPI_Datatype datatype, MPI_Status *status)
int File_write_ordered_c(MPI_File fh, const void *buf, MPI_Count count, MPI_Datatype datatype, MPI_Status *status)
int File_write_at_all_c(MPI_File fh, MPI_Offset offset, const void *buf, MPI_Count count, MPI_Datatype datatype, MPI_Status *status)
int Recv_c(void *buf, MPI_Count count, MPI_Datatype datatype, int source, int tag, MPI_Comm comm, MPI_Status *status)
int File_read_at_c(MPI_File fh, MPI_Offset offset, void *buf, MPI_Count count, MPI_Datatype datatype, MPI_Status *status)
int Type_contiguous_c(MPI_Count count, MPI_Datatype oldtype, MPI_Datatype *newtype)
int Bcast_c(void *buf, MPI_Count count, MPI_Datatype datatype, unsigned int root_mpi_rank, MPI_Comm comm)
int Send_c(const void *buf, MPI_Count count, MPI_Datatype datatype, int dest, int tag, MPI_Comm comm)
int File_read_at_all_c(MPI_File fh, MPI_Offset offset, void *buf, MPI_Count count, MPI_Datatype datatype, MPI_Status *status)
static constexpr MPI_Count mpi_max_int_count
Definition: types.h:32
const MPI_Comm & comm