00001
00002
00003
00004
00005
00006
00007
00008
00009
00010
00011
00012
00013
00014
00015
00016
00017
00018
00019
00020
00021
00022
00023 #ifndef BL_PARALLELDESCRIPTOR_H
00024 #define BL_PARALLELDESCRIPTOR_H
00025
00026
00027
00028
00029 #include <winstd.H>
00030
00031 #if defined(BL_OLD_STL)
00032 #include <stdio.h>
00033 #include <ctype.h>
00034 #include <assert.h>
00035 #include <math.h>
00036 #else
00037 #include <cstdio>
00038 #include <cctype>
00039 #include <cassert>
00040 #include <cmath>
00041 #endif
00042
00043 #include <iostream>
00044 #include <fstream>
00045 #include <iomanip>
00046 #include <map>
00047 #include <algorithm>
00048 #include <vector>
00049 #include <stack>
00050 #include <limits>
00051
00052 #include <Thread.H>
00053
00054 #include <BLassert.H>
00055 #include <REAL.H>
00056 #include <Box.H>
00057 #include <ccse-mpi.H>
00058
00059
00060
00061 struct FabComTag
00062 {
00063 int fromProc;
00064 int toProc;
00065 int fabIndex;
00066 int fineIndex;
00067 int srcComp;
00068 int destComp;
00069 int nComp;
00070 int face;
00071 int fabArrayId;
00072 int fillBoxId;
00073 int procThatNeedsData;
00074 int procThatHasData;
00075 Box box;
00076
00077 FabComTag ();
00078 };
00079
00080
00081
00082
00083
00084
00085
00086
00087
00088
00089 struct CommData
00090 {
00091
00092
00093
00094 enum { DIM = 3*BL_SPACEDIM+7 };
00095
00096 int m_data[DIM];
00097
00098 CommData ();
00099 CommData (int face,
00100 int fabindex,
00101 int fromproc,
00102 int id,
00103 int ncomp,
00104 int srccomp,
00105 int fabarrayid,
00106 const Box& box);
00107
00108 CommData (const CommData& rhs);
00109
00110 CommData& operator= (const CommData& rhs);
00111
00112
00113
00114 bool operator== (const CommData& rhs) const;
00115
00116 bool operator!= (const CommData& rhs) const { return !operator==(rhs); }
00117
00118
00119
00120 int length () const { return DIM; }
00121
00122
00123
00124 int* dataPtr() { return &m_data[0]; }
00125
00126
00127
00128 int face () const { return m_data[0]; }
00129
00130
00131
00132 int fabindex () const { return m_data[1]; }
00133
00134
00135
00136 int fromproc () const { return m_data[2]; }
00137
00138
00139
00140
00141
00142
00143 int id () const { return m_data[3]; }
00144
00145
00146
00147 int nComp () const { return m_data[4]; }
00148
00149
00150
00151 int srcComp () const { return m_data[5]; }
00152
00153
00154
00155 int fabarrayid () const { return m_data[6]; }
00156
00157
00158
00159 Box box () const
00160 {
00161 return Box(IntVect(&m_data[7]),
00162 IntVect(&m_data[7+BL_SPACEDIM]),
00163 IntVect(&m_data[7+2*BL_SPACEDIM]));
00164 }
00165 };
00166
00167
00168
00169
00170 std::ostream& operator<< (std::ostream& os, const CommData& cd);
00171
00172 std::ostream& operator<< (std::ostream& os, const Array<CommData>& cd);
00173
00174
00175
00177
00182 namespace ParallelDescriptor
00183 {
00184 class Message
00185 {
00186 public:
00187
00188 Message ();
00189 Message (MPI_Request req_, MPI_Datatype type_);
00190 Message (MPI_Status stat_, MPI_Datatype type_);
00191 void wait ();
00192 bool test ();
00193 size_t count () const;
00194 int tag () const;
00195 int pid () const;
00196 MPI_Datatype type () const;
00197 MPI_Request req () const;
00198
00199 private:
00200
00201 bool m_finished;
00202 MPI_Datatype m_type;
00203 MPI_Request m_req;
00204 mutable MPI_Status m_stat;
00205 };
00206
00210 void StartParallel (int* argc = 0,
00211 char*** argv = 0);
00212
00216 void EndParallel ();
00217
00219
00220 int MyProc ();
00221
00223
00224 int NProcs ();
00225
00227
00228 int NProcsCFD ();
00229
00231
00232 MPI_Comm Communicator ();
00233
00235
00236 void Barrier ();
00237
00238 void Barrier (MPI_Comm comm);
00239
00241
00242 void Test (MPI_Request& request, int& flag, MPI_Status& status);
00243
00245
00246 void Comm_dup (MPI_Comm comm, MPI_Comm& newcomm);
00247
00249
00250 bool IOProcessor ();
00251
00253
00254 int IOProcessorNumber ();
00255
00257
00258 void Abort ();
00259
00261
00262 void Abort (int errorcode);
00263
00265
00266 const char* ErrorString (int errcode);
00267
00269
00270 double second ();
00271
00273
00274 void ReduceBoolAnd (bool& rvar);
00275
00277
00278 void ReduceBoolAnd (bool& rvar, int cpu);
00279
00281
00282 void ReduceBoolOr (bool& rvar);
00283
00285
00286 void ReduceBoolOr (bool& rvar, int cpu);
00287
00289
00290 void ReduceRealSum (Real& rvar);
00291
00293
00294 void ReduceRealSum (Real& rvar, int cpu);
00295
00297
00298 void ReduceRealMax (Real& rvar);
00299
00301
00302 void ReduceRealMax (Real& rvar, int cpu);
00303
00305
00306 void ReduceRealMin (Real& rvar);
00307
00309
00310 void ReduceRealMin (Real& rvar, int cpu);
00311
00313
00314 void ReduceIntSum (int& rvar);
00315
00317
00318 void ReduceIntSum (int& rvar, int cpu);
00319
00321
00322 void ReduceIntMax (int& rvar);
00323
00325
00326 void ReduceIntMax (int& rvar, int cpu);
00327
00329
00330 void ReduceIntMin (int& rvar);
00331
00333
00334 void ReduceIntMin (int& rvar, int cpu);
00335
00337
00338 void ReduceLongSum (long& rvar);
00339
00341
00342 void ReduceLongSum (long& rvar, int cpu);
00343
00345
00346 void ReduceLongMax (long& rvar);
00347
00349
00350 void ReduceLongMax (long& rvar, int cpu);
00351
00353
00354 void ReduceLongMin (long& rvar);
00355
00357
00358 void ReduceLongMin (long& rvar, int cpu);
00359
00361
00362 void ReduceLongAnd (long& rvar);
00363
00365
00366 void ReduceLongAnd (long& rvar, int cpu);
00367
00369
00370 void Gather (Real* sendbuf,
00371 int sendcount,
00372 Real* recvbuf,
00373 int root);
00374
00376
00377 int SeqNum ();
00378
00379 template <class T> Message Asend(const T*, size_t n, int pid, int tag);
00380 template <class T> Message Asend(const T*, size_t n, int pid, int tag, MPI_Comm comm);
00381 template <class T> Message Asend(const std::vector<T>& buf, int pid, int tag);
00382
00383 template <class T> Message Arecv(T*, size_t n, int pid, int tag);
00384 template <class T> Message Arecv(T*, size_t n, int pid, int tag, MPI_Comm comm);
00385 template <class T> Message Arecv(std::vector<T>& buf, int pid, int tag);
00386
00387 template <class T> Message Send(const T* buf, size_t n, int dst_pid, int tag);
00388 Message Send(const std::vector<size_t>& buf, int dst_pid, int tag);
00389
00390 template <class T> Message Recv(T*, size_t n, int pid, int tag);
00391 Message Recv(std::vector<size_t>& t, int pid, int tag);
00392 Message Recv(std::vector<char>& t, int pid, int tag);
00393
00394 template <class T> void Bcast(T*, size_t n, int root = 0);
00395
00396 template <class Op, class T> T Reduce(const T& t);
00397
00398 template <class T, class T1> void Scatter(T*, size_t n, const T1*, size_t n1, int root);
00399
00400 template <class T, class T1> void Gather(const T*, size_t n, T1*, size_t n1, int root);
00401 std::vector<size_t> Gather(const size_t&, int root);
00402
00403 void Waitsome (Array<MPI_Request>&, int&, Array<int>&, Array<MPI_Status>&);
00404
00405 void MPI_Error(const char* file, int line, const char* msg, int rc);
00406 }
00407
00408 #define BL_MPI_REQUIRE(x) \
00409 do \
00410 { \
00411 if ( int l_status_ = (x) ) \
00412 { \
00413 ParallelDescriptor::MPI_Error(__FILE__,__LINE__,#x, l_status_); \
00414 } \
00415 } \
00416 while ( false )
00417
00418 #if BL_USE_MPI
00419 template <class T>
00420 ParallelDescriptor::Message
00421 ParallelDescriptor::Asend (const T* buf,
00422 size_t n,
00423 int dst_pid,
00424 int tag)
00425 {
00426 MPI_Request req;
00427 BL_MPI_REQUIRE( MPI_Isend(const_cast<T*>(buf),
00428 n,
00429 Mpi_typemap<T>::type(),
00430 dst_pid,
00431 tag,
00432 Communicator(),
00433 &req) );
00434 return Message(req, Mpi_typemap<T>::type());
00435 }
00436
00437 template <class T>
00438 ParallelDescriptor::Message
00439 ParallelDescriptor::Asend (const T* buf,
00440 size_t n,
00441 int dst_pid,
00442 int tag,
00443 MPI_Comm comm)
00444 {
00445 MPI_Request req;
00446 BL_MPI_REQUIRE( MPI_Isend(const_cast<T*>(buf),
00447 n,
00448 Mpi_typemap<T>::type(),
00449 dst_pid,
00450 tag,
00451 comm,
00452 &req) );
00453 return Message(req, Mpi_typemap<T>::type());
00454 }
00455
00456 template <class T>
00457 ParallelDescriptor::Message
00458 ParallelDescriptor::Asend (const std::vector<T>& buf,
00459 int dst_pid,
00460 int tag)
00461 {
00462 MPI_Request req;
00463 BL_MPI_REQUIRE( MPI_Isend(const_cast<T*>(&buf[0]),
00464 buf.size(),
00465 Mpi_typemap<T>::type(),
00466 dst_pid,
00467 tag,
00468 Communicator(),
00469 &req) );
00470 return Message(req, Mpi_typemap<T>::type());
00471 }
00472
00473 template <class T>
00474 ParallelDescriptor::Message
00475 ParallelDescriptor::Send (const T* buf,
00476 size_t n,
00477 int dst_pid,
00478 int tag)
00479 {
00480 BL_MPI_REQUIRE( MPI_Send(const_cast<T*>(buf),
00481 n,
00482 Mpi_typemap<T>::type(),
00483 dst_pid,
00484 tag,
00485 Communicator()) );
00486 return Message();
00487 }
00488
00489 template <class T>
00490 ParallelDescriptor::Message
00491 ParallelDescriptor::Arecv (T* buf,
00492 size_t n,
00493 int src_pid,
00494 int tag)
00495 {
00496 MPI_Request req;
00497 BL_MPI_REQUIRE( MPI_Irecv(buf,
00498 n,
00499 Mpi_typemap<T>::type(),
00500 src_pid,
00501 tag,
00502 Communicator(),
00503 &req) );
00504 return Message(req, Mpi_typemap<T>::type());
00505 }
00506
00507 template <class T>
00508 ParallelDescriptor::Message
00509 ParallelDescriptor::Arecv (T* buf,
00510 size_t n,
00511 int src_pid,
00512 int tag,
00513 MPI_Comm comm)
00514 {
00515 MPI_Request req;
00516 BL_MPI_REQUIRE( MPI_Irecv(buf,
00517 n,
00518 Mpi_typemap<T>::type(),
00519 src_pid,
00520 tag,
00521 comm,
00522 &req) );
00523 return Message(req, Mpi_typemap<T>::type());
00524 }
00525
00526 template <class T>
00527 ParallelDescriptor::Message
00528 ParallelDescriptor::Arecv (std::vector<T>& buf,
00529 int src_pid,
00530 int tag)
00531 {
00532 MPI_Request req;
00533 BL_MPI_REQUIRE( MPI_Irecv(&buf[0],
00534 buf.size(),
00535 Mpi_typemap<T>::type(),
00536 src_pid,
00537 tag,
00538 Communicator(),
00539 &req) );
00540 return Message(req, Mpi_typemap<T>::type());
00541 }
00542
00543 template <class T>
00544 ParallelDescriptor::Message
00545 ParallelDescriptor::Recv (T* buf,
00546 size_t n,
00547 int src_pid,
00548 int tag)
00549 {
00550 MPI_Status stat;
00551 BL_MPI_REQUIRE( MPI_Recv(buf,
00552 n,
00553 Mpi_typemap<T>::type(),
00554 src_pid,
00555 tag,
00556 Communicator(),
00557 &stat) );
00558 return Message(stat, Mpi_typemap<T>::type());
00559 }
00560
00561 template <class Op, class T>
00562 T
00563 ParallelDescriptor::Reduce (const T& t)
00564 {
00565 T recv;
00566 BL_MPI_REQUIRE( MPI_Allreduce(const_cast<T*>(&t),
00567 &recv,
00568 1,
00569 Mpi_typemap<T>::type(),
00570 Op::op(),
00571 Communicator()) );
00572 return recv;
00573 }
00574
00575 template <class T>
00576 void
00577 ParallelDescriptor::Bcast (T* t,
00578 size_t n,
00579 int root)
00580 {
00581 BL_MPI_REQUIRE( MPI_Bcast(t,
00582 n,
00583 Mpi_typemap<T>::type(),
00584 root,
00585 Communicator()) );
00586 }
00587
00588 template <class T, class T1>
00589 void
00590 ParallelDescriptor::Gather (const T* t,
00591 size_t n,
00592 T1* t1,
00593 size_t n1,
00594 int root)
00595 {
00596 BL_MPI_REQUIRE( MPI_Gather(const_cast<T*>(t),
00597 n,
00598 Mpi_typemap<T>::type(),
00599 t1,
00600 n1,
00601 Mpi_typemap<T1>::type(),
00602 root,
00603 Communicator()) );
00604 }
00605
00606 template <class T, class T1>
00607 void
00608 ParallelDescriptor::Scatter (T* t,
00609 size_t n,
00610 const T1* t1,
00611 size_t n1,
00612 int root)
00613 {
00614 BL_MPI_REQUIRE( MPI_Scatter(const_cast<T1*>(t1),
00615 n1,
00616 Mpi_typemap<T1>::type(),
00617 t,
00618 n,
00619 Mpi_typemap<T>::type(),
00620 root,
00621 Communicator()) );
00622 }
00623
00624 #else
00625
00626 namespace ParallelDescriptor
00627 {
00628 template <class T>
00629 Message
00630 Asend(const T* buf, size_t n, int dst_pid, int tag)
00631 {
00632 return Message();
00633 }
00634
00635 template <class T>
00636 Message
00637 Asend(const T* buf, size_t n, int dst_pid, int tag, MPI_Comm comm)
00638 {
00639 return Message();
00640 }
00641
00642 template <class T>
00643 Message
00644 Asend(const std::vector<T>& buf, int dst_pid, int tag)
00645 {
00646 return Message();
00647 }
00648
00649 template <class T>
00650 Message
00651 Send(const T* buf, size_t n, int dst_pid, int tag)
00652 {
00653 return Message();
00654 }
00655
00656 template <class T>
00657 Message
00658 Arecv(T* buf, size_t n, int src_pid, int tag)
00659 {
00660 return Message();
00661 }
00662
00663 template <class T>
00664 Message
00665 Arecv(T* buf, size_t n, int src_pid, int tag, MPI_Comm comm)
00666 {
00667 return Message();
00668 }
00669
00670 template <class T>
00671 Message
00672 Arecv(std::vector<T>& buf, int src_pid, int tag)
00673 {
00674 return Message();
00675 }
00676
00677 template <class T>
00678 Message
00679 Recv(T* buf, size_t n, int src_pid, int tag)
00680 {
00681 return Message();
00682 }
00683
00684 template <class Op, class T>
00685 T
00686 Reduce(const T& t)
00687 {
00688 return t;
00689 }
00690
00691 template <class T>
00692 void
00693 Bcast(T* t, size_t n, int root)
00694 {}
00695
00696 template <class T, class T1>
00697 void
00698 Gather(const T* t, size_t n, T1* t1, size_t n1, int root)
00699 {}
00700
00701 template <class T, class T1>
00702 void
00703 Scatter(T* t, size_t n, const T1* t1, size_t n1, int root)
00704 {}
00705
00706 }
00707 #endif
00708
00709 #endif
