diff --git a/src/foam/Make/files b/src/foam/Make/files index d506e7a3f..f0f928e7d 100644 --- a/src/foam/Make/files +++ b/src/foam/Make/files @@ -141,6 +141,7 @@ $(StringStreams)/StringStreamsPrint.C Pstreams = $(Streams)/Pstreams $(Pstreams)/Pstream.C +$(Pstreams)/PstreamReduceOps.C $(Pstreams)/PstreamCommsStruct.C $(Pstreams)/PstreamGlobals.C $(Pstreams)/IPstream.C diff --git a/src/foam/db/IOstreams/Pstreams/Pstream.C b/src/foam/db/IOstreams/Pstreams/Pstream.C index e87e9836d..cb78615d0 100644 --- a/src/foam/db/IOstreams/Pstreams/Pstream.C +++ b/src/foam/db/IOstreams/Pstreams/Pstream.C @@ -32,21 +32,11 @@ License #include "OSspecific.H" #include "PstreamGlobals.H" #include "SubList.H" -#include "allReduce.H" #include #include #include -#if defined(WM_SP) -# define MPI_SCALAR MPI_FLOAT -#elif defined(WM_DP) -# define MPI_SCALAR MPI_DOUBLE -#elif defined(WM_LDP) -# define MPI_SCALAR MPI_LONG_DOUBLE -#endif - - // * * * * * * * * * * * * * * Static Data Members * * * * * * * * * * * * * // defineTypeNameAndDebug(Foam::Pstream, 0); @@ -548,8 +538,6 @@ Foam::label Foam::Pstream::procNo } - - // * * * * * * * * * * * * * * * Member Functions * * * * * * * * * * * * * // @@ -680,230 +668,6 @@ void Foam::Pstream::abort() } -void Foam::reduce -( - bool& Value, - const andOp& bop, - const int tag, - const label comm -) -{ - if (Pstream::warnComm != -1 && comm != Pstream::warnComm) - { - Pout<< "** reducing:" << Value << " with comm:" << comm - << " warnComm:" << Pstream::warnComm - << endl; - error::printStack(Pout); - } - - // Note: C++ bool is a type separate from C and cannot be cast - // For safety and compatibility with compilers, convert bool to int - // to comply with MPI types. HJ, 23/Sep/2016 - - int intBool = 0; - - if (Value) - { - intBool = 1; - } - - allReduce(intBool, 1, MPI_INT, MPI_LAND, bop, tag, comm); - - if (intBool > 0) - { - Value = true; - } - else - { - Value = false; - } -} - - -void Foam::reduce -( - bool& Value, - const orOp& bop, - const int tag, - const label comm -) -{ - if (Pstream::warnComm != -1 && comm != Pstream::warnComm) - { - Pout<< "** reducing:" << Value << " with comm:" << comm - << " warnComm:" << Pstream::warnComm - << endl; - error::printStack(Pout); - } - - // Note: C++ bool is a type separate from C and cannot be cast - // For safety and compatibility with compilers, convert bool to int - // to comply with MPI types. HJ, 23/Sep/2016 - - int intBool = 0; - - if (Value) - { - intBool = 1; - } - - allReduce(intBool, 1, MPI_INT, MPI_LOR, bop, tag, comm); - - if (intBool > 0) - { - Value = true; - } - else - { - Value = false; - } -} - - -void Foam::reduce -( - scalar& Value, - const minOp& bop, - const int tag, - const label comm -) -{ - if (Pstream::warnComm != -1 && comm != Pstream::warnComm) - { - Pout<< "** reducing:" << Value << " with comm:" << comm - << " warnComm:" << Pstream::warnComm - << endl; - error::printStack(Pout); - } - allReduce(Value, 1, MPI_SCALAR, MPI_MIN, bop, tag, comm); -} - - -void Foam::reduce -( - scalar& Value, - const maxOp& bop, - const int tag, - const label comm -) -{ - if (Pstream::warnComm != -1 && comm != Pstream::warnComm) - { - Pout<< "** reducing:" << Value << " with comm:" << comm - << " warnComm:" << Pstream::warnComm - << endl; - error::printStack(Pout); - } - allReduce(Value, 1, MPI_SCALAR, MPI_MAX, bop, tag, comm); -} - - -void Foam::reduce -( - scalar& Value, - const sumOp& bop, - const int tag, - const label comm -) -{ - if (Pstream::warnComm != -1 && comm != Pstream::warnComm) - { - Pout<< "** reducing:" << Value << " with comm:" << comm - << " warnComm:" << Pstream::warnComm - << endl; - error::printStack(Pout); - } - - allReduce(Value, 1, MPI_SCALAR, MPI_SUM, bop, tag, comm); -} - - -void Foam::reduce -( - vector2D& Value, - const sumOp& bop, - const int tag, - const label comm -) -{ - if (Pstream::warnComm != -1 && comm != Pstream::warnComm) - { - Pout<< "** reducing:" << Value << " with comm:" << comm - << " warnComm:" << Pstream::warnComm - << endl; - error::printStack(Pout); - } - allReduce(Value, 2, MPI_SCALAR, MPI_SUM, bop, tag, comm); -} - - -void Foam::sumReduce -( - scalar& Value, - label& Count, - const int tag, - const label comm -) -{ - if (Pstream::warnComm != -1 && comm != Pstream::warnComm) - { - Pout<< "** reducing:" << Value << " with comm:" << comm - << " warnComm:" << Pstream::warnComm - << endl; - error::printStack(Pout); - } - vector2D twoScalars(Value, scalar(Count)); - reduce(twoScalars, sumOp(), tag, comm); - - Value = twoScalars.x(); - Count = twoScalars.y(); -} - - -void Foam::reduce -( - scalar& Value, - const sumOp& bop, - const int tag, - const label comm, - label& requestID -) -{ -#ifdef MPIX_COMM_TYPE_SHARED - // Assume mpich2 with non-blocking collectives extensions. Once mpi3 - // is available this will change. - MPI_Request request; - scalar v = Value; - MPIX_Ireduce - ( - &v, - &Value, - 1, - MPI_SCALAR, - MPI_SUM, - 0, // root - PstreamGlobals::MPICommunicators_[comm], - &request - ); - - requestID = PstreamGlobals::outstandingRequests_.size(); - PstreamGlobals::outstandingRequests_.append(request); - - if (debug) - { - Pout<< "Pstream::allocateRequest for non-blocking reduce" - << " : request:" << requestID - << endl; - } -#else - // Non-blocking not yet implemented in mpi - reduce(Value, bop, tag, comm); - - requestID = -1; -#endif -} - - Foam::label Foam::Pstream::nRequests() { return PstreamGlobals::outstandingRequests_.size(); diff --git a/src/foam/db/IOstreams/Pstreams/PstreamReduceOps.C b/src/foam/db/IOstreams/Pstreams/PstreamReduceOps.C new file mode 100644 index 000000000..1ccf5ed27 --- /dev/null +++ b/src/foam/db/IOstreams/Pstreams/PstreamReduceOps.C @@ -0,0 +1,357 @@ +/*---------------------------------------------------------------------------*\ + ========= | + \\ / F ield | foam-extend: Open Source CFD + \\ / O peration | Version: 4.0 + \\ / A nd | Web: http://www.foam-extend.org + \\/ M anipulation | For copyright notice see file Copyright +------------------------------------------------------------------------------- +License + This file is part of foam-extend. + + foam-extend is free software: you can redistribute it and/or modify it + under the terms of the GNU General Public License as published by the + Free Software Foundation, either version 3 of the License, or (at your + option) any later version. + + foam-extend is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + You should have received a copy of the GNU General Public License + along with foam-extend. If not, see . + +\*---------------------------------------------------------------------------*/ + +#include "mpi.h" + +#include "label.H" +#include "Pstream.H" +#include "PstreamReduceOps.H" +#include "allReduce.H" + +// Check type of label for use in MPI calls +#if defined(WM_INT) +# define MPI_LABEL MPI_INT +#elif defined(WM_LONG) +# define MPI_LABEL MPI_LONG +#elif defined(WM_LLONG) +# define MPI_LABEL MPI_LONG_LONG +#endif + +// Check type of scalar for use in MPI calls +#if defined(WM_SP) +# define MPI_SCALAR MPI_FLOAT +#elif defined(WM_DP) +# define MPI_SCALAR MPI_DOUBLE +#elif defined(WM_LDP) +# define MPI_SCALAR MPI_LONG_DOUBLE +#endif + + +// * * * * * * * * * * * * * * * Member Functions * * * * * * * * * * * * * // + +// Optimizing template specialisations using MPI_REDUCE +void Foam::reduce +( + bool& Value, + const andOp& bop, + const int tag, + const label comm +) +{ + if (Pstream::warnComm != -1 && comm != Pstream::warnComm) + { + Pout<< "** reducing:" << Value << " with comm:" << comm + << " warnComm:" << Pstream::warnComm + << endl; + error::printStack(Pout); + } + + // Note: C++ bool is a type separate from C and cannot be cast + // For safety and compatibility with compilers, convert bool to int + // to comply with MPI types. HJ, 23/Sep/2016 + + int intBool = 0; + + if (Value) + { + intBool = 1; + } + + allReduce(intBool, 1, MPI_INT, MPI_LAND, bop, tag, comm); + + if (intBool > 0) + { + Value = true; + } + else + { + Value = false; + } +} + + +void Foam::reduce +( + bool& Value, + const orOp& bop, + const int tag, + const label comm +) +{ + if (Pstream::warnComm != -1 && comm != Pstream::warnComm) + { + Pout<< "** reducing:" << Value << " with comm:" << comm + << " warnComm:" << Pstream::warnComm + << endl; + error::printStack(Pout); + } + + // Note: C++ bool is a type separate from C and cannot be cast + // For safety and compatibility with compilers, convert bool to int + // to comply with MPI types. HJ, 23/Sep/2016 + + int intBool = 0; + + if (Value) + { + intBool = 1; + } + + allReduce(intBool, 1, MPI_INT, MPI_LOR, bop, tag, comm); + + if (intBool > 0) + { + Value = true; + } + else + { + Value = false; + } +} + + +void Foam::reduce +( + label& Value, + const minOp