exaFOAM/foam-extend4.1-coherent-io
Archived
1
1
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

ESI Comms optimisation: send to neighbours in reverse order

Browse source
This commit is contained in:
Hrvoje Jasak 2016-09-19 15:42:09 +01:00
parent e73480637b
commit 358c854945
4 changed files with 60 additions and 24 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

49
src/foam/db/IOstreams/Pstreams/Pstream.C
View file

@ -67,22 +67,43 @@ const Foam::NamedEnum<Foam::Pstream::commsTypes, 3>
void Foam::Pstream::setParRun(const label nProcs) void Foam::Pstream::setParRun(const label nProcs)
{ {
parRun_ = true; if (nProcs == 0)
// Redo worldComm communicator (created at static initialisation)
freeCommunicator(Pstream::worldComm);
label comm = allocateCommunicator(-1, identity(nProcs), true);
if (comm != Pstream::worldComm)
{ {
FatalErrorIn("Pstream::setParRun(const label)") parRun_ = false;
<< "problem : comm:" << comm freeCommunicator(Pstream::worldComm);
<< " Pstream::worldComm:" << Pstream::worldComm
<< Foam::exit(FatalError);
}
Pout.prefix() = '[' + name(myProcNo()) + "] "; label comm = allocateCommunicator(-1, labelList(1, label(0)), false);
Perr.prefix() = '[' + name(myProcNo()) + "] ";
if (comm != Pstream::worldComm)
{
FatalErrorIn("Pstream::setParRun(const label)")
<< "problem : comm:" << comm
<< " Pstream::worldComm:" << Pstream::worldComm
<< Foam::exit(FatalError);
}
Pout.prefix() = "";
Perr.prefix() = "";
}
else
{
parRun_ = true;
// Redo worldComm communicator (created at static initialisation)
freeCommunicator(Pstream::worldComm);
label comm = allocateCommunicator(-1, identity(nProcs), true);
if (comm != Pstream::worldComm)
{
FatalErrorIn("Pstream::setParRun(const label)")
<< "problem : comm:" << comm
<< " Pstream::worldComm:" << Pstream::worldComm
<< Foam::exit(FatalError);
}
Pout.prefix() = '[' + name(myProcNo()) + "] ";
Perr.prefix() = '[' + name(myProcNo()) + "] ";
}
} }

21
src/foam/db/IOstreams/Pstreams/combineGatherScatter.C
View file

@ -222,8 +222,11 @@ void Pstream::combineScatter
} }
} }
// Send to my downstairs neighbours // Send to my downstairs neighbours. Note reverse order (compared to
forAll (myComm.below(), belowI) // receiving). This is to make sure to send to the critical path
// (only when using a tree schedule!) first.
// This is ESI Comms optimisation, v16.06. HJ, 19/Sep/2016
forAllReverse (myComm.below(), belowI)
{ {
label belowID = myComm.below()[belowI]; label belowID = myComm.below()[belowI];
@ -461,8 +464,11 @@ void Pstream::listCombineScatter
} }
} }
// Send to my downstairs neighbours // Send to my downstairs neighbours. Note reverse order (compared to
forAll (myComm.below(), belowI) // receiving). This is to make sure to send to the critical path
// (only when using a tree schedule!) first.
// This is ESI Comms optimisation, v16.06. HJ, 19/Sep/2016
forAllReverse (myComm.below(), belowI)
{ {
label belowID = myComm.below()[belowI]; label belowID = myComm.below()[belowI];
@ -662,8 +668,11 @@ void Pstream::mapCombineScatter
} }
} }
// Send to my downstairs neighbours // Send to my downstairs neighbours. Note reverse order (compared to
forAll (myComm.below(), belowI) // receiving). This is to make sure to send to the critical path
// (only when using a tree schedule!) first.
// This is ESI Comms optimisation, v16.06. HJ, 19/Sep/2016
forAllReverse (myComm.below(), belowI)
{ {
label belowID = myComm.below()[belowI]; label belowID = myComm.below()[belowI];

7
src/foam/db/IOstreams/Pstreams/gatherScatter.C
View file

@ -183,8 +183,11 @@ void Pstream::scatter
} }
} }
// Send to my downstairs neighbours // Send to my downstairs neighbours. Note reverse order (compared to
forAll (myComm.below(), belowI) // receiving). This is to make sure to send to the critical path
// (only when using a tree schedule!) first.
// This is ESI Comms optimisation, v16.06. HJ, 19/Sep/2016
forAllReverse (myComm.below(), belowI)
{ {
if (contiguous<T>()) if (contiguous<T>())
{ {

7
src/foam/db/IOstreams/Pstreams/gatherScatterList.C
View file

@ -290,8 +290,11 @@ void Pstream::scatterList
} }
} }
// Send to my downstairs neighbours // Send to my downstairs neighbours. Note reverse order (compared to
forAll (myComm.below(), belowI) // receiving). This is to make sure to send to the critical path
// (only when using a tree schedule!) first.
// This is ESI Comms optimisation, v16.06. HJ, 19/Sep/2016
forAllReverse (myComm.below(), belowI)
{ {
label belowID = myComm.below()[belowI]; label belowID = myComm.below()[belowI];
const labelList& notBelowLeaves = comms[belowID].allNotBelow(); const labelList& notBelowLeaves = comms[belowID].allNotBelow();