ESI Comms optimisation: send to neighbours in reverse order

src/foam/db/IOstreams/Pstreams/Pstream.C

@@ -67,6 +67,26 @@ const Foam::NamedEnum<Foam::Pstream::commsTypes, 3>
 
 void Foam::Pstream::setParRun(const label nProcs)
 {
+    if (nProcs == 0)
+    {
+        parRun_ = false;
+        freeCommunicator(Pstream::worldComm);
+
+        label comm = allocateCommunicator(-1, labelList(1, label(0)), false);
+
+        if (comm != Pstream::worldComm)
+        {
+            FatalErrorIn("Pstream::setParRun(const label)")
+                << "problem : comm:" << comm
+                << "  Pstream::worldComm:" << Pstream::worldComm
+                << Foam::exit(FatalError);
+        }
+
+        Pout.prefix() = "";
+        Perr.prefix() = "";
+    }
+    else
+    {
         parRun_ = true;
 
         // Redo worldComm communicator (created at static initialisation)
@@ -83,6 +103,7 @@ void Foam::Pstream::setParRun(const label nProcs)
 
         Pout.prefix() = '[' +  name(myProcNo()) + "] ";
         Perr.prefix() = '[' +  name(myProcNo()) + "] ";
+    }
 }
 
 

src/foam/db/IOstreams/Pstreams/combineGatherScatter.C

@@ -222,8 +222,11 @@ void Pstream::combineScatter
             }
         }
 
-        // Send to my downstairs neighbours
+        // Send to my downstairs neighbours.  Note reverse order (compared to
-        forAll (myComm.below(), belowI)
+        // receiving). This is to make sure to send to the critical path
+        // (only when using a tree schedule!) first.
+        // This is ESI Comms optimisation, v16.06.  HJ, 19/Sep/2016
+        forAllReverse (myComm.below(), belowI)
         {
             label belowID = myComm.below()[belowI];
 
@@ -461,8 +464,11 @@ void Pstream::listCombineScatter
             }
         }
 
-        // Send to my downstairs neighbours
+        // Send to my downstairs neighbours.  Note reverse order (compared to
-        forAll (myComm.below(), belowI)
+        // receiving). This is to make sure to send to the critical path
+        // (only when using a tree schedule!) first.
+        // This is ESI Comms optimisation, v16.06.  HJ, 19/Sep/2016
+        forAllReverse (myComm.below(), belowI)
         {
             label belowID = myComm.below()[belowI];
 
@@ -662,8 +668,11 @@ void Pstream::mapCombineScatter
             }
         }
 
-        // Send to my downstairs neighbours
+        // Send to my downstairs neighbours.  Note reverse order (compared to
-        forAll (myComm.below(), belowI)
+        // receiving). This is to make sure to send to the critical path
+        // (only when using a tree schedule!) first.
+        // This is ESI Comms optimisation, v16.06.  HJ, 19/Sep/2016
+        forAllReverse (myComm.below(), belowI)
         {
             label belowID = myComm.below()[belowI];
 

src/foam/db/IOstreams/Pstreams/gatherScatter.C

@@ -183,8 +183,11 @@ void Pstream::scatter
             }
         }
 
-        // Send to my downstairs neighbours
+        // Send to my downstairs neighbours.  Note reverse order (compared to
-        forAll (myComm.below(), belowI)
+        // receiving). This is to make sure to send to the critical path
+        // (only when using a tree schedule!) first.
+        // This is ESI Comms optimisation, v16.06.  HJ, 19/Sep/2016
+        forAllReverse (myComm.below(), belowI)
         {
             if (contiguous<T>())
             {

src/foam/db/IOstreams/Pstreams/gatherScatterList.C

@@ -290,8 +290,11 @@ void Pstream::scatterList
             }
         }
 
-        // Send to my downstairs neighbours
+        // Send to my downstairs neighbours.  Note reverse order (compared to
-        forAll (myComm.below(), belowI)
+        // receiving). This is to make sure to send to the critical path
+        // (only when using a tree schedule!) first.
+        // This is ESI Comms optimisation, v16.06.  HJ, 19/Sep/2016
+        forAllReverse (myComm.below(), belowI)
         {
             label belowID = myComm.below()[belowI];
             const labelList& notBelowLeaves = comms[belowID].allNotBelow();