From 1460d200879b6eb63b4df17afd516314c9b6fafd Mon Sep 17 00:00:00 2001
From: Hrvoje Jasak <h.jasak@wikki.co.uk>
Date: Tue, 25 Oct 2016 11:15:33 +0100
Subject: [PATCH] Barrier messaging

---
 .../mapPolyMesh/mapDistribute/mapDistribute.C     | 15 ++++++---------
 .../mapDistribute/mapDistributeTemplates.C        |  9 ++++++---
 2 files changed, 12 insertions(+), 12 deletions(-)

diff --git a/src/foam/meshes/polyMesh/mapPolyMesh/mapDistribute/mapDistribute.C b/src/foam/meshes/polyMesh/mapPolyMesh/mapDistribute/mapDistribute.C
index e21511762..a4840f6d8 100644
--- a/src/foam/meshes/polyMesh/mapPolyMesh/mapDistribute/mapDistribute.C
+++ b/src/foam/meshes/polyMesh/mapPolyMesh/mapDistribute/mapDistribute.C
@@ -77,8 +77,7 @@ Foam::List<Foam::labelPair> Foam::mapDistribute::schedule
         {
             IPstream fromSlave
             (
-                Pstream::defaultComms(),
-//                Pstream::scheduled,
+                Pstream::scheduled,
                 slave,
                 0,
                 tag
@@ -106,8 +105,7 @@ Foam::List<Foam::labelPair> Foam::mapDistribute::schedule
         {
             OPstream toSlave
             (
-                Pstream::defaultComms(),
-//                Pstream::scheduled,
+                Pstream::scheduled,
                 slave,
                 0,
                 tag
@@ -121,8 +119,7 @@ Foam::List<Foam::labelPair> Foam::mapDistribute::schedule
         {
             OPstream toMaster
             (
-                Pstream::defaultComms(),
-//                Pstream::scheduled,
+                Pstream::scheduled,
                 Pstream::masterNo(),
                 0,
                 tag
@@ -133,8 +130,7 @@ Foam::List<Foam::labelPair> Foam::mapDistribute::schedule
         {
             IPstream fromMaster
             (
-                Pstream::defaultComms(),
-//                Pstream::scheduled,
+                Pstream::scheduled,
                 Pstream::masterNo(),
                 0,
                 tag
@@ -420,7 +416,8 @@ void Foam::mapDistribute::compact(const boolList& elemIsUsed, const int tag)
 
 
         // Wait for all to finish
-//         Pstream::waitRequests(startOfRequests);
+        Pout<< "BARRIER in compact" << endl;
+        Pstream::waitRequests(startOfRequests);
 
 
         // Compact out all submap entries that are referring to unused elements
diff --git a/src/foam/meshes/polyMesh/mapPolyMesh/mapDistribute/mapDistributeTemplates.C b/src/foam/meshes/polyMesh/mapPolyMesh/mapDistribute/mapDistributeTemplates.C
index 43aa235a2..cec2f0fc0 100644
--- a/src/foam/meshes/polyMesh/mapPolyMesh/mapDistribute/mapDistributeTemplates.C
+++ b/src/foam/meshes/polyMesh/mapPolyMesh/mapDistribute/mapDistributeTemplates.C
@@ -237,6 +237,7 @@ void Foam::mapDistribute::distribute
             }
 
             // Block ourselves, waiting only for the current comms
+            Pout<< "TEMPLATE BARRIER in distribute, non-blocking non-contiguous" << endl;
             Pstream::waitRequests(nOutstanding);
 
             // Consume
@@ -258,7 +259,7 @@ void Foam::mapDistribute::distribute
                 }
             }
         }
-        else
+        else // contiguous data
         {
             // Set up sends to neighbours
 
@@ -343,7 +344,7 @@ void Foam::mapDistribute::distribute
 
 
             // Wait for all to finish
-
+            Pout<< "TEMPLATE BARRIER in distribute, non-blocking, contiguous" << endl;
             Pstream::waitRequests(nOutstanding);
 
 
@@ -584,6 +585,7 @@ void Foam::mapDistribute::distribute
             }
 
             // Block ourselves, waiting only for the current comms
+            Pout<< "TEMPLATE BARRIER in distribute, non-blocking non-contiguous 2" << endl;
             Pstream::waitRequests(nOutstanding);
 
             // Consume
@@ -605,7 +607,7 @@ void Foam::mapDistribute::distribute
                 }
             }
         }
-        else
+        else // Contiguous data
         {
             // Set up sends to neighbours
 
@@ -689,6 +691,7 @@ void Foam::mapDistribute::distribute
 
 
             // Wait for all to finish
+            Pout<< "TEMPLATE BARRIER in distribute, non-blocking contiguous 2" << endl;
             Pstream::waitRequests(nOutstanding);