From b32edafe894d64d22d6495d7704b932ed2b987d8 Mon Sep 17 00:00:00 2001
From: Hrvoje Jasak <h.jasak@wikki.co.uk>
Date: Wed, 22 Jun 2016 17:29:31 +0100
Subject: [PATCH 001/130] Work in progress

---
 .../fvPatches/constraint/ggi/ggiFvPatch.C     |   7 +
 .../fvPatches/constraint/ggi/ggiFvPatch.H     |   5 +
 .../regionCouple/regionCoupleFvPatch.C        |  10 +
 .../regionCouple/regionCoupleFvPatch.H        |   5 +
 .../ggiLduInterface/ggiLduInterface.H         |   4 +
 .../overlapGGILduInterface.H                  |   2 +-
 .../interfaces/AMGInterface/AMGInterface.H    |  12 +
 .../ggiAMGInterface/ggiAMGInterface.C         | 388 ++++++++++++------
 .../ggiAMGInterface/ggiAMGInterface.H         |  20 +-
 .../ggiAMGInterfaceTemplates.C                |  68 ++-
 .../mixingPlaneAMGInterface.C                 |   1 +
 .../processorAMGInterface.C                   |   3 +-
 12 files changed, 387 insertions(+), 138 deletions(-)

diff --git a/src/finiteVolume/fvMesh/fvPatches/constraint/ggi/ggiFvPatch.C b/src/finiteVolume/fvMesh/fvPatches/constraint/ggi/ggiFvPatch.C
index 05ad430ce..09501ab16 100644
--- a/src/finiteVolume/fvMesh/fvPatches/constraint/ggi/ggiFvPatch.C
+++ b/src/finiteVolume/fvMesh/fvPatches/constraint/ggi/ggiFvPatch.C
@@ -271,6 +271,12 @@ const Foam::scalarListList& Foam::ggiFvPatch::weights() const
 }
 
 
+void Foam::ggiFvPatch::expandAddrToZone(labelField& lf) const
+{
+    lf = ggiPolyPatch_.fastExpand(lf);
+}
+
+
 Foam::tmp<Foam::labelField> Foam::ggiFvPatch::interfaceInternalField
 (
     const unallocLabelList& internalData
@@ -306,6 +312,7 @@ void Foam::ggiFvPatch::initInternalFieldTransfer
     const unallocLabelList& iF
 ) const
 {
+    // Label transfer is local without global reduction
     labelTransferBuffer_ = patchInternalField(iF);
 }
 
diff --git a/src/finiteVolume/fvMesh/fvPatches/constraint/ggi/ggiFvPatch.H b/src/finiteVolume/fvMesh/fvPatches/constraint/ggi/ggiFvPatch.H
index 6aa101dd3..740f0f208 100644
--- a/src/finiteVolume/fvMesh/fvPatches/constraint/ggi/ggiFvPatch.H
+++ b/src/finiteVolume/fvMesh/fvPatches/constraint/ggi/ggiFvPatch.H
@@ -187,6 +187,11 @@ public:
                 return coupledFvPatch::reverseT();
             }
 
+            //- Expand addressing to zone
+            //  Used in optimised AMG coarsening
+            virtual void expandAddrToZone(labelField&) const;
+
+
             //- Return the values of the given internal data adjacent to
             //  the interface as a field
             virtual tmp<labelField> interfaceInternalField
diff --git a/src/finiteVolume/fvMesh/fvPatches/constraint/regionCouple/regionCoupleFvPatch.C b/src/finiteVolume/fvMesh/fvPatches/constraint/regionCouple/regionCoupleFvPatch.C
index 41e8a58c1..38f5ba27d 100644
--- a/src/finiteVolume/fvMesh/fvPatches/constraint/regionCouple/regionCoupleFvPatch.C
+++ b/src/finiteVolume/fvMesh/fvPatches/constraint/regionCouple/regionCoupleFvPatch.C
@@ -315,6 +315,16 @@ const Foam::scalarListList& Foam::regionCoupleFvPatch::weights() const
 }
 
 
+void Foam::regionCoupleFvPatch::expandAddrToZone(labelField& lf) const
+{
+    // Missing code.  Activate for AMG solvers across regionCoupleFvPatch
+    notImplemented
+    (
+        "void regionCoupleFvPatch::expandAddrToZone(labelField& lf) const"
+    );
+}
+
+
 Foam::tmp<Foam::labelField> Foam::regionCoupleFvPatch::interfaceInternalField
 (
     const unallocLabelList& internalData
diff --git a/src/finiteVolume/fvMesh/fvPatches/constraint/regionCouple/regionCoupleFvPatch.H b/src/finiteVolume/fvMesh/fvPatches/constraint/regionCouple/regionCoupleFvPatch.H
index f09721e88..85a0fba14 100644
--- a/src/finiteVolume/fvMesh/fvPatches/constraint/regionCouple/regionCoupleFvPatch.H
+++ b/src/finiteVolume/fvMesh/fvPatches/constraint/regionCouple/regionCoupleFvPatch.H
@@ -196,6 +196,11 @@ public:
                 return coupledFvPatch::reverseT();
             }
 
+            //- Expand addressing to zone
+            //  Used in optimised AMG coarsening
+            virtual void expandAddrToZone(labelField&) const;
+
+
             //- Return the values of the given internal data adjacent to
             //  the interface as a field
             virtual tmp<labelField> interfaceInternalField
diff --git a/src/foam/matrices/lduMatrix/lduAddressing/lduInterfaces/ggiLduInterface/ggiLduInterface.H b/src/foam/matrices/lduMatrix/lduAddressing/lduInterfaces/ggiLduInterface/ggiLduInterface.H
index 4ad62d264..f13268ce0 100644
--- a/src/foam/matrices/lduMatrix/lduAddressing/lduInterfaces/ggiLduInterface/ggiLduInterface.H
+++ b/src/foam/matrices/lduMatrix/lduAddressing/lduInterfaces/ggiLduInterface/ggiLduInterface.H
@@ -121,6 +121,10 @@ public:
         //- Return face reverse transformation tensor
         virtual const tensorField& reverseT() const = 0;
 
+        //- Expand addressing to zone
+        //  Used in optimised AMG coarsening
+        virtual void expandAddrToZone(labelField&) const = 0;
+
 
         // Transfer buffer access
 
diff --git a/src/foam/matrices/lduMatrix/lduAddressing/lduInterfaces/overlapGGILduInterface/overlapGGILduInterface.H b/src/foam/matrices/lduMatrix/lduAddressing/lduInterfaces/overlapGGILduInterface/overlapGGILduInterface.H
index c7a4b181b..ddaa2890c 100644
--- a/src/foam/matrices/lduMatrix/lduAddressing/lduInterfaces/overlapGGILduInterface/overlapGGILduInterface.H
+++ b/src/foam/matrices/lduMatrix/lduAddressing/lduInterfaces/overlapGGILduInterface/overlapGGILduInterface.H
@@ -47,7 +47,7 @@ namespace Foam
 {
 
 /*---------------------------------------------------------------------------*\
-                       Class overlapGGILduInterface Declaration
+                   Class overlapGGILduInterface Declaration
 \*---------------------------------------------------------------------------*/
 
 class overlapGGILduInterface
diff --git a/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/AMGInterface/AMGInterface.H b/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/AMGInterface/AMGInterface.H
index f5b2557a9..3ef010832 100644
--- a/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/AMGInterface/AMGInterface.H
+++ b/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/AMGInterface/AMGInterface.H
@@ -82,9 +82,21 @@ protected:
         labelField faceCells_;
 
         //- Fine addressing
+        //  On GGI interfaces, a single fine coefficient may contribute to
+        //  multiple coarse coefficients using different weights.
+        //  To hanld this, a fine coefficient may be visited multiple times
+        //  which is recorded in fineAddressing.
+        //  For simple (matching) interfaces, fineAddressing_[i] = i
+        //  HJ, 21/Jun/2016
         labelField fineAddressing_;
 
         //- Restrict addressing
+        //  For each fine coefficient, list coarse cluster index it will be
+        //  agglomerated into
+        //  For cases where the fineAddressing is used, restrict addressing
+        //  and weights are expanded to match multiple hits for a single
+        //  fine coefficient, as dictated by fineAddressing
+        //  HJ, 21/Jun/2016
         labelField restrictAddressing_;
 
         //- Fine level agglomeration weights
diff --git a/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/ggiAMGInterface/ggiAMGInterface.C b/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/ggiAMGInterface/ggiAMGInterface.C
index 1f10a8724..82527393c 100644
--- a/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/ggiAMGInterface/ggiAMGInterface.C
+++ b/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/ggiAMGInterface/ggiAMGInterface.C
@@ -203,11 +203,6 @@ Foam::ggiAMGInterface::ggiAMGInterface
     zoneAddressing_(),
     mapPtr_(NULL)
 {
-    // Note.
-    // All processors will do the same coarsening and then filter
-    // the addressing to the local processor
-    // HJ, 1/Apr/2009
-
     // Note.
     // Signalling in global clustering requires me to recognise clustering
     // from separate processors as separate.  In the first phase, this will be
@@ -222,7 +217,15 @@ Foam::ggiAMGInterface::ggiAMGInterface
     // larger max int, which can be changed on request
     // HJ, 1/Apr/2009
 
-    // Expand the local and neighbour addressing to full zone size
+    // New algorithm will assemble local clusters and then create a global
+    // zone ordering by collecting all faces (coarse pairs) from proc0,
+    // followed by proc 1 etc.  This avoids global communication and allows
+    // each processor only to perform the analysis on locally created coarse
+    // faces
+    // HJ, 13/Jun/2016
+
+    // To help with analysis, expand the local and neighbour addressing
+    // to full zone size
     labelField localExpandAddressing(fineGgiInterface_.zoneSize(), 0);
 
     // Memory management, local
@@ -235,60 +238,70 @@ Foam::ggiAMGInterface::ggiAMGInterface
                 localRestrictAddressing[i] + procOffset*Pstream::myProcNo();
         }
 
-        if (!localParallel())
-        {
-            // Optimisation of this comms call is needed
-            // HJ, 9/Jun/2016
-            reduce(localExpandAddressing, sumOp<labelField>());
-        }
+        // Removed global reduce.  Only local faces will be analysed.
+        // HJ, 13/Jun/2016
     }
 
+    // Create addressing for neighbour faces.  Note: expandAddrToZone will
+    // expand the addressing to zone size.  HJ, 13/Jun/2016
     labelField neighbourExpandAddressing
     (
-        fineGgiInterface_.shadowInterface().zoneSize(),
-        0
+        fineGgiInterface_.shadowInterface().interfaceSize()
     );
 
+    // Fill local cluster ID with a combination of a local ID and processor
+    // offset
     // Memory management, neighbour
     {
-        const labelList& addr =
-            fineGgiInterface_.shadowInterface().zoneAddressing();
-
-        forAll (addr, i)
+        forAll (neighbourExpandAddressing, i)
         {
-            neighbourExpandAddressing[addr[i]] =
+            neighbourExpandAddressing[i] =
                 neighbourRestrictAddressing[i]
               + procOffset*Pstream::myProcNo();
         }
 
-        if (!localParallel())
-        {
-            // Optimisation of this comms call is needed
-            // HJ, 9/Jun/2016
-            reduce(neighbourExpandAddressing, sumOp<labelField>());
-        }
+        // Expand neighbour side to get all the data required from other
+        // processors.  Note: neigbour is now the size of remote zone
+        fineGgiInterface_.shadowInterface().expandAddrToZone
+        (
+            neighbourExpandAddressing
+        );
     }
 
+    // DEBUG: Check that all sizes are at zone size.
+    Info<< "Sizes check: local zone size "
+        << fineGgiInterface_.zoneSize()
+        << " " << localExpandAddressing << nl
+        << "shadow zone size "
+        << fineGgiInterface_.shadowInterface().zoneSize()
+        << " " << neighbourExpandAddressing
+        << endl;
+
+
+    // Note: neighbourExpandAddressing will be filled with NaNs for faces which
+    // not local
+
+    Info<< "End of reduce" << endl;
     // Make a lookup table of entries for owner/neighbour.
     // All sizes are guessed at the size of fine interface
     // HJ, 19/Feb/2009
 
     HashTable<SLList<label>, label, Hash<label> > neighboursTable
     (
-        localExpandAddressing.size()
+        fineGgiInterface_.interfaceSize()
     );
 
     // Table of face-sets to be agglomerated
     HashTable<SLList<SLList<label> >, label, Hash<label> > faceFaceTable
     (
-        localExpandAddressing.size()
+        fineGgiInterface_.interfaceSize()
     );
 
     // Table of face-sets weights to be agglomerated
     HashTable<SLList<SLList<scalar> >, label, Hash<label> >
         faceFaceWeightsTable
         (
-            localExpandAddressing.size()
+            fineGgiInterface_.interfaceSize()
         );
 
     // Count the number of coarse faces
@@ -300,11 +313,24 @@ Foam::ggiAMGInterface::ggiAMGInterface
     // On the fine level, addressing is made in a labelListList
     if (fineGgiInterface_.fineLevel())
     {
+        // This addressing defines how to interpolate for all zone faces
+        // across the interface
         const labelListList& fineAddr = fineGgiInterface_.addressing();
         const scalarListList& fineWeights = fineGgiInterface_.weights();
 
-        forAll (fineAddr, ffI)
+        // Note: cluster only locally live faces
+        // HJ, 13/Jun/2016
+
+        // This addressing defines which faces from zone are local
+        const labelList& fineZa = fineGgiInterface_.zoneAddressing();
+
+        // Perform analysis only for local faces
+        // HJ, 22/Jun/2016
+        forAll (fineZa, fineZaI)
         {
+            // Get the local face (from zone) to analyse
+            const label ffI = fineZa[fineZaI];
+
             const labelList& curFineNbrs = fineAddr[ffI];
             const scalarList& curFineWeigts = fineWeights[ffI];
 
@@ -372,6 +398,9 @@ Foam::ggiAMGInterface::ggiAMGInterface
                             nbrFound = true;
                             faceFacesIter().append(ffI);
                             faceFaceWeightsIter().append(curNW);
+
+                            // New agglomeration pair found in already
+                            // existing pair
                             nAgglomPairs++;
 
                             break;
@@ -384,7 +413,7 @@ Foam::ggiAMGInterface::ggiAMGInterface
                         curFaceFaces.append(SLList<label>(ffI));
                         curFaceWeights.append(SLList<scalar>(curNW));
 
-                        // New coarse face created
+                        // New coarse face created for an existing master
                         nCoarseFaces++;
                         nAgglomPairs++;
                     }
@@ -407,7 +436,7 @@ Foam::ggiAMGInterface::ggiAMGInterface
                         SLList<SLList<scalar> >(SLList<scalar>(curNW))
                     );
 
-                    // New coarse face created
+                    // New coarse face created for a new master
                     nCoarseFaces++;
                     nAgglomPairs++;
                 }
@@ -417,8 +446,16 @@ Foam::ggiAMGInterface::ggiAMGInterface
     else
     {
         // Coarse level, addressing is stored in faceCells
-        forAll (localExpandAddressing, ffI)
+        // This addressing defines whicf faces from zone are local
+        const labelList& fineZa = fineGgiInterface_.zoneAddressing();
+
+        // Perform analysis only for local faces
+        // HJ, 22/Jun/2016
+        forAll (fineZa, fineZaI)
         {
+            // Get the local face (from zone) to analyse
+            const label ffI = fineZa[fineZaI];
+
             label curMaster = -1;
             label curSlave = -1;
 
@@ -480,6 +517,9 @@ Foam::ggiAMGInterface::ggiAMGInterface
                         faceFacesIter().append(ffI);
                         // Add dummy weight
                         faceFaceWeightsIter().append(1.0);
+
+                        // New agglomeration pair found in already
+                        // existing pair
                         nAgglomPairs++;
                         break;
                     }
@@ -492,7 +532,7 @@ Foam::ggiAMGInterface::ggiAMGInterface
                     // Add dummy weight
                     curFaceWeights.append(SLList<scalar>(1.0));
 
-                    // New coarse face created
+                    // New coarse face created for an existing master
                     nCoarseFaces++;
                     nAgglomPairs++;
                 }
@@ -516,116 +556,193 @@ Foam::ggiAMGInterface::ggiAMGInterface
                     SLList<SLList<scalar> >(SLList<scalar>(1.0))
                 );
 
-                // New coarse face created
+                // New coarse face created for a new master
                 nCoarseFaces++;
                 nAgglomPairs++;
             }
         } // end for all fine faces
     }
 
-    faceCells_.setSize(nCoarseFaces, -1);
-    fineAddressing_.setSize(nAgglomPairs, -1);
-    restrictAddressing_.setSize(nAgglomPairs, -1);
-    restrictWeights_.setSize(nAgglomPairs);
+    // In order to assemble the coarse global face zone, find out
+    // how many faces have been created on each processor.
+    // Note that masters and slaves both count faces so we will only ask master
+    // sizes to count
+    labelList nCoarseFacesPerProc(Pstream::nProcs(), 0);
+
+    if (master())
+    {
+        nCoarseFacesPerProc[Pstream::myProcNo()] = nCoarseFaces;
+    }
+
+    reduce(nCoarseFacesPerProc, sumOp<labelList>());
+
+    Info<< "Number of faces per processor: " << nCoarseFacesPerProc
+        << endl;
+
+    // Coarse global face zone is assembled by adding all faces from proc0,
+    // followed by all faces from proc1 etc.
+    // Therefore, on procN, my master offset
+    // will be equal to the sum of numbers of coarse faces on all processors
+    // before mine
+    // HJ, 13/Jun/2016
+
+    label coarseGlobalFaceOffset = 0;
+
+    for (label i = 0; i < Pstream::myProcNo(); i++)
+    {
+        coarseGlobalFaceOffset += nCoarseFacesPerProc[i];
+    }
+
+    Pout<< "coarseGlobalFaceOffset: " << coarseGlobalFaceOffset << endl;
+
+    Info<< "End of contents assembly" << endl;
+    labelField masterFaceCells(nCoarseFaces, -1);
+    labelField masterZoneAddressing(nCoarseFaces, -1);
+    labelField masterFineAddressing(nCoarseFaces, -1);
+    labelField masterRestrictAddressing(nAgglomPairs, -1);
+    scalarField masterRestrictWeights(nAgglomPairs);
+
+    // Note: in multiple agglomeration
 
     labelList contents = neighboursTable.toc();
 
+    // Global faces shall be assembled by the increasing label of master
+    // cluster ID.
+
     // Sort makes sure the order is identical on both sides.
-    // Since the global zone is defined by this sort, the neighboursTable
-    // must be complete on all processors
     // HJ, 20/Feb/2009 and 6/Jun/2016
     sort(contents);
 
     // Grab zone size and create zone addressing
-    zoneSize_ = nCoarseFaces;
+    zoneSize_ = sum(nCoarseFacesPerProc);
+    Info<< "zoneSize_: " << zoneSize_ << endl;
 
-    zoneAddressing_.setSize(nCoarseFaces);
+    // Note:
+    // When I am agglomerating the master, I know faces are stacked up in order
+    // but on the slave side, all I know is the master cluster index and
+    // not a master coarse face index.  Therefore:
+    // - master needs to be agglomerated first
+    // - once master is agglomerated, I need to signal to the slave side
+    //   the global coarse face zone index
+
+
+    // Note: zone addressing will be assembled only for local clusters
+    // using the coarseGlobalFaceOffset
+    // HJ, 13/Jun/2016
     label nProcFaces = 0;
 
     // Reset face counter for re-use
     nCoarseFaces = 0;
     nAgglomPairs = 0;
 
-    if (master())
+    // Note:
+    // Since clustering has now happened only on local faces, addressing and
+    // all other array work on local indices and not on the coarse global zone
+    // HJ, 13/Jun/2016
+
+    // Establish zone addressing on the master side and communicate
+    // it to the shadow
+
+    // On master side, the owner addressing is stored in table of contents
+    forAll (contents, masterI)
     {
-        // On master side, the owner addressing is stored in table of contents
-        forAll (contents, masterI)
+        SLList<label>& curNbrs =
+            neighboursTable.find(contents[masterI])();
+
+        // Note: neighbour processor index is irrelevant.  HJ, 1/Apr/2009
+
+        SLList<SLList<label> >& curFaceFaces =
+            faceFaceTable.find(contents[masterI])();
+
+        SLList<SLList<scalar> >& curFaceWeights =
+            faceFaceWeightsTable.find(contents[masterI])();
+
+        SLList<label>::iterator nbrsIter = curNbrs.begin();
+        SLList<SLList<label> >::iterator faceFacesIter =
+            curFaceFaces.begin();
+
+        SLList<SLList<scalar> >::iterator faceFaceWeightsIter =
+            curFaceWeights.begin();
+
+        for
+        (
+            ;
+            nbrsIter != curNbrs.end()
+         && faceFacesIter != curFaceFaces.end()
+         && faceFaceWeightsIter != curFaceWeights.end();
+            ++nbrsIter, ++faceFacesIter, ++faceFaceWeightsIter
+        )
         {
-            SLList<label>& curNbrs =
-                neighboursTable.find(contents[masterI])();
+            // Check if master is on local processor: no longer needed,
+            // as only local processor is being searched.  HJ, 13/Jun/2016
 
-            // Note: neighbour processor index is irrelevant.  HJ, 1/Apr/2009
+            // Record that this face belongs locally
+            // Use offset to indicate its position in the list
+            masterZoneAddressing[nProcFaces] =
+                nProcFaces + coarseGlobalFaceOffset;
 
-            SLList<SLList<label> >& curFaceFaces =
-                faceFaceTable.find(contents[masterI])();
+            masterFaceCells[nProcFaces] =
+                contents[masterI] - procOffset*Pstream::myProcNo();
 
-            SLList<SLList<scalar> >& curFaceWeights =
-                faceFaceWeightsTable.find(contents[masterI])();
+            SLList<label>::iterator facesIter =
+                faceFacesIter().begin();
 
-            SLList<label>::iterator nbrsIter = curNbrs.begin();
-            SLList<SLList<label> >::iterator faceFacesIter =
-                curFaceFaces.begin();
-
-            SLList<SLList<scalar> >::iterator faceFaceWeightsIter =
-                curFaceWeights.begin();
+            SLList<scalar>::iterator weightsIter =
+                faceFaceWeightsIter().begin();
 
             for
             (
                 ;
-                nbrsIter != curNbrs.end()
-             && faceFacesIter != curFaceFaces.end()
-             && faceFaceWeightsIter != curFaceWeights.end();
-                ++nbrsIter, ++faceFacesIter, ++faceFaceWeightsIter
+                facesIter != faceFacesIter().end()
+             && weightsIter != faceFaceWeightsIter().end();
+                ++facesIter, ++weightsIter
             )
             {
-                // Check if master is on local processor
-                if
-                (
-                    contents[masterI] >= procOffset*Pstream::myProcNo()
-                 && contents[masterI] < procOffset*(Pstream::myProcNo() + 1)
-                )
-                {
-                    // Record that this face belongs locally
-                    zoneAddressing_[nProcFaces] = nCoarseFaces;
-                    faceCells_[nProcFaces] =
-                        contents[masterI] - procOffset*Pstream::myProcNo();
-                    nProcFaces++;
+                masterFineAddressing[nAgglomPairs] = facesIter();
 
-                    SLList<label>::iterator facesIter =
-                        faceFacesIter().begin();
-                    SLList<scalar>::iterator weightsIter =
-                        faceFaceWeightsIter().begin();
-
-                    for
-                    (
-                        ;
-                        facesIter != faceFacesIter().end()
-                     && weightsIter != faceFaceWeightsIter().end();
-                        ++facesIter, ++weightsIter
-                    )
-                    {
-                        fineAddressing_[nAgglomPairs] = facesIter();
-                        restrictAddressing_[nAgglomPairs] = nCoarseFaces;
-                        restrictWeights_[nAgglomPairs] = weightsIter();
-                        nAgglomPairs++;
-                    }
-                }
-
-                // Not a local face, but still created in global zone
-                nCoarseFaces++;
+                // Master processor zone face is calculated from
+                masterRestrictAddressing[nAgglomPairs] =
+                    nProcFaces + coarseGlobalFaceOffset;
+                masterRestrictWeights[nAgglomPairs] = weightsIter();
+                nAgglomPairs++;
             }
+
+            nProcFaces++;
         }
+    }
 
-        // Resize arrays: not all of ggi is used locally
-        faceCells_.setSize(nProcFaces);
-        zoneAddressing_.setSize(nProcFaces);
+    // Resize arrays: not all of ggi is used locally
+    masterFaceCells.setSize(nProcFaces);
+    masterZoneAddressing.setSize(nProcFaces);
 
-        fineAddressing_.setSize(nAgglomPairs);
-        restrictAddressing_.setSize(nAgglomPairs);
-        restrictWeights_.setSize(nAgglomPairs);
+    masterFineAddressing.setSize(nAgglomPairs);
+    masterRestrictAddressing.setSize(nAgglomPairs);
+    masterRestrictWeights.setSize(nAgglomPairs);
+
+    // Note: Both master and slave have done the same agglomeration up to here
+
+    if (master())
+    {
+        // Master has completed the clustering
+        faceCells_ = masterFaceCells;
+        zoneAddressing_ = masterZoneAddressing;
+        fineAddressing_ = masterFineAddressing;
+        restrictAddressing_ = masterRestrictAddressing;
+        restrictWeights_ = masterRestrictWeights;
     }
     else
     {
+        // Note: shadowRestrictAddressing contains the
+
+        // Note: zone addressing will be assembled only for local clusters
+        // using the coarseGlobalFaceOffset
+        // HJ, 13/Jun/2016
+        label nProcFaces = 0;
+
+        // Reset face counter for re-use
+        nCoarseFaces = 0;
+        nAgglomPairs = 0;
+
         // On slave side, the owner addressing is stored in linked lists
         forAll (contents, masterI)
         {
@@ -646,6 +763,7 @@ Foam::ggiAMGInterface::ggiAMGInterface
 
             SLList<SLList<scalar> >::iterator faceFaceWeightsIter =
                 curFaceWeights.begin();
+
             for
             (
                 ;
@@ -655,41 +773,37 @@ Foam::ggiAMGInterface::ggiAMGInterface
                 ++nbrsIter, ++faceFacesIter, ++faceFaceWeightsIter
             )
             {
-                // Check if the face is on local processor
-                if
+                // Check if the face is on local processor: no longer needed,
+                // as only local processor is being searched.  HJ, 13/Jun/2016
+
+                // Record that this face belongs locally.
+
+//HJ, HERE: I need to find out the global face index for the face that was created from the master side
+
+                zoneAddressing_[nProcFaces] = nCoarseFaces;
+                faceCells_[nProcFaces] =
+                    nbrsIter() - procOffset*Pstream::myProcNo();
+                nProcFaces++;
+
+                SLList<label>::iterator facesIter =
+                    faceFacesIter().begin();
+
+                SLList<scalar>::iterator weightsIter =
+                    faceFaceWeightsIter().begin();
+
+                for
                 (
-                    nbrsIter() >= procOffset*Pstream::myProcNo()
-                 && nbrsIter() < procOffset*(Pstream::myProcNo() + 1)
+                    ;
+                    facesIter != faceFacesIter().end()
+                 && weightsIter != faceFaceWeightsIter().end();
+                    ++facesIter, ++weightsIter
                 )
                 {
-                    // Record that this face belongs locally.
-                    zoneAddressing_[nProcFaces] = nCoarseFaces;
-                    faceCells_[nProcFaces] =
-                        nbrsIter() - procOffset*Pstream::myProcNo();
-                    nProcFaces++;
-
-                    SLList<label>::iterator facesIter =
-                        faceFacesIter().begin();
-
-                    SLList<scalar>::iterator weightsIter =
-                        faceFaceWeightsIter().begin();
-
-                    for
-                    (
-                        ;
-                        facesIter != faceFacesIter().end()
-                     && weightsIter != faceFaceWeightsIter().end();
-                        ++facesIter, ++weightsIter
-                    )
-                    {
-                        fineAddressing_[nAgglomPairs] = facesIter();
-                        restrictAddressing_[nAgglomPairs] = nCoarseFaces;
-                        restrictWeights_[nAgglomPairs] = weightsIter();
-                        nAgglomPairs++;
-                    }
+                    fineAddressing_[nAgglomPairs] = facesIter();
+                    restrictAddressing_[nAgglomPairs] = nCoarseFaces;
+                    restrictWeights_[nAgglomPairs] = weightsIter();
+                    nAgglomPairs++;
                 }
-
-                nCoarseFaces++;
             }
         }
 
@@ -718,6 +832,8 @@ Foam::tmp<Foam::scalarField> Foam::ggiAMGInterface::agglomerateCoeffs
     const scalarField& fineCoeffs
 ) const
 {
+    // HJ, HERE THIS SHOULD BE REMOVED: NO LONGER NEEDED BECAUSE ALL ADDRESSING IS LOCAL
+
     // Note: reconsider better parallel communication here.
     // Currently expanding to full zone size
     // HJ, 16/Mar/2016
@@ -917,4 +1033,10 @@ Foam::tmp<Foam::scalarField> Foam::ggiAMGInterface::internalFieldTransfer
 }
 
 
+void Foam::ggiAMGInterface::expandAddrToZone(labelField& lf) const
+{
+    lf = fastExpand(lf);
+}
+
+
 // ************************************************************************* //
diff --git a/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/ggiAMGInterface/ggiAMGInterface.H b/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/ggiAMGInterface/ggiAMGInterface.H
index 973b0467a..507909026 100644
--- a/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/ggiAMGInterface/ggiAMGInterface.H
+++ b/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/ggiAMGInterface/ggiAMGInterface.H
@@ -143,9 +143,23 @@ public:
 
         // Interface transfer functions
 
+            //- Fast expand
+            //  Note: contains global communications
+            //  This function will expand data to zone size with fast comms
+            //  filling in only the part of the zone that is required
+            //  on the shadow side
+            template<class Type>
+            tmp<Field<Type> > fastExpand(const UList<Type>&) const;
+
             //- Fast reduce
             //  Note: contains global communications
-            //  New, HJ, 24/Jun/2011
+            //  This function will expand data to zone size with fast comms
+            //  and filter the field to cover the need of a shadow zone
+            //  This makes sense on coarse levels because all addressing
+            //  is one-on-one (one master, one slave).  On the fine level
+            //  each face in the zone may contribute to multiple faces meaning
+            // that the data needs to be expanded to zone size
+            //  HJ, 24/Jun/2011 and 13/Jun/2016
             template<class Type>
             tmp<Field<Type> > fastReduce(const UList<Type>&) const;
 
@@ -234,6 +248,10 @@ public:
 
             //- Return neighbour-cell transformation tensor
             virtual const tensorField& reverseT() const;
+
+            //- Expand addressing to zone
+            //  Used in optimised AMG coarsening
+            virtual void expandAddrToZone(labelField&) const;
 };
 
 
diff --git a/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/ggiAMGInterface/ggiAMGInterfaceTemplates.C b/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/ggiAMGInterface/ggiAMGInterfaceTemplates.C
index 9c468bbb0..f0cbf2b11 100644
--- a/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/ggiAMGInterface/ggiAMGInterfaceTemplates.C
+++ b/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/ggiAMGInterface/ggiAMGInterfaceTemplates.C
@@ -32,10 +32,61 @@ namespace Foam
 
 // * * * * * * * * * * * * * * * Member Functions  * * * * * * * * * * * * * //
 
+template<class Type>
+tmp<Field<Type> > ggiAMGInterface::fastExpand(const UList<Type>& ff) const
+{
+    // Rewrite, 1/Jun/2016
+    // To avoid creating zone-sized data and gather-scatter communication
+    // to the master, the optimised map-distribute call is implemented.
+    // The field is filled with local data which is then sent where needed
+    // through map-distribute.
+    // On return, the field is expanded to zone size but only filled with
+    // the data which is needed for the shadow
+    // HJ, 1/Jun/2016
+
+    if (ff.size() != this->size())
+    {
+        FatalErrorIn
+        (
+            "tmp<Field<Type> > ggiAMGInterface::fastExpand"
+            "("
+            "    const UList<Type>& ff"
+            ") const"
+        )   << "Wrong field size.  ff: " << ff.size()
+            << " interface: " << this->size()
+            << abort(FatalError);
+    }
+
+    if (localParallel() || !Pstream::parRun())
+    {
+        // Field remains identical: no parallel communications required
+        tmp<Field<Type> > tresult(new Field<Type>(ff));
+
+        return tresult;
+    }
+    else
+    {
+        // Optimised mapDistribute
+
+        // Execute init reduce to calculate addressing if not already done
+        map();
+
+        // Prepare for distribute.  Note: field will be expanded to zone size
+        // during the distribute operation
+        tmp<Field<Type> > tresult(new Field<Type>(ff));
+        List<Type>& expand = tresult();
+
+        map().distribute(expand);
+
+        return tresult;
+    }
+}
+
+
 template<class Type>
 tmp<Field<Type> > ggiAMGInterface::fastReduce(const UList<Type>& ff) const
 {
-    // Algorithm
+    // Old algorithm: OBOSLETE
     // Local processor contains faceCells part of the zone and requires
     // zoneAddressing part.
     // For fast communications, each processor will send the faceCells and
@@ -44,6 +95,18 @@ tmp<Field<Type> > ggiAMGInterface::fastReduce(const UList<Type>& ff) const
     // the required data
     // HJ, 24/Jun/2011
 
+    // Rewrite, 1/Jun/2016
+    // To avoid creating zone-sized data and gather-scatter communication
+    // to the master, the optimised map-distribute call is implemented.
+    // The field is filled with local data which is then sent where needed
+    // through map-distribute.
+    // On return, the field is expanded to zone size but only filled with
+    // the data which is needed for the shadow
+    // Having received the zone data, shadow data is extracted from the
+    // field size.  Note: this works only on coarse levels, where one-on-one
+    // mapping applies
+    // HJ, 1/Jun/2016
+
     if (ff.size() != this->size())
     {
         FatalErrorIn
@@ -71,7 +134,8 @@ tmp<Field<Type> > ggiAMGInterface::fastReduce(const UList<Type>& ff) const
         // Execute init reduce to calculate addressing if not already done
         map();
 
-        // Prepare for distribute: field will be expanded to zone size
+        // Prepare for distribute.  Note: field will be expanded to zone size
+        // during the distribute operation
         List<Type> expand = ff;
 
         map().distribute(expand);
diff --git a/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/mixingPlaneAMGInterface/mixingPlaneAMGInterface.C b/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/mixingPlaneAMGInterface/mixingPlaneAMGInterface.C
index dd1841562..9a766bae4 100644
--- a/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/mixingPlaneAMGInterface/mixingPlaneAMGInterface.C
+++ b/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/mixingPlaneAMGInterface/mixingPlaneAMGInterface.C
@@ -168,6 +168,7 @@ void Foam::mixingPlaneAMGInterface::initInternalFieldTransfer
     const unallocLabelList& iF
 ) const
 {
+    // NOTE: Change this: requires fast reduce.  HJ, 13/Jun/20106
     labelTransferBuffer_ = interfaceInternalField(iF);
 }
 
diff --git a/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/processorAMGInterface/processorAMGInterface.C b/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/processorAMGInterface/processorAMGInterface.C
index 7b346a099..35921b46f 100644
--- a/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/processorAMGInterface/processorAMGInterface.C
+++ b/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/processorAMGInterface/processorAMGInterface.C
@@ -189,7 +189,8 @@ Foam::processorAMGInterface::processorAMGInterface
 
                 for
                 (
-                    SLList<label>::iterator facesIter = faceFacesIter().begin();
+                    SLList<label>::iterator facesIter =
+                        faceFacesIter().begin();
                     facesIter != faceFacesIter().end();
                     ++facesIter
                 )

From 568e0c9042f5f67ab9444911c96fca76f576ec6f Mon Sep 17 00:00:00 2001
From: Hrvoje Jasak <h.jasak@wikki.co.uk>
Date: Wed, 22 Jun 2016 17:29:43 +0100
Subject: [PATCH 002/130] Debug info

---
 .../BlockMatrixAgglomeration.C                | 24 +++++++++++++++----
 .../BlockMatrixAgglomeration.H                |  3 +++
 2 files changed, 23 insertions(+), 4 deletions(-)

diff --git a/src/foam/matrices/blockLduMatrix/BlockAMG/BlockMatrixCoarsening/BlockMatrixAgglomeration/BlockMatrixAgglomeration.C b/src/foam/matrices/blockLduMatrix/BlockAMG/BlockMatrixCoarsening/BlockMatrixAgglomeration/BlockMatrixAgglomeration.C
index 2eb4bd548..9278ad63b 100644
--- a/src/foam/matrices/blockLduMatrix/BlockAMG/BlockMatrixCoarsening/BlockMatrixAgglomeration/BlockMatrixAgglomeration.C
+++ b/src/foam/matrices/blockLduMatrix/BlockAMG/BlockMatrixCoarsening/BlockMatrixAgglomeration/BlockMatrixAgglomeration.C
@@ -74,6 +74,7 @@ void Foam::BlockMatrixAgglomeration<Type>::calcAgglomeration()
     // Create row-based addressing
 
     const label nRows = matrix_.lduAddr().size();
+    Info<< "Start, coarse size: " << nRows << endl;
 
     const unallocLabelList& upperAddr = matrix_.lduAddr().upperAddr();
     const unallocLabelList& lowerAddr = matrix_.lduAddr().lowerAddr();
@@ -142,6 +143,7 @@ void Foam::BlockMatrixAgglomeration<Type>::calcAgglomeration()
         // Reset agglomeration index array
         agglomIndex_ = -1;
     }
+    Info<< "End prepare addressing: " << lTime_.elapsedClockTime() << endl;
 
 
     // Calculate agglomeration
@@ -177,6 +179,7 @@ void Foam::BlockMatrixAgglomeration<Type>::calcAgglomeration()
 
         return;
     }
+    Info<< "End coeff mag: " << lTime_.elapsedClockTime() << endl;
 
     labelList sizeOfGroups(nRows, 0);
 
@@ -232,6 +235,7 @@ void Foam::BlockMatrixAgglomeration<Type>::calcAgglomeration()
             }
         }
     }
+    Info<< "End weak connections: " << lTime_.elapsedClockTime() << endl;
 
     // Go through the off-diagonal and create clusters, marking the child array
     label indexUngrouped, indexGrouped;
@@ -339,6 +343,7 @@ void Foam::BlockMatrixAgglomeration<Type>::calcAgglomeration()
             }
         }
     }
+    Info<< "End agglom index: " << lTime_.elapsedClockTime() << endl;
 
     // The decision on parallel agglomeration needs to be made for the
     // whole gang of processes; otherwise I may end up with a different
@@ -368,6 +373,7 @@ void Foam::BlockMatrixAgglomeration<Type>::calcAgglomeration()
             Pout << ".  Rejected" << endl;
         }
     }
+    Info<< "End calcAgglomeration: " << lTime_.elapsedClockTime() << endl;
 }
 
 
@@ -379,7 +385,7 @@ void Foam::BlockMatrixAgglomeration<Type>::restrictDiag
 ) const
 {
     typedef CoeffField<Type> TypeCoeffField;
-
+    Info<< "Start restrictDiag: " << lTime_.elapsedClockTime() << endl;
     if
     (
         Coeff.activeType() == blockCoeffBase::SQUARE
@@ -454,6 +460,7 @@ void Foam::BlockMatrixAgglomeration<Type>::restrictDiag
         )   << "Problem in coeff type morphing"
             << abort(FatalError);
     }
+    Info<< "End restrictDiag: " << lTime_.elapsedClockTime() << endl;
 }
 
 
@@ -470,7 +477,7 @@ void Foam::BlockMatrixAgglomeration<Type>::agglomerateCoeffs
 {
     // Does the matrix have solo equations
     bool soloEqns = nSolo_ > 0;
-
+    Info<< "Start agglomerateCoeffs symm: " << lTime_.elapsedClockTime() << endl;
     // Get addressing
     const unallocLabelList& upperAddr = matrix_.lduAddr().upperAddr();
     const unallocLabelList& lowerAddr = matrix_.lduAddr().lowerAddr();
@@ -503,6 +510,7 @@ void Foam::BlockMatrixAgglomeration<Type>::agglomerateCoeffs
               + activeFineUpperTranspose[fineCoeffI];
         }
     }
+    Info<< "End agglomerateCoeffs symm: " << lTime_.elapsedClockTime() << endl;
 }
 
 
@@ -520,7 +528,7 @@ void Foam::BlockMatrixAgglomeration<Type>::agglomerateCoeffs
 {
     // Does the matrix have solo equations
     bool soloEqns = nSolo_ > 0;
-
+    Info<< "Start agglomerateCoeffs asymm: " << lTime_.elapsedClockTime() << endl;
     // Get addressing
     const unallocLabelList& upperAddr = matrix_.lduAddr().upperAddr();
     const unallocLabelList& lowerAddr = matrix_.lduAddr().lowerAddr();
@@ -552,6 +560,7 @@ void Foam::BlockMatrixAgglomeration<Type>::agglomerateCoeffs
               + activeFineLower[fineCoeffI];
         }
     }
+    Info<< "End agglomerateCoeffs asymm: " << lTime_.elapsedClockTime() << endl;
 }
 
 
@@ -638,7 +647,8 @@ Foam::BlockMatrixAgglomeration<Type>::BlockMatrixAgglomeration
     groupSize_(groupSize),
     nSolo_(0),
     nCoarseEqns_(0),
-    coarsen_(false)
+    coarsen_(false),
+    lTime_()
 {
     calcAgglomeration();
 }
@@ -679,6 +689,7 @@ Foam::BlockMatrixAgglomeration<Type>::restrictMatrix() const
     //    insert the coeffs in the upper order. At the same time, collect the
     //    owner and neighbour addressing.
     // 4) Agglomerate the diagonal by summing up the fine diagonal
+    Info<< "Start restrictMatrix: " << lTime_.elapsedClockTime() << endl;
 
     // Get addressing
     const unallocLabelList& upperAddr = matrix_.lduAddr().upperAddr();
@@ -853,6 +864,7 @@ Foam::BlockMatrixAgglomeration<Type>::restrictMatrix() const
     blockNbrsData.setSize(0);
     initCoarseNeighb.setSize(0);
     coarseCoeffMap.setSize(0);
+    Info<< "End restrictMatrix core: " << lTime_.elapsedClockTime() << endl;
 
 
     // Create coarse-level coupled interfaces
@@ -885,6 +897,7 @@ Foam::BlockMatrixAgglomeration<Type>::restrictMatrix() const
     );
 
     // Initialise transfer of restrict addressing on the interface
+    // HJ, consider blocking comms.  HJ, 9/Jun/2016
     forAll (interfaceFields, intI)
     {
         if (interfaceFields.set(intI))
@@ -955,6 +968,7 @@ Foam::BlockMatrixAgglomeration<Type>::restrictMatrix() const
             coarseInterfaceAddr[intI] = coarseInterface.faceCells();
         }
     }
+    Info<< "End restrictMatrix interfaces: " << lTime_.elapsedClockTime() << endl;
 
     // Add interfaces
     coarseAddrPtr->addInterfaces
@@ -1031,6 +1045,7 @@ Foam::BlockMatrixAgglomeration<Type>::restrictMatrix() const
     }
 
     // Matrix restriction done!
+    Info<< "End restrictMatrix: " << lTime_.elapsedClockTime() << endl;
 
     typedef CoeffField<Type> TypeCoeffField;
 
@@ -1227,6 +1242,7 @@ Foam::BlockMatrixAgglomeration<Type>::restrictMatrix() const
                 << abort(FatalError);
         }
     }
+    Info<< "End restrictMatrix agglomerateCoeffs: " << lTime_.elapsedClockTime() << endl;
 
     // Create and return BlockAMGLevel
     return autoPtr<BlockAMGLevel<Type> >
diff --git a/src/foam/matrices/blockLduMatrix/BlockAMG/BlockMatrixCoarsening/BlockMatrixAgglomeration/BlockMatrixAgglomeration.H b/src/foam/matrices/blockLduMatrix/BlockAMG/BlockMatrixCoarsening/BlockMatrixAgglomeration/BlockMatrixAgglomeration.H
index aca68f389..65ac5eb8f 100644
--- a/src/foam/matrices/blockLduMatrix/BlockAMG/BlockMatrixCoarsening/BlockMatrixAgglomeration/BlockMatrixAgglomeration.H
+++ b/src/foam/matrices/blockLduMatrix/BlockAMG/BlockMatrixCoarsening/BlockMatrixAgglomeration/BlockMatrixAgglomeration.H
@@ -43,6 +43,7 @@ SourceFiles
 #include "BlockCoeffNorm.H"
 #include "BlockCoeff.H"
 #include "tolerancesSwitch.H"
+#include "clock.H"
 
 // * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
 
@@ -84,6 +85,8 @@ class BlockMatrixAgglomeration
         //- Can a coarse level be constructed?
         bool coarsen_;
 
+        clock lTime_;
+
 
     // Private Member Functions
 

From 92b36c74e739d61133ac56b94aaeb3f8c858317c Mon Sep 17 00:00:00 2001
From: Hrvoje Jasak <h.jasak@wikki.co.uk>
Date: Tue, 9 Aug 2016 13:21:53 +0100
Subject: [PATCH 003/130] Formatting

---
 .../mapPolyMesh/mapDistribute/mapDistributeTemplates.C    | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/foam/meshes/polyMesh/mapPolyMesh/mapDistribute/mapDistributeTemplates.C b/src/foam/meshes/polyMesh/mapPolyMesh/mapDistribute/mapDistributeTemplates.C
index d980e38c2..cccdfbe5d 100644
--- a/src/foam/meshes/polyMesh/mapPolyMesh/mapDistribute/mapDistributeTemplates.C
+++ b/src/foam/meshes/polyMesh/mapPolyMesh/mapDistribute/mapDistributeTemplates.C
@@ -166,7 +166,7 @@ void Foam::mapDistribute::distribute
 
         // Set up sends to neighbours
 
-        List<List<T > > sendFields(Pstream::nProcs());
+        List<List<T> > sendFields(Pstream::nProcs());
 
         for (label domain = 0; domain < Pstream::nProcs(); domain++)
         {
@@ -195,7 +195,7 @@ void Foam::mapDistribute::distribute
 
         // Set up receives from neighbours
 
-        List<List<T > > recvFields(Pstream::nProcs());
+        List<List<T> > recvFields(Pstream::nProcs());
 
         for (label domain = 0; domain < Pstream::nProcs(); domain++)
         {
@@ -460,7 +460,7 @@ void Foam::mapDistribute::distribute
 
         // Set up sends to neighbours
 
-        List<List<T > > sendFields(Pstream::nProcs());
+        List<List<T> > sendFields(Pstream::nProcs());
 
         for (label domain = 0; domain < Pstream::nProcs(); domain++)
         {
@@ -487,7 +487,7 @@ void Foam::mapDistribute::distribute
 
         // Set up receives from neighbours
 
-        List<List<T > > recvFields(Pstream::nProcs());
+        List<List<T> > recvFields(Pstream::nProcs());
 
         for (label domain = 0; domain < Pstream::nProcs(); domain++)
         {

From 9ee815914ddac62ceb2aa05dc30f3a99820a7cda Mon Sep 17 00:00:00 2001
From: Hrvoje Jasak <h.jasak@wikki.co.uk>
Date: Tue, 9 Aug 2016 13:22:26 +0100
Subject: [PATCH 004/130] Formatting

---
 src/foam/db/IOstreams/Pstreams/PstreamCombineReduceOps.H | 1 -
 src/foam/db/IOstreams/Pstreams/PstreamGlobals.H          | 2 +-
 2 files changed, 1 insertion(+), 2 deletions(-)

diff --git a/src/foam/db/IOstreams/Pstreams/PstreamCombineReduceOps.H b/src/foam/db/IOstreams/Pstreams/PstreamCombineReduceOps.H
index bade7b796..2e30e0962 100644
--- a/src/foam/db/IOstreams/Pstreams/PstreamCombineReduceOps.H
+++ b/src/foam/db/IOstreams/Pstreams/PstreamCombineReduceOps.H
@@ -30,7 +30,6 @@ Description
     combined using the given combination function and the result is
     broadcast to all nodes
 
-
 \*---------------------------------------------------------------------------*/
 
 #ifndef PstreamCombineReduceOps_H
diff --git a/src/foam/db/IOstreams/Pstreams/PstreamGlobals.H b/src/foam/db/IOstreams/Pstreams/PstreamGlobals.H
index 816fef8ed..c68ea965e 100644
--- a/src/foam/db/IOstreams/Pstreams/PstreamGlobals.H
+++ b/src/foam/db/IOstreams/Pstreams/PstreamGlobals.H
@@ -46,7 +46,7 @@ namespace Foam
 {
 
 /*---------------------------------------------------------------------------*\
-                           Class PstreamGlobals Declaration
+                     Class PstreamGlobals Declaration
 \*---------------------------------------------------------------------------*/
 
 namespace PstreamGlobals

From 5d4512cee20f6b054feaf4cf6d704a6c4da120c8 Mon Sep 17 00:00:00 2001
From: Hrvoje Jasak <h.jasak@wikki.co.uk>
Date: Tue, 9 Aug 2016 13:22:54 +0100
Subject: [PATCH 005/130] Formatting

---
 .../lduInterfaces/ggiLduInterface/ggiLduInterface.H          | 5 ++---
 .../lduAddressing/lduInterfaces/lduInterface/lduInterface.H  | 5 ++---
 2 files changed, 4 insertions(+), 6 deletions(-)

diff --git a/src/foam/matrices/lduMatrix/lduAddressing/lduInterfaces/ggiLduInterface/ggiLduInterface.H b/src/foam/matrices/lduMatrix/lduAddressing/lduInterfaces/ggiLduInterface/ggiLduInterface.H
index f13268ce0..a4fc7b2c7 100644
--- a/src/foam/matrices/lduMatrix/lduAddressing/lduInterfaces/ggiLduInterface/ggiLduInterface.H
+++ b/src/foam/matrices/lduMatrix/lduAddressing/lduInterfaces/ggiLduInterface/ggiLduInterface.H
@@ -78,9 +78,8 @@ public:
         {}
 
 
-    // Destructor
-
-        virtual ~ggiLduInterface();
+    //- Destructor
+    virtual ~ggiLduInterface();
 
 
     // Member Functions
diff --git a/src/foam/matrices/lduMatrix/lduAddressing/lduInterfaces/lduInterface/lduInterface.H b/src/foam/matrices/lduMatrix/lduAddressing/lduInterfaces/lduInterface/lduInterface.H
index 3586aa66c..dfe101f50 100644
--- a/src/foam/matrices/lduMatrix/lduAddressing/lduInterfaces/lduInterface/lduInterface.H
+++ b/src/foam/matrices/lduMatrix/lduAddressing/lduInterfaces/lduInterface/lduInterface.H
@@ -74,9 +74,8 @@ public:
         {}
 
 
-    // Destructor
-
-        virtual ~lduInterface();
+    //- Destructor
+    virtual ~lduInterface();
 
 
     // Member Functions

From 6e9ce7b547583880e63e63471f290af24036bff7 Mon Sep 17 00:00:00 2001
From: Hrvoje Jasak <h.jasak@wikki.co.uk>
Date: Tue, 9 Aug 2016 13:25:14 +0100
Subject: [PATCH 006/130] Feature: Rewrite of GGI comms on AMG agglomeration

---
 .../BlockMatrixAgglomeration.C                |   1 +
 .../GAMGAgglomerateLduAddressing.C            |   1 +
 .../interfaces/AMGInterface/AMGInterface.H    |   8 +
 .../interfaces/AMGInterface/newAMGInterface.C |  15 +-
 .../cyclicAMGInterface/cyclicAMGInterface.C   |   1 +
 .../cyclicAMGInterface/cyclicAMGInterface.H   |   1 +
 .../cyclicGGIAMGInterface.C                   |   2 +
 .../cyclicGGIAMGInterface.H                   |   1 +
 .../ggiAMGInterface/ggiAMGInterface.C         | 734 +++++++++++-------
 .../ggiAMGInterface/ggiAMGInterface.H         |  14 +-
 .../mixingPlaneAMGInterface.C                 |   1 +
 .../mixingPlaneAMGInterface.H                 |   1 +
 .../processorAMGInterface.C                   |   1 +
 .../processorAMGInterface.H                   |   1 +
 .../regionCoupleAMGInterface.C                |   2 +
 .../regionCoupleAMGInterface.H                |   1 +
 src/lduSolvers/amg/amgPolicy/pamgPolicy.C     |   1 +
 17 files changed, 504 insertions(+), 282 deletions(-)

diff --git a/src/foam/matrices/blockLduMatrix/BlockAMG/BlockMatrixCoarsening/BlockMatrixAgglomeration/BlockMatrixAgglomeration.C b/src/foam/matrices/blockLduMatrix/BlockAMG/BlockMatrixCoarsening/BlockMatrixAgglomeration/BlockMatrixAgglomeration.C
index 9278ad63b..792a39c0f 100644
--- a/src/foam/matrices/blockLduMatrix/BlockAMG/BlockMatrixCoarsening/BlockMatrixAgglomeration/BlockMatrixAgglomeration.C
+++ b/src/foam/matrices/blockLduMatrix/BlockAMG/BlockMatrixCoarsening/BlockMatrixAgglomeration/BlockMatrixAgglomeration.C
@@ -950,6 +950,7 @@ Foam::BlockMatrixAgglomeration<Type>::restrictMatrix() const
                 AMGInterface::New
                 (
                     coarseAddrPtr(),
+                    coarseInterfaces,
                     fineInterface,
                     fineInterface.interfaceInternalField(agglomIndex_),
                     fineInterfaceAddr[intI]
diff --git a/src/foam/matrices/lduMatrix/solvers/AMG/AMGAgglomerations/GAMGAgglomeration/GAMGAgglomerateLduAddressing.C b/src/foam/matrices/lduMatrix/solvers/AMG/AMGAgglomerations/GAMGAgglomeration/GAMGAgglomerateLduAddressing.C
index e93172b11..9ba20ce1f 100644
--- a/src/foam/matrices/lduMatrix/solvers/AMG/AMGAgglomerations/GAMGAgglomeration/GAMGAgglomerateLduAddressing.C
+++ b/src/foam/matrices/lduMatrix/solvers/AMG/AMGAgglomerations/GAMGAgglomeration/GAMGAgglomerateLduAddressing.C
@@ -292,6 +292,7 @@ void Foam::GAMGAgglomeration::agglomerateLduAddressing
                     AMGInterface::New
                     (
                         meshLevels_[fineLevelIndex],
+                        coarseInterfaces,
                         fineInterfaces[inti],
                         fineInterfaces[inti].interfaceInternalField
                         (
diff --git a/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/AMGInterface/AMGInterface.H b/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/AMGInterface/AMGInterface.H
index 3ef010832..639a04314 100644
--- a/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/AMGInterface/AMGInterface.H
+++ b/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/AMGInterface/AMGInterface.H
@@ -129,12 +129,14 @@ public:
             lduInterface,
             (
                 const lduPrimitiveMesh& lduMesh,
+                const lduInterfacePtrsList& coarseInterfaces,
                 const lduInterface& fineInterface,
                 const labelField& localRestrictAddressing,
                 const labelField& neighbourRestrictAddressing
             ),
             (
                 lduMesh,
+                coarseInterfaces,
                 fineInterface,
                 localRestrictAddressing,
                 neighbourRestrictAddressing
@@ -149,6 +151,7 @@ public:
         static autoPtr<AMGInterface> New
         (
             const lduPrimitiveMesh& lduMesh,
+            const lduInterfacePtrsList& coarseInterfaces,
             const lduInterface& fineInterface,
             const labelField& localRestrictAddressing,
             const labelField& neighbourRestrictAddressing
@@ -165,6 +168,11 @@ public:
         {}
 
 
+    //- Destructor
+    virtual ~AMGInterface()
+    {}
+
+
     // Member Functions
 
         // Access
diff --git a/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/AMGInterface/newAMGInterface.C b/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/AMGInterface/newAMGInterface.C
index 4a8f5236f..2f6dc2548 100644
--- a/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/AMGInterface/newAMGInterface.C
+++ b/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/AMGInterface/newAMGInterface.C
@@ -32,6 +32,7 @@ License
 Foam::autoPtr<Foam::AMGInterface> Foam::AMGInterface::New
 (
     const lduPrimitiveMesh& lduMesh,
+    const lduInterfacePtrsList& coarseInterfaces,
     const lduInterface& fineInterface,
     const labelField& localRestrictAddressing,
     const labelField& neighbourRestrictAddressing
@@ -46,11 +47,14 @@ Foam::autoPtr<Foam::AMGInterface> Foam::AMGInterface::New
     {
         FatalErrorIn
         (
-            "AMGInterface::New"
-            "(const lduPrimitiveMesh& lduMesh,"
-            "const lduInterface& fineInterface,"
-            "const labelField& localRestrictAddressing,"
-            "const labelField& neighbourRestrictAddressing)"
+            "AMGInterface::New\n"
+            "(\n"
+            "    const lduPrimitiveMesh& lduMesh,\n"
+            "    const lduInterfacePtrsList& coarseInterfaces,\n"
+            "    const lduInterface& fineInterface,\n"
+            "    const labelField& localRestrictAddressing,\n"
+            "    const labelField& neighbourRestrictAddressing\n"
+            ")"
         )   << "Unknown AMGInterface type " << coupleType << ".\n"
             << "Valid AMGInterface types are :"
             << lduInterfaceConstructorTablePtr_->sortedToc()
@@ -62,6 +66,7 @@ Foam::autoPtr<Foam::AMGInterface> Foam::AMGInterface::New
         cstrIter()
         (
             lduMesh,
+            coarseInterfaces,
             fineInterface,
             localRestrictAddressing,
             neighbourRestrictAddressing
diff --git a/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/cyclicAMGInterface/cyclicAMGInterface.C b/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/cyclicAMGInterface/cyclicAMGInterface.C
index 2ae72dbaa..72fe4cf0b 100644
--- a/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/cyclicAMGInterface/cyclicAMGInterface.C
+++ b/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/cyclicAMGInterface/cyclicAMGInterface.C
@@ -45,6 +45,7 @@ namespace Foam
 Foam::cyclicAMGInterface::cyclicAMGInterface
 (
     const lduPrimitiveMesh& lduMesh,
+    const lduInterfacePtrsList& coarseInterfaces,
     const lduInterface& fineInterface,
     const labelField& localRestrictAddressing,
     const labelField& neighbourRestrictAddressing
diff --git a/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/cyclicAMGInterface/cyclicAMGInterface.H b/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/cyclicAMGInterface/cyclicAMGInterface.H
index 3e983f19b..0a3978e47 100644
--- a/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/cyclicAMGInterface/cyclicAMGInterface.H
+++ b/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/cyclicAMGInterface/cyclicAMGInterface.H
@@ -81,6 +81,7 @@ public:
         cyclicAMGInterface
         (
             const lduPrimitiveMesh& lduMesh,
+            const lduInterfacePtrsList& coarseInterfaces,
             const lduInterface& fineInterface,
             const labelField& localRestrictAddressing,
             const labelField& neighbourRestrictAddressing
diff --git a/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/cyclicGGIAMGInterface/cyclicGGIAMGInterface.C b/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/cyclicGGIAMGInterface/cyclicGGIAMGInterface.C
index 6f1df8b3d..c64227099 100644
--- a/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/cyclicGGIAMGInterface/cyclicGGIAMGInterface.C
+++ b/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/cyclicGGIAMGInterface/cyclicGGIAMGInterface.C
@@ -45,6 +45,7 @@ namespace Foam
 Foam::cyclicGGIAMGInterface::cyclicGGIAMGInterface
 (
     const lduPrimitiveMesh& lduMesh,
+    const lduInterfacePtrsList& coarseInterfaces,
     const lduInterface& fineInterface,
     const labelField& localRestrictAddressing,
     const labelField& neighbourRestrictAddressing
@@ -53,6 +54,7 @@ Foam::cyclicGGIAMGInterface::cyclicGGIAMGInterface
     ggiAMGInterface
     (
         lduMesh,
+        coarseInterfaces,
         fineInterface,
         localRestrictAddressing,
         neighbourRestrictAddressing
diff --git a/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/cyclicGGIAMGInterface/cyclicGGIAMGInterface.H b/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/cyclicGGIAMGInterface/cyclicGGIAMGInterface.H
index 19f2b92d7..2498c2282 100644
--- a/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/cyclicGGIAMGInterface/cyclicGGIAMGInterface.H
+++ b/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/cyclicGGIAMGInterface/cyclicGGIAMGInterface.H
@@ -67,6 +67,7 @@ public:
         cyclicGGIAMGInterface
         (
             const lduPrimitiveMesh& lduMesh,
+            const lduInterfacePtrsList& coarseInterfaces,
             const lduInterface& fineInterface,
             const labelField& localRestrictAddressing,
             const labelField& neighbourRestrictAddressing
diff --git a/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/ggiAMGInterface/ggiAMGInterface.C b/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/ggiAMGInterface/ggiAMGInterface.C
index 82527393c..899ee6768 100644
--- a/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/ggiAMGInterface/ggiAMGInterface.C
+++ b/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/ggiAMGInterface/ggiAMGInterface.C
@@ -192,6 +192,7 @@ void Foam::ggiAMGInterface::initFastReduce() const
 Foam::ggiAMGInterface::ggiAMGInterface
 (
     const lduPrimitiveMesh& lduMesh,
+    const lduInterfacePtrsList& coarseInterfaces,
     const lduInterface& fineInterface,
     const labelField& localRestrictAddressing,
     const labelField& neighbourRestrictAddressing
@@ -201,107 +202,87 @@ Foam::ggiAMGInterface::ggiAMGInterface
     fineGgiInterface_(refCast<const ggiLduInterface>(fineInterface)),
     zoneSize_(0),
     zoneAddressing_(),
+    procMasterFaces_(),
     mapPtr_(NULL)
 {
-    // Note.
-    // Signalling in global clustering requires me to recognise clustering
-    // from separate processors as separate.  In the first phase, this will be
-    // used to recognise cluster from each processor as separate and in the
-    // second phase it will be used to filter local processor faces from
-    // the global patch.
-    // Currently, I am calculating unique cluster index as:
-    //
-    // id = cluster + procOffset*myProcID
-    // With procOffset = 1 million, this should be sufficient for 2000 CPUs
-    // with 2 million coarse cells each.  For larger numbers, I need a
-    // larger max int, which can be changed on request
-    // HJ, 1/Apr/2009
-
-    // New algorithm will assemble local clusters and then create a global
-    // zone ordering by collecting all faces (coarse pairs) from proc0,
+    // New algorithm will assemble local clusters on the master side and
+    // create zone ordering by collecting all faces (coarse pairs) from proc0,
     // followed by proc 1 etc.  This avoids global communication and allows
     // each processor only to perform the analysis on locally created coarse
     // faces
     // HJ, 13/Jun/2016
 
-    // To help with analysis, expand the local and neighbour addressing
-    // to full zone size
-    labelField localExpandAddressing(fineGgiInterface_.zoneSize(), 0);
-
-    // Memory management, local
-    {
-        const labelList& addr = fineGgiInterface_.zoneAddressing();
-
-        forAll (addr, i)
-        {
-            localExpandAddressing[addr[i]] =
-                localRestrictAddressing[i] + procOffset*Pstream::myProcNo();
-        }
-
-        // Removed global reduce.  Only local faces will be analysed.
-        // HJ, 13/Jun/2016
-    }
+    // Note: local addressing contains only local faces
+    const labelList& fineZa =  fineGgiInterface_.zoneAddressing();
 
     // Create addressing for neighbour faces.  Note: expandAddrToZone will
-    // expand the addressing to zone size.  HJ, 13/Jun/2016
-    labelField neighbourExpandAddressing
+    // expand the addressing to zone size, including communications.
+    // Faces which are not used locally will be marked by NaNs
+    // HJ, 13/Jun/2016
+    labelField neighbourExpandAddressing = neighbourRestrictAddressing;
+
+    // Expand neighbour side to get all the data required from other
+    // processors.
+    // Note: neigbour is now the size of remote zone
+    fineGgiInterface_.shadowInterface().expandAddrToZone
     (
-        fineGgiInterface_.shadowInterface().interfaceSize()
+        neighbourExpandAddressing
     );
 
-    // Fill local cluster ID with a combination of a local ID and processor
-    // offset
-    // Memory management, neighbour
-    {
-        forAll (neighbourExpandAddressing, i)
-        {
-            neighbourExpandAddressing[i] =
-                neighbourRestrictAddressing[i]
-              + procOffset*Pstream::myProcNo();
-        }
+    // Create addressing for neighbour processors.  Note: expandAddrToZone will
+    // expand the addressing to zone size.  HJ, 13/Jun/2016
+    labelField neighbourExpandProc
+    (
+        fineGgiInterface_.shadowInterface().interfaceSize(),
+        Pstream::myProcNo()
+    );
 
-        // Expand neighbour side to get all the data required from other
-        // processors.  Note: neigbour is now the size of remote zone
-        fineGgiInterface_.shadowInterface().expandAddrToZone
-        (
-            neighbourExpandAddressing
-        );
-    }
+    // Expand neighbour side to get all the data required from other
+    // processors.
+    // Note: neigbour is now the size of remote zone
+    fineGgiInterface_.shadowInterface().expandAddrToZone
+    (
+        neighbourExpandProc
+    );
 
-    // DEBUG: Check that all sizes are at zone size.
-    Info<< "Sizes check: local zone size "
-        << fineGgiInterface_.zoneSize()
-        << " " << localExpandAddressing << nl
-        << "shadow zone size "
-        << fineGgiInterface_.shadowInterface().zoneSize()
-        << " " << neighbourExpandAddressing
-        << endl;
+    // Note: neighbourExpandAddressing and neighbourExpandProc
+    // will be filled with NaNs for faces which are not local
 
-
-    // Note: neighbourExpandAddressing will be filled with NaNs for faces which
-    // not local
-
-    Info<< "End of reduce" << endl;
     // Make a lookup table of entries for owner/neighbour.
     // All sizes are guessed at the size of fine interface
     // HJ, 19/Feb/2009
 
-    HashTable<SLList<label>, label, Hash<label> > neighboursTable
+    // Note: Guessing size of HashTable to fine interface size
+
+    // Coded neighbour index. Note: using long int to simplify encoding
+    // HJ, 1/Aug/2016
+    HashTable<SLList<long>, long, Hash<long> > neighboursTable
     (
-        fineGgiInterface_.interfaceSize()
+        Foam::max(128, fineGgiInterface_.interfaceSize()/4)
     );
 
-    // Table of face-sets to be agglomerated
+    // Neignbour processor index
+    HashTable<SLList<label>, label, Hash<label> > nbrsProcTable
+    (
+        Foam::max(128, fineGgiInterface_.interfaceSize()/4)
+    );
+
+    // Neighbour face-faces addressing for a face with split neighbours
     HashTable<SLList<SLList<label> >, label, Hash<label> > faceFaceTable
     (
-        fineGgiInterface_.interfaceSize()
+        Foam::max(128, fineGgiInterface_.interfaceSize()/4)
     );
 
-    // Table of face-sets weights to be agglomerated
+    HashTable<SLList<SLList<label> >, label, Hash<label> > faceFaceNbrTable
+    (
+        Foam::max(128, fineGgiInterface_.interfaceSize()/4)
+    );
+
+    // Neighbour face-faces weights for a face with split neighbours
     HashTable<SLList<SLList<scalar> >, label, Hash<label> >
         faceFaceWeightsTable
         (
-            fineGgiInterface_.interfaceSize()
+            Foam::max(128, fineGgiInterface_.interfaceSize()/4)
         );
 
     // Count the number of coarse faces
@@ -318,12 +299,6 @@ Foam::ggiAMGInterface::ggiAMGInterface
         const labelListList& fineAddr = fineGgiInterface_.addressing();
         const scalarListList& fineWeights = fineGgiInterface_.weights();
 
-        // Note: cluster only locally live faces
-        // HJ, 13/Jun/2016
-
-        // This addressing defines which faces from zone are local
-        const labelList& fineZa = fineGgiInterface_.zoneAddressing();
-
         // Perform analysis only for local faces
         // HJ, 22/Jun/2016
         forAll (fineZa, fineZaI)
@@ -336,8 +311,27 @@ Foam::ggiAMGInterface::ggiAMGInterface
 
             forAll (curFineNbrs, nbrI)
             {
-                label curMaster = -1;
-                label curSlave = -1;
+                long curMaster = -1;
+                label curMasterProc = -1;
+                long curSlave = -1;
+                label curSlaveProc = -1;
+
+                // Note.  Signalling in global clustering requires
+                // me to recognise clustering from separate
+                // processors as separate.  In the first phase,
+                // this will be used to recognise cluster from
+                // each processor as separate and in the second
+                // phase it will be used to filter local processor
+                // faces from the global patch.  Currently, I am
+                // calculating unique cluster index as:
+                //
+                // id = cluster + procOffset*myProcID
+                //
+                // With procOffset = 1 million, this should be
+                // sufficient for 2000 CPUs with 2 million coarse
+                // cells each.  For larger numbers, I need a
+                // larger max int, which can be changed on request
+                // HJ, 1/Apr/2009
 
                 // My label = ffI
                 // Nbr label = nnI
@@ -347,49 +341,76 @@ Foam::ggiAMGInterface::ggiAMGInterface
                 if (fineGgiInterface_.master())
                 {
                     // Master side
-                    curMaster = localExpandAddressing[ffI];
+                    curMaster = localRestrictAddressing[fineZaI];
+                    curMasterProc = Pstream::myProcNo();
                     curSlave = neighbourExpandAddressing[nnI];
+                    curSlaveProc = neighbourExpandProc[nnI];
                 }
                 else
                 {
-                    // Slave side
                     curMaster = neighbourExpandAddressing[nnI];
-                    curSlave = localExpandAddressing[ffI];
+                    curMasterProc = neighbourExpandProc[nnI];
+                    curSlave = localRestrictAddressing[fineZaI];
+                    curSlaveProc = Pstream::myProcNo();
                 }
 
+                // Code in current master and slave
+                curMaster += procOffset*curMasterProc;
+                curSlave += procOffset*curSlaveProc;
+
                 // Look for the master cell.  If it has already got a face,
                 // add the coefficient to the face.  If not, create a new
-                // face.
+                // face
                 if (neighboursTable.found(curMaster))
                 {
+                    // This master side face already exists
+
                     // Check all current neighbours to see if the current
                     // slave already exists.  If so, add the coefficient.
 
-                    SLList<label>& curNbrs = neighboursTable.find(curMaster)();
+                    SLList<long>& curNbrs =
+                        neighboursTable.find(curMaster)();
+
+                    SLList<label>& curNbrsProc =
+                        nbrsProcTable.find(curMaster)();
 
                     SLList<SLList<label> >& curFaceFaces =
                         faceFaceTable.find(curMaster)();
 
+                    SLList<SLList<label> >& curFaceFaceNbrs =
+                        faceFaceNbrTable.find(curMaster)();
+
                     SLList<SLList<scalar> >& curFaceWeights =
                         faceFaceWeightsTable.find(curMaster)();
 
+                    // Search for coded neighbour
                     bool nbrFound = false;
 
-                    SLList<label>::iterator nbrsIter = curNbrs.begin();
+                    SLList<long>::iterator nbrsIter = curNbrs.begin();
+
+                    SLList<label>::iterator nbrsProcIter =
+                        curNbrsProc.begin();
 
                     SLList<SLList<label> >::iterator faceFacesIter =
                         curFaceFaces.begin();
 
+                    SLList<SLList<label> >::iterator faceFaceNbrsIter =
+                        curFaceFaceNbrs.begin();
+
                     SLList<SLList<scalar> >::iterator faceFaceWeightsIter =
-                        curFaceWeights.begin();
+                    curFaceWeights.begin();
 
                     for
                     (
                         ;
                         nbrsIter != curNbrs.end()
+                     && nbrsProcIter != curNbrsProc.end()
                      && faceFacesIter != curFaceFaces.end()
+                     && faceFaceNbrsIter != curFaceFaceNbrs.end()
                      && faceFaceWeightsIter != curFaceWeights.end();
-                        ++nbrsIter, ++faceFacesIter, ++faceFaceWeightsIter
+                        ++nbrsIter, ++nbrsProcIter,
+                        ++faceFacesIter, ++faceFaceNbrsIter,
+                        ++faceFaceWeightsIter
                     )
                     {
                         // Check neighbour slave
@@ -397,6 +418,7 @@ Foam::ggiAMGInterface::ggiAMGInterface
                         {
                             nbrFound = true;
                             faceFacesIter().append(ffI);
+                            faceFaceNbrsIter().append(nbrI);
                             faceFaceWeightsIter().append(curNW);
 
                             // New agglomeration pair found in already
@@ -410,7 +432,9 @@ Foam::ggiAMGInterface::ggiAMGInterface
                     if (!nbrFound)
                     {
                         curNbrs.append(curSlave);
+                        curNbrsProc.append(curSlaveProc);
                         curFaceFaces.append(SLList<label>(ffI));
+                        curFaceFaceNbrs.append(SLList<label>(nbrI));
                         curFaceWeights.append(SLList<scalar>(curNW));
 
                         // New coarse face created for an existing master
@@ -420,9 +444,20 @@ Foam::ggiAMGInterface::ggiAMGInterface
                 }
                 else
                 {
-                    // This master has got no neighbours yet.  Add a neighbour
-                    // and a coefficient, thus creating a new face
-                    neighboursTable.insert(curMaster, SLList<label>(curSlave));
+                    // This master has got no neighbours yet.
+                    // Add a neighbour, proc and a coefficient as a
+                    // new list, thus creating a new face
+                    neighboursTable.insert
+                    (
+                        curMaster,
+                        SLList<long>(curSlave)
+                    );
+
+                    nbrsProcTable.insert
+                    (
+                        curMaster,
+                        SLList<label>(curSlaveProc)
+                    );
 
                     faceFaceTable.insert
                     (
@@ -430,6 +465,12 @@ Foam::ggiAMGInterface::ggiAMGInterface
                         SLList<SLList<label> >(SLList<label>(ffI))
                     );
 
+                    faceFaceNbrTable.insert
+                    (
+                        curMaster,
+                        SLList<SLList<label> >(SLList<label>(nbrI))
+                    );
+
                     faceFaceWeightsTable.insert
                     (
                         curMaster,
@@ -446,8 +487,7 @@ Foam::ggiAMGInterface::ggiAMGInterface
     else
     {
         // Coarse level, addressing is stored in faceCells
-        // This addressing defines whicf faces from zone are local
-        const labelList& fineZa = fineGgiInterface_.zoneAddressing();
+        // This addressing defines which faces from zone are local
 
         // Perform analysis only for local faces
         // HJ, 22/Jun/2016
@@ -456,48 +496,85 @@ Foam::ggiAMGInterface::ggiAMGInterface
             // Get the local face (from zone) to analyse
             const label ffI = fineZa[fineZaI];
 
-            label curMaster = -1;
-            label curSlave = -1;
+            long curMaster = -1;
+            label curMasterProc = -1;
+            long curSlave = -1;
+            label curSlaveProc = -1;
 
-            // Do switching on master/slave indices based on the
-            // owner/neighbour of the processor index such that
-            // both sides get the same answer.
-            if (master())
+            // Note.  Signalling in global clustering requires
+            // me to recognise clustering from separate
+            // processors as separate.  In the first phase,
+            // this will be used to recognise cluster from
+            // each processor as separate and in the second
+            // phase it will be used to filter local processor
+            // faces from the global patch.  Currently, I am
+            // calculating unique cluster index as:
+            //
+            // id = cluster + procOffset*myProcID
+            //
+            // With procOffset = 1 million, this should be
+            // sufficient for 2000 CPUs with 2 million coarse
+            // cells each.  For larger numbers, I need a
+            // larger max int, which can be changed on request
+            // HJ, 1/Apr/2009
+
+            if (fineGgiInterface_.master())
             {
                 // Master side
-                curMaster = localExpandAddressing[ffI];
+                curMaster = localRestrictAddressing[fineZaI];
+                curMasterProc = Pstream::myProcNo();
                 curSlave = neighbourExpandAddressing[ffI];
+                curSlaveProc = neighbourExpandProc[ffI];
             }
             else
             {
-                // Slave side
                 curMaster = neighbourExpandAddressing[ffI];
-                curSlave = localExpandAddressing[ffI];
+                curMasterProc = neighbourExpandProc[ffI];
+                curSlave = localRestrictAddressing[fineZaI];
+                curSlaveProc = Pstream::myProcNo();
             }
 
+            // Code in current master and slave
+            curMaster += procOffset*curMasterProc;
+            curSlave += procOffset*curSlaveProc;
+
             // Look for the master cell.  If it has already got a face,
             // add the coefficient to the face.  If not, create a new face.
             if (neighboursTable.found(curMaster))
             {
+                // This master side face already exists
+
                 // Check all current neighbours to see if the current slave
                 // already exists and if so, add the fine face
                 // to the agglomeration.
 
-                SLList<label>& curNbrs = neighboursTable.find(curMaster)();
+                SLList<long>& curNbrs = neighboursTable.find(curMaster)();
+
+                SLList<label>& curNbrsProc =
+                    nbrsProcTable.find(curMaster)();
 
                 SLList<SLList<label> >& curFaceFaces =
                     faceFaceTable.find(curMaster)();
 
-                    SLList<SLList<scalar> >& curFaceWeights =
-                        faceFaceWeightsTable.find(curMaster)();
+                SLList<SLList<label> >& curFaceFaceNbrs =
+                    faceFaceNbrTable.find(curMaster)();
+
+                SLList<SLList<scalar> >& curFaceWeights =
+                    faceFaceWeightsTable.find(curMaster)();
 
                 bool nbrFound = false;
 
-                SLList<label>::iterator nbrsIter = curNbrs.begin();
+                SLList<long>::iterator nbrsIter = curNbrs.begin();
+
+                SLList<label>::iterator nbrsProcIter =
+                    curNbrsProc.begin();
 
                 SLList<SLList<label> >::iterator faceFacesIter =
                     curFaceFaces.begin();
 
+                SLList<SLList<label> >::iterator faceFaceNbrsIter =
+                    curFaceFaceNbrs.begin();
+
                 SLList<SLList<scalar> >::iterator faceFaceWeightsIter =
                     curFaceWeights.begin();
 
@@ -506,8 +583,10 @@ Foam::ggiAMGInterface::ggiAMGInterface
                     ;
                     nbrsIter != curNbrs.end()
                  && faceFacesIter != curFaceFaces.end()
+                 && faceFaceNbrsIter != curFaceFaceNbrs.end()
                  && faceFaceWeightsIter != curFaceWeights.end();
-                    ++nbrsIter, ++faceFacesIter, ++faceFaceWeightsIter
+                    ++nbrsIter, ++nbrsProcIter,
+                    ++faceFacesIter, ++faceFaceNbrsIter, ++faceFaceWeightsIter
                 )
                 {
                     // Check neighbour slave
@@ -515,12 +594,15 @@ Foam::ggiAMGInterface::ggiAMGInterface
                     {
                         nbrFound = true;
                         faceFacesIter().append(ffI);
+                        // Add dummy nbr
+                        faceFaceNbrsIter().append(0);
                         // Add dummy weight
                         faceFaceWeightsIter().append(1.0);
 
                         // New agglomeration pair found in already
                         // existing pair
                         nAgglomPairs++;
+
                         break;
                     }
                 }
@@ -528,10 +610,13 @@ Foam::ggiAMGInterface::ggiAMGInterface
                 if (!nbrFound)
                 {
                     curNbrs.append(curSlave);
+                    curNbrsProc.append(curSlaveProc);
                     curFaceFaces.append(SLList<label>(ffI));
+                    // Add dummy nbr
+                    curFaceFaceNbrs.append(SLList<label>(0));
                     // Add dummy weight
                     curFaceWeights.append(SLList<scalar>(1.0));
-
+                    
                     // New coarse face created for an existing master
                     nCoarseFaces++;
                     nAgglomPairs++;
@@ -541,7 +626,17 @@ Foam::ggiAMGInterface::ggiAMGInterface
             {
                 // This master has got no neighbours yet.  Add a neighbour
                 // and a coefficient, thus creating a new face
-                neighboursTable.insert(curMaster, SLList<label>(curSlave));
+                neighboursTable.insert
+                (
+                    curMaster,
+                    SLList<long>(curSlave)
+                );
+
+                nbrsProcTable.insert
+                (
+                    curMaster,
+                    SLList<label>(curSlaveProc)
+                );
 
                 faceFaceTable.insert
                 (
@@ -549,41 +644,48 @@ Foam::ggiAMGInterface::ggiAMGInterface
                     SLList<SLList<label> >(SLList<label>(ffI))
                 );
 
+                // Add dummy nbr
+                faceFaceNbrTable.insert
+                (
+                    curMaster,
+                    SLList<SLList<label> >(SLList<label>(0))
+                );
+
                 // Add dummy weight
                 faceFaceWeightsTable.insert
                 (
                     curMaster,
                     SLList<SLList<scalar> >(SLList<scalar>(1.0))
                 );
-
+                
                 // New coarse face created for a new master
                 nCoarseFaces++;
                 nAgglomPairs++;
             }
         } // end for all fine faces
-    }
+    } // end of else in fine level (coarse level)
+
+    // Since only local faces are analysed, lists can now be resized
+    faceCells_.setSize(nCoarseFaces, -1);
+    fineAddressing_.setSize(nAgglomPairs, -1);
+    restrictAddressing_.setSize(nAgglomPairs, -1);
+    restrictWeights_.setSize(nAgglomPairs);
 
     // In order to assemble the coarse global face zone, find out
     // how many faces have been created on each processor.
-    // Note that masters and slaves both count faces so we will only ask master
-    // sizes to count
+    // Note that masters and slaves both count faces so we will
+    //  only ask master sizes to count
     labelList nCoarseFacesPerProc(Pstream::nProcs(), 0);
 
-    if (master())
-    {
-        nCoarseFacesPerProc[Pstream::myProcNo()] = nCoarseFaces;
-    }
+    nCoarseFacesPerProc[Pstream::myProcNo()] = nCoarseFaces;
 
     reduce(nCoarseFacesPerProc, sumOp<labelList>());
 
-    Info<< "Number of faces per processor: " << nCoarseFacesPerProc
-        << endl;
-
     // Coarse global face zone is assembled by adding all faces from proc0,
     // followed by all faces from proc1 etc.
     // Therefore, on procN, my master offset
-    // will be equal to the sum of numbers of coarse faces on all processors
-    // before mine
+    // will be equal to the sum of numbers of coarse faces on all
+    // processors before mine
     // HJ, 13/Jun/2016
 
     label coarseGlobalFaceOffset = 0;
@@ -592,147 +694,43 @@ Foam::ggiAMGInterface::ggiAMGInterface
     {
         coarseGlobalFaceOffset += nCoarseFacesPerProc[i];
     }
+    
+    // Grab zone size and create zone addressing
+    zoneSize_ = sum(nCoarseFacesPerProc);
 
-    Pout<< "coarseGlobalFaceOffset: " << coarseGlobalFaceOffset << endl;
+    zoneAddressing_.setSize(nCoarseFaces);
 
-    Info<< "End of contents assembly" << endl;
-    labelField masterFaceCells(nCoarseFaces, -1);
-    labelField masterZoneAddressing(nCoarseFaces, -1);
-    labelField masterFineAddressing(nCoarseFaces, -1);
-    labelField masterRestrictAddressing(nAgglomPairs, -1);
-    scalarField masterRestrictWeights(nAgglomPairs);
-
-    // Note: in multiple agglomeration
-
-    labelList contents = neighboursTable.toc();
+    // Both master and slave have done agglomeration, but only master
+    // will construct the global faces index.
+    // To avoid searching, master will prepare a list of global face
+    // indices that appear on each processor in order and communicate them
+    // to the slave
+    // The slave will then know which is the next global face from which
+    // processor and pick them out in the same order
 
     // Global faces shall be assembled by the increasing label of master
     // cluster ID.
+    List<long> contents = neighboursTable.toc();
 
     // Sort makes sure the order is identical on both sides.
     // HJ, 20/Feb/2009 and 6/Jun/2016
     sort(contents);
 
-    // Grab zone size and create zone addressing
-    zoneSize_ = sum(nCoarseFacesPerProc);
-    Info<< "zoneSize_: " << zoneSize_ << endl;
-
-    // Note:
-    // When I am agglomerating the master, I know faces are stacked up in order
-    // but on the slave side, all I know is the master cluster index and
-    // not a master coarse face index.  Therefore:
-    // - master needs to be agglomerated first
-    // - once master is agglomerated, I need to signal to the slave side
-    //   the global coarse face zone index
-
-
-    // Note: zone addressing will be assembled only for local clusters
-    // using the coarseGlobalFaceOffset
-    // HJ, 13/Jun/2016
-    label nProcFaces = 0;
-
-    // Reset face counter for re-use
-    nCoarseFaces = 0;
-    nAgglomPairs = 0;
-
-    // Note:
-    // Since clustering has now happened only on local faces, addressing and
-    // all other array work on local indices and not on the coarse global zone
-    // HJ, 13/Jun/2016
-
-    // Establish zone addressing on the master side and communicate
-    // it to the shadow
-
-    // On master side, the owner addressing is stored in table of contents
-    forAll (contents, masterI)
-    {
-        SLList<label>& curNbrs =
-            neighboursTable.find(contents[masterI])();
-
-        // Note: neighbour processor index is irrelevant.  HJ, 1/Apr/2009
-
-        SLList<SLList<label> >& curFaceFaces =
-            faceFaceTable.find(contents[masterI])();
-
-        SLList<SLList<scalar> >& curFaceWeights =
-            faceFaceWeightsTable.find(contents[masterI])();
-
-        SLList<label>::iterator nbrsIter = curNbrs.begin();
-        SLList<SLList<label> >::iterator faceFacesIter =
-            curFaceFaces.begin();
-
-        SLList<SLList<scalar> >::iterator faceFaceWeightsIter =
-            curFaceWeights.begin();
-
-        for
-        (
-            ;
-            nbrsIter != curNbrs.end()
-         && faceFacesIter != curFaceFaces.end()
-         && faceFaceWeightsIter != curFaceWeights.end();
-            ++nbrsIter, ++faceFacesIter, ++faceFaceWeightsIter
-        )
-        {
-            // Check if master is on local processor: no longer needed,
-            // as only local processor is being searched.  HJ, 13/Jun/2016
-
-            // Record that this face belongs locally
-            // Use offset to indicate its position in the list
-            masterZoneAddressing[nProcFaces] =
-                nProcFaces + coarseGlobalFaceOffset;
-
-            masterFaceCells[nProcFaces] =
-                contents[masterI] - procOffset*Pstream::myProcNo();
-
-            SLList<label>::iterator facesIter =
-                faceFacesIter().begin();
-
-            SLList<scalar>::iterator weightsIter =
-                faceFaceWeightsIter().begin();
-
-            for
-            (
-                ;
-                facesIter != faceFacesIter().end()
-             && weightsIter != faceFaceWeightsIter().end();
-                ++facesIter, ++weightsIter
-            )
-            {
-                masterFineAddressing[nAgglomPairs] = facesIter();
-
-                // Master processor zone face is calculated from
-                masterRestrictAddressing[nAgglomPairs] =
-                    nProcFaces + coarseGlobalFaceOffset;
-                masterRestrictWeights[nAgglomPairs] = weightsIter();
-                nAgglomPairs++;
-            }
-
-            nProcFaces++;
-        }
-    }
-
-    // Resize arrays: not all of ggi is used locally
-    masterFaceCells.setSize(nProcFaces);
-    masterZoneAddressing.setSize(nProcFaces);
-
-    masterFineAddressing.setSize(nAgglomPairs);
-    masterRestrictAddressing.setSize(nAgglomPairs);
-    masterRestrictWeights.setSize(nAgglomPairs);
-
-    // Note: Both master and slave have done the same agglomeration up to here
-
+    // Note: Restriction is done on master side only because this is where
+    // the local zone is created.  HJ, 1/Aug/2016
     if (master())
     {
-        // Master has completed the clustering
-        faceCells_ = masterFaceCells;
-        zoneAddressing_ = masterZoneAddressing;
-        fineAddressing_ = masterFineAddressing;
-        restrictAddressing_ = masterRestrictAddressing;
-        restrictWeights_ = masterRestrictWeights;
-    }
-    else
-    {
-        // Note: shadowRestrictAddressing contains the
+        // Note:
+        // When I am agglomerating the master, faces are stacked up in order
+        // but on the slave side, all I know is the master cluster index and
+        // not a master coarse face index.  Therefore:
+        // - master needs to be agglomerated first
+        // - once master is agglomerated, I need to signal to the slave side
+        //   the global coarse face zone index
+
+        // For each new global face created on master proc,
+        // record its index under the slave proc array
+        List<SLList<label> > procMasterFacesLL(Pstream::nProcs());
 
         // Note: zone addressing will be assembled only for local clusters
         // using the coarseGlobalFaceOffset
@@ -743,12 +741,20 @@ Foam::ggiAMGInterface::ggiAMGInterface
         nCoarseFaces = 0;
         nAgglomPairs = 0;
 
-        // On slave side, the owner addressing is stored in linked lists
+        // Note:
+        // Since clustering has now happened only on local faces,
+        // addressing and all other array work on local indices and
+        // not on the coarse global zone
+        // HJ, 13/Jun/2016
+
+        // On master side, the owner addressing is stored in table of contents
         forAll (contents, masterI)
         {
-            // Note: master processor index is irrelevant.  HJ, 1/Apr/2009
+            SLList<long>& curNbrs =
+                neighboursTable.find(contents[masterI])();
 
-            SLList<label>& curNbrs = neighboursTable.find(contents[masterI])();
+            SLList<label>& curNbrsProc =
+                nbrsProcTable.find(contents[masterI])();
 
             SLList<SLList<label> >& curFaceFaces =
                 faceFaceTable.find(contents[masterI])();
@@ -756,7 +762,10 @@ Foam::ggiAMGInterface::ggiAMGInterface
             SLList<SLList<scalar> >& curFaceWeights =
                 faceFaceWeightsTable.find(contents[masterI])();
 
-            SLList<label>::iterator nbrsIter = curNbrs.begin();
+            SLList<long>::iterator nbrsIter = curNbrs.begin();
+
+            SLList<label>::iterator nbrsProcIter =
+                curNbrsProc.begin();
 
             SLList<SLList<label> >::iterator faceFacesIter =
                 curFaceFaces.begin();
@@ -768,29 +777,195 @@ Foam::ggiAMGInterface::ggiAMGInterface
             (
                 ;
                 nbrsIter != curNbrs.end()
+             && nbrsProcIter != curNbrsProc.end()
              && faceFacesIter != curFaceFaces.end()
              && faceFaceWeightsIter != curFaceWeights.end();
-                ++nbrsIter, ++faceFacesIter, ++faceFaceWeightsIter
+                ++nbrsIter, ++nbrsProcIter,
+                ++faceFacesIter, ++faceFaceWeightsIter
             )
             {
-                // Check if the face is on local processor: no longer needed,
+                // Check if master is on local processor: no longer needed,
                 // as only local processor is being searched.  HJ, 13/Jun/2016
 
-                // Record that this face belongs locally.
-
-//HJ, HERE: I need to find out the global face index for the face that was created from the master side
-
-                zoneAddressing_[nProcFaces] = nCoarseFaces;
-                faceCells_[nProcFaces] =
-                    nbrsIter() - procOffset*Pstream::myProcNo();
-                nProcFaces++;
-
+                // Get faces and weights
                 SLList<label>::iterator facesIter =
                     faceFacesIter().begin();
 
                 SLList<scalar>::iterator weightsIter =
                     faceFaceWeightsIter().begin();
 
+                // Record that this face belongs locally
+                // Use offset to indicate its position in the list
+                zoneAddressing_[nProcFaces] =
+                    nProcFaces + coarseGlobalFaceOffset;
+
+                // Record master cluster index
+                faceCells_[nProcFaces] =
+                    contents[masterI] - procOffset*Pstream::myProcNo();
+
+                // Record global processor face
+                procMasterFacesLL[nbrsProcIter()].append
+                (
+                    nProcFaces + coarseGlobalFaceOffset
+                );
+
+                // Collect agglomeration data
+                for
+                (
+                    ;
+                    facesIter != faceFacesIter().end()
+                 && weightsIter != faceFaceWeightsIter().end();
+                    ++facesIter, ++weightsIter
+                )
+                {
+                    fineAddressing_[nAgglomPairs] = facesIter();
+
+                    // Master processor zone face is calculated from
+                    // global offset
+                    restrictAddressing_[nAgglomPairs] =
+                        nProcFaces + coarseGlobalFaceOffset;
+
+                    restrictWeights_[nAgglomPairs] = weightsIter();
+                    nAgglomPairs++;
+                }
+
+                nProcFaces++;
+            }
+        }
+
+        // No need to resize arrays only local faces are used
+        // HJ, 1/Aug/2016
+
+        // Re-pack singly linked list of processor master faces
+        // and pass to other processors
+
+        // First index: master proc
+        // Second index: slave proc
+        // List contents: global faces in order
+        labelListListList crissCrossList(Pstream::nProcs());
+
+        labelListList& crissList = crissCrossList[Pstream::myProcNo()];
+
+        crissList.setSize(Pstream::nProcs());
+
+        forAll (crissList, procI)
+        {
+            crissList[procI] = procMasterFacesLL[procI];
+        }
+
+        Pstream::gatherList(crissCrossList);
+        Pstream::scatterList(crissCrossList);
+
+        procMasterFaces_.setSize(Pstream::nProcs());
+
+        forAll (procMasterFaces_, procI)
+        {
+            procMasterFaces_[procI] =
+                crissCrossList[procI][Pstream::myProcNo()];
+        }
+    }
+    // Agglomerate slave
+    else
+    {
+        // Note: zone addressing will be assembled only for local clusters
+        // using the coarseGlobalFaceOffset
+        // HJ, 13/Jun/2016
+        label nProcFaces = 0;
+
+        // Get master side procMasterFaces
+        // Note: this needs to be picked up from coarse interfaces rather than
+        // the matrix, as the coarse matrix assembly is not complete yet.
+        // Further, master has got a lower index in the list, meaning that
+        // it has already completed the assembly
+        const ggiAMGInterface& shadowGGI =
+            refCast<const ggiAMGInterface>(coarseInterfaces[shadowIndex()]);
+
+        const labelListList& masterProcMasterFaces =
+            shadowGGI.procMasterFaces();
+
+        // Reset face counter for re-use
+        nCoarseFaces = 0;
+        nAgglomPairs = 0;
+
+        // Count how many global faces are used for each proc on the other side
+        labelList npmf(Pstream::nProcs(), 0);
+
+        // On slave side, the owner addressing is stored in linked lists
+        forAll (contents, masterI)
+        {
+            SLList<long>& curNbrs = neighboursTable.find(contents[masterI])();
+
+            SLList<SLList<label> >& curFaceFaces =
+                faceFaceTable.find(contents[masterI])();
+
+            SLList<SLList<label> >& curFaceFaceNbrs =
+                faceFaceNbrTable.find(contents[masterI])();
+
+            SLList<SLList<scalar> >& curFaceWeights =
+                faceFaceWeightsTable.find(contents[masterI])();
+
+            SLList<long>::iterator nbrsIter = curNbrs.begin();
+
+            SLList<SLList<label> >::iterator faceFacesIter =
+                curFaceFaces.begin();
+
+            SLList<SLList<label> >::iterator faceFaceNbrsIter =
+                curFaceFaceNbrs.begin();
+
+            SLList<SLList<scalar> >::iterator faceFaceWeightsIter =
+                curFaceWeights.begin();
+
+            for
+            (
+                ;
+                nbrsIter != curNbrs.end()
+             && faceFacesIter != curFaceFaces.end()
+             && faceFaceNbrsIter != curFaceFaceNbrs.end()
+             && faceFaceWeightsIter != curFaceWeights.end();
+                ++nbrsIter, ++faceFacesIter, ++faceFaceNbrsIter,
+                ++faceFaceWeightsIter
+            )
+            {
+                // Check if the face is on local processor: no longer needed,
+                // as only local processor is being searched.  HJ, 13/Jun/2016
+
+                SLList<label>::iterator facesIter =
+                    faceFacesIter().begin();
+
+                SLList<label>::iterator faceNbrsIter =
+                    faceFaceNbrsIter().begin();
+
+                SLList<scalar>::iterator weightsIter =
+                    faceFaceWeightsIter().begin();
+
+                // Find neighbour proc index from the first face
+                // on the other side
+                label nbrProc;
+
+                if (fineGgiInterface_.fineLevel())
+                {
+                    const labelListList& fineAddr =
+                        fineGgiInterface_.addressing();
+
+                    nbrProc = neighbourExpandProc
+                        [fineAddr[facesIter()][faceNbrsIter()]];
+                }
+                else
+                {
+                    nbrProc = neighbourExpandProc[facesIter()];
+                }
+
+                // Read coarse face index
+                const label coarseFace =
+                    masterProcMasterFaces[nbrProc][npmf[nbrProc]];
+
+                // and mark it as used
+                npmf[nbrProc]++;
+
+                zoneAddressing_[nProcFaces] = coarseFace;
+                faceCells_[nProcFaces] =
+                    nbrsIter() - procOffset*Pstream::myProcNo();
+
                 for
                 (
                     ;
@@ -800,19 +975,15 @@ Foam::ggiAMGInterface::ggiAMGInterface
                 )
                 {
                     fineAddressing_[nAgglomPairs] = facesIter();
-                    restrictAddressing_[nAgglomPairs] = nCoarseFaces;
+                    restrictAddressing_[nAgglomPairs] = coarseFace;
                     restrictWeights_[nAgglomPairs] = weightsIter();
+
                     nAgglomPairs++;
                 }
+
+                nProcFaces++;
             }
         }
-
-        // Resize arrays: not all of ggi is used locally
-        faceCells_.setSize(nProcFaces);
-        zoneAddressing_.setSize(nProcFaces);
-        fineAddressing_.setSize(nAgglomPairs);
-        restrictAddressing_.setSize(nAgglomPairs);
-        restrictWeights_.setSize(nAgglomPairs);
     }
 }
 
@@ -934,6 +1105,21 @@ bool Foam::ggiAMGInterface::localParallel() const
 }
 
 
+const Foam::labelListList& Foam::ggiAMGInterface::procMasterFaces() const
+{
+    if (!master())
+    {
+        FatalErrorIn
+        (
+            "const labelListList& ggiGAMGInterface::procMasterFaces() const"
+        )   << "Requester procMasterFaces from a slave.  This is not allowed"
+            << abort(FatalError);
+    }
+
+    return procMasterFaces_;
+}
+
+
 const Foam::mapDistribute& Foam::ggiAMGInterface::map() const
 {
     if (!mapPtr_)
diff --git a/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/ggiAMGInterface/ggiAMGInterface.H b/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/ggiAMGInterface/ggiAMGInterface.H
index 507909026..b3c3087e8 100644
--- a/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/ggiAMGInterface/ggiAMGInterface.H
+++ b/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/ggiAMGInterface/ggiAMGInterface.H
@@ -68,6 +68,11 @@ class ggiAMGInterface
         //- Zone addressing
         labelList zoneAddressing_;
 
+        //- Processor master faces
+        //  Per-processor insertion list of local faces into global zone
+        //  The list is created on the master side and passed onto the slave
+        //  to allow the slave to insert faces in the same order
+        labelListList procMasterFaces_;
 
         // Parallel communication
 
@@ -110,15 +115,15 @@ public:
         ggiAMGInterface
         (
             const lduPrimitiveMesh& lduMesh,
+            const lduInterfacePtrsList& coarseInterfaces,
             const lduInterface& fineInterface,
             const labelField& localRestrictAddressing,
             const labelField& neighbourRestrictAddressing
         );
 
 
-    // Destructor
-
-        virtual ~ggiAMGInterface();
+    //- Destructor
+    virtual ~ggiAMGInterface();
 
 
     // Member Functions
@@ -240,6 +245,9 @@ public:
             //- Is the patch localised on a single processor
             virtual bool localParallel() const;
 
+            //- Processor master face insertion list
+            const labelListList& procMasterFaces() const;
+
             //- Return weights
             virtual const scalarListList& weights() const;
 
diff --git a/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/mixingPlaneAMGInterface/mixingPlaneAMGInterface.C b/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/mixingPlaneAMGInterface/mixingPlaneAMGInterface.C
index 9a766bae4..e8f95a673 100644
--- a/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/mixingPlaneAMGInterface/mixingPlaneAMGInterface.C
+++ b/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/mixingPlaneAMGInterface/mixingPlaneAMGInterface.C
@@ -51,6 +51,7 @@ namespace Foam
 Foam::mixingPlaneAMGInterface::mixingPlaneAMGInterface
 (
     const lduPrimitiveMesh& lduMesh,
+    const lduInterfacePtrsList& coarseInterfaces,
     const lduInterface& fineInterface,
     const labelField& localRestrictAddressing,
     const labelField& neighbourRestrictAddressing
diff --git a/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/mixingPlaneAMGInterface/mixingPlaneAMGInterface.H b/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/mixingPlaneAMGInterface/mixingPlaneAMGInterface.H
index 71e29dc7b..16b1ade06 100644
--- a/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/mixingPlaneAMGInterface/mixingPlaneAMGInterface.H
+++ b/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/mixingPlaneAMGInterface/mixingPlaneAMGInterface.H
@@ -93,6 +93,7 @@ public:
         mixingPlaneAMGInterface
         (
             const lduPrimitiveMesh& lduMesh,
+            const lduInterfacePtrsList& coarseInterfaces,
             const lduInterface& fineInterface,
             const labelField& localRestrictAddressing,
             const labelField& neighbourRestrictAddressing
diff --git a/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/processorAMGInterface/processorAMGInterface.C b/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/processorAMGInterface/processorAMGInterface.C
index 35921b46f..8f60ee4b9 100644
--- a/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/processorAMGInterface/processorAMGInterface.C
+++ b/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/processorAMGInterface/processorAMGInterface.C
@@ -45,6 +45,7 @@ namespace Foam
 Foam::processorAMGInterface::processorAMGInterface
 (
     const lduPrimitiveMesh& lduMesh,
+    const lduInterfacePtrsList& coarseInterfaces,
     const lduInterface& fineInterface,
     const labelField& localRestrictAddressing,
     const labelField& neighbourRestrictAddressing
diff --git a/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/processorAMGInterface/processorAMGInterface.H b/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/processorAMGInterface/processorAMGInterface.H
index 7f1613c60..50db8b0f7 100644
--- a/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/processorAMGInterface/processorAMGInterface.H
+++ b/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/processorAMGInterface/processorAMGInterface.H
@@ -81,6 +81,7 @@ public:
         processorAMGInterface
         (
             const lduPrimitiveMesh& lduMesh,
+            const lduInterfacePtrsList& coarseInterfaces,
             const lduInterface& fineInterface,
             const labelField& localRestrictAddressing,
             const labelField& neighbourRestrictAddressing
diff --git a/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/regionCoupleAMGInterface/regionCoupleAMGInterface.C b/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/regionCoupleAMGInterface/regionCoupleAMGInterface.C
index 47bfdaacc..4f1c1f3b8 100644
--- a/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/regionCoupleAMGInterface/regionCoupleAMGInterface.C
+++ b/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/regionCoupleAMGInterface/regionCoupleAMGInterface.C
@@ -45,6 +45,7 @@ namespace Foam
 Foam::regionCoupleAMGInterface::regionCoupleAMGInterface
 (
     const lduPrimitiveMesh& lduMesh,
+    const lduInterfacePtrsList& coarseInterfaces,
     const lduInterface& fineInterface,
     const labelField& localRestrictAddressing,
     const labelField& neighbourRestrictAddressing
@@ -53,6 +54,7 @@ Foam::regionCoupleAMGInterface::regionCoupleAMGInterface
     ggiAMGInterface
     (
         lduMesh,
+        coarseInterfaces,
         fineInterface,
         localRestrictAddressing,
         neighbourRestrictAddressing
diff --git a/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/regionCoupleAMGInterface/regionCoupleAMGInterface.H b/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/regionCoupleAMGInterface/regionCoupleAMGInterface.H
index f7f2f7503..d132df7ae 100644
--- a/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/regionCoupleAMGInterface/regionCoupleAMGInterface.H
+++ b/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/regionCoupleAMGInterface/regionCoupleAMGInterface.H
@@ -73,6 +73,7 @@ public:
         regionCoupleAMGInterface
         (
             const lduPrimitiveMesh& lduMesh,
+            const lduInterfacePtrsList& coarseInterfaces,
             const lduInterface& fineInterface,
             const labelField& localRestrictAddressing,
             const labelField& neighbourRestrictAddressing
diff --git a/src/lduSolvers/amg/amgPolicy/pamgPolicy.C b/src/lduSolvers/amg/amgPolicy/pamgPolicy.C
index a34c36c48..a4d302e32 100644
--- a/src/lduSolvers/amg/amgPolicy/pamgPolicy.C
+++ b/src/lduSolvers/amg/amgPolicy/pamgPolicy.C
@@ -648,6 +648,7 @@ Foam::autoPtr<Foam::amgMatrix> Foam::pamgPolicy::restrictMatrix
                 AMGInterface::New
                 (
                     *coarseAddrPtr,
+                    coarseInterfaces,
                     fineInterface,
                     fineInterface.interfaceInternalField(child_),
                     fineInterfaceAddr[intI]

From 80da2652883fb08597b7c5f6e911a78ada80030e Mon Sep 17 00:00:00 2001
From: Hrvoje Jasak <h.jasak@wikki.co.uk>
Date: Tue, 9 Aug 2016 13:25:49 +0100
Subject: [PATCH 007/130] Write forces dat file

---
 bin/plotForces.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/bin/plotForces.py b/bin/plotForces.py
index ff59ca41a..fd8ff938d 100755
--- a/bin/plotForces.py
+++ b/bin/plotForces.py
@@ -35,7 +35,6 @@ for line in lines:
                 mvy.append(float(match.group(12)))
                 mvz.append(float(match.group(13)))
 
-
 # combine pressure and viscous forces and moments
 fx = fpx
 fy = fpy
@@ -53,6 +52,12 @@ for i in range(1,len(t)):
         my[i] += mvy[i]
         mz[i] += mvz[i]
 
+# write clean data file
+outfile=open('forces.dat','w')
+
+for data in zip(t,fx,fy,fz):
+        outfile.write(' '.join([str(d) for d in data])+'\n')
+
 # plot forces
 import pylab
 pylab.xlabel('iteration')

From 4f12739deac75b9423c0c2093e6d243b8167568b Mon Sep 17 00:00:00 2001
From: Hrvoje Jasak <h.jasak@wikki.co.uk>
Date: Tue, 9 Aug 2016 13:25:57 +0100
Subject: [PATCH 008/130] Formatting

---
 bin/plotResidual.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/bin/plotResidual.py b/bin/plotResidual.py
index 22149b34e..5122812d4 100755
--- a/bin/plotResidual.py
+++ b/bin/plotResidual.py
@@ -62,6 +62,7 @@ for line in lines:
                 tepsilon.append(iepsilon)
                 epsilon.append(float(matchepsilon.group(2)))
 
+# write clean data file
 outfile=open('residual.dat','w')
 
 if iomega > 0:

From 70c6e0aa048f34f8a6669a1a2c85d9fb130f6ce9 Mon Sep 17 00:00:00 2001
From: Hrvoje Jasak <h.jasak@wikki.co.uk>
Date: Wed, 10 Aug 2016 12:10:41 +0100
Subject: [PATCH 009/130] Formatting

---
 src/foam/containers/Lists/List/List.H | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/foam/containers/Lists/List/List.H b/src/foam/containers/Lists/List/List.H
index ea96501d2..96119c4af 100644
--- a/src/foam/containers/Lists/List/List.H
+++ b/src/foam/containers/Lists/List/List.H
@@ -99,6 +99,7 @@ public:
         //- Return a null list
         inline static const List<T>& null();
 
+
     // Constructors
 
         //- Null constructor.

From 2cdc9de13d4e9a8b56cc1f6e46d74c571df5bb3c Mon Sep 17 00:00:00 2001
From: Hrvoje Jasak <h.jasak@wikki.co.uk>
Date: Wed, 10 Aug 2016 14:08:15 +0100
Subject: [PATCH 010/130] Comms update on GGI AMG interface

---
 .../fvPatches/constraint/ggi/ggiFvPatch.C     |   6 +
 .../fvPatches/constraint/ggi/ggiFvPatch.H     |   3 +
 .../regionCouple/regionCoupleFvPatch.C        |   7 +
 .../regionCouple/regionCoupleFvPatch.H        |   3 +
 .../ggiLduInterface/ggiLduInterface.H         |   7 +
 .../ggiAMGInterface/ggiAMGInterface.C         |  77 ++++--
 .../polyPatches/constraint/ggi/ggiPolyPatch.C |  27 +-
 .../polyPatches/constraint/ggi/ggiPolyPatch.H |   6 +-
 .../regionCouple/regionCouplePolyPatch.C      | 238 ++++++++++++------
 .../regionCouple/regionCouplePolyPatch.H      |  59 ++---
 .../regionCouplePolyPatchTemplates.C          | 147 +++--------
 11 files changed, 322 insertions(+), 258 deletions(-)

diff --git a/src/finiteVolume/fvMesh/fvPatches/constraint/ggi/ggiFvPatch.C b/src/finiteVolume/fvMesh/fvPatches/constraint/ggi/ggiFvPatch.C
index 09501ab16..3b1addbde 100644
--- a/src/finiteVolume/fvMesh/fvPatches/constraint/ggi/ggiFvPatch.C
+++ b/src/finiteVolume/fvMesh/fvPatches/constraint/ggi/ggiFvPatch.C
@@ -258,6 +258,12 @@ bool Foam::ggiFvPatch::localParallel() const
 }
 
 
+const Foam::mapDistribute& Foam::ggiFvPatch::map() const
+{
+    return ggiPolyPatch_.map();
+}
+
+
 const Foam::scalarListList& Foam::ggiFvPatch::weights() const
 {
     if (ggiPolyPatch_.master())
diff --git a/src/finiteVolume/fvMesh/fvPatches/constraint/ggi/ggiFvPatch.H b/src/finiteVolume/fvMesh/fvPatches/constraint/ggi/ggiFvPatch.H
index 740f0f208..3006723c8 100644
--- a/src/finiteVolume/fvMesh/fvPatches/constraint/ggi/ggiFvPatch.H
+++ b/src/finiteVolume/fvMesh/fvPatches/constraint/ggi/ggiFvPatch.H
@@ -171,6 +171,9 @@ public:
             //- Is the patch localised on a single processor
             virtual bool localParallel() const;
 
+            //- Return mapDistribute
+            virtual const mapDistribute& map() const;
+
             //- Return weights.  Master side returns own weights and
             //  slave side returns weights from master
             virtual const scalarListList& weights() const;
diff --git a/src/finiteVolume/fvMesh/fvPatches/constraint/regionCouple/regionCoupleFvPatch.C b/src/finiteVolume/fvMesh/fvPatches/constraint/regionCouple/regionCoupleFvPatch.C
index 38f5ba27d..6da84ef02 100644
--- a/src/finiteVolume/fvMesh/fvPatches/constraint/regionCouple/regionCoupleFvPatch.C
+++ b/src/finiteVolume/fvMesh/fvPatches/constraint/regionCouple/regionCoupleFvPatch.C
@@ -34,6 +34,7 @@ Author
 #include "fvMesh.H"
 #include "fvBoundaryMesh.H"
 #include "foamTime.H"
+#include "mapDistribute.H"
 
 // * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
 
@@ -302,6 +303,12 @@ bool Foam::regionCoupleFvPatch::localParallel() const
 }
 
 
+const Foam::mapDistribute& Foam::regionCoupleFvPatch::map() const
+{
+    return rcPolyPatch_.map();
+}
+
+
 const Foam::scalarListList& Foam::regionCoupleFvPatch::weights() const
 {
     if (rcPolyPatch_.master())
diff --git a/src/finiteVolume/fvMesh/fvPatches/constraint/regionCouple/regionCoupleFvPatch.H b/src/finiteVolume/fvMesh/fvPatches/constraint/regionCouple/regionCoupleFvPatch.H
index 85a0fba14..9603b54a1 100644
--- a/src/finiteVolume/fvMesh/fvPatches/constraint/regionCouple/regionCoupleFvPatch.H
+++ b/src/finiteVolume/fvMesh/fvPatches/constraint/regionCouple/regionCoupleFvPatch.H
@@ -180,6 +180,9 @@ public:
             //- Is the patch localised on a single processor
             virtual bool localParallel() const;
 
+            //- Return mapDistribute
+            const mapDistribute& map() const;
+
             //- Return weights.  Master side returns own weights and
             //  slave side returns weights from master
             virtual const scalarListList& weights() const;
diff --git a/src/foam/matrices/lduMatrix/lduAddressing/lduInterfaces/ggiLduInterface/ggiLduInterface.H b/src/foam/matrices/lduMatrix/lduAddressing/lduInterfaces/ggiLduInterface/ggiLduInterface.H
index a4fc7b2c7..3e5b815f7 100644
--- a/src/foam/matrices/lduMatrix/lduAddressing/lduInterfaces/ggiLduInterface/ggiLduInterface.H
+++ b/src/foam/matrices/lduMatrix/lduAddressing/lduInterfaces/ggiLduInterface/ggiLduInterface.H
@@ -46,6 +46,10 @@ SourceFiles
 namespace Foam
 {
 
+// Forward declaration of classes
+
+class mapDistribute;
+
 /*---------------------------------------------------------------------------*\
                        Class ggiLduInterface Declaration
 \*---------------------------------------------------------------------------*/
@@ -111,6 +115,9 @@ public:
         //- Is the patch localised on a single processor
         virtual bool localParallel() const = 0;
 
+        //- Return mapDistribute
+        virtual const mapDistribute& map() const = 0;
+
         //- Return weights
         virtual const scalarListList& weights() const = 0;
 
diff --git a/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/ggiAMGInterface/ggiAMGInterface.C b/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/ggiAMGInterface/ggiAMGInterface.C
index 899ee6768..9332078f2 100644
--- a/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/ggiAMGInterface/ggiAMGInterface.C
+++ b/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/ggiAMGInterface/ggiAMGInterface.C
@@ -839,30 +839,67 @@ Foam::ggiAMGInterface::ggiAMGInterface
         // Re-pack singly linked list of processor master faces
         // and pass to other processors
 
-        // First index: master proc
-        // Second index: slave proc
-        // List contents: global faces in order
-        labelListListList crissCrossList(Pstream::nProcs());
-
-        labelListList& crissList = crissCrossList[Pstream::myProcNo()];
-
-        crissList.setSize(Pstream::nProcs());
-
-        forAll (crissList, procI)
-        {
-            crissList[procI] = procMasterFacesLL[procI];
-        }
-
-        Pstream::gatherList(crissCrossList);
-        Pstream::scatterList(crissCrossList);
-
         procMasterFaces_.setSize(Pstream::nProcs());
 
-        forAll (procMasterFaces_, procI)
+        // Copy self
+        procMasterFaces_[Pstream::myProcNo()] =
+            procMasterFacesLL[Pstream::myProcNo()];
+
+        if (Pstream::parRun())
         {
-            procMasterFaces_[procI] =
-                crissCrossList[procI][Pstream::myProcNo()];
+            const List<labelPair> schedule =
+                fineGgiInterface_.map().schedule();
+
+            // Do the comms
+            forAll (schedule, i)
+            {
+                const label sendProc = schedule[i].first();
+                const label recvProc = schedule[i].second();
+
+                if (Pstream::myProcNo() == sendProc)
+                {
+                    OPstream toNbr(Pstream::scheduled, recvProc);
+                    toNbr << labelList(procMasterFacesLL[recvProc]);
+                }
+                else if (Pstream::myProcNo() == recvProc)
+                {
+                    IPstream fromNbr(Pstream::scheduled, sendProc);
+                    
+                    procMasterFaces_[sendProc] = labelList(fromNbr);
+                }
+                else
+                {
+                    FatalErrorIn("...")
+                        << "My proc number " << Pstream::myProcNo()
+                            << " is neither a sender nor a receiver: "
+                            << schedule[i]
+                            << abort(FatalError);
+                }
+            }
         }
+
+//         // First index: master proc
+//         // Second index: slave proc
+//         // List contents: global faces in order
+//         labelListListList crissCrossList(Pstream::nProcs());
+
+//         labelListList& crissList = crissCrossList[Pstream::myProcNo()];
+
+//         crissList.setSize(Pstream::nProcs());
+
+//         forAll (crissList, procI)
+//         {
+//             crissList[procI] = procMasterFacesLL[procI];
+//         }
+
+//         Pstream::gatherList(crissCrossList);
+//         Pstream::scatterList(crissCrossList);
+
+//         forAll (procMasterFaces_, procI)
+//         {
+//             procMasterFaces_[procI] =
+//                 crissCrossList[procI][Pstream::myProcNo()];
+//         }
     }
     // Agglomerate slave
     else
diff --git a/src/foam/meshes/polyMesh/polyPatches/constraint/ggi/ggiPolyPatch.C b/src/foam/meshes/polyMesh/polyPatches/constraint/ggi/ggiPolyPatch.C
index de8012c7d..c555f4998 100644
--- a/src/foam/meshes/polyMesh/polyPatches/constraint/ggi/ggiPolyPatch.C
+++ b/src/foam/meshes/polyMesh/polyPatches/constraint/ggi/ggiPolyPatch.C
@@ -509,17 +509,6 @@ void Foam::ggiPolyPatch::calcSendReceive() const
 }
 
 
-const Foam::mapDistribute& Foam::ggiPolyPatch::map() const
-{
-    if (!mapPtr_)
-    {
-        calcSendReceive();
-    }
-
-    return *mapPtr_;
-}
-
-
 void Foam::ggiPolyPatch::clearGeom() const
 {
     deleteDemandDrivenData(reconFaceCellCentresPtr_);
@@ -530,6 +519,10 @@ void Foam::ggiPolyPatch::clearGeom() const
     // HJ, 23/Jun/2011
     deleteDemandDrivenData(remoteZoneAddressingPtr_);
 
+    // localParallel depends on geometry - must be cleared!
+    // HR, 11/Jul/2013
+    deleteDemandDrivenData(localParallelPtr_);
+
     deleteDemandDrivenData(mapPtr_);
 }
 
@@ -543,7 +536,6 @@ void Foam::ggiPolyPatch::clearOut() const
 
     deleteDemandDrivenData(zoneAddressingPtr_);
     deleteDemandDrivenData(patchToPatchPtr_);
-    deleteDemandDrivenData(localParallelPtr_);
 }
 
 
@@ -826,6 +818,17 @@ const Foam::ggiZoneInterpolation& Foam::ggiPolyPatch::patchToPatch() const
 }
 
 
+const Foam::mapDistribute& Foam::ggiPolyPatch::map() const
+{
+    if (!mapPtr_)
+    {
+        calcSendReceive();
+    }
+
+    return *mapPtr_;
+}
+
+
 const Foam::vectorField& Foam::ggiPolyPatch::reconFaceCellCentres() const
 {
     if (!reconFaceCellCentresPtr_)
diff --git a/src/foam/meshes/polyMesh/polyPatches/constraint/ggi/ggiPolyPatch.H b/src/foam/meshes/polyMesh/polyPatches/constraint/ggi/ggiPolyPatch.H
index fb3fc403e..ab6ca49b0 100644
--- a/src/foam/meshes/polyMesh/polyPatches/constraint/ggi/ggiPolyPatch.H
+++ b/src/foam/meshes/polyMesh/polyPatches/constraint/ggi/ggiPolyPatch.H
@@ -138,9 +138,6 @@ class ggiPolyPatch
             //- Calculate send and receive addressing
             void calcSendReceive() const;
 
-            //- Return mapDistribute
-            const mapDistribute& map() const;
-
 
         // Memory management
 
@@ -335,6 +332,9 @@ public:
             //- Is the patch localised on a single processor
             bool localParallel() const;
 
+            //- Return mapDistribute
+            const mapDistribute& map() const;
+
             //- Return reference to patch-to-patch interpolation
             //  Used only for addressing
             const ggiZoneInterpolation& patchToPatch() const;
diff --git a/src/foam/meshes/polyMesh/polyPatches/constraint/regionCouple/regionCouplePolyPatch.C b/src/foam/meshes/polyMesh/polyPatches/constraint/regionCouple/regionCouplePolyPatch.C
index 3363afb2d..3f07d7175 100644
--- a/src/foam/meshes/polyMesh/polyPatches/constraint/regionCouple/regionCouplePolyPatch.C
+++ b/src/foam/meshes/polyMesh/polyPatches/constraint/regionCouple/regionCouplePolyPatch.C
@@ -226,7 +226,7 @@ void Foam::regionCouplePolyPatch::calcPatchToPatch() const
                 SMALL,         // Non-overlapping face tolerances
                 SMALL,
                 true,          // Rescale weighting factors
-                ggiInterpolation::BB_OCTREE  // Octree search, MB.
+                reject_        // Quick rejection algorithm, default BB_OCTREE
             );
 
         // Abort immediately if uncovered faces are present and the option
@@ -378,7 +378,7 @@ void Foam::regionCouplePolyPatch::calcSendReceive() const
     // (of the calc-call) they will be set to zero-sized array
     // HJ, 4/Jun/2011
 
-    if (receiveAddrPtr_ || sendAddrPtr_)
+    if (mapPtr_)
     {
         FatalErrorIn("void regionCouplePolyPatch::calcSendReceive() const")
             << "Send-receive addressing already calculated"
@@ -399,74 +399,129 @@ void Foam::regionCouplePolyPatch::calcSendReceive() const
             << abort(FatalError);
     }
 
-    // Master will receive and store the maps
-    if (Pstream::master())
+    // Gather send and receive addressing (to master)
+
+    // Get patch-to-zone addressing
+    const labelList& za = zoneAddressing();
+
+    // Make a zone-sized field and fill it in with proc markings for processor
+    // that holds and requires the data
+    labelField zoneProcID(zone().size(), -1);
+
+    forAll (za, zaI)
     {
-        receiveAddrPtr_ = new labelListList(Pstream::nProcs());
-        labelListList& rAddr = *receiveAddrPtr_;
+        zoneProcID[za[zaI]] = Pstream::myProcNo();
+    }
 
-        sendAddrPtr_ = new labelListList(Pstream::nProcs());
-        labelListList& sAddr = *sendAddrPtr_;
+    reduce(zoneProcID, maxOp<labelField>());
 
-        // Insert master
-        rAddr[0] = zoneAddressing();
+    const labelList& shadowRza = shadow().remoteZoneAddressing();
 
-        for (label procI = 1; procI < Pstream::nProcs(); procI++)
+    // Find out where my zone data is coming from
+    labelList nRecv(Pstream::nProcs(), 0);
+
+    // Note: only visit the data from the local zone
+    forAll (shadowRza, shadowRzaI)
+    {
+        nRecv[zoneProcID[shadowRza[shadowRzaI]]]++;
+    }
+
+    // Make a receiving sub-map
+    // It tells me which data I will receive from which processor and
+    // where I need to put it into the remoteZone data before the mapping
+    labelListList constructMap(Pstream::nProcs());
+
+    // Size the receiving list
+    forAll (nRecv, procI)
+    {
+        constructMap[procI].setSize(nRecv[procI]);
+    }
+
+    // Reset counters for processors
+    nRecv = 0;
+
+    forAll (shadowRza, shadowRzaI)
+    {
+        label recvProc = zoneProcID[shadowRza[shadowRzaI]];
+
+        constructMap[recvProc][nRecv[recvProc]] = shadowRza[shadowRzaI];
+
+        nRecv[recvProc]++;
+    }
+
+    // Make the sending sub-map
+    // It tells me which data is required from me to be sent to which
+    // processor
+
+    // Algorithm
+    // - expand the local zone faces with indices into a size of local zone
+    // - go through remote zone addressing on all processors
+    // - find out who hits my faces
+    labelList localZoneIndices(zone().size(), -1);
+
+    forAll (za, zaI)
+    {
+        localZoneIndices[za[zaI]] = zaI;
+    }
+
+    labelListList shadowToReceiveAddr(Pstream::nProcs());
+
+    // Get the list of what my shadow needs to receive from my zone
+    // on all other processors
+    shadowToReceiveAddr[Pstream::myProcNo()] = shadowRza;
+    Pstream::gatherList(shadowToReceiveAddr);
+    Pstream::scatterList(shadowToReceiveAddr);
+
+    // Now local zone indices contain the index of a local face that will
+    // provide the data.  For faces that are not local, the index will be -1
+
+    // Find out where my zone data is going to
+
+    // Make a sending sub-map
+    // It tells me which data I will send to which processor
+    labelListList sendMap(Pstream::nProcs());
+
+    // Collect local labels to be sent to each processor
+    forAll (shadowToReceiveAddr, procI)
+    {
+        const labelList& curProcSend = shadowToReceiveAddr[procI];
+
+        // Find out how much of my data is going to this processor
+        label nProcSend = 0;
+
+        forAll (curProcSend, sendI)
         {
-            // Note: must use normal comms because the size of the
-            // communicated lists is unknown on the receiving side
-            // HJ, 4/Jun/2011
+            if (localZoneIndices[curProcSend[sendI]] > -1)
+            {
+                nProcSend++;
+            }
+        }
 
-            // Opt: reconsider mode of communication
-            IPstream ip(Pstream::scheduled, procI);
+        if (nProcSend > 0)
+        {
+            // Collect the indices
+            labelList& curSendMap = sendMap[procI];
 
-            rAddr[procI] = labelList(ip);
+            curSendMap.setSize(nProcSend);
 
-            sAddr[procI] = labelList(ip);
+            // Reset counter
+            nProcSend = 0;
+
+            forAll (curProcSend, sendI)
+            {
+                if (localZoneIndices[curProcSend[sendI]] > -1)
+                {
+                    curSendMap[nProcSend] =
+                        localZoneIndices[curProcSend[sendI]];
+                    nProcSend++;
+                }
+            }
         }
     }
-    else
-    {
-        // Create dummy pointers: only master processor stores maps
-        receiveAddrPtr_ = new labelListList();
-        sendAddrPtr_ = new labelListList();
 
-        // Send information to master
-        const labelList& za = zoneAddressing();
-        const labelList& ra = remoteZoneAddressing();
-
-        // Note: must use normal comms because the size of the
-        // communicated lists is unknown on the receiving side
-        // HJ, 4/Jun/2011
-
-        // Opt: reconsider mode of communication
-        OPstream op(Pstream::scheduled, Pstream::masterNo());
-
-        // Send local and remote addressing to master
-        op << za << ra;
-    }
-}
-
-
-const Foam::labelListList& Foam::regionCouplePolyPatch::receiveAddr() const
-{
-    if (!receiveAddrPtr_)
-    {
-        calcSendReceive();
-    }
-
-    return *receiveAddrPtr_;
-}
-
-
-const Foam::labelListList& Foam::regionCouplePolyPatch::sendAddr() const
-{
-    if (!sendAddrPtr_)
-    {
-        calcSendReceive();
-    }
-
-    return *sendAddrPtr_;
+    // Map will return the object of the size of remote zone
+    // HJ, 9/May/2016
+    mapPtr_ = new mapDistribute(zone().size(), sendMap, constructMap);
 }
 
 
@@ -492,18 +547,19 @@ void Foam::regionCouplePolyPatch::clearGeom() const
 {
     clearDeltas();
 
+    deleteDemandDrivenData(reconFaceCellCentresPtr_);
+
     // Remote addressing and send-receive maps depend on the local
     // position.  Therefore, it needs to be recalculated at mesh motion.
     // Local zone addressing does not change with mesh motion
     // HJ, 23/Jun/2011
     deleteDemandDrivenData(remoteZoneAddressingPtr_);
 
-    deleteDemandDrivenData(receiveAddrPtr_);
-    deleteDemandDrivenData(sendAddrPtr_);
-
     // localParallel depends on geometry - must be cleared!
     // HR, 11/Jul/2013
     deleteDemandDrivenData(localParallelPtr_);
+
+    deleteDemandDrivenData(mapPtr_);
 }
 
 
@@ -511,6 +567,9 @@ void Foam::regionCouplePolyPatch::clearOut() const
 {
     clearGeom();
 
+    shadowIndex_ = -1;
+    zoneIndex_ = -1;
+
     deleteDemandDrivenData(zoneAddressingPtr_);
     deleteDemandDrivenData(patchToPatchPtr_);
 }
@@ -535,6 +594,7 @@ Foam::regionCouplePolyPatch::regionCouplePolyPatch
     master_(false),
     isWall_(false),
     bridgeOverlap_(false),
+    reject_(ggiZoneInterpolation::BB_OCTREE),
     shadowIndex_(-1),
     zoneIndex_(-1),
     patchToPatchPtr_(NULL),
@@ -542,8 +602,7 @@ Foam::regionCouplePolyPatch::regionCouplePolyPatch
     remoteZoneAddressingPtr_(NULL),
     reconFaceCellCentresPtr_(NULL),
     localParallelPtr_(NULL),
-    receiveAddrPtr_(NULL),
-    sendAddrPtr_(NULL)
+    mapPtr_(NULL)
 {}
 
 
@@ -560,7 +619,8 @@ Foam::regionCouplePolyPatch::regionCouplePolyPatch
     const bool attached,
     const bool master,
     const bool isWall,
-    const bool bridgeOverlap
+    const bool bridgeOverlap,
+    const ggiZoneInterpolation::quickReject reject
 )
 :
     coupledPolyPatch(name, size, start, index, bm),
@@ -571,6 +631,7 @@ Foam::regionCouplePolyPatch::regionCouplePolyPatch
     master_(master),
     isWall_(isWall),
     bridgeOverlap_(bridgeOverlap),
+    reject_(reject),
     shadowIndex_(-1),
     zoneIndex_(-1),
     patchToPatchPtr_(NULL),
@@ -578,8 +639,7 @@ Foam::regionCouplePolyPatch::regionCouplePolyPatch
     remoteZoneAddressingPtr_(NULL),
     reconFaceCellCentresPtr_(NULL),
     localParallelPtr_(NULL),
-    receiveAddrPtr_(NULL),
-    sendAddrPtr_(NULL)
+    mapPtr_(NULL)
 {}
 
 
@@ -599,6 +659,7 @@ Foam::regionCouplePolyPatch::regionCouplePolyPatch
     master_(dict.lookup("master")),
     isWall_(dict.lookup("isWall")),
     bridgeOverlap_(dict.lookup("bridgeOverlap")),
+    reject_(ggiZoneInterpolation::BB_OCTREE),
     shadowIndex_(-1),
     zoneIndex_(-1),
     patchToPatchPtr_(NULL),
@@ -606,9 +667,16 @@ Foam::regionCouplePolyPatch::regionCouplePolyPatch
     remoteZoneAddressingPtr_(NULL),
     reconFaceCellCentresPtr_(NULL),
     localParallelPtr_(NULL),
-    receiveAddrPtr_(NULL),
-    sendAddrPtr_(NULL)
-{}
+    mapPtr_(NULL)
+{
+    if (dict.found("quickReject"))
+    {
+        reject_ = ggiZoneInterpolation::quickRejectNames_.read
+        (
+            dict.lookup("quickReject")
+        );
+    }
+}
 
 
 Foam::regionCouplePolyPatch::regionCouplePolyPatch
@@ -624,6 +692,8 @@ Foam::regionCouplePolyPatch::regionCouplePolyPatch
     attached_(pp.attached_),
     master_(pp.master_),
     isWall_(pp.isWall_),
+    bridgeOverlap_(pp.bridgeOverlap_),
+    reject_(pp.reject_),
     shadowIndex_(-1),
     zoneIndex_(-1),
     patchToPatchPtr_(NULL),
@@ -631,8 +701,7 @@ Foam::regionCouplePolyPatch::regionCouplePolyPatch
     remoteZoneAddressingPtr_(NULL),
     reconFaceCellCentresPtr_(NULL),
     localParallelPtr_(NULL),
-    receiveAddrPtr_(NULL),
-    sendAddrPtr_(NULL)
+    mapPtr_(NULL)
 {}
 
 
@@ -652,6 +721,8 @@ Foam::regionCouplePolyPatch::regionCouplePolyPatch
     attached_(pp.attached_),
     master_(pp.master_),
     isWall_(pp.isWall_),
+    bridgeOverlap_(pp.bridgeOverlap_),
+    reject_(pp.reject_),
     shadowIndex_(-1),
     zoneIndex_(-1),
     patchToPatchPtr_(NULL),
@@ -659,8 +730,7 @@ Foam::regionCouplePolyPatch::regionCouplePolyPatch
     remoteZoneAddressingPtr_(NULL),
     reconFaceCellCentresPtr_(NULL),
     localParallelPtr_(NULL),
-    receiveAddrPtr_(NULL),
-    sendAddrPtr_(NULL)
+    mapPtr_(NULL)
 {}
 
 
@@ -911,6 +981,17 @@ Foam::regionCouplePolyPatch::patchToPatch() const
 }
 
 
+const Foam::mapDistribute& Foam::regionCouplePolyPatch::map() const
+{
+    if (!mapPtr_)
+    {
+        calcSendReceive();
+    }
+
+    return *mapPtr_;
+}
+
+
 const Foam::vectorField&
 Foam::regionCouplePolyPatch::reconFaceCellCentres() const
 {
@@ -940,6 +1021,11 @@ void Foam::regionCouplePolyPatch::initAddressing()
         // Calculate transforms for correct GGI cut
         calcTransforms();
 
+        if (master())
+        {
+            shadow().calcTransforms();
+        }
+
         // Force zone addressing and remote zone addressing
         // (uses GGI interpolator)
         zoneAddressing();
@@ -949,7 +1035,7 @@ void Foam::regionCouplePolyPatch::initAddressing()
         if (Pstream::parRun() && !localParallel())
         {
             // Calculate send addressing
-            sendAddr();
+            map();
         }
     }
 
@@ -1019,7 +1105,7 @@ void Foam::regionCouplePolyPatch::initMovePoints(const pointField& p)
         if (Pstream::parRun() && !localParallel())
         {
             // Calculate send addressing
-            sendAddr();
+            map();
         }
     }
 
diff --git a/src/foam/meshes/polyMesh/polyPatches/constraint/regionCouple/regionCouplePolyPatch.H b/src/foam/meshes/polyMesh/polyPatches/constraint/regionCouple/regionCouplePolyPatch.H
index 5816af204..5a9dacee6 100644
--- a/src/foam/meshes/polyMesh/polyPatches/constraint/regionCouple/regionCouplePolyPatch.H
+++ b/src/foam/meshes/polyMesh/polyPatches/constraint/regionCouple/regionCouplePolyPatch.H
@@ -42,6 +42,7 @@ SourceFiles
 #include "ggiInterpolation.H"
 #include "faceZone.H"
 #include "Switch.H"
+#include "mapDistribute.H"
 
 // * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
 
@@ -81,24 +82,30 @@ class regionCouplePolyPatch
         //- Use bridging to fix overlap error in interpolation
         Switch bridgeOverlap_;
 
-        //- Shadow patch index.  Delayed evaluation for construction
-        mutable label shadowIndex_;
-
-        //- Interpolation zone index.  Delayed evaluation for construction
-        mutable label zoneIndex_;
+        //- Quick reject algorithm
+        ggiZoneInterpolation::quickReject reject_;
 
 
-        //- Patch-to-patch interpolation
-        mutable ggiZoneInterpolation* patchToPatchPtr_;
+        // Demand-driven data
 
-        //- Patch-to-zone addressing
-        mutable labelList* zoneAddressingPtr_;
+            //- Shadow patch index.  Delayed evaluation for construction
+            mutable label shadowIndex_;
 
-        //- Remote zone addressing: data needed for interpolation
-        mutable labelList* remoteZoneAddressingPtr_;
+            //- Interpolation zone index.  Delayed evaluation for construction
+            mutable label zoneIndex_;
 
-        //- Reconstructed patch neighbour cell centres
-        mutable vectorField* reconFaceCellCentresPtr_;
+
+            //- Patch-to-patch interpolation
+            mutable ggiZoneInterpolation* patchToPatchPtr_;
+
+            //- Patch-to-zone addressing
+            mutable labelList* zoneAddressingPtr_;
+
+            //- Remote zone addressing: data needed for interpolation
+            mutable labelList* remoteZoneAddressingPtr_;
+
+            //- Reconstructed patch neighbour cell centres
+            mutable vectorField* reconFaceCellCentresPtr_;
 
 
         // Parallel communication optimisation, stored on master processor
@@ -108,11 +115,8 @@ class regionCouplePolyPatch
             //  Used for parallel optimisation
             mutable bool* localParallelPtr_;
 
-            //- List of zone faces indices received from each processor
-            mutable labelListList* receiveAddrPtr_;
-
-            //- List of zone faces indices to send to each processor
-            mutable labelListList* sendAddrPtr_;
+            //- Map-distribute comms tool
+            mutable mapDistribute* mapPtr_;
 
 
     // Private member functions
@@ -145,13 +149,6 @@ class regionCouplePolyPatch
             void calcSendReceive() const;
 
 
-            //- Return receive addressing
-            const labelListList& receiveAddr() const;
-
-            //- Return send addressing
-            const labelListList& sendAddr() const;
-
-
         // Memory management
 
             //- Clear delta coefficients
@@ -232,7 +229,9 @@ public:
             const bool attached,
             const bool master,
             const bool isWall,
-            const bool bridgeOverlap
+            const bool bridgeOverlap,
+            const ggiZoneInterpolation::quickReject
+                reject = ggiZoneInterpolation::BB_OCTREE
         );
 
         //- Construct from dictionary
@@ -292,9 +291,8 @@ public:
         }
 
 
-    // Destructor
-
-        virtual ~regionCouplePolyPatch();
+    //-Destructor
+    virtual ~regionCouplePolyPatch();
 
 
     // Member functions
@@ -382,6 +380,9 @@ public:
             //- Is the patch localised on a single processor
             bool localParallel() const;
 
+            //- Return mapDistribute
+            const mapDistribute& map() const;
+
             //- Return reference to patch-to-patch interpolation
             const ggiZoneInterpolation& patchToPatch() const;
 
diff --git a/src/foam/meshes/polyMesh/polyPatches/constraint/regionCouple/regionCouplePolyPatchTemplates.C b/src/foam/meshes/polyMesh/polyPatches/constraint/regionCouple/regionCouplePolyPatchTemplates.C
index 043e4720b..51423dde3 100644
--- a/src/foam/meshes/polyMesh/polyPatches/constraint/regionCouple/regionCouplePolyPatchTemplates.C
+++ b/src/foam/meshes/polyMesh/polyPatches/constraint/regionCouple/regionCouplePolyPatchTemplates.C
@@ -39,22 +39,6 @@ Foam::tmp<Foam::Field<Type> > Foam::regionCouplePolyPatch::fastExpand
 {
     // Check and expand the field from patch size to zone size
     // with communication
-
-    // Algorithm:
-    // 1) Master processor holds maps of all zone addressing (data provided)
-    // and all remote zone addressing (data required)
-    // 2) Each processor will send the locally active data to the master
-    // 3) Master assembles all the data
-    // 4) Master sends to all processors the data they need to receive
-    //
-    // Notes:
-    // A) If the size of zone addressing is zero, data is not sent
-    // B) Communicated data on each processor has the size of live faces
-    // C) Expanded data will be equal to actual data from other processors
-    //    only for the faces marked in remote; for other faces, it will be
-    //    equal to zero
-    // D) On processor zero, complete data is available
-    // HJ, 4/Jun/2011
     if (ff.size() != size())
     {
         FatalErrorIn
@@ -80,127 +64,54 @@ Foam::tmp<Foam::Field<Type> > Foam::regionCouplePolyPatch::fastExpand
             << abort(FatalError);
     }
 
+    // Replaced old comms algorithm.  HJ, 31/May/2016
+    // HJ, 4/Jun/2011
+
     // This function requires send-receive-addressing, but usage is not
     // symmetric across processors. Hence trigger re-calculate at this point
     // HR, 10/Jul/2013
     if (Pstream::parRun() && !localParallel())
     {
-        receiveAddr();
-        shadow().receiveAddr();
+        map();
+        shadow().map();
     }
 
-    // Expand the field to zone size
-    tmp<Field<Type> > texpandField
-    (
-        new Field<Type>(zone().size(), pTraits<Type>::zero)
-    );
+    // New version: mapDistribute
 
-    Field<Type>& expandField = texpandField();
-
-    if (Pstream::master())
+    if (Pstream::parRun())
     {
-        // Insert master processor
-        const labelList& za = zoneAddressing();
+        // Optimised mapDistribute
 
-        forAll (za, i)
-        {
-            expandField[za[i]] = ff[i];
-        }
+        // Prepare return field: expand the field to zone size
+        tmp<Field<Type> > texpandField
+        (
+            new Field<Type>(ff)
+        );
+        Field<Type>& expandField = texpandField();
 
-        // Master receives and inserts data from all processors for which
-        // receiveAddr contains entries
-        for (label procI = 1; procI < Pstream::nProcs(); procI++)
-        {
-            const labelList& curRAddr = receiveAddr()[procI];
+        map().distribute(expandField);
 
-            if (!curRAddr.empty())
-            {
-                Field<Type> receiveBuf(curRAddr.size());
-
-                // Opt: reconsider mode of communication
-                IPstream::read
-                (
-                    Pstream::blocking,
-                    procI,
-                    reinterpret_cast<char*>(receiveBuf.begin()),
-                    receiveBuf.byteSize()
-                );
-
-                // Insert received information
-                forAll (curRAddr, i)
-                {
-                    expandField[curRAddr[i]] = receiveBuf[i];
-                }
-            }
-        }
-
-        // Expanded field complete, send required data to other processors
-        for (label procI = 1; procI < Pstream::nProcs(); procI++)
-        {
-            const labelList& curSAddr = shadow().sendAddr()[procI];
-
-            if (!curSAddr.empty())
-            {
-                Field<Type> sendBuf(curSAddr.size());
-
-                forAll (curSAddr, i)
-                {
-                    sendBuf[i] = expandField[curSAddr[i]];
-                }
-
-                // Opt: reconsider mode of communication
-                OPstream::write
-                (
-                    Pstream::blocking,
-                    procI,
-                    reinterpret_cast<const char*>(sendBuf.begin()),
-                    sendBuf.byteSize()
-                );
-            }
-        }
+        return texpandField;
     }
     else
     {
-        // Send local data to master and receive remote data
-        // If patch is empty, communication is avoided
-        // HJ, 4/Jun/2011
-        if (size())
+        // Serial.  Expand the field to zone size
+
+        tmp<Field<Type> > texpandField
+        (
+            new Field<Type>(zone().size())  // filled with nans
+        );
+        Field<Type>& expandField = texpandField();
+
+        const labelList& zAddr = zoneAddressing();
+
+        forAll (zAddr, i)
         {
-            // Opt: reconsider mode of communication
-            OPstream::write
-            (
-                Pstream::blocking,
-                Pstream::masterNo(),
-                reinterpret_cast<const char*>(ff.begin()),
-                ff.byteSize()
-            );
+            expandField[zAddr[i]] = ff[i];
         }
 
-        // Prepare to receive remote data
-        const labelList& rza = shadow().remoteZoneAddressing();
-
-        if (!rza.empty())
-        {
-            Field<Type> receiveBuf(rza.size());
-
-            // Opt: reconsider mode of communication
-            IPstream::read
-            (
-                Pstream::blocking,
-                Pstream::masterNo(),
-                reinterpret_cast<char*>(receiveBuf.begin()),
-                receiveBuf.byteSize()
-            );
-
-            // Insert the data into expanded field
-            forAll (rza, i)
-            {
-                expandField[rza[i]] = receiveBuf[i];
-            }
-        }
+        return texpandField;
     }
-
-    return texpandField;
 }
 
 

From d4721bd6d1e8e64774c7ce5a03817bd2ea008403 Mon Sep 17 00:00:00 2001
From: Hrvoje Jasak <h.jasak@wikki.co.uk>
Date: Wed, 22 Jun 2016 17:29:31 +0100
Subject: [PATCH 011/130] Work in progress

---
 .../fvPatches/constraint/ggi/ggiFvPatch.C     |   7 +
 .../fvPatches/constraint/ggi/ggiFvPatch.H     |   5 +
 .../regionCouple/regionCoupleFvPatch.C        |  10 +
 .../regionCouple/regionCoupleFvPatch.H        |   5 +
 .../ggiLduInterface/ggiLduInterface.H         |   4 +
 .../overlapGGILduInterface.H                  |   2 +-
 .../interfaces/AMGInterface/AMGInterface.H    |  12 +
 .../ggiAMGInterface/ggiAMGInterface.C         | 388 ++++++++++++------
 .../ggiAMGInterface/ggiAMGInterface.H         |  20 +-
 .../ggiAMGInterfaceTemplates.C                |  68 ++-
 .../mixingPlaneAMGInterface.C                 |   1 +
 .../processorAMGInterface.C                   |   3 +-
 12 files changed, 387 insertions(+), 138 deletions(-)

diff --git a/src/finiteVolume/fvMesh/fvPatches/constraint/ggi/ggiFvPatch.C b/src/finiteVolume/fvMesh/fvPatches/constraint/ggi/ggiFvPatch.C
index 05ad430ce..09501ab16 100644
--- a/src/finiteVolume/fvMesh/fvPatches/constraint/ggi/ggiFvPatch.C
+++ b/src/finiteVolume/fvMesh/fvPatches/constraint/ggi/ggiFvPatch.C
@@ -271,6 +271,12 @@ const Foam::scalarListList& Foam::ggiFvPatch::weights() const
 }
 
 
+void Foam::ggiFvPatch::expandAddrToZone(labelField& lf) const
+{
+    lf = ggiPolyPatch_.fastExpand(lf);
+}
+
+
 Foam::tmp<Foam::labelField> Foam::ggiFvPatch::interfaceInternalField
 (
     const unallocLabelList& internalData
@@ -306,6 +312,7 @@ void Foam::ggiFvPatch::initInternalFieldTransfer
     const unallocLabelList& iF
 ) const
 {
+    // Label transfer is local without global reduction
     labelTransferBuffer_ = patchInternalField(iF);
 }
 
diff --git a/src/finiteVolume/fvMesh/fvPatches/constraint/ggi/ggiFvPatch.H b/src/finiteVolume/fvMesh/fvPatches/constraint/ggi/ggiFvPatch.H
index 6aa101dd3..740f0f208 100644
--- a/src/finiteVolume/fvMesh/fvPatches/constraint/ggi/ggiFvPatch.H
+++ b/src/finiteVolume/fvMesh/fvPatches/constraint/ggi/ggiFvPatch.H
@@ -187,6 +187,11 @@ public:
                 return coupledFvPatch::reverseT();
             }
 
+            //- Expand addressing to zone
+            //  Used in optimised AMG coarsening
+            virtual void expandAddrToZone(labelField&) const;
+
+
             //- Return the values of the given internal data adjacent to
             //  the interface as a field
             virtual tmp<labelField> interfaceInternalField
diff --git a/src/finiteVolume/fvMesh/fvPatches/constraint/regionCouple/regionCoupleFvPatch.C b/src/finiteVolume/fvMesh/fvPatches/constraint/regionCouple/regionCoupleFvPatch.C
index 41e8a58c1..38f5ba27d 100644
--- a/src/finiteVolume/fvMesh/fvPatches/constraint/regionCouple/regionCoupleFvPatch.C
+++ b/src/finiteVolume/fvMesh/fvPatches/constraint/regionCouple/regionCoupleFvPatch.C
@@ -315,6 +315,16 @@ const Foam::scalarListList& Foam::regionCoupleFvPatch::weights() const
 }
 
 
+void Foam::regionCoupleFvPatch::expandAddrToZone(labelField& lf) const
+{
+    // Missing code.  Activate for AMG solvers across regionCoupleFvPatch
+    notImplemented
+    (
+        "void regionCoupleFvPatch::expandAddrToZone(labelField& lf) const"
+    );
+}
+
+
 Foam::tmp<Foam::labelField> Foam::regionCoupleFvPatch::interfaceInternalField
 (
     const unallocLabelList& internalData
diff --git a/src/finiteVolume/fvMesh/fvPatches/constraint/regionCouple/regionCoupleFvPatch.H b/src/finiteVolume/fvMesh/fvPatches/constraint/regionCouple/regionCoupleFvPatch.H
index f09721e88..85a0fba14 100644
--- a/src/finiteVolume/fvMesh/fvPatches/constraint/regionCouple/regionCoupleFvPatch.H
+++ b/src/finiteVolume/fvMesh/fvPatches/constraint/regionCouple/regionCoupleFvPatch.H
@@ -196,6 +196,11 @@ public:
                 return coupledFvPatch::reverseT();
             }
 
+            //- Expand addressing to zone
+            //  Used in optimised AMG coarsening
+            virtual void expandAddrToZone(labelField&) const;
+
+
             //- Return the values of the given internal data adjacent to
             //  the interface as a field
             virtual tmp<labelField> interfaceInternalField
diff --git a/src/foam/matrices/lduMatrix/lduAddressing/lduInterfaces/ggiLduInterface/ggiLduInterface.H b/src/foam/matrices/lduMatrix/lduAddressing/lduInterfaces/ggiLduInterface/ggiLduInterface.H
index 4ad62d264..f13268ce0 100644
--- a/src/foam/matrices/lduMatrix/lduAddressing/lduInterfaces/ggiLduInterface/ggiLduInterface.H
+++ b/src/foam/matrices/lduMatrix/lduAddressing/lduInterfaces/ggiLduInterface/ggiLduInterface.H
@@ -121,6 +121,10 @@ public:
         //- Return face reverse transformation tensor
         virtual const tensorField& reverseT() const = 0;
 
+        //- Expand addressing to zone
+        //  Used in optimised AMG coarsening
+        virtual void expandAddrToZone(labelField&) const = 0;
+
 
         // Transfer buffer access
 
diff --git a/src/foam/matrices/lduMatrix/lduAddressing/lduInterfaces/overlapGGILduInterface/overlapGGILduInterface.H b/src/foam/matrices/lduMatrix/lduAddressing/lduInterfaces/overlapGGILduInterface/overlapGGILduInterface.H
index c7a4b181b..ddaa2890c 100644
--- a/src/foam/matrices/lduMatrix/lduAddressing/lduInterfaces/overlapGGILduInterface/overlapGGILduInterface.H
+++ b/src/foam/matrices/lduMatrix/lduAddressing/lduInterfaces/overlapGGILduInterface/overlapGGILduInterface.H
@@ -47,7 +47,7 @@ namespace Foam
 {
 
 /*---------------------------------------------------------------------------*\
-                       Class overlapGGILduInterface Declaration
+                   Class overlapGGILduInterface Declaration
 \*---------------------------------------------------------------------------*/
 
 class overlapGGILduInterface
diff --git a/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/AMGInterface/AMGInterface.H b/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/AMGInterface/AMGInterface.H
index f5b2557a9..3ef010832 100644
--- a/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/AMGInterface/AMGInterface.H
+++ b/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/AMGInterface/AMGInterface.H
@@ -82,9 +82,21 @@ protected:
         labelField faceCells_;
 
         //- Fine addressing
+        //  On GGI interfaces, a single fine coefficient may contribute to
+        //  multiple coarse coefficients using different weights.
+        //  To hanld this, a fine coefficient may be visited multiple times
+        //  which is recorded in fineAddressing.
+        //  For simple (matching) interfaces, fineAddressing_[i] = i
+        //  HJ, 21/Jun/2016
         labelField fineAddressing_;
 
         //- Restrict addressing
+        //  For each fine coefficient, list coarse cluster index it will be
+        //  agglomerated into
+        //  For cases where the fineAddressing is used, restrict addressing
+        //  and weights are expanded to match multiple hits for a single
+        //  fine coefficient, as dictated by fineAddressing
+        //  HJ, 21/Jun/2016
         labelField restrictAddressing_;
 
         //- Fine level agglomeration weights
diff --git a/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/ggiAMGInterface/ggiAMGInterface.C b/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/ggiAMGInterface/ggiAMGInterface.C
index 1f10a8724..82527393c 100644
--- a/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/ggiAMGInterface/ggiAMGInterface.C
+++ b/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/ggiAMGInterface/ggiAMGInterface.C
@@ -203,11 +203,6 @@ Foam::ggiAMGInterface::ggiAMGInterface
     zoneAddressing_(),
     mapPtr_(NULL)
 {
-    // Note.
-    // All processors will do the same coarsening and then filter
-    // the addressing to the local processor
-    // HJ, 1/Apr/2009
-
     // Note.
     // Signalling in global clustering requires me to recognise clustering
     // from separate processors as separate.  In the first phase, this will be
@@ -222,7 +217,15 @@ Foam::ggiAMGInterface::ggiAMGInterface
     // larger max int, which can be changed on request
     // HJ, 1/Apr/2009
 
-    // Expand the local and neighbour addressing to full zone size
+    // New algorithm will assemble local clusters and then create a global
+    // zone ordering by collecting all faces (coarse pairs) from proc0,
+    // followed by proc 1 etc.  This avoids global communication and allows
+    // each processor only to perform the analysis on locally created coarse
+    // faces
+    // HJ, 13/Jun/2016
+
+    // To help with analysis, expand the local and neighbour addressing
+    // to full zone size
     labelField localExpandAddressing(fineGgiInterface_.zoneSize(), 0);
 
     // Memory management, local
@@ -235,60 +238,70 @@ Foam::ggiAMGInterface::ggiAMGInterface
                 localRestrictAddressing[i] + procOffset*Pstream::myProcNo();
         }
 
-        if (!localParallel())
-        {
-            // Optimisation of this comms call is needed
-            // HJ, 9/Jun/2016
-            reduce(localExpandAddressing, sumOp<labelField>());
-        }
+        // Removed global reduce.  Only local faces will be analysed.
+        // HJ, 13/Jun/2016
     }
 
+    // Create addressing for neighbour faces.  Note: expandAddrToZone will
+    // expand the addressing to zone size.  HJ, 13/Jun/2016
     labelField neighbourExpandAddressing
     (
-        fineGgiInterface_.shadowInterface().zoneSize(),
-        0
+        fineGgiInterface_.shadowInterface().interfaceSize()
     );
 
+    // Fill local cluster ID with a combination of a local ID and processor
+    // offset
     // Memory management, neighbour
     {
-        const labelList& addr =
-            fineGgiInterface_.shadowInterface().zoneAddressing();
-
-        forAll (addr, i)
+        forAll (neighbourExpandAddressing, i)
         {
-            neighbourExpandAddressing[addr[i]] =
+            neighbourExpandAddressing[i] =
                 neighbourRestrictAddressing[i]
               + procOffset*Pstream::myProcNo();
         }
 
-        if (!localParallel())
-        {
-            // Optimisation of this comms call is needed
-            // HJ, 9/Jun/2016
-            reduce(neighbourExpandAddressing, sumOp<labelField>());
-        }
+        // Expand neighbour side to get all the data required from other
+        // processors.  Note: neigbour is now the size of remote zone
+        fineGgiInterface_.shadowInterface().expandAddrToZone
+        (
+            neighbourExpandAddressing
+        );
     }
 
+    // DEBUG: Check that all sizes are at zone size.
+    Info<< "Sizes check: local zone size "
+        << fineGgiInterface_.zoneSize()
+        << " " << localExpandAddressing << nl
+        << "shadow zone size "
+        << fineGgiInterface_.shadowInterface().zoneSize()
+        << " " << neighbourExpandAddressing
+        << endl;
+
+
+    // Note: neighbourExpandAddressing will be filled with NaNs for faces which
+    // not local
+
+    Info<< "End of reduce" << endl;
     // Make a lookup table of entries for owner/neighbour.
     // All sizes are guessed at the size of fine interface
     // HJ, 19/Feb/2009
 
     HashTable<SLList<label>, label, Hash<label> > neighboursTable
     (
-        localExpandAddressing.size()
+        fineGgiInterface_.interfaceSize()
     );
 
     // Table of face-sets to be agglomerated
     HashTable<SLList<SLList<label> >, label, Hash<label> > faceFaceTable
     (
-        localExpandAddressing.size()
+        fineGgiInterface_.interfaceSize()
     );
 
     // Table of face-sets weights to be agglomerated
     HashTable<SLList<SLList<scalar> >, label, Hash<label> >
         faceFaceWeightsTable
         (
-            localExpandAddressing.size()
+            fineGgiInterface_.interfaceSize()
         );
 
     // Count the number of coarse faces
@@ -300,11 +313,24 @@ Foam::ggiAMGInterface::ggiAMGInterface
     // On the fine level, addressing is made in a labelListList
     if (fineGgiInterface_.fineLevel())
     {
+        // This addressing defines how to interpolate for all zone faces
+        // across the interface
         const labelListList& fineAddr = fineGgiInterface_.addressing();
         const scalarListList& fineWeights = fineGgiInterface_.weights();
 
-        forAll (fineAddr, ffI)
+        // Note: cluster only locally live faces
+        // HJ, 13/Jun/2016
+
+        // This addressing defines which faces from zone are local
+        const labelList& fineZa = fineGgiInterface_.zoneAddressing();
+
+        // Perform analysis only for local faces
+        // HJ, 22/Jun/2016
+        forAll (fineZa, fineZaI)
         {
+            // Get the local face (from zone) to analyse
+            const label ffI = fineZa[fineZaI];
+
             const labelList& curFineNbrs = fineAddr[ffI];
             const scalarList& curFineWeigts = fineWeights[ffI];
 
@@ -372,6 +398,9 @@ Foam::ggiAMGInterface::ggiAMGInterface
                             nbrFound = true;
                             faceFacesIter().append(ffI);
                             faceFaceWeightsIter().append(curNW);
+
+                            // New agglomeration pair found in already
+                            // existing pair
                             nAgglomPairs++;
 
                             break;
@@ -384,7 +413,7 @@ Foam::ggiAMGInterface::ggiAMGInterface
                         curFaceFaces.append(SLList<label>(ffI));
                         curFaceWeights.append(SLList<scalar>(curNW));
 
-                        // New coarse face created
+                        // New coarse face created for an existing master
                         nCoarseFaces++;
                         nAgglomPairs++;
                     }
@@ -407,7 +436,7 @@ Foam::ggiAMGInterface::ggiAMGInterface
                         SLList<SLList<scalar> >(SLList<scalar>(curNW))
                     );
 
-                    // New coarse face created
+                    // New coarse face created for a new master
                     nCoarseFaces++;
                     nAgglomPairs++;
                 }
@@ -417,8 +446,16 @@ Foam::ggiAMGInterface::ggiAMGInterface
     else
     {
         // Coarse level, addressing is stored in faceCells
-        forAll (localExpandAddressing, ffI)
+        // This addressing defines whicf faces from zone are local
+        const labelList& fineZa = fineGgiInterface_.zoneAddressing();
+
+        // Perform analysis only for local faces
+        // HJ, 22/Jun/2016
+        forAll (fineZa, fineZaI)
         {
+            // Get the local face (from zone) to analyse
+            const label ffI = fineZa[fineZaI];
+
             label curMaster = -1;
             label curSlave = -1;
 
@@ -480,6 +517,9 @@ Foam::ggiAMGInterface::ggiAMGInterface
                         faceFacesIter().append(ffI);
                         // Add dummy weight
                         faceFaceWeightsIter().append(1.0);
+
+                        // New agglomeration pair found in already
+                        // existing pair
                         nAgglomPairs++;
                         break;
                     }
@@ -492,7 +532,7 @@ Foam::ggiAMGInterface::ggiAMGInterface
                     // Add dummy weight
                     curFaceWeights.append(SLList<scalar>(1.0));
 
-                    // New coarse face created
+                    // New coarse face created for an existing master
                     nCoarseFaces++;
                     nAgglomPairs++;
                 }
@@ -516,116 +556,193 @@ Foam::ggiAMGInterface::ggiAMGInterface
                     SLList<SLList<scalar> >(SLList<scalar>(1.0))
                 );
 
-                // New coarse face created
+                // New coarse face created for a new master
                 nCoarseFaces++;
                 nAgglomPairs++;
             }
         } // end for all fine faces
     }
 
-    faceCells_.setSize(nCoarseFaces, -1);
-    fineAddressing_.setSize(nAgglomPairs, -1);
-    restrictAddressing_.setSize(nAgglomPairs, -1);
-    restrictWeights_.setSize(nAgglomPairs);
+    // In order to assemble the coarse global face zone, find out
+    // how many faces have been created on each processor.
+    // Note that masters and slaves both count faces so we will only ask master
+    // sizes to count
+    labelList nCoarseFacesPerProc(Pstream::nProcs(), 0);
+
+    if (master())
+    {
+        nCoarseFacesPerProc[Pstream::myProcNo()] = nCoarseFaces;
+    }
+
+    reduce(nCoarseFacesPerProc, sumOp<labelList>());
+
+    Info<< "Number of faces per processor: " << nCoarseFacesPerProc
+        << endl;
+
+    // Coarse global face zone is assembled by adding all faces from proc0,
+    // followed by all faces from proc1 etc.
+    // Therefore, on procN, my master offset
+    // will be equal to the sum of numbers of coarse faces on all processors
+    // before mine
+    // HJ, 13/Jun/2016
+
+    label coarseGlobalFaceOffset = 0;
+
+    for (label i = 0; i < Pstream::myProcNo(); i++)
+    {
+        coarseGlobalFaceOffset += nCoarseFacesPerProc[i];
+    }
+
+    Pout<< "coarseGlobalFaceOffset: " << coarseGlobalFaceOffset << endl;
+
+    Info<< "End of contents assembly" << endl;
+    labelField masterFaceCells(nCoarseFaces, -1);
+    labelField masterZoneAddressing(nCoarseFaces, -1);
+    labelField masterFineAddressing(nCoarseFaces, -1);
+    labelField masterRestrictAddressing(nAgglomPairs, -1);
+    scalarField masterRestrictWeights(nAgglomPairs);
+
+    // Note: in multiple agglomeration
 
     labelList contents = neighboursTable.toc();
 
+    // Global faces shall be assembled by the increasing label of master
+    // cluster ID.
+
     // Sort makes sure the order is identical on both sides.
-    // Since the global zone is defined by this sort, the neighboursTable
-    // must be complete on all processors
     // HJ, 20/Feb/2009 and 6/Jun/2016
     sort(contents);
 
     // Grab zone size and create zone addressing
-    zoneSize_ = nCoarseFaces;
+    zoneSize_ = sum(nCoarseFacesPerProc);
+    Info<< "zoneSize_: " << zoneSize_ << endl;
 
-    zoneAddressing_.setSize(nCoarseFaces);
+    // Note:
+    // When I am agglomerating the master, I know faces are stacked up in order
+    // but on the slave side, all I know is the master cluster index and
+    // not a master coarse face index.  Therefore:
+    // - master needs to be agglomerated first
+    // - once master is agglomerated, I need to signal to the slave side
+    //   the global coarse face zone index
+
+
+    // Note: zone addressing will be assembled only for local clusters
+    // using the coarseGlobalFaceOffset
+    // HJ, 13/Jun/2016
     label nProcFaces = 0;
 
     // Reset face counter for re-use
     nCoarseFaces = 0;
     nAgglomPairs = 0;
 
-    if (master())
+    // Note:
+    // Since clustering has now happened only on local faces, addressing and
+    // all other array work on local indices and not on the coarse global zone
+    // HJ, 13/Jun/2016
+
+    // Establish zone addressing on the master side and communicate
+    // it to the shadow
+
+    // On master side, the owner addressing is stored in table of contents
+    forAll (contents, masterI)
     {
-        // On master side, the owner addressing is stored in table of contents
-        forAll (contents, masterI)
+        SLList<label>& curNbrs =
+            neighboursTable.find(contents[masterI])();
+
+        // Note: neighbour processor index is irrelevant.  HJ, 1/Apr/2009
+
+        SLList<SLList<label> >& curFaceFaces =
+            faceFaceTable.find(contents[masterI])();
+
+        SLList<SLList<scalar> >& curFaceWeights =
+            faceFaceWeightsTable.find(contents[masterI])();
+
+        SLList<label>::iterator nbrsIter = curNbrs.begin();
+        SLList<SLList<label> >::iterator faceFacesIter =
+            curFaceFaces.begin();
+
+        SLList<SLList<scalar> >::iterator faceFaceWeightsIter =
+            curFaceWeights.begin();
+
+        for
+        (
+            ;
+            nbrsIter != curNbrs.end()
+         && faceFacesIter != curFaceFaces.end()
+         && faceFaceWeightsIter != curFaceWeights.end();
+            ++nbrsIter, ++faceFacesIter, ++faceFaceWeightsIter
+        )
         {
-            SLList<label>& curNbrs =
-                neighboursTable.find(contents[masterI])();
+            // Check if master is on local processor: no longer needed,
+            // as only local processor is being searched.  HJ, 13/Jun/2016
 
-            // Note: neighbour processor index is irrelevant.  HJ, 1/Apr/2009
+            // Record that this face belongs locally
+            // Use offset to indicate its position in the list
+            masterZoneAddressing[nProcFaces] =
+                nProcFaces + coarseGlobalFaceOffset;
 
-            SLList<SLList<label> >& curFaceFaces =
-                faceFaceTable.find(contents[masterI])();
+            masterFaceCells[nProcFaces] =
+                contents[masterI] - procOffset*Pstream::myProcNo();
 
-            SLList<SLList<scalar> >& curFaceWeights =
-                faceFaceWeightsTable.find(contents[masterI])();
+            SLList<label>::iterator facesIter =
+                faceFacesIter().begin();
 
-            SLList<label>::iterator nbrsIter = curNbrs.begin();
-            SLList<SLList<label> >::iterator faceFacesIter =
-                curFaceFaces.begin();
-
-            SLList<SLList<scalar> >::iterator faceFaceWeightsIter =
-                curFaceWeights.begin();
+            SLList<scalar>::iterator weightsIter =
+                faceFaceWeightsIter().begin();
 
             for
             (
                 ;
-                nbrsIter != curNbrs.end()
-             && faceFacesIter != curFaceFaces.end()
-             && faceFaceWeightsIter != curFaceWeights.end();
-                ++nbrsIter, ++faceFacesIter, ++faceFaceWeightsIter
+                facesIter != faceFacesIter().end()
+             && weightsIter != faceFaceWeightsIter().end();
+                ++facesIter, ++weightsIter
             )
             {
-                // Check if master is on local processor
-                if
-                (
-                    contents[masterI] >= procOffset*Pstream::myProcNo()
-                 && contents[masterI] < procOffset*(Pstream::myProcNo() + 1)
-                )
-                {
-                    // Record that this face belongs locally
-                    zoneAddressing_[nProcFaces] = nCoarseFaces;
-                    faceCells_[nProcFaces] =
-                        contents[masterI] - procOffset*Pstream::myProcNo();
-                    nProcFaces++;
+                masterFineAddressing[nAgglomPairs] = facesIter();
 
-                    SLList<label>::iterator facesIter =
-                        faceFacesIter().begin();
-                    SLList<scalar>::iterator weightsIter =
-                        faceFaceWeightsIter().begin();
-
-                    for
-                    (
-                        ;
-                        facesIter != faceFacesIter().end()
-                     && weightsIter != faceFaceWeightsIter().end();
-                        ++facesIter, ++weightsIter
-                    )
-                    {
-                        fineAddressing_[nAgglomPairs] = facesIter();
-                        restrictAddressing_[nAgglomPairs] = nCoarseFaces;
-                        restrictWeights_[nAgglomPairs] = weightsIter();
-                        nAgglomPairs++;
-                    }
-                }
-
-                // Not a local face, but still created in global zone
-                nCoarseFaces++;
+                // Master processor zone face is calculated from
+                masterRestrictAddressing[nAgglomPairs] =
+                    nProcFaces + coarseGlobalFaceOffset;
+                masterRestrictWeights[nAgglomPairs] = weightsIter();
+                nAgglomPairs++;
             }
+
+            nProcFaces++;
         }
+    }
 
-        // Resize arrays: not all of ggi is used locally
-        faceCells_.setSize(nProcFaces);
-        zoneAddressing_.setSize(nProcFaces);
+    // Resize arrays: not all of ggi is used locally
+    masterFaceCells.setSize(nProcFaces);
+    masterZoneAddressing.setSize(nProcFaces);
 
-        fineAddressing_.setSize(nAgglomPairs);
-        restrictAddressing_.setSize(nAgglomPairs);
-        restrictWeights_.setSize(nAgglomPairs);
+    masterFineAddressing.setSize(nAgglomPairs);
+    masterRestrictAddressing.setSize(nAgglomPairs);
+    masterRestrictWeights.setSize(nAgglomPairs);
+
+    // Note: Both master and slave have done the same agglomeration up to here
+
+    if (master())
+    {
+        // Master has completed the clustering
+        faceCells_ = masterFaceCells;
+        zoneAddressing_ = masterZoneAddressing;
+        fineAddressing_ = masterFineAddressing;
+        restrictAddressing_ = masterRestrictAddressing;
+        restrictWeights_ = masterRestrictWeights;
     }
     else
     {
+        // Note: shadowRestrictAddressing contains the
+
+        // Note: zone addressing will be assembled only for local clusters
+        // using the coarseGlobalFaceOffset
+        // HJ, 13/Jun/2016
+        label nProcFaces = 0;
+
+        // Reset face counter for re-use
+        nCoarseFaces = 0;
+        nAgglomPairs = 0;
+
         // On slave side, the owner addressing is stored in linked lists
         forAll (contents, masterI)
         {
@@ -646,6 +763,7 @@ Foam::ggiAMGInterface::ggiAMGInterface
 
             SLList<SLList<scalar> >::iterator faceFaceWeightsIter =
                 curFaceWeights.begin();
+
             for
             (
                 ;
@@ -655,41 +773,37 @@ Foam::ggiAMGInterface::ggiAMGInterface
                 ++nbrsIter, ++faceFacesIter, ++faceFaceWeightsIter
             )
             {
-                // Check if the face is on local processor
-                if
+                // Check if the face is on local processor: no longer needed,
+                // as only local processor is being searched.  HJ, 13/Jun/2016
+
+                // Record that this face belongs locally.
+
+//HJ, HERE: I need to find out the global face index for the face that was created from the master side
+
+                zoneAddressing_[nProcFaces] = nCoarseFaces;
+                faceCells_[nProcFaces] =
+                    nbrsIter() - procOffset*Pstream::myProcNo();
+                nProcFaces++;
+
+                SLList<label>::iterator facesIter =
+                    faceFacesIter().begin();
+
+                SLList<scalar>::iterator weightsIter =
+                    faceFaceWeightsIter().begin();
+
+                for
                 (
-                    nbrsIter() >= procOffset*Pstream::myProcNo()
-                 && nbrsIter() < procOffset*(Pstream::myProcNo() + 1)
+                    ;
+                    facesIter != faceFacesIter().end()
+                 && weightsIter != faceFaceWeightsIter().end();
+                    ++facesIter, ++weightsIter
                 )
                 {
-                    // Record that this face belongs locally.
-                    zoneAddressing_[nProcFaces] = nCoarseFaces;
-                    faceCells_[nProcFaces] =
-                        nbrsIter() - procOffset*Pstream::myProcNo();
-                    nProcFaces++;
-
-                    SLList<label>::iterator facesIter =
-                        faceFacesIter().begin();
-
-                    SLList<scalar>::iterator weightsIter =
-                        faceFaceWeightsIter().begin();
-
-                    for
-                    (
-                        ;
-                        facesIter != faceFacesIter().end()
-                     && weightsIter != faceFaceWeightsIter().end();
-                        ++facesIter, ++weightsIter
-                    )
-                    {
-                        fineAddressing_[nAgglomPairs] = facesIter();
-                        restrictAddressing_[nAgglomPairs] = nCoarseFaces;
-                        restrictWeights_[nAgglomPairs] = weightsIter();
-                        nAgglomPairs++;
-                    }
+                    fineAddressing_[nAgglomPairs] = facesIter();
+                    restrictAddressing_[nAgglomPairs] = nCoarseFaces;
+                    restrictWeights_[nAgglomPairs] = weightsIter();
+                    nAgglomPairs++;
                 }
-
-                nCoarseFaces++;
             }
         }
 
@@ -718,6 +832,8 @@ Foam::tmp<Foam::scalarField> Foam::ggiAMGInterface::agglomerateCoeffs
     const scalarField& fineCoeffs
 ) const
 {
+    // HJ, HERE THIS SHOULD BE REMOVED: NO LONGER NEEDED BECAUSE ALL ADDRESSING IS LOCAL
+
     // Note: reconsider better parallel communication here.
     // Currently expanding to full zone size
     // HJ, 16/Mar/2016
@@ -917,4 +1033,10 @@ Foam::tmp<Foam::scalarField> Foam::ggiAMGInterface::internalFieldTransfer
 }
 
 
+void Foam::ggiAMGInterface::expandAddrToZone(labelField& lf) const
+{
+    lf = fastExpand(lf);
+}
+
+
 // ************************************************************************* //
diff --git a/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/ggiAMGInterface/ggiAMGInterface.H b/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/ggiAMGInterface/ggiAMGInterface.H
index 973b0467a..507909026 100644
--- a/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/ggiAMGInterface/ggiAMGInterface.H
+++ b/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/ggiAMGInterface/ggiAMGInterface.H
@@ -143,9 +143,23 @@ public:
 
         // Interface transfer functions
 
+            //- Fast expand
+            //  Note: contains global communications
+            //  This function will expand data to zone size with fast comms
+            //  filling in only the part of the zone that is required
+            //  on the shadow side
+            template<class Type>
+            tmp<Field<Type> > fastExpand(const UList<Type>&) const;
+
             //- Fast reduce
             //  Note: contains global communications
-            //  New, HJ, 24/Jun/2011
+            //  This function will expand data to zone size with fast comms
+            //  and filter the field to cover the need of a shadow zone
+            //  This makes sense on coarse levels because all addressing
+            //  is one-on-one (one master, one slave).  On the fine level
+            //  each face in the zone may contribute to multiple faces meaning
+            // that the data needs to be expanded to zone size
+            //  HJ, 24/Jun/2011 and 13/Jun/2016
             template<class Type>
             tmp<Field<Type> > fastReduce(const UList<Type>&) const;
 
@@ -234,6 +248,10 @@ public:
 
             //- Return neighbour-cell transformation tensor
             virtual const tensorField& reverseT() const;
+
+            //- Expand addressing to zone
+            //  Used in optimised AMG coarsening
+            virtual void expandAddrToZone(labelField&) const;
 };
 
 
diff --git a/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/ggiAMGInterface/ggiAMGInterfaceTemplates.C b/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/ggiAMGInterface/ggiAMGInterfaceTemplates.C
index 9c468bbb0..f0cbf2b11 100644
--- a/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/ggiAMGInterface/ggiAMGInterfaceTemplates.C
+++ b/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/ggiAMGInterface/ggiAMGInterfaceTemplates.C
@@ -32,10 +32,61 @@ namespace Foam
 
 // * * * * * * * * * * * * * * * Member Functions  * * * * * * * * * * * * * //
 
+template<class Type>
+tmp<Field<Type> > ggiAMGInterface::fastExpand(const UList<Type>& ff) const
+{
+    // Rewrite, 1/Jun/2016
+    // To avoid creating zone-sized data and gather-scatter communication
+    // to the master, the optimised map-distribute call is implemented.
+    // The field is filled with local data which is then sent where needed
+    // through map-distribute.
+    // On return, the field is expanded to zone size but only filled with
+    // the data which is needed for the shadow
+    // HJ, 1/Jun/2016
+
+    if (ff.size() != this->size())
+    {
+        FatalErrorIn
+        (
+            "tmp<Field<Type> > ggiAMGInterface::fastExpand"
+            "("
+            "    const UList<Type>& ff"
+            ") const"
+        )   << "Wrong field size.  ff: " << ff.size()
+            << " interface: " << this->size()
+            << abort(FatalError);
+    }
+
+    if (localParallel() || !Pstream::parRun())
+    {
+        // Field remains identical: no parallel communications required
+        tmp<Field<Type> > tresult(new Field<Type>(ff));
+
+        return tresult;
+    }
+    else
+    {
+        // Optimised mapDistribute
+
+        // Execute init reduce to calculate addressing if not already done
+        map();
+
+        // Prepare for distribute.  Note: field will be expanded to zone size
+        // during the distribute operation
+        tmp<Field<Type> > tresult(new Field<Type>(ff));
+        List<Type>& expand = tresult();
+
+        map().distribute(expand);
+
+        return tresult;
+    }
+}
+
+
 template<class Type>
 tmp<Field<Type> > ggiAMGInterface::fastReduce(const UList<Type>& ff) const
 {
-    // Algorithm
+    // Old algorithm: OBOSLETE
     // Local processor contains faceCells part of the zone and requires
     // zoneAddressing part.
     // For fast communications, each processor will send the faceCells and
@@ -44,6 +95,18 @@ tmp<Field<Type> > ggiAMGInterface::fastReduce(const UList<Type>& ff) const
     // the required data
     // HJ, 24/Jun/2011
 
+    // Rewrite, 1/Jun/2016
+    // To avoid creating zone-sized data and gather-scatter communication
+    // to the master, the optimised map-distribute call is implemented.
+    // The field is filled with local data which is then sent where needed
+    // through map-distribute.
+    // On return, the field is expanded to zone size but only filled with
+    // the data which is needed for the shadow
+    // Having received the zone data, shadow data is extracted from the
+    // field size.  Note: this works only on coarse levels, where one-on-one
+    // mapping applies
+    // HJ, 1/Jun/2016
+
     if (ff.size() != this->size())
     {
         FatalErrorIn
@@ -71,7 +134,8 @@ tmp<Field<Type> > ggiAMGInterface::fastReduce(const UList<Type>& ff) const
         // Execute init reduce to calculate addressing if not already done
         map();
 
-        // Prepare for distribute: field will be expanded to zone size
+        // Prepare for distribute.  Note: field will be expanded to zone size
+        // during the distribute operation
         List<Type> expand = ff;
 
         map().distribute(expand);
diff --git a/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/mixingPlaneAMGInterface/mixingPlaneAMGInterface.C b/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/mixingPlaneAMGInterface/mixingPlaneAMGInterface.C
index dd1841562..9a766bae4 100644
--- a/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/mixingPlaneAMGInterface/mixingPlaneAMGInterface.C
+++ b/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/mixingPlaneAMGInterface/mixingPlaneAMGInterface.C
@@ -168,6 +168,7 @@ void Foam::mixingPlaneAMGInterface::initInternalFieldTransfer
     const unallocLabelList& iF
 ) const
 {
+    // NOTE: Change this: requires fast reduce.  HJ, 13/Jun/20106
     labelTransferBuffer_ = interfaceInternalField(iF);
 }
 
diff --git a/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/processorAMGInterface/processorAMGInterface.C b/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/processorAMGInterface/processorAMGInterface.C
index 7b346a099..35921b46f 100644
--- a/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/processorAMGInterface/processorAMGInterface.C
+++ b/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/processorAMGInterface/processorAMGInterface.C
@@ -189,7 +189,8 @@ Foam::processorAMGInterface::processorAMGInterface
 
                 for
                 (
-                    SLList<label>::iterator facesIter = faceFacesIter().begin();
+                    SLList<label>::iterator facesIter =
+                        faceFacesIter().begin();
                     facesIter != faceFacesIter().end();
                     ++facesIter
                 )

From 734b4de88355f7dbe502df205d4de0a80d463acf Mon Sep 17 00:00:00 2001
From: Hrvoje Jasak <h.jasak@wikki.co.uk>
Date: Wed, 22 Jun 2016 17:29:43 +0100
Subject: [PATCH 012/130] Debug info

---
 .../BlockMatrixAgglomeration.C                | 24 +++++++++++++++----
 .../BlockMatrixAgglomeration.H                |  3 +++
 2 files changed, 23 insertions(+), 4 deletions(-)

diff --git a/src/foam/matrices/blockLduMatrix/BlockAMG/BlockMatrixCoarsening/BlockMatrixAgglomeration/BlockMatrixAgglomeration.C b/src/foam/matrices/blockLduMatrix/BlockAMG/BlockMatrixCoarsening/BlockMatrixAgglomeration/BlockMatrixAgglomeration.C
index 2eb4bd548..9278ad63b 100644
--- a/src/foam/matrices/blockLduMatrix/BlockAMG/BlockMatrixCoarsening/BlockMatrixAgglomeration/BlockMatrixAgglomeration.C
+++ b/src/foam/matrices/blockLduMatrix/BlockAMG/BlockMatrixCoarsening/BlockMatrixAgglomeration/BlockMatrixAgglomeration.C
@@ -74,6 +74,7 @@ void Foam::BlockMatrixAgglomeration<Type>::calcAgglomeration()
     // Create row-based addressing
 
     const label nRows = matrix_.lduAddr().size();
+    Info<< "Start, coarse size: " << nRows << endl;
 
     const unallocLabelList& upperAddr = matrix_.lduAddr().upperAddr();
     const unallocLabelList& lowerAddr = matrix_.lduAddr().lowerAddr();
@@ -142,6 +143,7 @@ void Foam::BlockMatrixAgglomeration<Type>::calcAgglomeration()
         // Reset agglomeration index array
         agglomIndex_ = -1;
     }
+    Info<< "End prepare addressing: " << lTime_.elapsedClockTime() << endl;
 
 
     // Calculate agglomeration
@@ -177,6 +179,7 @@ void Foam::BlockMatrixAgglomeration<Type>::calcAgglomeration()
 
         return;
     }
+    Info<< "End coeff mag: " << lTime_.elapsedClockTime() << endl;
 
     labelList sizeOfGroups(nRows, 0);
 
@@ -232,6 +235,7 @@ void Foam::BlockMatrixAgglomeration<Type>::calcAgglomeration()
             }
         }
     }
+    Info<< "End weak connections: " << lTime_.elapsedClockTime() << endl;
 
     // Go through the off-diagonal and create clusters, marking the child array
     label indexUngrouped, indexGrouped;
@@ -339,6 +343,7 @@ void Foam::BlockMatrixAgglomeration<Type>::calcAgglomeration()
             }
         }
     }
+    Info<< "End agglom index: " << lTime_.elapsedClockTime() << endl;
 
     // The decision on parallel agglomeration needs to be made for the
     // whole gang of processes; otherwise I may end up with a different
@@ -368,6 +373,7 @@ void Foam::BlockMatrixAgglomeration<Type>::calcAgglomeration()
             Pout << ".  Rejected" << endl;
         }
     }
+    Info<< "End calcAgglomeration: " << lTime_.elapsedClockTime() << endl;
 }
 
 
@@ -379,7 +385,7 @@ void Foam::BlockMatrixAgglomeration<Type>::restrictDiag
 ) const
 {
     typedef CoeffField<Type> TypeCoeffField;
-
+    Info<< "Start restrictDiag: " << lTime_.elapsedClockTime() << endl;
     if
     (
         Coeff.activeType() == blockCoeffBase::SQUARE
@@ -454,6 +460,7 @@ void Foam::BlockMatrixAgglomeration<Type>::restrictDiag
         )   << "Problem in coeff type morphing"
             << abort(FatalError);
     }
+    Info<< "End restrictDiag: " << lTime_.elapsedClockTime() << endl;
 }
 
 
@@ -470,7 +477,7 @@ void Foam::BlockMatrixAgglomeration<Type>::agglomerateCoeffs
 {
     // Does the matrix have solo equations
     bool soloEqns = nSolo_ > 0;
-
+    Info<< "Start agglomerateCoeffs symm: " << lTime_.elapsedClockTime() << endl;
     // Get addressing
     const unallocLabelList& upperAddr = matrix_.lduAddr().upperAddr();
     const unallocLabelList& lowerAddr = matrix_.lduAddr().lowerAddr();
@@ -503,6 +510,7 @@ void Foam::BlockMatrixAgglomeration<Type>::agglomerateCoeffs
               + activeFineUpperTranspose[fineCoeffI];
         }
     }
+    Info<< "End agglomerateCoeffs symm: " << lTime_.elapsedClockTime() << endl;
 }
 
 
@@ -520,7 +528,7 @@ void Foam::BlockMatrixAgglomeration<Type>::agglomerateCoeffs
 {
     // Does the matrix have solo equations
     bool soloEqns = nSolo_ > 0;
-
+    Info<< "Start agglomerateCoeffs asymm: " << lTime_.elapsedClockTime() << endl;
     // Get addressing
     const unallocLabelList& upperAddr = matrix_.lduAddr().upperAddr();
     const unallocLabelList& lowerAddr = matrix_.lduAddr().lowerAddr();
@@ -552,6 +560,7 @@ void Foam::BlockMatrixAgglomeration<Type>::agglomerateCoeffs
               + activeFineLower[fineCoeffI];
         }
     }
+    Info<< "End agglomerateCoeffs asymm: " << lTime_.elapsedClockTime() << endl;
 }
 
 
@@ -638,7 +647,8 @@ Foam::BlockMatrixAgglomeration<Type>::BlockMatrixAgglomeration
     groupSize_(groupSize),
     nSolo_(0),
     nCoarseEqns_(0),
-    coarsen_(false)
+    coarsen_(false),
+    lTime_()
 {
     calcAgglomeration();
 }
@@ -679,6 +689,7 @@ Foam::BlockMatrixAgglomeration<Type>::restrictMatrix() const
     //    insert the coeffs in the upper order. At the same time, collect the
     //    owner and neighbour addressing.
     // 4) Agglomerate the diagonal by summing up the fine diagonal
+    Info<< "Start restrictMatrix: " << lTime_.elapsedClockTime() << endl;
 
     // Get addressing
     const unallocLabelList& upperAddr = matrix_.lduAddr().upperAddr();
@@ -853,6 +864,7 @@ Foam::BlockMatrixAgglomeration<Type>::restrictMatrix() const
     blockNbrsData.setSize(0);
     initCoarseNeighb.setSize(0);
     coarseCoeffMap.setSize(0);
+    Info<< "End restrictMatrix core: " << lTime_.elapsedClockTime() << endl;
 
 
     // Create coarse-level coupled interfaces
@@ -885,6 +897,7 @@ Foam::BlockMatrixAgglomeration<Type>::restrictMatrix() const
     );
 
     // Initialise transfer of restrict addressing on the interface
+    // HJ, consider blocking comms.  HJ, 9/Jun/2016
     forAll (interfaceFields, intI)
     {
         if (interfaceFields.set(intI))
@@ -955,6 +968,7 @@ Foam::BlockMatrixAgglomeration<Type>::restrictMatrix() const
             coarseInterfaceAddr[intI] = coarseInterface.faceCells();
         }
     }
+    Info<< "End restrictMatrix interfaces: " << lTime_.elapsedClockTime() << endl;
 
     // Add interfaces
     coarseAddrPtr->addInterfaces
@@ -1031,6 +1045,7 @@ Foam::BlockMatrixAgglomeration<Type>::restrictMatrix() const
     }
 
     // Matrix restriction done!
+    Info<< "End restrictMatrix: " << lTime_.elapsedClockTime() << endl;
 
     typedef CoeffField<Type> TypeCoeffField;
 
@@ -1227,6 +1242,7 @@ Foam::BlockMatrixAgglomeration<Type>::restrictMatrix() const
                 << abort(FatalError);
         }
     }
+    Info<< "End restrictMatrix agglomerateCoeffs: " << lTime_.elapsedClockTime() << endl;
 
     // Create and return BlockAMGLevel
     return autoPtr<BlockAMGLevel<Type> >
diff --git a/src/foam/matrices/blockLduMatrix/BlockAMG/BlockMatrixCoarsening/BlockMatrixAgglomeration/BlockMatrixAgglomeration.H b/src/foam/matrices/blockLduMatrix/BlockAMG/BlockMatrixCoarsening/BlockMatrixAgglomeration/BlockMatrixAgglomeration.H
index aca68f389..65ac5eb8f 100644
--- a/src/foam/matrices/blockLduMatrix/BlockAMG/BlockMatrixCoarsening/BlockMatrixAgglomeration/BlockMatrixAgglomeration.H
+++ b/src/foam/matrices/blockLduMatrix/BlockAMG/BlockMatrixCoarsening/BlockMatrixAgglomeration/BlockMatrixAgglomeration.H
@@ -43,6 +43,7 @@ SourceFiles
 #include "BlockCoeffNorm.H"
 #include "BlockCoeff.H"
 #include "tolerancesSwitch.H"
+#include "clock.H"
 
 // * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
 
@@ -84,6 +85,8 @@ class BlockMatrixAgglomeration
         //- Can a coarse level be constructed?
         bool coarsen_;
 
+        clock lTime_;
+
 
     // Private Member Functions
 

From a1b190ed76948e19360b8deced12a25a37f471c3 Mon Sep 17 00:00:00 2001
From: Hrvoje Jasak <h.jasak@wikki.co.uk>
Date: Tue, 9 Aug 2016 13:21:53 +0100
Subject: [PATCH 013/130] Formatting

---
 .../mapPolyMesh/mapDistribute/mapDistributeTemplates.C    | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/foam/meshes/polyMesh/mapPolyMesh/mapDistribute/mapDistributeTemplates.C b/src/foam/meshes/polyMesh/mapPolyMesh/mapDistribute/mapDistributeTemplates.C
index d980e38c2..cccdfbe5d 100644
--- a/src/foam/meshes/polyMesh/mapPolyMesh/mapDistribute/mapDistributeTemplates.C
+++ b/src/foam/meshes/polyMesh/mapPolyMesh/mapDistribute/mapDistributeTemplates.C
@@ -166,7 +166,7 @@ void Foam::mapDistribute::distribute
 
         // Set up sends to neighbours
 
-        List<List<T > > sendFields(Pstream::nProcs());
+        List<List<T> > sendFields(Pstream::nProcs());
 
         for (label domain = 0; domain < Pstream::nProcs(); domain++)
         {
@@ -195,7 +195,7 @@ void Foam::mapDistribute::distribute
 
         // Set up receives from neighbours
 
-        List<List<T > > recvFields(Pstream::nProcs());
+        List<List<T> > recvFields(Pstream::nProcs());
 
         for (label domain = 0; domain < Pstream::nProcs(); domain++)
         {
@@ -460,7 +460,7 @@ void Foam::mapDistribute::distribute
 
         // Set up sends to neighbours
 
-        List<List<T > > sendFields(Pstream::nProcs());
+        List<List<T> > sendFields(Pstream::nProcs());
 
         for (label domain = 0; domain < Pstream::nProcs(); domain++)
         {
@@ -487,7 +487,7 @@ void Foam::mapDistribute::distribute
 
         // Set up receives from neighbours
 
-        List<List<T > > recvFields(Pstream::nProcs());
+        List<List<T> > recvFields(Pstream::nProcs());
 
         for (label domain = 0; domain < Pstream::nProcs(); domain++)
         {

From 60d46509a86276a1e12df6deb702f48d68156d25 Mon Sep 17 00:00:00 2001
From: Hrvoje Jasak <h.jasak@wikki.co.uk>
Date: Tue, 9 Aug 2016 13:22:26 +0100
Subject: [PATCH 014/130] Formatting

---
 src/foam/db/IOstreams/Pstreams/PstreamCombineReduceOps.H | 1 -
 src/foam/db/IOstreams/Pstreams/PstreamGlobals.H          | 2 +-
 2 files changed, 1 insertion(+), 2 deletions(-)

diff --git a/src/foam/db/IOstreams/Pstreams/PstreamCombineReduceOps.H b/src/foam/db/IOstreams/Pstreams/PstreamCombineReduceOps.H
index bade7b796..2e30e0962 100644
--- a/src/foam/db/IOstreams/Pstreams/PstreamCombineReduceOps.H
+++ b/src/foam/db/IOstreams/Pstreams/PstreamCombineReduceOps.H
@@ -30,7 +30,6 @@ Description
     combined using the given combination function and the result is
     broadcast to all nodes
 
-
 \*---------------------------------------------------------------------------*/
 
 #ifndef PstreamCombineReduceOps_H
diff --git a/src/foam/db/IOstreams/Pstreams/PstreamGlobals.H b/src/foam/db/IOstreams/Pstreams/PstreamGlobals.H
index 816fef8ed..c68ea965e 100644
--- a/src/foam/db/IOstreams/Pstreams/PstreamGlobals.H
+++ b/src/foam/db/IOstreams/Pstreams/PstreamGlobals.H
@@ -46,7 +46,7 @@ namespace Foam
 {
 
 /*---------------------------------------------------------------------------*\
-                           Class PstreamGlobals Declaration
+                     Class PstreamGlobals Declaration
 \*---------------------------------------------------------------------------*/
 
 namespace PstreamGlobals

From c076f28ce1388870428834364adb3aa6ea675680 Mon Sep 17 00:00:00 2001
From: Hrvoje Jasak <h.jasak@wikki.co.uk>
Date: Tue, 9 Aug 2016 13:22:54 +0100
Subject: [PATCH 015/130] Formatting

---
 .../lduInterfaces/ggiLduInterface/ggiLduInterface.H          | 5 ++---
 .../lduAddressing/lduInterfaces/lduInterface/lduInterface.H  | 5 ++---
 2 files changed, 4 insertions(+), 6 deletions(-)

diff --git a/src/foam/matrices/lduMatrix/lduAddressing/lduInterfaces/ggiLduInterface/ggiLduInterface.H b/src/foam/matrices/lduMatrix/lduAddressing/lduInterfaces/ggiLduInterface/ggiLduInterface.H
index f13268ce0..a4fc7b2c7 100644
--- a/src/foam/matrices/lduMatrix/lduAddressing/lduInterfaces/ggiLduInterface/ggiLduInterface.H
+++ b/src/foam/matrices/lduMatrix/lduAddressing/lduInterfaces/ggiLduInterface/ggiLduInterface.H
@@ -78,9 +78,8 @@ public:
         {}
 
 
-    // Destructor
-
-        virtual ~ggiLduInterface();
+    //- Destructor
+    virtual ~ggiLduInterface();
 
 
     // Member Functions
diff --git a/src/foam/matrices/lduMatrix/lduAddressing/lduInterfaces/lduInterface/lduInterface.H b/src/foam/matrices/lduMatrix/lduAddressing/lduInterfaces/lduInterface/lduInterface.H
index 3586aa66c..dfe101f50 100644
--- a/src/foam/matrices/lduMatrix/lduAddressing/lduInterfaces/lduInterface/lduInterface.H
+++ b/src/foam/matrices/lduMatrix/lduAddressing/lduInterfaces/lduInterface/lduInterface.H
@@ -74,9 +74,8 @@ public:
         {}
 
 
-    // Destructor
-
-        virtual ~lduInterface();
+    //- Destructor
+    virtual ~lduInterface();
 
 
     // Member Functions

From 3143f456cc3d3e16d424b2397deb14de0062a5cc Mon Sep 17 00:00:00 2001
From: Hrvoje Jasak <h.jasak@wikki.co.uk>
Date: Tue, 9 Aug 2016 13:25:14 +0100
Subject: [PATCH 016/130] Feature: Rewrite of GGI comms on AMG agglomeration

---
 .../BlockMatrixAgglomeration.C                |   1 +
 .../GAMGAgglomerateLduAddressing.C            |   1 +
 .../interfaces/AMGInterface/AMGInterface.H    |   8 +
 .../interfaces/AMGInterface/newAMGInterface.C |  15 +-
 .../cyclicAMGInterface/cyclicAMGInterface.C   |   1 +
 .../cyclicAMGInterface/cyclicAMGInterface.H   |   1 +
 .../cyclicGGIAMGInterface.C                   |   2 +
 .../cyclicGGIAMGInterface.H                   |   1 +
 .../ggiAMGInterface/ggiAMGInterface.C         | 728 +++++++++++-------
 .../ggiAMGInterface/ggiAMGInterface.H         |  14 +-
 .../mixingPlaneAMGInterface.C                 |   1 +
 .../mixingPlaneAMGInterface.H                 |   1 +
 .../processorAMGInterface.C                   |   1 +
 .../processorAMGInterface.H                   |   1 +
 .../regionCoupleAMGInterface.C                |   2 +
 .../regionCoupleAMGInterface.H                |   1 +
 src/lduSolvers/amg/amgPolicy/pamgPolicy.C     |   1 +
 17 files changed, 501 insertions(+), 279 deletions(-)

diff --git a/src/foam/matrices/blockLduMatrix/BlockAMG/BlockMatrixCoarsening/BlockMatrixAgglomeration/BlockMatrixAgglomeration.C b/src/foam/matrices/blockLduMatrix/BlockAMG/BlockMatrixCoarsening/BlockMatrixAgglomeration/BlockMatrixAgglomeration.C
index 9278ad63b..792a39c0f 100644
--- a/src/foam/matrices/blockLduMatrix/BlockAMG/BlockMatrixCoarsening/BlockMatrixAgglomeration/BlockMatrixAgglomeration.C
+++ b/src/foam/matrices/blockLduMatrix/BlockAMG/BlockMatrixCoarsening/BlockMatrixAgglomeration/BlockMatrixAgglomeration.C
@@ -950,6 +950,7 @@ Foam::BlockMatrixAgglomeration<Type>::restrictMatrix() const
                 AMGInterface::New
                 (
                     coarseAddrPtr(),
+                    coarseInterfaces,
                     fineInterface,
                     fineInterface.interfaceInternalField(agglomIndex_),
                     fineInterfaceAddr[intI]
diff --git a/src/foam/matrices/lduMatrix/solvers/AMG/AMGAgglomerations/GAMGAgglomeration/GAMGAgglomerateLduAddressing.C b/src/foam/matrices/lduMatrix/solvers/AMG/AMGAgglomerations/GAMGAgglomeration/GAMGAgglomerateLduAddressing.C
index e93172b11..9ba20ce1f 100644
--- a/src/foam/matrices/lduMatrix/solvers/AMG/AMGAgglomerations/GAMGAgglomeration/GAMGAgglomerateLduAddressing.C
+++ b/src/foam/matrices/lduMatrix/solvers/AMG/AMGAgglomerations/GAMGAgglomeration/GAMGAgglomerateLduAddressing.C
@@ -292,6 +292,7 @@ void Foam::GAMGAgglomeration::agglomerateLduAddressing
                     AMGInterface::New
                     (
                         meshLevels_[fineLevelIndex],
+                        coarseInterfaces,
                         fineInterfaces[inti],
                         fineInterfaces[inti].interfaceInternalField
                         (
diff --git a/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/AMGInterface/AMGInterface.H b/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/AMGInterface/AMGInterface.H
index 3ef010832..639a04314 100644
--- a/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/AMGInterface/AMGInterface.H
+++ b/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/AMGInterface/AMGInterface.H
@@ -129,12 +129,14 @@ public:
             lduInterface,
             (
                 const lduPrimitiveMesh& lduMesh,
+                const lduInterfacePtrsList& coarseInterfaces,
                 const lduInterface& fineInterface,
                 const labelField& localRestrictAddressing,
                 const labelField& neighbourRestrictAddressing
             ),
             (
                 lduMesh,
+                coarseInterfaces,
                 fineInterface,
                 localRestrictAddressing,
                 neighbourRestrictAddressing
@@ -149,6 +151,7 @@ public:
         static autoPtr<AMGInterface> New
         (
             const lduPrimitiveMesh& lduMesh,
+            const lduInterfacePtrsList& coarseInterfaces,
             const lduInterface& fineInterface,
             const labelField& localRestrictAddressing,
             const labelField& neighbourRestrictAddressing
@@ -165,6 +168,11 @@ public:
         {}
 
 
+    //- Destructor
+    virtual ~AMGInterface()
+    {}
+
+
     // Member Functions
 
         // Access
diff --git a/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/AMGInterface/newAMGInterface.C b/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/AMGInterface/newAMGInterface.C
index 4a8f5236f..2f6dc2548 100644
--- a/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/AMGInterface/newAMGInterface.C
+++ b/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/AMGInterface/newAMGInterface.C
@@ -32,6 +32,7 @@ License
 Foam::autoPtr<Foam::AMGInterface> Foam::AMGInterface::New
 (
     const lduPrimitiveMesh& lduMesh,
+    const lduInterfacePtrsList& coarseInterfaces,
     const lduInterface& fineInterface,
     const labelField& localRestrictAddressing,
     const labelField& neighbourRestrictAddressing
@@ -46,11 +47,14 @@ Foam::autoPtr<Foam::AMGInterface> Foam::AMGInterface::New
     {
         FatalErrorIn
         (
-            "AMGInterface::New"
-            "(const lduPrimitiveMesh& lduMesh,"
-            "const lduInterface& fineInterface,"
-            "const labelField& localRestrictAddressing,"
-            "const labelField& neighbourRestrictAddressing)"
+            "AMGInterface::New\n"
+            "(\n"
+            "    const lduPrimitiveMesh& lduMesh,\n"
+            "    const lduInterfacePtrsList& coarseInterfaces,\n"
+            "    const lduInterface& fineInterface,\n"
+            "    const labelField& localRestrictAddressing,\n"
+            "    const labelField& neighbourRestrictAddressing\n"
+            ")"
         )   << "Unknown AMGInterface type " << coupleType << ".\n"
             << "Valid AMGInterface types are :"
             << lduInterfaceConstructorTablePtr_->sortedToc()
@@ -62,6 +66,7 @@ Foam::autoPtr<Foam::AMGInterface> Foam::AMGInterface::New
         cstrIter()
         (
             lduMesh,
+            coarseInterfaces,
             fineInterface,
             localRestrictAddressing,
             neighbourRestrictAddressing
diff --git a/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/cyclicAMGInterface/cyclicAMGInterface.C b/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/cyclicAMGInterface/cyclicAMGInterface.C
index 2ae72dbaa..72fe4cf0b 100644
--- a/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/cyclicAMGInterface/cyclicAMGInterface.C
+++ b/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/cyclicAMGInterface/cyclicAMGInterface.C
@@ -45,6 +45,7 @@ namespace Foam
 Foam::cyclicAMGInterface::cyclicAMGInterface
 (
     const lduPrimitiveMesh& lduMesh,
+    const lduInterfacePtrsList& coarseInterfaces,
     const lduInterface& fineInterface,
     const labelField& localRestrictAddressing,
     const labelField& neighbourRestrictAddressing
diff --git a/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/cyclicAMGInterface/cyclicAMGInterface.H b/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/cyclicAMGInterface/cyclicAMGInterface.H
index 3e983f19b..0a3978e47 100644
--- a/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/cyclicAMGInterface/cyclicAMGInterface.H
+++ b/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/cyclicAMGInterface/cyclicAMGInterface.H
@@ -81,6 +81,7 @@ public:
         cyclicAMGInterface
         (
             const lduPrimitiveMesh& lduMesh,
+            const lduInterfacePtrsList& coarseInterfaces,
             const lduInterface& fineInterface,
             const labelField& localRestrictAddressing,
             const labelField& neighbourRestrictAddressing
diff --git a/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/cyclicGGIAMGInterface/cyclicGGIAMGInterface.C b/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/cyclicGGIAMGInterface/cyclicGGIAMGInterface.C
index 6f1df8b3d..c64227099 100644
--- a/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/cyclicGGIAMGInterface/cyclicGGIAMGInterface.C
+++ b/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/cyclicGGIAMGInterface/cyclicGGIAMGInterface.C
@@ -45,6 +45,7 @@ namespace Foam
 Foam::cyclicGGIAMGInterface::cyclicGGIAMGInterface
 (
     const lduPrimitiveMesh& lduMesh,
+    const lduInterfacePtrsList& coarseInterfaces,
     const lduInterface& fineInterface,
     const labelField& localRestrictAddressing,
     const labelField& neighbourRestrictAddressing
@@ -53,6 +54,7 @@ Foam::cyclicGGIAMGInterface::cyclicGGIAMGInterface
     ggiAMGInterface
     (
         lduMesh,
+        coarseInterfaces,
         fineInterface,
         localRestrictAddressing,
         neighbourRestrictAddressing
diff --git a/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/cyclicGGIAMGInterface/cyclicGGIAMGInterface.H b/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/cyclicGGIAMGInterface/cyclicGGIAMGInterface.H
index 19f2b92d7..2498c2282 100644
--- a/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/cyclicGGIAMGInterface/cyclicGGIAMGInterface.H
+++ b/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/cyclicGGIAMGInterface/cyclicGGIAMGInterface.H
@@ -67,6 +67,7 @@ public:
         cyclicGGIAMGInterface
         (
             const lduPrimitiveMesh& lduMesh,
+            const lduInterfacePtrsList& coarseInterfaces,
             const lduInterface& fineInterface,
             const labelField& localRestrictAddressing,
             const labelField& neighbourRestrictAddressing
diff --git a/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/ggiAMGInterface/ggiAMGInterface.C b/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/ggiAMGInterface/ggiAMGInterface.C
index 82527393c..cc77d00b0 100644
--- a/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/ggiAMGInterface/ggiAMGInterface.C
+++ b/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/ggiAMGInterface/ggiAMGInterface.C
@@ -192,6 +192,7 @@ void Foam::ggiAMGInterface::initFastReduce() const
 Foam::ggiAMGInterface::ggiAMGInterface
 (
     const lduPrimitiveMesh& lduMesh,
+    const lduInterfacePtrsList& coarseInterfaces,
     const lduInterface& fineInterface,
     const labelField& localRestrictAddressing,
     const labelField& neighbourRestrictAddressing
@@ -201,107 +202,87 @@ Foam::ggiAMGInterface::ggiAMGInterface
     fineGgiInterface_(refCast<const ggiLduInterface>(fineInterface)),
     zoneSize_(0),
     zoneAddressing_(),
+    procMasterFaces_(),
     mapPtr_(NULL)
 {
-    // Note.
-    // Signalling in global clustering requires me to recognise clustering
-    // from separate processors as separate.  In the first phase, this will be
-    // used to recognise cluster from each processor as separate and in the
-    // second phase it will be used to filter local processor faces from
-    // the global patch.
-    // Currently, I am calculating unique cluster index as:
-    //
-    // id = cluster + procOffset*myProcID
-    // With procOffset = 1 million, this should be sufficient for 2000 CPUs
-    // with 2 million coarse cells each.  For larger numbers, I need a
-    // larger max int, which can be changed on request
-    // HJ, 1/Apr/2009
-
-    // New algorithm will assemble local clusters and then create a global
-    // zone ordering by collecting all faces (coarse pairs) from proc0,
+    // New algorithm will assemble local clusters on the master side and
+    // create zone ordering by collecting all faces (coarse pairs) from proc0,
     // followed by proc 1 etc.  This avoids global communication and allows
     // each processor only to perform the analysis on locally created coarse
     // faces
     // HJ, 13/Jun/2016
 
-    // To help with analysis, expand the local and neighbour addressing
-    // to full zone size
-    labelField localExpandAddressing(fineGgiInterface_.zoneSize(), 0);
-
-    // Memory management, local
-    {
-        const labelList& addr = fineGgiInterface_.zoneAddressing();
-
-        forAll (addr, i)
-        {
-            localExpandAddressing[addr[i]] =
-                localRestrictAddressing[i] + procOffset*Pstream::myProcNo();
-        }
-
-        // Removed global reduce.  Only local faces will be analysed.
-        // HJ, 13/Jun/2016
-    }
+    // Note: local addressing contains only local faces
+    const labelList& fineZa =  fineGgiInterface_.zoneAddressing();
 
     // Create addressing for neighbour faces.  Note: expandAddrToZone will
-    // expand the addressing to zone size.  HJ, 13/Jun/2016
-    labelField neighbourExpandAddressing
+    // expand the addressing to zone size, including communications.
+    // Faces which are not used locally will be marked by NaNs
+    // HJ, 13/Jun/2016
+    labelField neighbourExpandAddressing = neighbourRestrictAddressing;
+
+    // Expand neighbour side to get all the data required from other
+    // processors.
+    // Note: neigbour is now the size of remote zone
+    fineGgiInterface_.shadowInterface().expandAddrToZone
     (
-        fineGgiInterface_.shadowInterface().interfaceSize()
+        neighbourExpandAddressing
     );
 
-    // Fill local cluster ID with a combination of a local ID and processor
-    // offset
-    // Memory management, neighbour
-    {
-        forAll (neighbourExpandAddressing, i)
-        {
-            neighbourExpandAddressing[i] =
-                neighbourRestrictAddressing[i]
-              + procOffset*Pstream::myProcNo();
-        }
+    // Create addressing for neighbour processors.  Note: expandAddrToZone will
+    // expand the addressing to zone size.  HJ, 13/Jun/2016
+    labelField neighbourExpandProc
+    (
+        fineGgiInterface_.shadowInterface().interfaceSize(),
+        Pstream::myProcNo()
+    );
 
-        // Expand neighbour side to get all the data required from other
-        // processors.  Note: neigbour is now the size of remote zone
-        fineGgiInterface_.shadowInterface().expandAddrToZone
-        (
-            neighbourExpandAddressing
-        );
-    }
+    // Expand neighbour side to get all the data required from other
+    // processors.
+    // Note: neigbour is now the size of remote zone
+    fineGgiInterface_.shadowInterface().expandAddrToZone
+    (
+        neighbourExpandProc
+    );
 
-    // DEBUG: Check that all sizes are at zone size.
-    Info<< "Sizes check: local zone size "
-        << fineGgiInterface_.zoneSize()
-        << " " << localExpandAddressing << nl
-        << "shadow zone size "
-        << fineGgiInterface_.shadowInterface().zoneSize()
-        << " " << neighbourExpandAddressing
-        << endl;
+    // Note: neighbourExpandAddressing and neighbourExpandProc
+    // will be filled with NaNs for faces which are not local
 
-
-    // Note: neighbourExpandAddressing will be filled with NaNs for faces which
-    // not local
-
-    Info<< "End of reduce" << endl;
     // Make a lookup table of entries for owner/neighbour.
     // All sizes are guessed at the size of fine interface
     // HJ, 19/Feb/2009
 
-    HashTable<SLList<label>, label, Hash<label> > neighboursTable
+    // Note: Guessing size of HashTable to fine interface size
+
+    // Coded neighbour index. Note: using long int to simplify encoding
+    // HJ, 1/Aug/2016
+    HashTable<SLList<long>, long, Hash<long> > neighboursTable
     (
-        fineGgiInterface_.interfaceSize()
+        Foam::max(128, fineGgiInterface_.interfaceSize()/4)
     );
 
-    // Table of face-sets to be agglomerated
+    // Neignbour processor index
+    HashTable<SLList<label>, label, Hash<label> > nbrsProcTable
+    (
+        Foam::max(128, fineGgiInterface_.interfaceSize()/4)
+    );
+
+    // Neighbour face-faces addressing for a face with split neighbours
     HashTable<SLList<SLList<label> >, label, Hash<label> > faceFaceTable
     (
-        fineGgiInterface_.interfaceSize()
+        Foam::max(128, fineGgiInterface_.interfaceSize()/4)
     );
 
-    // Table of face-sets weights to be agglomerated
+    HashTable<SLList<SLList<label> >, label, Hash<label> > faceFaceNbrTable
+    (
+        Foam::max(128, fineGgiInterface_.interfaceSize()/4)
+    );
+
+    // Neighbour face-faces weights for a face with split neighbours
     HashTable<SLList<SLList<scalar> >, label, Hash<label> >
         faceFaceWeightsTable
         (
-            fineGgiInterface_.interfaceSize()
+            Foam::max(128, fineGgiInterface_.interfaceSize()/4)
         );
 
     // Count the number of coarse faces
@@ -318,12 +299,6 @@ Foam::ggiAMGInterface::ggiAMGInterface
         const labelListList& fineAddr = fineGgiInterface_.addressing();
         const scalarListList& fineWeights = fineGgiInterface_.weights();
 
-        // Note: cluster only locally live faces
-        // HJ, 13/Jun/2016
-
-        // This addressing defines which faces from zone are local
-        const labelList& fineZa = fineGgiInterface_.zoneAddressing();
-
         // Perform analysis only for local faces
         // HJ, 22/Jun/2016
         forAll (fineZa, fineZaI)
@@ -336,8 +311,27 @@ Foam::ggiAMGInterface::ggiAMGInterface
 
             forAll (curFineNbrs, nbrI)
             {
-                label curMaster = -1;
-                label curSlave = -1;
+                long curMaster = -1;
+                label curMasterProc = -1;
+                long curSlave = -1;
+                label curSlaveProc = -1;
+
+                // Note.  Signalling in global clustering requires
+                // me to recognise clustering from separate
+                // processors as separate.  In the first phase,
+                // this will be used to recognise cluster from
+                // each processor as separate and in the second
+                // phase it will be used to filter local processor
+                // faces from the global patch.  Currently, I am
+                // calculating unique cluster index as:
+                //
+                // id = cluster + procOffset*myProcID
+                //
+                // With procOffset = 1 million, this should be
+                // sufficient for 2000 CPUs with 2 million coarse
+                // cells each.  For larger numbers, I need a
+                // larger max int, which can be changed on request
+                // HJ, 1/Apr/2009
 
                 // My label = ffI
                 // Nbr label = nnI
@@ -347,49 +341,76 @@ Foam::ggiAMGInterface::ggiAMGInterface
                 if (fineGgiInterface_.master())
                 {
                     // Master side
-                    curMaster = localExpandAddressing[ffI];
+                    curMaster = localRestrictAddressing[fineZaI];
+                    curMasterProc = Pstream::myProcNo();
                     curSlave = neighbourExpandAddressing[nnI];
+                    curSlaveProc = neighbourExpandProc[nnI];
                 }
                 else
                 {
-                    // Slave side
                     curMaster = neighbourExpandAddressing[nnI];
-                    curSlave = localExpandAddressing[ffI];
+                    curMasterProc = neighbourExpandProc[nnI];
+                    curSlave = localRestrictAddressing[fineZaI];
+                    curSlaveProc = Pstream::myProcNo();
                 }
 
+                // Code in current master and slave
+                curMaster += procOffset*curMasterProc;
+                curSlave += procOffset*curSlaveProc;
+
                 // Look for the master cell.  If it has already got a face,
                 // add the coefficient to the face.  If not, create a new
-                // face.
+                // face
                 if (neighboursTable.found(curMaster))
                 {
+                    // This master side face already exists
+
                     // Check all current neighbours to see if the current
                     // slave already exists.  If so, add the coefficient.
 
-                    SLList<label>& curNbrs = neighboursTable.find(curMaster)();
+                    SLList<long>& curNbrs =
+                        neighboursTable.find(curMaster)();
+
+                    SLList<label>& curNbrsProc =
+                        nbrsProcTable.find(curMaster)();
 
                     SLList<SLList<label> >& curFaceFaces =
                         faceFaceTable.find(curMaster)();
 
+                    SLList<SLList<label> >& curFaceFaceNbrs =
+                        faceFaceNbrTable.find(curMaster)();
+
                     SLList<SLList<scalar> >& curFaceWeights =
                         faceFaceWeightsTable.find(curMaster)();
 
+                    // Search for coded neighbour
                     bool nbrFound = false;
 
-                    SLList<label>::iterator nbrsIter = curNbrs.begin();
+                    SLList<long>::iterator nbrsIter = curNbrs.begin();
+
+                    SLList<label>::iterator nbrsProcIter =
+                        curNbrsProc.begin();
 
                     SLList<SLList<label> >::iterator faceFacesIter =
                         curFaceFaces.begin();
 
+                    SLList<SLList<label> >::iterator faceFaceNbrsIter =
+                        curFaceFaceNbrs.begin();
+
                     SLList<SLList<scalar> >::iterator faceFaceWeightsIter =
-                        curFaceWeights.begin();
+                    curFaceWeights.begin();
 
                     for
                     (
                         ;
                         nbrsIter != curNbrs.end()
+                     && nbrsProcIter != curNbrsProc.end()
                      && faceFacesIter != curFaceFaces.end()
+                     && faceFaceNbrsIter != curFaceFaceNbrs.end()
                      && faceFaceWeightsIter != curFaceWeights.end();
-                        ++nbrsIter, ++faceFacesIter, ++faceFaceWeightsIter
+                        ++nbrsIter, ++nbrsProcIter,
+                        ++faceFacesIter, ++faceFaceNbrsIter,
+                        ++faceFaceWeightsIter
                     )
                     {
                         // Check neighbour slave
@@ -397,6 +418,7 @@ Foam::ggiAMGInterface::ggiAMGInterface
                         {
                             nbrFound = true;
                             faceFacesIter().append(ffI);
+                            faceFaceNbrsIter().append(nbrI);
                             faceFaceWeightsIter().append(curNW);
 
                             // New agglomeration pair found in already
@@ -410,7 +432,9 @@ Foam::ggiAMGInterface::ggiAMGInterface
                     if (!nbrFound)
                     {
                         curNbrs.append(curSlave);
+                        curNbrsProc.append(curSlaveProc);
                         curFaceFaces.append(SLList<label>(ffI));
+                        curFaceFaceNbrs.append(SLList<label>(nbrI));
                         curFaceWeights.append(SLList<scalar>(curNW));
 
                         // New coarse face created for an existing master
@@ -420,9 +444,20 @@ Foam::ggiAMGInterface::ggiAMGInterface
                 }
                 else
                 {
-                    // This master has got no neighbours yet.  Add a neighbour
-                    // and a coefficient, thus creating a new face
-                    neighboursTable.insert(curMaster, SLList<label>(curSlave));
+                    // This master has got no neighbours yet.
+                    // Add a neighbour, proc and a coefficient as a
+                    // new list, thus creating a new face
+                    neighboursTable.insert
+                    (
+                        curMaster,
+                        SLList<long>(curSlave)
+                    );
+
+                    nbrsProcTable.insert
+                    (
+                        curMaster,
+                        SLList<label>(curSlaveProc)
+                    );
 
                     faceFaceTable.insert
                     (
@@ -430,6 +465,12 @@ Foam::ggiAMGInterface::ggiAMGInterface
                         SLList<SLList<label> >(SLList<label>(ffI))
                     );
 
+                    faceFaceNbrTable.insert
+                    (
+                        curMaster,
+                        SLList<SLList<label> >(SLList<label>(nbrI))
+                    );
+
                     faceFaceWeightsTable.insert
                     (
                         curMaster,
@@ -446,8 +487,7 @@ Foam::ggiAMGInterface::ggiAMGInterface
     else
     {
         // Coarse level, addressing is stored in faceCells
-        // This addressing defines whicf faces from zone are local
-        const labelList& fineZa = fineGgiInterface_.zoneAddressing();
+        // This addressing defines which faces from zone are local
 
         // Perform analysis only for local faces
         // HJ, 22/Jun/2016
@@ -456,48 +496,85 @@ Foam::ggiAMGInterface::ggiAMGInterface
             // Get the local face (from zone) to analyse
             const label ffI = fineZa[fineZaI];
 
-            label curMaster = -1;
-            label curSlave = -1;
+            long curMaster = -1;
+            label curMasterProc = -1;
+            long curSlave = -1;
+            label curSlaveProc = -1;
 
-            // Do switching on master/slave indices based on the
-            // owner/neighbour of the processor index such that
-            // both sides get the same answer.
-            if (master())
+            // Note.  Signalling in global clustering requires
+            // me to recognise clustering from separate
+            // processors as separate.  In the first phase,
+            // this will be used to recognise cluster from
+            // each processor as separate and in the second
+            // phase it will be used to filter local processor
+            // faces from the global patch.  Currently, I am
+            // calculating unique cluster index as:
+            //
+            // id = cluster + procOffset*myProcID
+            //
+            // With procOffset = 1 million, this should be
+            // sufficient for 2000 CPUs with 2 million coarse
+            // cells each.  For larger numbers, I need a
+            // larger max int, which can be changed on request
+            // HJ, 1/Apr/2009
+
+            if (fineGgiInterface_.master())
             {
                 // Master side
-                curMaster = localExpandAddressing[ffI];
+                curMaster = localRestrictAddressing[fineZaI];
+                curMasterProc = Pstream::myProcNo();
                 curSlave = neighbourExpandAddressing[ffI];
+                curSlaveProc = neighbourExpandProc[ffI];
             }
             else
             {
-                // Slave side
                 curMaster = neighbourExpandAddressing[ffI];
-                curSlave = localExpandAddressing[ffI];
+                curMasterProc = neighbourExpandProc[ffI];
+                curSlave = localRestrictAddressing[fineZaI];
+                curSlaveProc = Pstream::myProcNo();
             }
 
+            // Code in current master and slave
+            curMaster += procOffset*curMasterProc;
+            curSlave += procOffset*curSlaveProc;
+
             // Look for the master cell.  If it has already got a face,
             // add the coefficient to the face.  If not, create a new face.
             if (neighboursTable.found(curMaster))
             {
+                // This master side face already exists
+
                 // Check all current neighbours to see if the current slave
                 // already exists and if so, add the fine face
                 // to the agglomeration.
 
-                SLList<label>& curNbrs = neighboursTable.find(curMaster)();
+                SLList<long>& curNbrs = neighboursTable.find(curMaster)();
+
+                SLList<label>& curNbrsProc =
+                    nbrsProcTable.find(curMaster)();
 
                 SLList<SLList<label> >& curFaceFaces =
                     faceFaceTable.find(curMaster)();
 
-                    SLList<SLList<scalar> >& curFaceWeights =
-                        faceFaceWeightsTable.find(curMaster)();
+                SLList<SLList<label> >& curFaceFaceNbrs =
+                    faceFaceNbrTable.find(curMaster)();
+
+                SLList<SLList<scalar> >& curFaceWeights =
+                    faceFaceWeightsTable.find(curMaster)();
 
                 bool nbrFound = false;
 
-                SLList<label>::iterator nbrsIter = curNbrs.begin();
+                SLList<long>::iterator nbrsIter = curNbrs.begin();
+
+                SLList<label>::iterator nbrsProcIter =
+                    curNbrsProc.begin();
 
                 SLList<SLList<label> >::iterator faceFacesIter =
                     curFaceFaces.begin();
 
+                SLList<SLList<label> >::iterator faceFaceNbrsIter =
+                    curFaceFaceNbrs.begin();
+
                 SLList<SLList<scalar> >::iterator faceFaceWeightsIter =
                     curFaceWeights.begin();
 
@@ -506,8 +583,10 @@ Foam::ggiAMGInterface::ggiAMGInterface
                     ;
                     nbrsIter != curNbrs.end()
                  && faceFacesIter != curFaceFaces.end()
+                 && faceFaceNbrsIter != curFaceFaceNbrs.end()
                  && faceFaceWeightsIter != curFaceWeights.end();
-                    ++nbrsIter, ++faceFacesIter, ++faceFaceWeightsIter
+                    ++nbrsIter, ++nbrsProcIter,
+                    ++faceFacesIter, ++faceFaceNbrsIter, ++faceFaceWeightsIter
                 )
                 {
                     // Check neighbour slave
@@ -515,12 +594,15 @@ Foam::ggiAMGInterface::ggiAMGInterface
                     {
                         nbrFound = true;
                         faceFacesIter().append(ffI);
+                        // Add dummy nbr
+                        faceFaceNbrsIter().append(0);
                         // Add dummy weight
                         faceFaceWeightsIter().append(1.0);
 
                         // New agglomeration pair found in already
                         // existing pair
                         nAgglomPairs++;
+
                         break;
                     }
                 }
@@ -528,7 +610,10 @@ Foam::ggiAMGInterface::ggiAMGInterface
                 if (!nbrFound)
                 {
                     curNbrs.append(curSlave);
+                    curNbrsProc.append(curSlaveProc);
                     curFaceFaces.append(SLList<label>(ffI));
+                    // Add dummy nbr
+                    curFaceFaceNbrs.append(SLList<label>(0));
                     // Add dummy weight
                     curFaceWeights.append(SLList<scalar>(1.0));
 
@@ -541,7 +626,17 @@ Foam::ggiAMGInterface::ggiAMGInterface
             {
                 // This master has got no neighbours yet.  Add a neighbour
                 // and a coefficient, thus creating a new face
-                neighboursTable.insert(curMaster, SLList<label>(curSlave));
+                neighboursTable.insert
+                (
+                    curMaster,
+                    SLList<long>(curSlave)
+                );
+
+                nbrsProcTable.insert
+                (
+                    curMaster,
+                    SLList<label>(curSlaveProc)
+                );
 
                 faceFaceTable.insert
                 (
@@ -549,6 +644,13 @@ Foam::ggiAMGInterface::ggiAMGInterface
                     SLList<SLList<label> >(SLList<label>(ffI))
                 );
 
+                // Add dummy nbr
+                faceFaceNbrTable.insert
+                (
+                    curMaster,
+                    SLList<SLList<label> >(SLList<label>(0))
+                );
+
                 // Add dummy weight
                 faceFaceWeightsTable.insert
                 (
@@ -561,29 +663,29 @@ Foam::ggiAMGInterface::ggiAMGInterface
                 nAgglomPairs++;
             }
         } // end for all fine faces
-    }
+    } // end of else in fine level (coarse level)
+
+    // Since only local faces are analysed, lists can now be resized
+    faceCells_.setSize(nCoarseFaces, -1);
+    fineAddressing_.setSize(nAgglomPairs, -1);
+    restrictAddressing_.setSize(nAgglomPairs, -1);
+    restrictWeights_.setSize(nAgglomPairs);
 
     // In order to assemble the coarse global face zone, find out
     // how many faces have been created on each processor.
-    // Note that masters and slaves both count faces so we will only ask master
-    // sizes to count
+    // Note that masters and slaves both count faces so we will
+    //  only ask master sizes to count
     labelList nCoarseFacesPerProc(Pstream::nProcs(), 0);
 
-    if (master())
-    {
-        nCoarseFacesPerProc[Pstream::myProcNo()] = nCoarseFaces;
-    }
+    nCoarseFacesPerProc[Pstream::myProcNo()] = nCoarseFaces;
 
     reduce(nCoarseFacesPerProc, sumOp<labelList>());
 
-    Info<< "Number of faces per processor: " << nCoarseFacesPerProc
-        << endl;
-
     // Coarse global face zone is assembled by adding all faces from proc0,
     // followed by all faces from proc1 etc.
     // Therefore, on procN, my master offset
-    // will be equal to the sum of numbers of coarse faces on all processors
-    // before mine
+    // will be equal to the sum of numbers of coarse faces on all
+    // processors before mine
     // HJ, 13/Jun/2016
 
     label coarseGlobalFaceOffset = 0;
@@ -593,146 +695,42 @@ Foam::ggiAMGInterface::ggiAMGInterface
         coarseGlobalFaceOffset += nCoarseFacesPerProc[i];
     }
 
-    Pout<< "coarseGlobalFaceOffset: " << coarseGlobalFaceOffset << endl;
+    // Grab zone size and create zone addressing
+    zoneSize_ = sum(nCoarseFacesPerProc);
 
-    Info<< "End of contents assembly" << endl;
-    labelField masterFaceCells(nCoarseFaces, -1);
-    labelField masterZoneAddressing(nCoarseFaces, -1);
-    labelField masterFineAddressing(nCoarseFaces, -1);
-    labelField masterRestrictAddressing(nAgglomPairs, -1);
-    scalarField masterRestrictWeights(nAgglomPairs);
+    zoneAddressing_.setSize(nCoarseFaces);
 
-    // Note: in multiple agglomeration
-
-    labelList contents = neighboursTable.toc();
+    // Both master and slave have done agglomeration, but only master
+    // will construct the global faces index.
+    // To avoid searching, master will prepare a list of global face
+    // indices that appear on each processor in order and communicate them
+    // to the slave
+    // The slave will then know which is the next global face from which
+    // processor and pick them out in the same order
 
     // Global faces shall be assembled by the increasing label of master
     // cluster ID.
+    List<long> contents = neighboursTable.toc();
 
     // Sort makes sure the order is identical on both sides.
     // HJ, 20/Feb/2009 and 6/Jun/2016
     sort(contents);
 
-    // Grab zone size and create zone addressing
-    zoneSize_ = sum(nCoarseFacesPerProc);
-    Info<< "zoneSize_: " << zoneSize_ << endl;
-
-    // Note:
-    // When I am agglomerating the master, I know faces are stacked up in order
-    // but on the slave side, all I know is the master cluster index and
-    // not a master coarse face index.  Therefore:
-    // - master needs to be agglomerated first
-    // - once master is agglomerated, I need to signal to the slave side
-    //   the global coarse face zone index
-
-
-    // Note: zone addressing will be assembled only for local clusters
-    // using the coarseGlobalFaceOffset
-    // HJ, 13/Jun/2016
-    label nProcFaces = 0;
-
-    // Reset face counter for re-use
-    nCoarseFaces = 0;
-    nAgglomPairs = 0;
-
-    // Note:
-    // Since clustering has now happened only on local faces, addressing and
-    // all other array work on local indices and not on the coarse global zone
-    // HJ, 13/Jun/2016
-
-    // Establish zone addressing on the master side and communicate
-    // it to the shadow
-
-    // On master side, the owner addressing is stored in table of contents
-    forAll (contents, masterI)
-    {
-        SLList<label>& curNbrs =
-            neighboursTable.find(contents[masterI])();
-
-        // Note: neighbour processor index is irrelevant.  HJ, 1/Apr/2009
-
-        SLList<SLList<label> >& curFaceFaces =
-            faceFaceTable.find(contents[masterI])();
-
-        SLList<SLList<scalar> >& curFaceWeights =
-            faceFaceWeightsTable.find(contents[masterI])();
-
-        SLList<label>::iterator nbrsIter = curNbrs.begin();
-        SLList<SLList<label> >::iterator faceFacesIter =
-            curFaceFaces.begin();
-
-        SLList<SLList<scalar> >::iterator faceFaceWeightsIter =
-            curFaceWeights.begin();
-
-        for
-        (
-            ;
-            nbrsIter != curNbrs.end()
-         && faceFacesIter != curFaceFaces.end()
-         && faceFaceWeightsIter != curFaceWeights.end();
-            ++nbrsIter, ++faceFacesIter, ++faceFaceWeightsIter
-        )
-        {
-            // Check if master is on local processor: no longer needed,
-            // as only local processor is being searched.  HJ, 13/Jun/2016
-
-            // Record that this face belongs locally
-            // Use offset to indicate its position in the list
-            masterZoneAddressing[nProcFaces] =
-                nProcFaces + coarseGlobalFaceOffset;
-
-            masterFaceCells[nProcFaces] =
-                contents[masterI] - procOffset*Pstream::myProcNo();
-
-            SLList<label>::iterator facesIter =
-                faceFacesIter().begin();
-
-            SLList<scalar>::iterator weightsIter =
-                faceFaceWeightsIter().begin();
-
-            for
-            (
-                ;
-                facesIter != faceFacesIter().end()
-             && weightsIter != faceFaceWeightsIter().end();
-                ++facesIter, ++weightsIter
-            )
-            {
-                masterFineAddressing[nAgglomPairs] = facesIter();
-
-                // Master processor zone face is calculated from
-                masterRestrictAddressing[nAgglomPairs] =
-                    nProcFaces + coarseGlobalFaceOffset;
-                masterRestrictWeights[nAgglomPairs] = weightsIter();
-                nAgglomPairs++;
-            }
-
-            nProcFaces++;
-        }
-    }
-
-    // Resize arrays: not all of ggi is used locally
-    masterFaceCells.setSize(nProcFaces);
-    masterZoneAddressing.setSize(nProcFaces);
-
-    masterFineAddressing.setSize(nAgglomPairs);
-    masterRestrictAddressing.setSize(nAgglomPairs);
-    masterRestrictWeights.setSize(nAgglomPairs);
-
-    // Note: Both master and slave have done the same agglomeration up to here
-
+    // Note: Restriction is done on master side only because this is where
+    // the local zone is created.  HJ, 1/Aug/2016
     if (master())
     {
-        // Master has completed the clustering
-        faceCells_ = masterFaceCells;
-        zoneAddressing_ = masterZoneAddressing;
-        fineAddressing_ = masterFineAddressing;
-        restrictAddressing_ = masterRestrictAddressing;
-        restrictWeights_ = masterRestrictWeights;
-    }
-    else
-    {
-        // Note: shadowRestrictAddressing contains the
+        // Note:
+        // When I am agglomerating the master, faces are stacked up in order
+        // but on the slave side, all I know is the master cluster index and
+        // not a master coarse face index.  Therefore:
+        // - master needs to be agglomerated first
+        // - once master is agglomerated, I need to signal to the slave side
+        //   the global coarse face zone index
+
+        // For each new global face created on master proc,
+        // record its index under the slave proc array
+        List<SLList<label> > procMasterFacesLL(Pstream::nProcs());
 
         // Note: zone addressing will be assembled only for local clusters
         // using the coarseGlobalFaceOffset
@@ -743,12 +741,20 @@ Foam::ggiAMGInterface::ggiAMGInterface
         nCoarseFaces = 0;
         nAgglomPairs = 0;
 
-        // On slave side, the owner addressing is stored in linked lists
+        // Note:
+        // Since clustering has now happened only on local faces,
+        // addressing and all other array work on local indices and
+        // not on the coarse global zone
+        // HJ, 13/Jun/2016
+
+        // On master side, the owner addressing is stored in table of contents
         forAll (contents, masterI)
         {
-            // Note: master processor index is irrelevant.  HJ, 1/Apr/2009
+            SLList<long>& curNbrs =
+                neighboursTable.find(contents[masterI])();
 
-            SLList<label>& curNbrs = neighboursTable.find(contents[masterI])();
+            SLList<label>& curNbrsProc =
+                nbrsProcTable.find(contents[masterI])();
 
             SLList<SLList<label> >& curFaceFaces =
                 faceFaceTable.find(contents[masterI])();
@@ -756,7 +762,10 @@ Foam::ggiAMGInterface::ggiAMGInterface
             SLList<SLList<scalar> >& curFaceWeights =
                 faceFaceWeightsTable.find(contents[masterI])();
 
-            SLList<label>::iterator nbrsIter = curNbrs.begin();
+            SLList<long>::iterator nbrsIter = curNbrs.begin();
+
+            SLList<label>::iterator nbrsProcIter =
+                curNbrsProc.begin();
 
             SLList<SLList<label> >::iterator faceFacesIter =
                 curFaceFaces.begin();
@@ -768,29 +777,195 @@ Foam::ggiAMGInterface::ggiAMGInterface
             (
                 ;
                 nbrsIter != curNbrs.end()
+             && nbrsProcIter != curNbrsProc.end()
              && faceFacesIter != curFaceFaces.end()
              && faceFaceWeightsIter != curFaceWeights.end();
-                ++nbrsIter, ++faceFacesIter, ++faceFaceWeightsIter
+                ++nbrsIter, ++nbrsProcIter,
+                ++faceFacesIter, ++faceFaceWeightsIter
             )
             {
-                // Check if the face is on local processor: no longer needed,
+                // Check if master is on local processor: no longer needed,
                 // as only local processor is being searched.  HJ, 13/Jun/2016
 
-                // Record that this face belongs locally.
-
-//HJ, HERE: I need to find out the global face index for the face that was created from the master side
-
-                zoneAddressing_[nProcFaces] = nCoarseFaces;
-                faceCells_[nProcFaces] =
-                    nbrsIter() - procOffset*Pstream::myProcNo();
-                nProcFaces++;
-
+                // Get faces and weights
                 SLList<label>::iterator facesIter =
                     faceFacesIter().begin();
 
                 SLList<scalar>::iterator weightsIter =
                     faceFaceWeightsIter().begin();
 
+                // Record that this face belongs locally
+                // Use offset to indicate its position in the list
+                zoneAddressing_[nProcFaces] =
+                    nProcFaces + coarseGlobalFaceOffset;
+
+                // Record master cluster index
+                faceCells_[nProcFaces] =
+                    contents[masterI] - procOffset*Pstream::myProcNo();
+
+                // Record global processor face
+                procMasterFacesLL[nbrsProcIter()].append
+                (
+                    nProcFaces + coarseGlobalFaceOffset
+                );
+
+                // Collect agglomeration data
+                for
+                (
+                    ;
+                    facesIter != faceFacesIter().end()
+                 && weightsIter != faceFaceWeightsIter().end();
+                    ++facesIter, ++weightsIter
+                )
+                {
+                    fineAddressing_[nAgglomPairs] = facesIter();
+
+                    // Master processor zone face is calculated from
+                    // global offset
+                    restrictAddressing_[nAgglomPairs] =
+                        nProcFaces + coarseGlobalFaceOffset;
+
+                    restrictWeights_[nAgglomPairs] = weightsIter();
+                    nAgglomPairs++;
+                }
+
+                nProcFaces++;
+            }
+        }
+
+        // No need to resize arrays only local faces are used
+        // HJ, 1/Aug/2016
+
+        // Re-pack singly linked list of processor master faces
+        // and pass to other processors
+
+        // First index: master proc
+        // Second index: slave proc
+        // List contents: global faces in order
+        labelListListList crissCrossList(Pstream::nProcs());
+
+        labelListList& crissList = crissCrossList[Pstream::myProcNo()];
+
+        crissList.setSize(Pstream::nProcs());
+
+        forAll (crissList, procI)
+        {
+            crissList[procI] = procMasterFacesLL[procI];
+        }
+
+        Pstream::gatherList(crissCrossList);
+        Pstream::scatterList(crissCrossList);
+
+        procMasterFaces_.setSize(Pstream::nProcs());
+
+        forAll (procMasterFaces_, procI)
+        {
+            procMasterFaces_[procI] =
+                crissCrossList[procI][Pstream::myProcNo()];
+        }
+    }
+    // Agglomerate slave
+    else
+    {
+        // Note: zone addressing will be assembled only for local clusters
+        // using the coarseGlobalFaceOffset
+        // HJ, 13/Jun/2016
+        label nProcFaces = 0;
+
+        // Get master side procMasterFaces
+        // Note: this needs to be picked up from coarse interfaces rather than
+        // the matrix, as the coarse matrix assembly is not complete yet.
+        // Further, master has got a lower index in the list, meaning that
+        // it has already completed the assembly
+        const ggiAMGInterface& shadowGGI =
+            refCast<const ggiAMGInterface>(coarseInterfaces[shadowIndex()]);
+
+        const labelListList& masterProcMasterFaces =
+            shadowGGI.procMasterFaces();
+
+        // Reset face counter for re-use
+        nCoarseFaces = 0;
+        nAgglomPairs = 0;
+
+        // Count how many global faces are used for each proc on the other side
+        labelList npmf(Pstream::nProcs(), 0);
+
+        // On slave side, the owner addressing is stored in linked lists
+        forAll (contents, masterI)
+        {
+            SLList<long>& curNbrs = neighboursTable.find(contents[masterI])();
+
+            SLList<SLList<label> >& curFaceFaces =
+                faceFaceTable.find(contents[masterI])();
+
+            SLList<SLList<label> >& curFaceFaceNbrs =
+                faceFaceNbrTable.find(contents[masterI])();
+
+            SLList<SLList<scalar> >& curFaceWeights =
+                faceFaceWeightsTable.find(contents[masterI])();
+
+            SLList<long>::iterator nbrsIter = curNbrs.begin();
+
+            SLList<SLList<label> >::iterator faceFacesIter =
+                curFaceFaces.begin();
+
+            SLList<SLList<label> >::iterator faceFaceNbrsIter =
+                curFaceFaceNbrs.begin();
+
+            SLList<SLList<scalar> >::iterator faceFaceWeightsIter =
+                curFaceWeights.begin();
+
+            for
+            (
+                ;
+                nbrsIter != curNbrs.end()
+             && faceFacesIter != curFaceFaces.end()
+             && faceFaceNbrsIter != curFaceFaceNbrs.end()
+             && faceFaceWeightsIter != curFaceWeights.end();
+                ++nbrsIter, ++faceFacesIter, ++faceFaceNbrsIter,
+                ++faceFaceWeightsIter
+            )
+            {
+                // Check if the face is on local processor: no longer needed,
+                // as only local processor is being searched.  HJ, 13/Jun/2016
+
+                SLList<label>::iterator facesIter =
+                    faceFacesIter().begin();
+
+                SLList<label>::iterator faceNbrsIter =
+                    faceFaceNbrsIter().begin();
+
+                SLList<scalar>::iterator weightsIter =
+                    faceFaceWeightsIter().begin();
+
+                // Find neighbour proc index from the first face
+                // on the other side
+                label nbrProc;
+
+                if (fineGgiInterface_.fineLevel())
+                {
+                    const labelListList& fineAddr =
+                        fineGgiInterface_.addressing();
+
+                    nbrProc = neighbourExpandProc
+                        [fineAddr[facesIter()][faceNbrsIter()]];
+                }
+                else
+                {
+                    nbrProc = neighbourExpandProc[facesIter()];
+                }
+
+                // Read coarse face index
+                const label coarseFace =
+                    masterProcMasterFaces[nbrProc][npmf[nbrProc]];
+
+                // and mark it as used
+                npmf[nbrProc]++;
+
+                zoneAddressing_[nProcFaces] = coarseFace;
+                faceCells_[nProcFaces] =
+                    nbrsIter() - procOffset*Pstream::myProcNo();
+
                 for
                 (
                     ;
@@ -800,19 +975,15 @@ Foam::ggiAMGInterface::ggiAMGInterface
                 )
                 {
                     fineAddressing_[nAgglomPairs] = facesIter();
-                    restrictAddressing_[nAgglomPairs] = nCoarseFaces;
+                    restrictAddressing_[nAgglomPairs] = coarseFace;
                     restrictWeights_[nAgglomPairs] = weightsIter();
+
                     nAgglomPairs++;
                 }
+
+                nProcFaces++;
             }
         }
-
-        // Resize arrays: not all of ggi is used locally
-        faceCells_.setSize(nProcFaces);
-        zoneAddressing_.setSize(nProcFaces);
-        fineAddressing_.setSize(nAgglomPairs);
-        restrictAddressing_.setSize(nAgglomPairs);
-        restrictWeights_.setSize(nAgglomPairs);
     }
 }
 
@@ -934,6 +1105,21 @@ bool Foam::ggiAMGInterface::localParallel() const
 }
 
 
+const Foam::labelListList& Foam::ggiAMGInterface::procMasterFaces() const
+{
+    if (!master())
+    {
+        FatalErrorIn
+        (
+            "const labelListList& ggiGAMGInterface::procMasterFaces() const"
+        )   << "Requester procMasterFaces from a slave.  This is not allowed"
+            << abort(FatalError);
+    }
+
+    return procMasterFaces_;
+}
+
+
 const Foam::mapDistribute& Foam::ggiAMGInterface::map() const
 {
     if (!mapPtr_)
diff --git a/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/ggiAMGInterface/ggiAMGInterface.H b/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/ggiAMGInterface/ggiAMGInterface.H
index 507909026..b3c3087e8 100644
--- a/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/ggiAMGInterface/ggiAMGInterface.H
+++ b/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/ggiAMGInterface/ggiAMGInterface.H
@@ -68,6 +68,11 @@ class ggiAMGInterface
         //- Zone addressing
         labelList zoneAddressing_;
 
+        //- Processor master faces
+        //  Per-processor insertion list of local faces into global zone
+        //  The list is created on the master side and passed onto the slave
+        //  to allow the slave to insert faces in the same order
+        labelListList procMasterFaces_;
 
         // Parallel communication
 
@@ -110,15 +115,15 @@ public:
         ggiAMGInterface
         (
             const lduPrimitiveMesh& lduMesh,
+            const lduInterfacePtrsList& coarseInterfaces,
             const lduInterface& fineInterface,
             const labelField& localRestrictAddressing,
             const labelField& neighbourRestrictAddressing
         );
 
 
-    // Destructor
-
-        virtual ~ggiAMGInterface();
+    //- Destructor
+    virtual ~ggiAMGInterface();
 
 
     // Member Functions
@@ -240,6 +245,9 @@ public:
             //- Is the patch localised on a single processor
             virtual bool localParallel() const;
 
+            //- Processor master face insertion list
+            const labelListList& procMasterFaces() const;
+
             //- Return weights
             virtual const scalarListList& weights() const;
 
diff --git a/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/mixingPlaneAMGInterface/mixingPlaneAMGInterface.C b/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/mixingPlaneAMGInterface/mixingPlaneAMGInterface.C
index 9a766bae4..e8f95a673 100644
--- a/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/mixingPlaneAMGInterface/mixingPlaneAMGInterface.C
+++ b/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/mixingPlaneAMGInterface/mixingPlaneAMGInterface.C
@@ -51,6 +51,7 @@ namespace Foam
 Foam::mixingPlaneAMGInterface::mixingPlaneAMGInterface
 (
     const lduPrimitiveMesh& lduMesh,
+    const lduInterfacePtrsList& coarseInterfaces,
     const lduInterface& fineInterface,
     const labelField& localRestrictAddressing,
     const labelField& neighbourRestrictAddressing
diff --git a/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/mixingPlaneAMGInterface/mixingPlaneAMGInterface.H b/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/mixingPlaneAMGInterface/mixingPlaneAMGInterface.H
index 71e29dc7b..16b1ade06 100644
--- a/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/mixingPlaneAMGInterface/mixingPlaneAMGInterface.H
+++ b/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/mixingPlaneAMGInterface/mixingPlaneAMGInterface.H
@@ -93,6 +93,7 @@ public:
         mixingPlaneAMGInterface
         (
             const lduPrimitiveMesh& lduMesh,
+            const lduInterfacePtrsList& coarseInterfaces,
             const lduInterface& fineInterface,
             const labelField& localRestrictAddressing,
             const labelField& neighbourRestrictAddressing
diff --git a/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/processorAMGInterface/processorAMGInterface.C b/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/processorAMGInterface/processorAMGInterface.C
index 35921b46f..8f60ee4b9 100644
--- a/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/processorAMGInterface/processorAMGInterface.C
+++ b/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/processorAMGInterface/processorAMGInterface.C
@@ -45,6 +45,7 @@ namespace Foam
 Foam::processorAMGInterface::processorAMGInterface
 (
     const lduPrimitiveMesh& lduMesh,
+    const lduInterfacePtrsList& coarseInterfaces,
     const lduInterface& fineInterface,
     const labelField& localRestrictAddressing,
     const labelField& neighbourRestrictAddressing
diff --git a/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/processorAMGInterface/processorAMGInterface.H b/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/processorAMGInterface/processorAMGInterface.H
index 7f1613c60..50db8b0f7 100644
--- a/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/processorAMGInterface/processorAMGInterface.H
+++ b/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/processorAMGInterface/processorAMGInterface.H
@@ -81,6 +81,7 @@ public:
         processorAMGInterface
         (
             const lduPrimitiveMesh& lduMesh,
+            const lduInterfacePtrsList& coarseInterfaces,
             const lduInterface& fineInterface,
             const labelField& localRestrictAddressing,
             const labelField& neighbourRestrictAddressing
diff --git a/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/regionCoupleAMGInterface/regionCoupleAMGInterface.C b/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/regionCoupleAMGInterface/regionCoupleAMGInterface.C
index 47bfdaacc..4f1c1f3b8 100644
--- a/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/regionCoupleAMGInterface/regionCoupleAMGInterface.C
+++ b/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/regionCoupleAMGInterface/regionCoupleAMGInterface.C
@@ -45,6 +45,7 @@ namespace Foam
 Foam::regionCoupleAMGInterface::regionCoupleAMGInterface
 (
     const lduPrimitiveMesh& lduMesh,
+    const lduInterfacePtrsList& coarseInterfaces,
     const lduInterface& fineInterface,
     const labelField& localRestrictAddressing,
     const labelField& neighbourRestrictAddressing
@@ -53,6 +54,7 @@ Foam::regionCoupleAMGInterface::regionCoupleAMGInterface
     ggiAMGInterface
     (
         lduMesh,
+        coarseInterfaces,
         fineInterface,
         localRestrictAddressing,
         neighbourRestrictAddressing
diff --git a/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/regionCoupleAMGInterface/regionCoupleAMGInterface.H b/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/regionCoupleAMGInterface/regionCoupleAMGInterface.H
index f7f2f7503..d132df7ae 100644
--- a/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/regionCoupleAMGInterface/regionCoupleAMGInterface.H
+++ b/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/regionCoupleAMGInterface/regionCoupleAMGInterface.H
@@ -73,6 +73,7 @@ public:
         regionCoupleAMGInterface
         (
             const lduPrimitiveMesh& lduMesh,
+            const lduInterfacePtrsList& coarseInterfaces,
             const lduInterface& fineInterface,
             const labelField& localRestrictAddressing,
             const labelField& neighbourRestrictAddressing
diff --git a/src/lduSolvers/amg/amgPolicy/pamgPolicy.C b/src/lduSolvers/amg/amgPolicy/pamgPolicy.C
index a34c36c48..a4d302e32 100644
--- a/src/lduSolvers/amg/amgPolicy/pamgPolicy.C
+++ b/src/lduSolvers/amg/amgPolicy/pamgPolicy.C
@@ -648,6 +648,7 @@ Foam::autoPtr<Foam::amgMatrix> Foam::pamgPolicy::restrictMatrix
                 AMGInterface::New
                 (
                     *coarseAddrPtr,
+                    coarseInterfaces,
                     fineInterface,
                     fineInterface.interfaceInternalField(child_),
                     fineInterfaceAddr[intI]

From 520b77015da0cde6bc11aae2c05835c734a99dda Mon Sep 17 00:00:00 2001
From: Hrvoje Jasak <h.jasak@wikki.co.uk>
Date: Tue, 9 Aug 2016 13:25:49 +0100
Subject: [PATCH 017/130] Write forces dat file

---
 bin/plotForces.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/bin/plotForces.py b/bin/plotForces.py
index ff59ca41a..fd8ff938d 100755
--- a/bin/plotForces.py
+++ b/bin/plotForces.py
@@ -35,7 +35,6 @@ for line in lines:
                 mvy.append(float(match.group(12)))
                 mvz.append(float(match.group(13)))
 
-
 # combine pressure and viscous forces and moments
 fx = fpx
 fy = fpy
@@ -53,6 +52,12 @@ for i in range(1,len(t)):
         my[i] += mvy[i]
         mz[i] += mvz[i]
 
+# write clean data file
+outfile=open('forces.dat','w')
+
+for data in zip(t,fx,fy,fz):
+        outfile.write(' '.join([str(d) for d in data])+'\n')
+
 # plot forces
 import pylab
 pylab.xlabel('iteration')

From e7f0f5aa7b4fe58103d08511e07c82b3d3a57ac0 Mon Sep 17 00:00:00 2001
From: Hrvoje Jasak <h.jasak@wikki.co.uk>
Date: Tue, 9 Aug 2016 13:25:57 +0100
Subject: [PATCH 018/130] Formatting

---
 bin/plotResidual.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/bin/plotResidual.py b/bin/plotResidual.py
index 22149b34e..5122812d4 100755
--- a/bin/plotResidual.py
+++ b/bin/plotResidual.py
@@ -62,6 +62,7 @@ for line in lines:
                 tepsilon.append(iepsilon)
                 epsilon.append(float(matchepsilon.group(2)))
 
+# write clean data file
 outfile=open('residual.dat','w')
 
 if iomega > 0:

From 5f5860e3c44a6da1a8d42b6be2d675d8b952b2ad Mon Sep 17 00:00:00 2001
From: Hrvoje Jasak <h.jasak@wikki.co.uk>
Date: Wed, 10 Aug 2016 12:10:41 +0100
Subject: [PATCH 019/130] Formatting

---
 src/foam/containers/Lists/List/List.H | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/foam/containers/Lists/List/List.H b/src/foam/containers/Lists/List/List.H
index ea96501d2..96119c4af 100644
--- a/src/foam/containers/Lists/List/List.H
+++ b/src/foam/containers/Lists/List/List.H
@@ -99,6 +99,7 @@ public:
         //- Return a null list
         inline static const List<T>& null();
 
+
     // Constructors
 
         //- Null constructor.

From 07de8431c13b49d08d78db71e01bdd58ee7f3c97 Mon Sep 17 00:00:00 2001
From: Hrvoje Jasak <h.jasak@wikki.co.uk>
Date: Wed, 10 Aug 2016 14:08:15 +0100
Subject: [PATCH 020/130] Comms update on GGI AMG interface

---
 .../fvPatches/constraint/ggi/ggiFvPatch.C     |   6 +
 .../fvPatches/constraint/ggi/ggiFvPatch.H     |   3 +
 .../regionCouple/regionCoupleFvPatch.C        |   7 +
 .../regionCouple/regionCoupleFvPatch.H        |   3 +
 .../ggiLduInterface/ggiLduInterface.H         |   7 +
 .../ggiAMGInterface/ggiAMGInterface.C         |  77 ++++--
 .../polyPatches/constraint/ggi/ggiPolyPatch.C |  27 +-
 .../polyPatches/constraint/ggi/ggiPolyPatch.H |   6 +-
 .../regionCouple/regionCouplePolyPatch.C      | 238 ++++++++++++------
 .../regionCouple/regionCouplePolyPatch.H      |  59 ++---
 .../regionCouplePolyPatchTemplates.C          | 147 +++--------
 11 files changed, 322 insertions(+), 258 deletions(-)

diff --git a/src/finiteVolume/fvMesh/fvPatches/constraint/ggi/ggiFvPatch.C b/src/finiteVolume/fvMesh/fvPatches/constraint/ggi/ggiFvPatch.C
index 09501ab16..3b1addbde 100644
--- a/src/finiteVolume/fvMesh/fvPatches/constraint/ggi/ggiFvPatch.C
+++ b/src/finiteVolume/fvMesh/fvPatches/constraint/ggi/ggiFvPatch.C
@@ -258,6 +258,12 @@ bool Foam::ggiFvPatch::localParallel() const
 }
 
 
+const Foam::mapDistribute& Foam::ggiFvPatch::map() const
+{
+    return ggiPolyPatch_.map();
+}
+
+
 const Foam::scalarListList& Foam::ggiFvPatch::weights() const
 {
     if (ggiPolyPatch_.master())
diff --git a/src/finiteVolume/fvMesh/fvPatches/constraint/ggi/ggiFvPatch.H b/src/finiteVolume/fvMesh/fvPatches/constraint/ggi/ggiFvPatch.H
index 740f0f208..3006723c8 100644
--- a/src/finiteVolume/fvMesh/fvPatches/constraint/ggi/ggiFvPatch.H
+++ b/src/finiteVolume/fvMesh/fvPatches/constraint/ggi/ggiFvPatch.H
@@ -171,6 +171,9 @@ public:
             //- Is the patch localised on a single processor
             virtual bool localParallel() const;
 
+            //- Return mapDistribute
+            virtual const mapDistribute& map() const;
+
             //- Return weights.  Master side returns own weights and
             //  slave side returns weights from master
             virtual const scalarListList& weights() const;
diff --git a/src/finiteVolume/fvMesh/fvPatches/constraint/regionCouple/regionCoupleFvPatch.C b/src/finiteVolume/fvMesh/fvPatches/constraint/regionCouple/regionCoupleFvPatch.C
index 38f5ba27d..6da84ef02 100644
--- a/src/finiteVolume/fvMesh/fvPatches/constraint/regionCouple/regionCoupleFvPatch.C
+++ b/src/finiteVolume/fvMesh/fvPatches/constraint/regionCouple/regionCoupleFvPatch.C
@@ -34,6 +34,7 @@ Author
 #include "fvMesh.H"
 #include "fvBoundaryMesh.H"
 #include "foamTime.H"
+#include "mapDistribute.H"
 
 // * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
 
@@ -302,6 +303,12 @@ bool Foam::regionCoupleFvPatch::localParallel() const
 }
 
 
+const Foam::mapDistribute& Foam::regionCoupleFvPatch::map() const
+{
+    return rcPolyPatch_.map();
+}
+
+
 const Foam::scalarListList& Foam::regionCoupleFvPatch::weights() const
 {
     if (rcPolyPatch_.master())
diff --git a/src/finiteVolume/fvMesh/fvPatches/constraint/regionCouple/regionCoupleFvPatch.H b/src/finiteVolume/fvMesh/fvPatches/constraint/regionCouple/regionCoupleFvPatch.H
index 85a0fba14..9603b54a1 100644
--- a/src/finiteVolume/fvMesh/fvPatches/constraint/regionCouple/regionCoupleFvPatch.H
+++ b/src/finiteVolume/fvMesh/fvPatches/constraint/regionCouple/regionCoupleFvPatch.H
@@ -180,6 +180,9 @@ public:
             //- Is the patch localised on a single processor
             virtual bool localParallel() const;
 
+            //- Return mapDistribute
+            const mapDistribute& map() const;
+
             //- Return weights.  Master side returns own weights and
             //  slave side returns weights from master
             virtual const scalarListList& weights() const;
diff --git a/src/foam/matrices/lduMatrix/lduAddressing/lduInterfaces/ggiLduInterface/ggiLduInterface.H b/src/foam/matrices/lduMatrix/lduAddressing/lduInterfaces/ggiLduInterface/ggiLduInterface.H
index a4fc7b2c7..3e5b815f7 100644
--- a/src/foam/matrices/lduMatrix/lduAddressing/lduInterfaces/ggiLduInterface/ggiLduInterface.H
+++ b/src/foam/matrices/lduMatrix/lduAddressing/lduInterfaces/ggiLduInterface/ggiLduInterface.H
@@ -46,6 +46,10 @@ SourceFiles
 namespace Foam
 {
 
+// Forward declaration of classes
+
+class mapDistribute;
+
 /*---------------------------------------------------------------------------*\
                        Class ggiLduInterface Declaration
 \*---------------------------------------------------------------------------*/
@@ -111,6 +115,9 @@ public:
         //- Is the patch localised on a single processor
         virtual bool localParallel() const = 0;
 
+        //- Return mapDistribute
+        virtual const mapDistribute& map() const = 0;
+
         //- Return weights
         virtual const scalarListList& weights() const = 0;
 
diff --git a/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/ggiAMGInterface/ggiAMGInterface.C b/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/ggiAMGInterface/ggiAMGInterface.C
index cc77d00b0..84dfc8e86 100644
--- a/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/ggiAMGInterface/ggiAMGInterface.C
+++ b/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/ggiAMGInterface/ggiAMGInterface.C
@@ -839,30 +839,67 @@ Foam::ggiAMGInterface::ggiAMGInterface
         // Re-pack singly linked list of processor master faces
         // and pass to other processors
 
-        // First index: master proc
-        // Second index: slave proc
-        // List contents: global faces in order
-        labelListListList crissCrossList(Pstream::nProcs());
-
-        labelListList& crissList = crissCrossList[Pstream::myProcNo()];
-
-        crissList.setSize(Pstream::nProcs());
-
-        forAll (crissList, procI)
-        {
-            crissList[procI] = procMasterFacesLL[procI];
-        }
-
-        Pstream::gatherList(crissCrossList);
-        Pstream::scatterList(crissCrossList);
-
         procMasterFaces_.setSize(Pstream::nProcs());
 
-        forAll (procMasterFaces_, procI)
+        // Copy self
+        procMasterFaces_[Pstream::myProcNo()] =
+            procMasterFacesLL[Pstream::myProcNo()];
+
+        if (Pstream::parRun())
         {
-            procMasterFaces_[procI] =
-                crissCrossList[procI][Pstream::myProcNo()];
+            const List<labelPair> schedule =
+                fineGgiInterface_.map().schedule();
+
+            // Do the comms
+            forAll (schedule, i)
+            {
+                const label sendProc = schedule[i].first();
+                const label recvProc = schedule[i].second();
+
+                if (Pstream::myProcNo() == sendProc)
+                {
+                    OPstream toNbr(Pstream::scheduled, recvProc);
+                    toNbr << labelList(procMasterFacesLL[recvProc]);
+                }
+                else if (Pstream::myProcNo() == recvProc)
+                {
+                    IPstream fromNbr(Pstream::scheduled, sendProc);
+
+                    procMasterFaces_[sendProc] = labelList(fromNbr);
+                }
+                else
+                {
+                    FatalErrorIn("...")
+                        << "My proc number " << Pstream::myProcNo()
+                            << " is neither a sender nor a receiver: "
+                            << schedule[i]
+                            << abort(FatalError);
+                }
+            }
         }
+
+//         // First index: master proc
+//         // Second index: slave proc
+//         // List contents: global faces in order
+//         labelListListList crissCrossList(Pstream::nProcs());
+
+//         labelListList& crissList = crissCrossList[Pstream::myProcNo()];
+
+//         crissList.setSize(Pstream::nProcs());
+
+//         forAll (crissList, procI)
+//         {
+//             crissList[procI] = procMasterFacesLL[procI];
+//         }
+
+//         Pstream::gatherList(crissCrossList);
+//         Pstream::scatterList(crissCrossList);
+
+//         forAll (procMasterFaces_, procI)
+//         {
+//             procMasterFaces_[procI] =
+//                 crissCrossList[procI][Pstream::myProcNo()];
+//         }
     }
     // Agglomerate slave
     else
diff --git a/src/foam/meshes/polyMesh/polyPatches/constraint/ggi/ggiPolyPatch.C b/src/foam/meshes/polyMesh/polyPatches/constraint/ggi/ggiPolyPatch.C
index de8012c7d..c555f4998 100644
--- a/src/foam/meshes/polyMesh/polyPatches/constraint/ggi/ggiPolyPatch.C
+++ b/src/foam/meshes/polyMesh/polyPatches/constraint/ggi/ggiPolyPatch.C
@@ -509,17 +509,6 @@ void Foam::ggiPolyPatch::calcSendReceive() const
 }
 
 
-const Foam::mapDistribute& Foam::ggiPolyPatch::map() const
-{
-    if (!mapPtr_)
-    {
-        calcSendReceive();
-    }
-
-    return *mapPtr_;
-}
-
-
 void Foam::ggiPolyPatch::clearGeom() const
 {
     deleteDemandDrivenData(reconFaceCellCentresPtr_);
@@ -530,6 +519,10 @@ void Foam::ggiPolyPatch::clearGeom() const
     // HJ, 23/Jun/2011
     deleteDemandDrivenData(remoteZoneAddressingPtr_);
 
+    // localParallel depends on geometry - must be cleared!
+    // HR, 11/Jul/2013
+    deleteDemandDrivenData(localParallelPtr_);
+
     deleteDemandDrivenData(mapPtr_);
 }
 
@@ -543,7 +536,6 @@ void Foam::ggiPolyPatch::clearOut() const
 
     deleteDemandDrivenData(zoneAddressingPtr_);
     deleteDemandDrivenData(patchToPatchPtr_);
-    deleteDemandDrivenData(localParallelPtr_);
 }
 
 
@@ -826,6 +818,17 @@ const Foam::ggiZoneInterpolation& Foam::ggiPolyPatch::patchToPatch() const
 }
 
 
+const Foam::mapDistribute& Foam::ggiPolyPatch::map() const
+{
+    if (!mapPtr_)
+    {
+        calcSendReceive();
+    }
+
+    return *mapPtr_;
+}
+
+
 const Foam::vectorField& Foam::ggiPolyPatch::reconFaceCellCentres() const
 {
     if (!reconFaceCellCentresPtr_)
diff --git a/src/foam/meshes/polyMesh/polyPatches/constraint/ggi/ggiPolyPatch.H b/src/foam/meshes/polyMesh/polyPatches/constraint/ggi/ggiPolyPatch.H
index fb3fc403e..ab6ca49b0 100644
--- a/src/foam/meshes/polyMesh/polyPatches/constraint/ggi/ggiPolyPatch.H
+++ b/src/foam/meshes/polyMesh/polyPatches/constraint/ggi/ggiPolyPatch.H
@@ -138,9 +138,6 @@ class ggiPolyPatch
             //- Calculate send and receive addressing
             void calcSendReceive() const;
 
-            //- Return mapDistribute
-            const mapDistribute& map() const;
-
 
         // Memory management
 
@@ -335,6 +332,9 @@ public:
             //- Is the patch localised on a single processor
             bool localParallel() const;
 
+            //- Return mapDistribute
+            const mapDistribute& map() const;
+
             //- Return reference to patch-to-patch interpolation
             //  Used only for addressing
             const ggiZoneInterpolation& patchToPatch() const;
diff --git a/src/foam/meshes/polyMesh/polyPatches/constraint/regionCouple/regionCouplePolyPatch.C b/src/foam/meshes/polyMesh/polyPatches/constraint/regionCouple/regionCouplePolyPatch.C
index 3363afb2d..3f07d7175 100644
--- a/src/foam/meshes/polyMesh/polyPatches/constraint/regionCouple/regionCouplePolyPatch.C
+++ b/src/foam/meshes/polyMesh/polyPatches/constraint/regionCouple/regionCouplePolyPatch.C
@@ -226,7 +226,7 @@ void Foam::regionCouplePolyPatch::calcPatchToPatch() const
                 SMALL,         // Non-overlapping face tolerances
                 SMALL,
                 true,          // Rescale weighting factors
-                ggiInterpolation::BB_OCTREE  // Octree search, MB.
+                reject_        // Quick rejection algorithm, default BB_OCTREE
             );
 
         // Abort immediately if uncovered faces are present and the option
@@ -378,7 +378,7 @@ void Foam::regionCouplePolyPatch::calcSendReceive() const
     // (of the calc-call) they will be set to zero-sized array
     // HJ, 4/Jun/2011
 
-    if (receiveAddrPtr_ || sendAddrPtr_)
+    if (mapPtr_)
     {
         FatalErrorIn("void regionCouplePolyPatch::calcSendReceive() const")
             << "Send-receive addressing already calculated"
@@ -399,74 +399,129 @@ void Foam::regionCouplePolyPatch::calcSendReceive() const
             << abort(FatalError);
     }
 
-    // Master will receive and store the maps
-    if (Pstream::master())
+    // Gather send and receive addressing (to master)
+
+    // Get patch-to-zone addressing
+    const labelList& za = zoneAddressing();
+
+    // Make a zone-sized field and fill it in with proc markings for processor
+    // that holds and requires the data
+    labelField zoneProcID(zone().size(), -1);
+
+    forAll (za, zaI)
     {
-        receiveAddrPtr_ = new labelListList(Pstream::nProcs());
-        labelListList& rAddr = *receiveAddrPtr_;
+        zoneProcID[za[zaI]] = Pstream::myProcNo();
+    }
 
-        sendAddrPtr_ = new labelListList(Pstream::nProcs());
-        labelListList& sAddr = *sendAddrPtr_;
+    reduce(zoneProcID, maxOp<labelField>());
 
-        // Insert master
-        rAddr[0] = zoneAddressing();
+    const labelList& shadowRza = shadow().remoteZoneAddressing();
 
-        for (label procI = 1; procI < Pstream::nProcs(); procI++)
+    // Find out where my zone data is coming from
+    labelList nRecv(Pstream::nProcs(), 0);
+
+    // Note: only visit the data from the local zone
+    forAll (shadowRza, shadowRzaI)
+    {
+        nRecv[zoneProcID[shadowRza[shadowRzaI]]]++;
+    }
+
+    // Make a receiving sub-map
+    // It tells me which data I will receive from which processor and
+    // where I need to put it into the remoteZone data before the mapping
+    labelListList constructMap(Pstream::nProcs());
+
+    // Size the receiving list
+    forAll (nRecv, procI)
+    {
+        constructMap[procI].setSize(nRecv[procI]);
+    }
+
+    // Reset counters for processors
+    nRecv = 0;
+
+    forAll (shadowRza, shadowRzaI)
+    {
+        label recvProc = zoneProcID[shadowRza[shadowRzaI]];
+
+        constructMap[recvProc][nRecv[recvProc]] = shadowRza[shadowRzaI];
+
+        nRecv[recvProc]++;
+    }
+
+    // Make the sending sub-map
+    // It tells me which data is required from me to be sent to which
+    // processor
+
+    // Algorithm
+    // - expand the local zone faces with indices into a size of local zone
+    // - go through remote zone addressing on all processors
+    // - find out who hits my faces
+    labelList localZoneIndices(zone().size(), -1);
+
+    forAll (za, zaI)
+    {
+        localZoneIndices[za[zaI]] = zaI;
+    }
+
+    labelListList shadowToReceiveAddr(Pstream::nProcs());
+
+    // Get the list of what my shadow needs to receive from my zone
+    // on all other processors
+    shadowToReceiveAddr[Pstream::myProcNo()] = shadowRza;
+    Pstream::gatherList(shadowToReceiveAddr);
+    Pstream::scatterList(shadowToReceiveAddr);
+
+    // Now local zone indices contain the index of a local face that will
+    // provide the data.  For faces that are not local, the index will be -1
+
+    // Find out where my zone data is going to
+
+    // Make a sending sub-map
+    // It tells me which data I will send to which processor
+    labelListList sendMap(Pstream::nProcs());
+
+    // Collect local labels to be sent to each processor
+    forAll (shadowToReceiveAddr, procI)
+    {
+        const labelList& curProcSend = shadowToReceiveAddr[procI];
+
+        // Find out how much of my data is going to this processor
+        label nProcSend = 0;
+
+        forAll (curProcSend, sendI)
         {
-            // Note: must use normal comms because the size of the
-            // communicated lists is unknown on the receiving side
-            // HJ, 4/Jun/2011
+            if (localZoneIndices[curProcSend[sendI]] > -1)
+            {
+                nProcSend++;
+            }
+        }
 
-            // Opt: reconsider mode of communication
-            IPstream ip(Pstream::scheduled, procI);
+        if (nProcSend > 0)
+        {
+            // Collect the indices
+            labelList& curSendMap = sendMap[procI];
 
-            rAddr[procI] = labelList(ip);
+            curSendMap.setSize(nProcSend);
 
-            sAddr[procI] = labelList(ip);
+            // Reset counter
+            nProcSend = 0;
+
+            forAll (curProcSend, sendI)
+            {
+                if (localZoneIndices[curProcSend[sendI]] > -1)
+                {
+                    curSendMap[nProcSend] =
+                        localZoneIndices[curProcSend[sendI]];
+                    nProcSend++;
+                }
+            }
         }
     }
-    else
-    {
-        // Create dummy pointers: only master processor stores maps
-        receiveAddrPtr_ = new labelListList();
-        sendAddrPtr_ = new labelListList();
 
-        // Send information to master
-        const labelList& za = zoneAddressing();
-        const labelList& ra = remoteZoneAddressing();
-
-        // Note: must use normal comms because the size of the
-        // communicated lists is unknown on the receiving side
-        // HJ, 4/Jun/2011
-
-        // Opt: reconsider mode of communication
-        OPstream op(Pstream::scheduled, Pstream::masterNo());
-
-        // Send local and remote addressing to master
-        op << za << ra;
-    }
-}
-
-
-const Foam::labelListList& Foam::regionCouplePolyPatch::receiveAddr() const
-{
-    if (!receiveAddrPtr_)
-    {
-        calcSendReceive();
-    }
-
-    return *receiveAddrPtr_;
-}
-
-
-const Foam::labelListList& Foam::regionCouplePolyPatch::sendAddr() const
-{
-    if (!sendAddrPtr_)
-    {
-        calcSendReceive();
-    }
-
-    return *sendAddrPtr_;
+    // Map will return the object of the size of remote zone
+    // HJ, 9/May/2016
+    mapPtr_ = new mapDistribute(zone().size(), sendMap, constructMap);
 }
 
 
@@ -492,18 +547,19 @@ void Foam::regionCouplePolyPatch::clearGeom() const
 {
     clearDeltas();
 
+    deleteDemandDrivenData(reconFaceCellCentresPtr_);
+
     // Remote addressing and send-receive maps depend on the local
     // position.  Therefore, it needs to be recalculated at mesh motion.
     // Local zone addressing does not change with mesh motion
     // HJ, 23/Jun/2011
     deleteDemandDrivenData(remoteZoneAddressingPtr_);
 
-    deleteDemandDrivenData(receiveAddrPtr_);
-    deleteDemandDrivenData(sendAddrPtr_);
-
     // localParallel depends on geometry - must be cleared!
     // HR, 11/Jul/2013
     deleteDemandDrivenData(localParallelPtr_);
+
+    deleteDemandDrivenData(mapPtr_);
 }
 
 
@@ -511,6 +567,9 @@ void Foam::regionCouplePolyPatch::clearOut() const
 {
     clearGeom();
 
+    shadowIndex_ = -1;
+    zoneIndex_ = -1;
+
     deleteDemandDrivenData(zoneAddressingPtr_);
     deleteDemandDrivenData(patchToPatchPtr_);
 }
@@ -535,6 +594,7 @@ Foam::regionCouplePolyPatch::regionCouplePolyPatch
     master_(false),
     isWall_(false),
     bridgeOverlap_(false),
+    reject_(ggiZoneInterpolation::BB_OCTREE),
     shadowIndex_(-1),
     zoneIndex_(-1),
     patchToPatchPtr_(NULL),
@@ -542,8 +602,7 @@ Foam::regionCouplePolyPatch::regionCouplePolyPatch
     remoteZoneAddressingPtr_(NULL),
     reconFaceCellCentresPtr_(NULL),
     localParallelPtr_(NULL),
-    receiveAddrPtr_(NULL),
-    sendAddrPtr_(NULL)
+    mapPtr_(NULL)
 {}
 
 
@@ -560,7 +619,8 @@ Foam::regionCouplePolyPatch::regionCouplePolyPatch
     const bool attached,
     const bool master,
     const bool isWall,
-    const bool bridgeOverlap
+    const bool bridgeOverlap,
+    const ggiZoneInterpolation::quickReject reject
 )
 :
     coupledPolyPatch(name, size, start, index, bm),
@@ -571,6 +631,7 @@ Foam::regionCouplePolyPatch::regionCouplePolyPatch
     master_(master),
     isWall_(isWall),
     bridgeOverlap_(bridgeOverlap),
+    reject_(reject),
     shadowIndex_(-1),
     zoneIndex_(-1),
     patchToPatchPtr_(NULL),
@@ -578,8 +639,7 @@ Foam::regionCouplePolyPatch::regionCouplePolyPatch
     remoteZoneAddressingPtr_(NULL),
     reconFaceCellCentresPtr_(NULL),
     localParallelPtr_(NULL),
-    receiveAddrPtr_(NULL),
-    sendAddrPtr_(NULL)
+    mapPtr_(NULL)
 {}
 
 
@@ -599,6 +659,7 @@ Foam::regionCouplePolyPatch::regionCouplePolyPatch
     master_(dict.lookup("master")),
     isWall_(dict.lookup("isWall")),
     bridgeOverlap_(dict.lookup("bridgeOverlap")),
+    reject_(ggiZoneInterpolation::BB_OCTREE),
     shadowIndex_(-1),
     zoneIndex_(-1),
     patchToPatchPtr_(NULL),
@@ -606,9 +667,16 @@ Foam::regionCouplePolyPatch::regionCouplePolyPatch
     remoteZoneAddressingPtr_(NULL),
     reconFaceCellCentresPtr_(NULL),
     localParallelPtr_(NULL),
-    receiveAddrPtr_(NULL),
-    sendAddrPtr_(NULL)
-{}
+    mapPtr_(NULL)
+{
+    if (dict.found("quickReject"))
+    {
+        reject_ = ggiZoneInterpolation::quickRejectNames_.read
+        (
+            dict.lookup("quickReject")
+        );
+    }
+}
 
 
 Foam::regionCouplePolyPatch::regionCouplePolyPatch
@@ -624,6 +692,8 @@ Foam::regionCouplePolyPatch::regionCouplePolyPatch
     attached_(pp.attached_),
     master_(pp.master_),
     isWall_(pp.isWall_),
+    bridgeOverlap_(pp.bridgeOverlap_),
+    reject_(pp.reject_),
     shadowIndex_(-1),
     zoneIndex_(-1),
     patchToPatchPtr_(NULL),
@@ -631,8 +701,7 @@ Foam::regionCouplePolyPatch::regionCouplePolyPatch
     remoteZoneAddressingPtr_(NULL),
     reconFaceCellCentresPtr_(NULL),
     localParallelPtr_(NULL),
-    receiveAddrPtr_(NULL),
-    sendAddrPtr_(NULL)
+    mapPtr_(NULL)
 {}
 
 
@@ -652,6 +721,8 @@ Foam::regionCouplePolyPatch::regionCouplePolyPatch
     attached_(pp.attached_),
     master_(pp.master_),
     isWall_(pp.isWall_),
+    bridgeOverlap_(pp.bridgeOverlap_),
+    reject_(pp.reject_),
     shadowIndex_(-1),
     zoneIndex_(-1),
     patchToPatchPtr_(NULL),
@@ -659,8 +730,7 @@ Foam::regionCouplePolyPatch::regionCouplePolyPatch
     remoteZoneAddressingPtr_(NULL),
     reconFaceCellCentresPtr_(NULL),
     localParallelPtr_(NULL),
-    receiveAddrPtr_(NULL),
-    sendAddrPtr_(NULL)
+    mapPtr_(NULL)
 {}
 
 
@@ -911,6 +981,17 @@ Foam::regionCouplePolyPatch::patchToPatch() const
 }
 
 
+const Foam::mapDistribute& Foam::regionCouplePolyPatch::map() const
+{
+    if (!mapPtr_)
+    {
+        calcSendReceive();
+    }
+
+    return *mapPtr_;
+}
+
+
 const Foam::vectorField&
 Foam::regionCouplePolyPatch::reconFaceCellCentres() const
 {
@@ -940,6 +1021,11 @@ void Foam::regionCouplePolyPatch::initAddressing()
         // Calculate transforms for correct GGI cut
         calcTransforms();
 
+        if (master())
+        {
+            shadow().calcTransforms();
+        }
+
         // Force zone addressing and remote zone addressing
         // (uses GGI interpolator)
         zoneAddressing();
@@ -949,7 +1035,7 @@ void Foam::regionCouplePolyPatch::initAddressing()
         if (Pstream::parRun() && !localParallel())
         {
             // Calculate send addressing
-            sendAddr();
+            map();
         }
     }
 
@@ -1019,7 +1105,7 @@ void Foam::regionCouplePolyPatch::initMovePoints(const pointField& p)
         if (Pstream::parRun() && !localParallel())
         {
             // Calculate send addressing
-            sendAddr();
+            map();
         }
     }
 
diff --git a/src/foam/meshes/polyMesh/polyPatches/constraint/regionCouple/regionCouplePolyPatch.H b/src/foam/meshes/polyMesh/polyPatches/constraint/regionCouple/regionCouplePolyPatch.H
index 5816af204..5a9dacee6 100644
--- a/src/foam/meshes/polyMesh/polyPatches/constraint/regionCouple/regionCouplePolyPatch.H
+++ b/src/foam/meshes/polyMesh/polyPatches/constraint/regionCouple/regionCouplePolyPatch.H
@@ -42,6 +42,7 @@ SourceFiles
 #include "ggiInterpolation.H"
 #include "faceZone.H"
 #include "Switch.H"
+#include "mapDistribute.H"
 
 // * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
 
@@ -81,24 +82,30 @@ class regionCouplePolyPatch
         //- Use bridging to fix overlap error in interpolation
         Switch bridgeOverlap_;
 
-        //- Shadow patch index.  Delayed evaluation for construction
-        mutable label shadowIndex_;
-
-        //- Interpolation zone index.  Delayed evaluation for construction
-        mutable label zoneIndex_;
+        //- Quick reject algorithm
+        ggiZoneInterpolation::quickReject reject_;
 
 
-        //- Patch-to-patch interpolation
-        mutable ggiZoneInterpolation* patchToPatchPtr_;
+        // Demand-driven data
 
-        //- Patch-to-zone addressing
-        mutable labelList* zoneAddressingPtr_;
+            //- Shadow patch index.  Delayed evaluation for construction
+            mutable label shadowIndex_;
 
-        //- Remote zone addressing: data needed for interpolation
-        mutable labelList* remoteZoneAddressingPtr_;
+            //- Interpolation zone index.  Delayed evaluation for construction
+            mutable label zoneIndex_;
 
-        //- Reconstructed patch neighbour cell centres
-        mutable vectorField* reconFaceCellCentresPtr_;
+
+            //- Patch-to-patch interpolation
+            mutable ggiZoneInterpolation* patchToPatchPtr_;
+
+            //- Patch-to-zone addressing
+            mutable labelList* zoneAddressingPtr_;
+
+            //- Remote zone addressing: data needed for interpolation
+            mutable labelList* remoteZoneAddressingPtr_;
+
+            //- Reconstructed patch neighbour cell centres
+            mutable vectorField* reconFaceCellCentresPtr_;
 
 
         // Parallel communication optimisation, stored on master processor
@@ -108,11 +115,8 @@ class regionCouplePolyPatch
             //  Used for parallel optimisation
             mutable bool* localParallelPtr_;
 
-            //- List of zone faces indices received from each processor
-            mutable labelListList* receiveAddrPtr_;
-
-            //- List of zone faces indices to send to each processor
-            mutable labelListList* sendAddrPtr_;
+            //- Map-distribute comms tool
+            mutable mapDistribute* mapPtr_;
 
 
     // Private member functions
@@ -145,13 +149,6 @@ class regionCouplePolyPatch
             void calcSendReceive() const;
 
 
-            //- Return receive addressing
-            const labelListList& receiveAddr() const;
-
-            //- Return send addressing
-            const labelListList& sendAddr() const;
-
-
         // Memory management
 
             //- Clear delta coefficients
@@ -232,7 +229,9 @@ public:
             const bool attached,
             const bool master,
             const bool isWall,
-            const bool bridgeOverlap
+            const bool bridgeOverlap,
+            const ggiZoneInterpolation::quickReject
+                reject = ggiZoneInterpolation::BB_OCTREE
         );
 
         //- Construct from dictionary
@@ -292,9 +291,8 @@ public:
         }
 
 
-    // Destructor
-
-        virtual ~regionCouplePolyPatch();
+    //-Destructor
+    virtual ~regionCouplePolyPatch();
 
 
     // Member functions
@@ -382,6 +380,9 @@ public:
             //- Is the patch localised on a single processor
             bool localParallel() const;
 
+            //- Return mapDistribute
+            const mapDistribute& map() const;
+
             //- Return reference to patch-to-patch interpolation
             const ggiZoneInterpolation& patchToPatch() const;
 
diff --git a/src/foam/meshes/polyMesh/polyPatches/constraint/regionCouple/regionCouplePolyPatchTemplates.C b/src/foam/meshes/polyMesh/polyPatches/constraint/regionCouple/regionCouplePolyPatchTemplates.C
index 043e4720b..51423dde3 100644
--- a/src/foam/meshes/polyMesh/polyPatches/constraint/regionCouple/regionCouplePolyPatchTemplates.C
+++ b/src/foam/meshes/polyMesh/polyPatches/constraint/regionCouple/regionCouplePolyPatchTemplates.C
@@ -39,22 +39,6 @@ Foam::tmp<Foam::Field<Type> > Foam::regionCouplePolyPatch::fastExpand
 {
     // Check and expand the field from patch size to zone size
     // with communication
-
-    // Algorithm:
-    // 1) Master processor holds maps of all zone addressing (data provided)
-    // and all remote zone addressing (data required)
-    // 2) Each processor will send the locally active data to the master
-    // 3) Master assembles all the data
-    // 4) Master sends to all processors the data they need to receive
-    //
-    // Notes:
-    // A) If the size of zone addressing is zero, data is not sent
-    // B) Communicated data on each processor has the size of live faces
-    // C) Expanded data will be equal to actual data from other processors
-    //    only for the faces marked in remote; for other faces, it will be
-    //    equal to zero
-    // D) On processor zero, complete data is available
-    // HJ, 4/Jun/2011
     if (ff.size() != size())
     {
         FatalErrorIn
@@ -80,127 +64,54 @@ Foam::tmp<Foam::Field<Type> > Foam::regionCouplePolyPatch::fastExpand
             << abort(FatalError);
     }
 
+    // Replaced old comms algorithm.  HJ, 31/May/2016
+    // HJ, 4/Jun/2011
+
     // This function requires send-receive-addressing, but usage is not
     // symmetric across processors. Hence trigger re-calculate at this point
     // HR, 10/Jul/2013
     if (Pstream::parRun() && !localParallel())
     {
-        receiveAddr();
-        shadow().receiveAddr();
+        map();
+        shadow().map();
     }
 
-    // Expand the field to zone size
-    tmp<Field<Type> > texpandField
-    (
-        new Field<Type>(zone().size(), pTraits<Type>::zero)
-    );
+    // New version: mapDistribute
 
-    Field<Type>& expandField = texpandField();
-
-    if (Pstream::master())
+    if (Pstream::parRun())
     {
-        // Insert master processor
-        const labelList& za = zoneAddressing();
+        // Optimised mapDistribute
 
-        forAll (za, i)
-        {
-            expandField[za[i]] = ff[i];
-        }
+        // Prepare return field: expand the field to zone size
+        tmp<Field<Type> > texpandField
+        (
+            new Field<Type>(ff)
+        );
+        Field<Type>& expandField = texpandField();
 
-        // Master receives and inserts data from all processors for which
-        // receiveAddr contains entries
-        for (label procI = 1; procI < Pstream::nProcs(); procI++)
-        {
-            const labelList& curRAddr = receiveAddr()[procI];
+        map().distribute(expandField);
 
-            if (!curRAddr.empty())
-            {
-                Field<Type> receiveBuf(curRAddr.size());
-
-                // Opt: reconsider mode of communication
-                IPstream::read
-                (
-                    Pstream::blocking,
-                    procI,
-                    reinterpret_cast<char*>(receiveBuf.begin()),
-                    receiveBuf.byteSize()
-                );
-
-                // Insert received information
-                forAll (curRAddr, i)
-                {
-                    expandField[curRAddr[i]] = receiveBuf[i];
-                }
-            }
-        }
-
-        // Expanded field complete, send required data to other processors
-        for (label procI = 1; procI < Pstream::nProcs(); procI++)
-        {
-            const labelList& curSAddr = shadow().sendAddr()[procI];
-
-            if (!curSAddr.empty())
-            {
-                Field<Type> sendBuf(curSAddr.size());
-
-                forAll (curSAddr, i)
-                {
-                    sendBuf[i] = expandField[curSAddr[i]];
-                }
-
-                // Opt: reconsider mode of communication
-                OPstream::write
-                (
-                    Pstream::blocking,
-                    procI,
-                    reinterpret_cast<const char*>(sendBuf.begin()),
-                    sendBuf.byteSize()
-                );
-            }
-        }
+        return texpandField;
     }
     else
     {
-        // Send local data to master and receive remote data
-        // If patch is empty, communication is avoided
-        // HJ, 4/Jun/2011
-        if (size())
+        // Serial.  Expand the field to zone size
+
+        tmp<Field<Type> > texpandField
+        (
+            new Field<Type>(zone().size())  // filled with nans
+        );
+        Field<Type>& expandField = texpandField();
+
+        const labelList& zAddr = zoneAddressing();
+
+        forAll (zAddr, i)
         {
-            // Opt: reconsider mode of communication
-            OPstream::write
-            (
-                Pstream::blocking,
-                Pstream::masterNo(),
-                reinterpret_cast<const char*>(ff.begin()),
-                ff.byteSize()
-            );
+            expandField[zAddr[i]] = ff[i];
         }
 
-        // Prepare to receive remote data
-        const labelList& rza = shadow().remoteZoneAddressing();
-
-        if (!rza.empty())
-        {
-            Field<Type> receiveBuf(rza.size());
-
-            // Opt: reconsider mode of communication
-            IPstream::read
-            (
-                Pstream::blocking,
-                Pstream::masterNo(),
-                reinterpret_cast<char*>(receiveBuf.begin()),
-                receiveBuf.byteSize()
-            );
-
-            // Insert the data into expanded field
-            forAll (rza, i)
-            {
-                expandField[rza[i]] = receiveBuf[i];
-            }
-        }
+        return texpandField;
     }
-
-    return texpandField;
 }
 
 

From 604a3fe9ab2d79c8948546032e98adaecd08e657 Mon Sep 17 00:00:00 2001
From: Hrvoje Jasak <h.jasak@wikki.co.uk>
Date: Sun, 28 Aug 2016 20:36:48 +0100
Subject: [PATCH 021/130] Fix problem with localParallel in new GGI AMG

---
 .../ggiAMGInterface/ggiAMGInterface.C         | 27 ++++++++++++-------
 .../ggiAMGInterface/ggiAMGInterface.H         |  2 +-
 2 files changed, 19 insertions(+), 10 deletions(-)

diff --git a/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/ggiAMGInterface/ggiAMGInterface.C b/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/ggiAMGInterface/ggiAMGInterface.C
index 84dfc8e86..c529bab19 100644
--- a/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/ggiAMGInterface/ggiAMGInterface.C
+++ b/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/ggiAMGInterface/ggiAMGInterface.C
@@ -224,10 +224,13 @@ Foam::ggiAMGInterface::ggiAMGInterface
     // Expand neighbour side to get all the data required from other
     // processors.
     // Note: neigbour is now the size of remote zone
-    fineGgiInterface_.shadowInterface().expandAddrToZone
-    (
-        neighbourExpandAddressing
-    );
+    if (!fineGgiInterface_.shadowInterface().localParallel())
+    {
+        fineGgiInterface_.shadowInterface().expandAddrToZone
+        (
+            neighbourExpandAddressing
+        );
+    }
 
     // Create addressing for neighbour processors.  Note: expandAddrToZone will
     // expand the addressing to zone size.  HJ, 13/Jun/2016
@@ -240,10 +243,13 @@ Foam::ggiAMGInterface::ggiAMGInterface
     // Expand neighbour side to get all the data required from other
     // processors.
     // Note: neigbour is now the size of remote zone
-    fineGgiInterface_.shadowInterface().expandAddrToZone
-    (
-        neighbourExpandProc
-    );
+    if (!fineGgiInterface_.shadowInterface().localParallel())
+    {
+        fineGgiInterface_.shadowInterface().expandAddrToZone
+        (
+            neighbourExpandProc
+        );
+    }
 
     // Note: neighbourExpandAddressing and neighbourExpandProc
     // will be filled with NaNs for faces which are not local
@@ -1258,7 +1264,10 @@ Foam::tmp<Foam::scalarField> Foam::ggiAMGInterface::internalFieldTransfer
 
 void Foam::ggiAMGInterface::expandAddrToZone(labelField& lf) const
 {
-    lf = fastExpand(lf);
+    if (!localParallel())
+    {
+        lf = fastExpand(lf);
+    }
 }
 
 
diff --git a/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/ggiAMGInterface/ggiAMGInterface.H b/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/ggiAMGInterface/ggiAMGInterface.H
index b3c3087e8..a4ba6d917 100644
--- a/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/ggiAMGInterface/ggiAMGInterface.H
+++ b/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/ggiAMGInterface/ggiAMGInterface.H
@@ -99,7 +99,7 @@ class ggiAMGInterface
     // Private static data
 
         //- Processor cluster offset index
-        static const label procOffset = 4000000;
+        static const long procOffset = 12000000;
 
 
 public:

From 4b9edc42de2b687c56b2855f06db861e92d2023c Mon Sep 17 00:00:00 2001
From: Hrvoje Jasak <h.jasak@wikki.co.uk>
Date: Mon, 5 Sep 2016 11:17:11 +0100
Subject: [PATCH 022/130] Formatting

---
 ThirdParty/rpmBuild/SPECS/openmpi-1.8.8.spec | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ThirdParty/rpmBuild/SPECS/openmpi-1.8.8.spec b/ThirdParty/rpmBuild/SPECS/openmpi-1.8.8.spec
index 78530a615..913d2e1cc 100644
--- a/ThirdParty/rpmBuild/SPECS/openmpi-1.8.8.spec
+++ b/ThirdParty/rpmBuild/SPECS/openmpi-1.8.8.spec
@@ -138,7 +138,7 @@ Group: 			Development/Tools
         --enable-orterun-prefix-by-default \
         --enable-shared \
         --enable-mpi-cxx \
-	--disable-static \
+        --disable-static \
         --disable-mpi-f77 \
         --disable-mpi-f90 \
         --without-slurm \

From 8d059312202d7f30af8fcfd2a00d1d5d7d6d757e Mon Sep 17 00:00:00 2001
From: Hrvoje Jasak <h.jasak@wikki.co.uk>
Date: Mon, 5 Sep 2016 11:17:38 +0100
Subject: [PATCH 023/130] More verbose debug

---
 ThirdParty/tools/makeThirdPartyFunctionsForRPM | 1 +
 1 file changed, 1 insertion(+)

diff --git a/ThirdParty/tools/makeThirdPartyFunctionsForRPM b/ThirdParty/tools/makeThirdPartyFunctionsForRPM
index b256f3944..f2308f4e0 100755
--- a/ThirdParty/tools/makeThirdPartyFunctionsForRPM
+++ b/ThirdParty/tools/makeThirdPartyFunctionsForRPM
@@ -82,6 +82,7 @@ rpm_make()
     echo "Package name      : $_PACKAGE"
     echo "Package URL       : $_PACKAGE_URL"
     echo "RPM spec file name: $_SPECFILE"
+    echo "RPM file name     : $_RPMFILENAME"
     echo "Additional flags  : $_ADDITIONALFLAGS"
 
     # Shift options

From d5840f2ec15b855cb130812d7b802606567e27a0 Mon Sep 17 00:00:00 2001
From: Hrvoje Jasak <h.jasak@wikki.co.uk>
Date: Mon, 5 Sep 2016 11:18:02 +0100
Subject: [PATCH 024/130] Formatting

---
 .../fvPatchFields/constraint/wedge/wedgeFvPatchField.C       | 5 ++++-
 .../fvPatchFields/constraint/wedge/wedgeFvPatchField.H       | 2 +-
 2 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/src/finiteVolume/fields/fvPatchFields/constraint/wedge/wedgeFvPatchField.C b/src/finiteVolume/fields/fvPatchFields/constraint/wedge/wedgeFvPatchField.C
index f714fc01e..54b6d50e9 100644
--- a/src/finiteVolume/fields/fvPatchFields/constraint/wedge/wedgeFvPatchField.C
+++ b/src/finiteVolume/fields/fvPatchFields/constraint/wedge/wedgeFvPatchField.C
@@ -141,7 +141,10 @@ tmp<Field<Type> > wedgeFvPatchField<Type>::snGrad() const
     Field<Type> pif = this->patchInternalField();
     return
     (
-        transform(refCast<const wedgeFvPatch>(this->patch()).cellT(), pif) - pif
+        transform
+        (
+            refCast<const wedgeFvPatch>(this->patch()).cellT(), pif
+        ) - pif
     )*(0.5*this->patch().deltaCoeffs());
 }
 
diff --git a/src/finiteVolume/fields/fvPatchFields/constraint/wedge/wedgeFvPatchField.H b/src/finiteVolume/fields/fvPatchFields/constraint/wedge/wedgeFvPatchField.H
index 3dce3802d..a4ac779c5 100644
--- a/src/finiteVolume/fields/fvPatchFields/constraint/wedge/wedgeFvPatchField.H
+++ b/src/finiteVolume/fields/fvPatchFields/constraint/wedge/wedgeFvPatchField.H
@@ -44,7 +44,7 @@ namespace Foam
 {
 
 /*---------------------------------------------------------------------------*\
-                         Class wedgeFvPatch Declaration
+                     Class wedgeFvPatchField Declaration
 \*---------------------------------------------------------------------------*/
 
 template<class Type>

From feeffeec55d72a9bba352d8db84996ef28612f73 Mon Sep 17 00:00:00 2001
From: Hrvoje Jasak <h.jasak@wikki.co.uk>
Date: Mon, 5 Sep 2016 11:18:23 +0100
Subject: [PATCH 025/130] Fixed wrond include

---
 src/foam/primitives/VectorN/vector8/sphericalTensor8.H | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/foam/primitives/VectorN/vector8/sphericalTensor8.H b/src/foam/primitives/VectorN/vector8/sphericalTensor8.H
index 8ce925cd2..c7eeb12dc 100644
--- a/src/foam/primitives/VectorN/vector8/sphericalTensor8.H
+++ b/src/foam/primitives/VectorN/vector8/sphericalTensor8.H
@@ -33,7 +33,7 @@ Description
 #define sphericalTensor8_H
 
 #include "SphericalTensorN.H"
-#include "vector6.H"
+#include "vector8.H"
 #include "contiguous.H"
 
 // * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //

From 3fb876887da6356912982c3d2c440f8ab4212353 Mon Sep 17 00:00:00 2001
From: Hrvoje Jasak <h.jasak@wikki.co.uk>
Date: Mon, 5 Sep 2016 11:20:21 +0100
Subject: [PATCH 026/130] Better comments

---
 .../tetPolyPatchInterpolation/tetPolyPatchInterpolation.H    | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/tetFiniteElement/tetPolyPatchInterpolation/tetPolyPatchInterpolation.H b/src/tetFiniteElement/tetPolyPatchInterpolation/tetPolyPatchInterpolation.H
index df1fdd5bf..6f4e55c32 100644
--- a/src/tetFiniteElement/tetPolyPatchInterpolation/tetPolyPatchInterpolation.H
+++ b/src/tetFiniteElement/tetPolyPatchInterpolation/tetPolyPatchInterpolation.H
@@ -46,7 +46,7 @@ namespace Foam
 class faceTetPolyPatch;
 
 /*---------------------------------------------------------------------------*\
-              Class tetPolyPatchInterpolation Declaration
+                  Class tetPolyPatchInterpolation Declaration
 \*---------------------------------------------------------------------------*/
 
 class tetPolyPatchInterpolation
@@ -88,13 +88,14 @@ public:
 
     // Member Functions
 
-        //- Interpolate from faces to points
+        //- Interpolate from mesh faces to tet FEM points
         template<class Type>
         tmp<Field<Type> > faceToPointInterpolate
         (
             const Field<Type>& ff
         ) const;
 
+        //- Interpolate from mesh faces to tet FEM points
         template<class Type>
         tmp<Field<Type> > faceToPointInterpolate
         (

From 7e950dd6b084671e3a036377d53df74fadc4d600 Mon Sep 17 00:00:00 2001
From: Hrvoje Jasak <h.jasak@wikki.co.uk>
Date: Mon, 5 Sep 2016 11:21:02 +0100
Subject: [PATCH 027/130] Compilation fix

---
 .../immersedBoundaryFvPatchTriAddressing.C                   | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/src/immersedBoundary/immersedBoundary/immersedBoundaryFvPatch/immersedBoundaryFvPatchTriAddressing.C b/src/immersedBoundary/immersedBoundary/immersedBoundaryFvPatch/immersedBoundaryFvPatchTriAddressing.C
index c7f669bd2..cfe3320ef 100644
--- a/src/immersedBoundary/immersedBoundary/immersedBoundaryFvPatch/immersedBoundaryFvPatchTriAddressing.C
+++ b/src/immersedBoundary/immersedBoundary/immersedBoundaryFvPatch/immersedBoundaryFvPatchTriAddressing.C
@@ -129,10 +129,7 @@ void Foam::immersedBoundaryFvPatch::makeTriAddressing() const
                 }
                 while (visited[curTri]);
 
-                // Discard tri if already visited
-                if (visited[curTri]) continue;
-
-                visited.insert(curTri);
+                visited[curTri] = true;
 
                 const triFace& curTriPoints = triPatch[curTri];
 

From 94654bf821fb0c032a6cdc4dbb400b01ecf77b7c Mon Sep 17 00:00:00 2001
From: Hrvoje Jasak <h.jasak@wikki.co.uk>
Date: Mon, 5 Sep 2016 16:02:59 +0100
Subject: [PATCH 028/130] Changed decomposition for testing

---
 .../MRFSimpleFoam/mixerGgiMRF/system/decomposeParDict     | 8 +-------
 1 file changed, 1 insertion(+), 7 deletions(-)

diff --git a/tutorials/incompressible/MRFSimpleFoam/mixerGgiMRF/system/decomposeParDict b/tutorials/incompressible/MRFSimpleFoam/mixerGgiMRF/system/decomposeParDict
index c5263537a..ffddb7623 100644
--- a/tutorials/incompressible/MRFSimpleFoam/mixerGgiMRF/system/decomposeParDict
+++ b/tutorials/incompressible/MRFSimpleFoam/mixerGgiMRF/system/decomposeParDict
@@ -16,16 +16,10 @@ FoamFile
 
 numberOfSubdomains 4;
 
-method          simple;
+method          scotch;
 
 globalFaceZones ( insideZone outsideZone );
 
-simpleCoeffs
-{
-    n               (2 2 1);
-    delta           0.001;
-}
-
 distributed     no;
 
 roots

From 65fff701345a1e1f0d25c9d9826b8c81fd126e9f Mon Sep 17 00:00:00 2001
From: Hrvoje Jasak <h.jasak@wikki.co.uk>
Date: Mon, 5 Sep 2016 16:03:59 +0100
Subject: [PATCH 029/130] Fix compilation error

---
 applications/solvers/incompressible/simpleFoam/pEqn.H | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/applications/solvers/incompressible/simpleFoam/pEqn.H b/applications/solvers/incompressible/simpleFoam/pEqn.H
index 01afab709..7d14ec1e9 100644
--- a/applications/solvers/incompressible/simpleFoam/pEqn.H
+++ b/applications/solvers/incompressible/simpleFoam/pEqn.H
@@ -1,3 +1,4 @@
+{
     p.boundaryField().updateCoeffs();
 
     // Prepare clean 1/Ap without contribution from under-relaxation
@@ -14,6 +15,7 @@
 
     U = rAU*HUEqn().H();
     HUEqn.clear();
+
     phi = fvc::interpolate(U) & mesh.Sf();
     adjustPhi(phi, U, p);
 
@@ -22,7 +24,7 @@
     {
         fvScalarMatrix pEqn
         (
-            fvm::laplacian(rUA, p) == fvc::div(phi)
+            fvm::laplacian(rAU, p) == fvc::div(phi)
         );
 
         pEqn.setReference(pRefCell, pRefValue);
@@ -43,6 +45,6 @@
     // Momentum corrector
     // Note: since under-relaxation does not change aU, H/a in U can be
     // re-used.  HJ, 22/Jan/2016
-    U = UUrf*(U - rUA*fvc::grad(p)) + (1 - UUrf)*U.prevIter();
+    U = UUrf*(U - rAU*fvc::grad(p)) + (1 - UUrf)*U.prevIter();
     U.correctBoundaryConditions();
-
+}

From cb7afdb827c00f5c119e7ddc4d6233c3237b917d Mon Sep 17 00:00:00 2001
From: Hrvoje Jasak <h.jasak@wikki.co.uk>
Date: Mon, 5 Sep 2016 16:04:23 +0100
Subject: [PATCH 030/130] Added optional ramping to MRF

---
 .../cfdTools/general/MRF/MRFZone.C            | 151 ++++++++++--------
 .../cfdTools/general/MRF/MRFZone.H            |  46 ++++--
 .../cfdTools/general/MRF/MRFZoneTemplates.C   |  34 ++--
 3 files changed, 139 insertions(+), 92 deletions(-)

diff --git a/src/finiteVolume/cfdTools/general/MRF/MRFZone.C b/src/finiteVolume/cfdTools/general/MRF/MRFZone.C
index ed907ca08..09dfb6e25 100644
--- a/src/finiteVolume/cfdTools/general/MRF/MRFZone.C
+++ b/src/finiteVolume/cfdTools/general/MRF/MRFZone.C
@@ -62,7 +62,7 @@ void Foam::MRFZone::setMRFFaces()
     if (cellZoneID_ != -1)
     {
         const labelList& cellLabels = mesh_.cellZones()[cellZoneID_];
-        forAll(cellLabels, i)
+        forAll (cellLabels, i)
         {
             zoneCell[cellLabels[i]] = true;
         }
@@ -84,15 +84,15 @@ void Foam::MRFZone::setMRFFaces()
 
     labelHashSet excludedPatches(excludedPatchLabels_);
 
-    forAll(patches, patchI)
+    forAll (patches, patchI)
     {
         const polyPatch& pp = patches[patchI];
 
         if (pp.isWall() && !excludedPatches.found(patchI))
         {
-            forAll(pp, i)
+            forAll (pp, i)
             {
-                label faceI = pp.start()+i;
+                label faceI = pp.start() + i;
 
                 if (zoneCell[own[faceI]])
                 {
@@ -103,9 +103,9 @@ void Foam::MRFZone::setMRFFaces()
         }
         else if (!isA<emptyPolyPatch>(pp))
         {
-            forAll(pp, i)
+            forAll (pp, i)
             {
-                label faceI = pp.start()+i;
+                label faceI = pp.start() + i;
 
                 if (zoneCell[own[faceI]])
                 {
@@ -138,11 +138,11 @@ void Foam::MRFZone::setMRFFaces()
     labelList nIncludedFaces(patches.size(), 0);
     labelList nExcludedFaces(patches.size(), 0);
 
-    forAll(patches, patchi)
+    forAll (patches, patchi)
     {
         const polyPatch& pp = patches[patchi];
 
-        forAll(pp, patchFacei)
+        forAll (pp, patchFacei)
         {
             label faceI = pp.start()+patchFacei;
 
@@ -159,7 +159,7 @@ void Foam::MRFZone::setMRFFaces()
 
     includedFaces_.setSize(patches.size());
     excludedFaces_.setSize(patches.size());
-    forAll(nIncludedFaces, patchi)
+    forAll (nIncludedFaces, patchi)
     {
         includedFaces_[patchi].setSize(nIncludedFaces[patchi]);
         excludedFaces_[patchi].setSize(nExcludedFaces[patchi]);
@@ -167,11 +167,11 @@ void Foam::MRFZone::setMRFFaces()
     nIncludedFaces = 0;
     nExcludedFaces = 0;
 
-    forAll(patches, patchi)
+    forAll (patches, patchi)
     {
         const polyPatch& pp = patches[patchi];
 
-        forAll(pp, patchFacei)
+        forAll (pp, patchFacei)
         {
             label faceI = pp.start()+patchFacei;
 
@@ -201,9 +201,9 @@ void Foam::MRFZone::setMRFFaces()
         internalFaces.write();
 
         faceSet MRFFaces(mesh_, "includedFaces", 100);
-        forAll(includedFaces_, patchi)
+        forAll (includedFaces_, patchi)
         {
-            forAll(includedFaces_[patchi], i)
+            forAll (includedFaces_[patchi], i)
             {
                 label patchFacei = includedFaces_[patchi][i];
                 MRFFaces.insert(patches[patchi].start()+patchFacei);
@@ -215,12 +215,12 @@ void Foam::MRFZone::setMRFFaces()
         MRFFaces.write();
 
         faceSet excludedFaces(mesh_, "excludedFaces", 100);
-        forAll(excludedFaces_, patchi)
+        forAll (excludedFaces_, patchi)
         {
-            forAll(excludedFaces_[patchi], i)
+            forAll (excludedFaces_[patchi], i)
             {
                 label patchFacei = excludedFaces_[patchi][i];
-                excludedFaces.insert(patches[patchi].start()+patchFacei);
+                excludedFaces.insert(patches[patchi].start() + patchFacei);
             }
         }
         Pout<< "Writing " << excludedFaces.size()
@@ -231,12 +231,30 @@ void Foam::MRFZone::setMRFFaces()
 }
 
 
+Foam::vector Foam::MRFZone::Omega() const
+{
+    if (rampTime_ < SMALL)
+    {
+        return omega_.value()*axis_.value();
+    }
+    else
+    {
+        // Ramping
+        Info<< "MRF Ramp: " << Foam::min(mesh_.time().value()/rampTime_, 1)
+            << endl;
+
+        return Foam::min(mesh_.time().value()/rampTime_, 1)*
+            omega_.value()*axis_.value();
+    }
+}
+
+
 // * * * * * * * * * * * * * * * * Constructors  * * * * * * * * * * * * * * //
 
 Foam::MRFZone::MRFZone(const fvMesh& mesh, Istream& is)
 :
-    mesh_(mesh),
     name_(is),
+    mesh_(mesh),
     dict_(is),
     cellZoneID_(mesh_.cellZones().findZoneID(name_)),
     excludedPatchNames_
@@ -246,7 +264,7 @@ Foam::MRFZone::MRFZone(const fvMesh& mesh, Istream& is)
     origin_(dict_.lookup("origin")),
     axis_(dict_.lookup("axis")),
     omega_(dict_.lookup("omega")),
-    Omega_("Omega", omega_*axis_)
+    rampTime_(dict_.lookupOrDefault<scalar>("rampTime", 0))
 {
     if (dict_.found("patches"))
     {
@@ -260,12 +278,19 @@ Foam::MRFZone::MRFZone(const fvMesh& mesh, Istream& is)
 
     const polyBoundaryMesh& patches = mesh_.boundaryMesh();
 
+    if (mag(axis_.value()) < SMALL)
+    {
+        FatalErrorIn("MRFZone(const fvMesh&, Istream&)")
+            << "Axis vector has zero magnitude: " << axis_
+            << ".  This is not allowed"
+            << abort(FatalError);
+    }
+
     axis_ = axis_/mag(axis_);
-    Omega_ = omega_*axis_;
 
     excludedPatchLabels_.setSize(excludedPatchNames_.size());
 
-    forAll(excludedPatchNames_, i)
+    forAll (excludedPatchNames_, i)
     {
         excludedPatchLabels_[i] = patches.findPatchID(excludedPatchNames_[i]);
 
@@ -309,12 +334,12 @@ void Foam::MRFZone::addCoriolis(fvVectorMatrix& UEqn) const
     const scalarField& V = mesh_.V();
     vectorField& Usource = UEqn.source();
     const vectorField& U = UEqn.psi();
-    const vector& Omega = Omega_.value();
+    const vector rotVel = Omega();
 
-    forAll(cells, i)
+    forAll (cells, i)
     {
         label celli = cells[i];
-        Usource[celli] -= V[celli]*(Omega ^ U[celli]);
+        Usource[celli] -= V[celli]*(rotVel ^ U[celli]);
     }
 }
 
@@ -334,12 +359,12 @@ void Foam::MRFZone::addCoriolis
     const scalarField& V = mesh_.V();
     vectorField& Usource = UEqn.source();
     const vectorField& U = UEqn.psi();
-    const vector& Omega = Omega_.value();
+    const vector rotVel = Omega();
 
-    forAll(cells, i)
+    forAll (cells, i)
     {
         label celli = cells[i];
-        Usource[celli] -= V[celli]*rho[celli]*(Omega ^ U[celli]);
+        Usource[celli] -= V[celli]*rho[celli]*(rotVel ^ U[celli]);
     }
 }
 
@@ -349,20 +374,20 @@ void Foam::MRFZone::relativeVelocity(volVectorField& U) const
     const volVectorField& C = mesh_.C();
 
     const vector& origin = origin_.value();
-    const vector& Omega = Omega_.value();
+    const vector rotVel = Omega();
 
     const labelList& cells = mesh_.cellZones()[cellZoneID_];
 
-    forAll(cells, i)
+    forAll (cells, i)
     {
         label celli = cells[i];
-        U[celli] -= (Omega ^ (C[celli] - origin));
+        U[celli] -= (rotVel ^ (C[celli] - origin));
     }
 
     // Included patches
-    forAll(includedFaces_, patchi)
+    forAll (includedFaces_, patchi)
     {
-        forAll(includedFaces_[patchi], i)
+        forAll (includedFaces_[patchi], i)
         {
             label patchFacei = includedFaces_[patchi][i];
             U.boundaryField()[patchi][patchFacei] = vector::zero;
@@ -370,13 +395,13 @@ void Foam::MRFZone::relativeVelocity(volVectorField& U) const
     }
 
     // Excluded patches
-    forAll(excludedFaces_, patchi)
+    forAll (excludedFaces_, patchi)
     {
-        forAll(excludedFaces_[patchi], i)
+        forAll (excludedFaces_[patchi], i)
         {
             label patchFacei = excludedFaces_[patchi][i];
             U.boundaryField()[patchi][patchFacei] -=
-                (Omega ^ (C.boundaryField()[patchi][patchFacei] - origin));
+                (rotVel ^ (C.boundaryField()[patchi][patchFacei] - origin));
         }
     }
 }
@@ -387,35 +412,35 @@ void Foam::MRFZone::absoluteVelocity(volVectorField& U) const
     const volVectorField& C = mesh_.C();
 
     const vector& origin = origin_.value();
-    const vector& Omega = Omega_.value();
+    const vector rotVel = Omega();
 
     const labelList& cells = mesh_.cellZones()[cellZoneID_];
 
-    forAll(cells, i)
+    forAll (cells, i)
     {
         label celli = cells[i];
-        U[celli] += (Omega ^ (C[celli] - origin));
+        U[celli] += (rotVel ^ (C[celli] - origin));
     }
 
     // Included patches
-    forAll(includedFaces_, patchi)
+    forAll (includedFaces_, patchi)
     {
-        forAll(includedFaces_[patchi], i)
+        forAll (includedFaces_[patchi], i)
         {
             label patchFacei = includedFaces_[patchi][i];
             U.boundaryField()[patchi][patchFacei] =
-                (Omega ^ (C.boundaryField()[patchi][patchFacei] - origin));
+                (rotVel ^ (C.boundaryField()[patchi][patchFacei] - origin));
         }
     }
 
     // Excluded patches
-    forAll(excludedFaces_, patchi)
+    forAll (excludedFaces_, patchi)
     {
-        forAll(excludedFaces_[patchi], i)
+        forAll (excludedFaces_[patchi], i)
         {
             label patchFacei = excludedFaces_[patchi][i];
             U.boundaryField()[patchi][patchFacei] +=
-                (Omega ^ (C.boundaryField()[patchi][patchFacei] - origin));
+                (rotVel ^ (C.boundaryField()[patchi][patchFacei] - origin));
         }
     }
 }
@@ -460,36 +485,36 @@ void Foam::MRFZone::meshPhi
     const surfaceVectorField& Cf = mesh_.Cf();
 
     const vector& origin = origin_.value();
-    const vector& Omega = Omega_.value();
+    const vector rotVel = Omega();
 
     // Internal faces
-    forAll(internalFaces_, i)
+    forAll (internalFaces_, i)
     {
         label facei = internalFaces_[i];
-        phi[facei] = (Omega ^ (Cf[facei] - origin));
+        phi[facei] = (rotVel ^ (Cf[facei] - origin));
     }
 
     // Included patches
-    forAll(includedFaces_, patchi)
+    forAll (includedFaces_, patchi)
     {
-        forAll(includedFaces_[patchi], i)
+        forAll (includedFaces_[patchi], i)
         {
             label patchFacei = includedFaces_[patchi][i];
 
             phi.boundaryField()[patchi][patchFacei] =
-                (Omega ^ (Cf.boundaryField()[patchi][patchFacei] - origin));
+                (rotVel ^ (Cf.boundaryField()[patchi][patchFacei] - origin));
         }
     }
 
     // Excluded patches
-    forAll(excludedFaces_, patchi)
+    forAll (excludedFaces_, patchi)
     {
-        forAll(excludedFaces_[patchi], i)
+        forAll (excludedFaces_[patchi], i)
         {
             label patchFacei = excludedFaces_[patchi][i];
 
             phi.boundaryField()[patchi][patchFacei] =
-                (Omega ^ (Cf.boundaryField()[patchi][patchFacei] - origin));
+                (rotVel ^ (Cf.boundaryField()[patchi][patchFacei] - origin));
         }
     }
 }
@@ -497,20 +522,20 @@ void Foam::MRFZone::meshPhi
 void Foam::MRFZone::correctBoundaryVelocity(volVectorField& U) const
 {
     const vector& origin = origin_.value();
-    const vector& Omega = Omega_.value();
+    const vector rotVel = Omega();
 
     // Included patches
-    forAll(includedFaces_, patchi)
+    forAll (includedFaces_, patchi)
     {
         const vectorField& patchC = mesh_.Cf().boundaryField()[patchi];
 
         vectorField pfld(U.boundaryField()[patchi]);
 
-        forAll(includedFaces_[patchi], i)
+        forAll (includedFaces_[patchi], i)
         {
             label patchFacei = includedFaces_[patchi][i];
 
-            pfld[patchFacei] = (Omega ^ (patchC[patchFacei] - origin));
+            pfld[patchFacei] = (rotVel ^ (patchC[patchFacei] - origin));
         }
 
         U.boundaryField()[patchi] == pfld;
@@ -529,12 +554,12 @@ void Foam::MRFZone::Su
     const volVectorField& C = mesh_.C();
 
     const vector& origin = origin_.value();
-    const vector& Omega = Omega_.value();
+    const vector rotVel = Omega();
 
-    forAll(cells, i)
+    forAll (cells, i)
     {
         source[cells[i]] =
-            (Omega ^ (C[cells[i]] - origin)) & gradPhi[cells[i]];
+            (rotVel ^ (C[cells[i]] - origin)) & gradPhi[cells[i]];
     }
 }
 
@@ -550,13 +575,13 @@ void Foam::MRFZone::Su
     const volVectorField& C = mesh_.C();
 
     const vector& origin = origin_.value();
-    const vector& Omega = Omega_.value();
+    const vector rotVel = Omega();
 
-    forAll(cells, i)
+    forAll (cells, i)
     {
         source[cells[i]] =
-            ((Omega ^ (C[cells[i]] - origin)) & gradPhi[cells[i]])
-          - (Omega ^ phi[cells[i]]);
+            ((rotVel ^ (C[cells[i]] - origin)) & gradPhi[cells[i]])
+          - (rotVel ^ phi[cells[i]]);
     }
 }
 
diff --git a/src/finiteVolume/cfdTools/general/MRF/MRFZone.H b/src/finiteVolume/cfdTools/general/MRF/MRFZone.H
index 63cd0dd16..954bc85b1 100644
--- a/src/finiteVolume/cfdTools/general/MRF/MRFZone.H
+++ b/src/finiteVolume/cfdTools/general/MRF/MRFZone.H
@@ -29,7 +29,7 @@ Description
     obtained from a control dictionary constructed from the given stream.
 
     The rotation of the MRF region is defined by an origin and axis of
-    rotation and an angular speed.
+    rotation and an angular speed in rad/s.
 
 SourceFiles
     MRFZone.C
@@ -65,15 +65,22 @@ class MRFZone
 {
     // Private data
 
-        const fvMesh& mesh_;
-
+        //- Name of the MRF zone
         const word name_;
 
+        //- Mesh reference
+        const fvMesh& mesh_;
+
+        //- Reference to dictionary
         const dictionary dict_;
 
+        //- MRF cell zone ID: cells in MRF region
         label cellZoneID_;
 
+        //- List of patch names touching the MRF zone which do not move
         const wordList excludedPatchNames_;
+
+        //- List of patch labels touching the MRF zone which do not move
         labelList excludedPatchLabels_;
 
         //- Internal faces that are part of MRF
@@ -85,17 +92,34 @@ class MRFZone
         //- Excluded faces (per patch) that do not move with the MRF
         labelListList excludedFaces_;
 
+        //- Origin of rotation
         const dimensionedVector origin_;
+
+        //- Axis of rotation
         dimensionedVector axis_;
-        const dimensionedScalar omega_;
-        dimensionedVector Omega_;
+
+        //- Rotational velocity in rad/s
+        dimensionedScalar omega_;
+
+        //- Ramping time scale
+        scalar rampTime_;
 
 
     // Private Member Functions
 
+        //- Disallow default bitwise copy construct
+        MRFZone(const MRFZone&);
+
+        //- Disallow default bitwise assignment
+        void operator=(const MRFZone&);
+
+
         //- Divide faces in frame according to patch
         void setMRFFaces();
 
+        //- Return rotational vector
+        vector Omega() const;
+
         //- Make the given absolute mass/vol flux relative within MRF region
         template<class RhoFieldType>
         void relativeRhoFlux
@@ -112,12 +136,6 @@ class MRFZone
             surfaceScalarField& phi
         ) const;
 
-        //- Disallow default bitwise copy construct
-        MRFZone(const MRFZone&);
-
-        //- Disallow default bitwise assignment
-        void operator=(const MRFZone&);
-
 
 public:
 
@@ -170,7 +188,11 @@ public:
         void addCoriolis(fvVectorMatrix& UEqn) const;
 
         //- Add the Coriolis force contribution to the momentum equation
-        void addCoriolis(const volScalarField& rho, fvVectorMatrix& UEqn) const;
+        void addCoriolis
+        (
+            const volScalarField& rho,
+            fvVectorMatrix& UEqn
+        ) const;
 
         //- Make the given absolute velocity relative within the MRF region
         void relativeVelocity(volVectorField& U) const;
diff --git a/src/finiteVolume/cfdTools/general/MRF/MRFZoneTemplates.C b/src/finiteVolume/cfdTools/general/MRF/MRFZoneTemplates.C
index d8070148b..0ae07364a 100644
--- a/src/finiteVolume/cfdTools/general/MRF/MRFZoneTemplates.C
+++ b/src/finiteVolume/cfdTools/general/MRF/MRFZoneTemplates.C
@@ -41,19 +41,19 @@ void Foam::MRFZone::relativeRhoFlux
     const surfaceVectorField& Sf = mesh_.Sf();
 
     const vector& origin = origin_.value();
-    const vector& Omega = Omega_.value();
+    const vector rotVel = Omega();
 
     // Internal faces
-    forAll(internalFaces_, i)
+    forAll (internalFaces_, i)
     {
         label facei = internalFaces_[i];
-        phi[facei] -= rho[facei]*(Omega ^ (Cf[facei] - origin)) & Sf[facei];
+        phi[facei] -= rho[facei]*(rotVel ^ (Cf[facei] - origin)) & Sf[facei];
     }
 
     // Included patches
-    forAll(includedFaces_, patchi)
+    forAll (includedFaces_, patchi)
     {
-        forAll(includedFaces_[patchi], i)
+        forAll (includedFaces_[patchi], i)
         {
             label patchFacei = includedFaces_[patchi][i];
 
@@ -62,15 +62,15 @@ void Foam::MRFZone::relativeRhoFlux
     }
 
     // Excluded patches
-    forAll(excludedFaces_, patchi)
+    forAll (excludedFaces_, patchi)
     {
-        forAll(excludedFaces_[patchi], i)
+        forAll (excludedFaces_[patchi], i)
         {
             label patchFacei = excludedFaces_[patchi][i];
 
             phi.boundaryField()[patchi][patchFacei] -=
                 rho.boundaryField()[patchi][patchFacei]
-               *(Omega ^ (Cf.boundaryField()[patchi][patchFacei] - origin))
+               *(rotVel ^ (Cf.boundaryField()[patchi][patchFacei] - origin))
               & Sf.boundaryField()[patchi][patchFacei];
         }
     }
@@ -88,37 +88,37 @@ void Foam::MRFZone::absoluteRhoFlux
     const surfaceVectorField& Sf = mesh_.Sf();
 
     const vector& origin = origin_.value();
-    const vector& Omega = Omega_.value();
+    const vector rotVel = Omega();
 
     // Internal faces
-    forAll(internalFaces_, i)
+    forAll (internalFaces_, i)
     {
         label facei = internalFaces_[i];
-        phi[facei] += (Omega ^ (Cf[facei] - origin)) & Sf[facei];
+        phi[facei] += (rotVel ^ (Cf[facei] - origin)) & Sf[facei];
     }
 
     // Included patches
-    forAll(includedFaces_, patchi)
+    forAll (includedFaces_, patchi)
     {
-        forAll(includedFaces_[patchi], i)
+        forAll (includedFaces_[patchi], i)
         {
             label patchFacei = includedFaces_[patchi][i];
 
             phi.boundaryField()[patchi][patchFacei] +=
-                (Omega ^ (Cf.boundaryField()[patchi][patchFacei] - origin))
+                (rotVel ^ (Cf.boundaryField()[patchi][patchFacei] - origin))
               & Sf.boundaryField()[patchi][patchFacei];
         }
     }
 
     // Excluded patches
-    forAll(excludedFaces_, patchi)
+    forAll (excludedFaces_, patchi)
     {
-        forAll(excludedFaces_[patchi], i)
+        forAll (excludedFaces_[patchi], i)
         {
             label patchFacei = excludedFaces_[patchi][i];
 
             phi.boundaryField()[patchi][patchFacei] +=
-                (Omega ^ (Cf.boundaryField()[patchi][patchFacei] - origin))
+                (rotVel ^ (Cf.boundaryField()[patchi][patchFacei] - origin))
               & Sf.boundaryField()[patchi][patchFacei];
         }
     }

From c113c6b77467f7337eecde6ff8adfa18d1e067b6 Mon Sep 17 00:00:00 2001
From: Hrvoje Jasak <h.jasak@wikki.co.uk>
Date: Mon, 5 Sep 2016 16:14:51 +0100
Subject: [PATCH 031/130] Bugfix: consistent relaxation formulation in MRF

---
 .../incompressible/MRFSimpleFoam/UEqn.H       | 19 +++++++
 .../incompressible/MRFSimpleFoam/pEqn.H       | 50 +++++++++++++++++++
 2 files changed, 69 insertions(+)
 create mode 100644 applications/solvers/incompressible/MRFSimpleFoam/UEqn.H
 create mode 100644 applications/solvers/incompressible/MRFSimpleFoam/pEqn.H

diff --git a/applications/solvers/incompressible/MRFSimpleFoam/UEqn.H b/applications/solvers/incompressible/MRFSimpleFoam/UEqn.H
new file mode 100644
index 000000000..db0cdfb73
--- /dev/null
+++ b/applications/solvers/incompressible/MRFSimpleFoam/UEqn.H
@@ -0,0 +1,19 @@
+    // Solve the momentum equation
+
+    tmp<fvVectorMatrix> HUEqn
+    (
+        fvm::div(phi, U)
+      + turbulence->divDevReff()
+    );
+    mrfZones.addCoriolis(HUEqn());
+
+    // Get under-relaxation factor
+    const scalar UUrf = mesh.solutionDict().equationRelaxationFactor(U.name());
+
+    // Momentum solution
+    solve
+    (
+        relax(HUEqn(), UUrf)
+     ==
+        -fvc::grad(p)
+    );
diff --git a/applications/solvers/incompressible/MRFSimpleFoam/pEqn.H b/applications/solvers/incompressible/MRFSimpleFoam/pEqn.H
new file mode 100644
index 000000000..1752fc663
--- /dev/null
+++ b/applications/solvers/incompressible/MRFSimpleFoam/pEqn.H
@@ -0,0 +1,50 @@
+{
+    p.boundaryField().updateCoeffs();
+
+    // Prepare clean 1/Ap without contribution from under-relaxation
+    // HJ, 26/Oct/2015
+    volScalarField rAU
+    (
+        "(1|A(U))",
+        1/HUEqn().A()
+    );
+
+    // Store velocity under-relaxation point before using U for
+    // the flux precursor
+    U.storePrevIter();
+
+    U = rAU*HUEqn().H();
+    HUEqn.clear();
+
+     phi = fvc::interpolate(U) & mesh.Sf();
+     mrfZones.relativeFlux(phi);
+     adjustPhi(phi, U, p);
+
+     // Non-orthogonal pressure corrector loop
+     while (simple.correctNonOrthogonal())
+     {
+         fvScalarMatrix pEqn
+         (
+             fvm::laplacian(rAU, p) == fvc::div(phi)
+         );
+
+         pEqn.setReference(pRefCell, pRefValue);
+         pEqn.solve();
+
+         if (simple.finalNonOrthogonalIter())
+         {
+             phi -= pEqn.flux();
+         }
+     }
+
+#   include "continuityErrs.H"
+
+    // Explicitly relax pressure for momentum corrector
+    p.relax();
+
+    // Momentum corrector
+    // Note: since under-relaxation does not change aU, H/a in U can be
+    // re-used.  HJ, 22/Jan/2016
+    U = UUrf*(U - rAU*fvc::grad(p)) + (1 - UUrf)*U.prevIter();
+    U.correctBoundaryConditions();
+}

From b19ac9131838b799705274920cd4d9a4f54dc37b Mon Sep 17 00:00:00 2001
From: Hrvoje Jasak <h.jasak@wikki.co.uk>
Date: Mon, 5 Sep 2016 16:16:16 +0100
Subject: [PATCH 032/130] Bugfix: adjusted handling of coupled interfaces on
 decomposition and reconstruction

---
 .../polyMesh/polyPatches/constraint/ggi/ggiPolyPatch.C   | 9 +++++++++
 .../constraint/mixingPlane/mixingPlanePolyPatch.C        | 9 +++++++++
 .../constraint/overlapGgi/overlapGgiPolyPatch.C          | 9 +++++++++
 .../constraint/regionCouple/regionCouplePolyPatch.C      | 8 ++++----
 4 files changed, 31 insertions(+), 4 deletions(-)

diff --git a/src/foam/meshes/polyMesh/polyPatches/constraint/ggi/ggiPolyPatch.C b/src/foam/meshes/polyMesh/polyPatches/constraint/ggi/ggiPolyPatch.C
index c555f4998..71181bcb5 100644
--- a/src/foam/meshes/polyMesh/polyPatches/constraint/ggi/ggiPolyPatch.C
+++ b/src/foam/meshes/polyMesh/polyPatches/constraint/ggi/ggiPolyPatch.C
@@ -62,6 +62,15 @@ bool Foam::ggiPolyPatch::active() const
     polyPatchID shadow(shadowName_, boundaryMesh());
     faceZoneID zone(zoneName_, boundaryMesh().mesh().faceZones());
 
+    // For decomposition and reconstruction
+    // If not runing in parallel and the patch is not local, this is a serial
+    // operation on a piece of a parallel decomposition and is therefore
+    // inactive.  HJ, 5/Spe/2016
+    if (!Pstream::parRun() && !localParallel())
+    {
+        return false;
+    }
+
     return shadow.active() && zone.active();
 }
 
diff --git a/src/foam/meshes/polyMesh/polyPatches/constraint/mixingPlane/mixingPlanePolyPatch.C b/src/foam/meshes/polyMesh/polyPatches/constraint/mixingPlane/mixingPlanePolyPatch.C
index 18b04f674..298853762 100644
--- a/src/foam/meshes/polyMesh/polyPatches/constraint/mixingPlane/mixingPlanePolyPatch.C
+++ b/src/foam/meshes/polyMesh/polyPatches/constraint/mixingPlane/mixingPlanePolyPatch.C
@@ -58,6 +58,15 @@ bool Foam::mixingPlanePolyPatch::active() const
     polyPatchID shadow(shadowName_, boundaryMesh());
     faceZoneID zone(zoneName_, boundaryMesh().mesh().faceZones());
 
+    // For decomposition and reconstruction
+    // If not runing in parallel and the patch is not local, this is a serial
+    // operation on a piece of a parallel decomposition and is therefore
+    // inactive.  HJ, 5/Sep/2016
+    if (!Pstream::parRun() && !localParallel())
+    {
+        return false;
+    }
+
     return shadow.active() && zone.active();
 }
 
diff --git a/src/foam/meshes/polyMesh/polyPatches/constraint/overlapGgi/overlapGgiPolyPatch.C b/src/foam/meshes/polyMesh/polyPatches/constraint/overlapGgi/overlapGgiPolyPatch.C
index b21c3b87c..a174ffa7b 100644
--- a/src/foam/meshes/polyMesh/polyPatches/constraint/overlapGgi/overlapGgiPolyPatch.C
+++ b/src/foam/meshes/polyMesh/polyPatches/constraint/overlapGgi/overlapGgiPolyPatch.C
@@ -53,6 +53,15 @@ bool Foam::overlapGgiPolyPatch::active() const
     polyPatchID shadow(shadowName_, boundaryMesh());
     faceZoneID zone(zoneName_, boundaryMesh().mesh().faceZones());
 
+    // For decomposition and reconstruction
+    // If not runing in parallel and the patch is not local, this is a serial
+    // operation on a piece of a parallel decomposition and is therefore
+    // inactive.  HJ, 5/Sep/2016
+    if (!Pstream::parRun() && !localParallel())
+    {
+        return false;
+    }
+
     return shadow.active() && zone.active();
 }
 
diff --git a/src/foam/meshes/polyMesh/polyPatches/constraint/regionCouple/regionCouplePolyPatch.C b/src/foam/meshes/polyMesh/polyPatches/constraint/regionCouple/regionCouplePolyPatch.C
index 3f07d7175..7471e348e 100644
--- a/src/foam/meshes/polyMesh/polyPatches/constraint/regionCouple/regionCouplePolyPatch.C
+++ b/src/foam/meshes/polyMesh/polyPatches/constraint/regionCouple/regionCouplePolyPatch.C
@@ -70,10 +70,10 @@ bool Foam::regionCouplePolyPatch::active() const
     {
         // Shadow region present
         const polyMesh& sr = boundaryMesh().mesh().db().parent().
-        objectRegistry::lookupObject<polyMesh>
-        (
-            shadowRegionName_
-        );
+            objectRegistry::lookupObject<polyMesh>
+            (
+                shadowRegionName_
+            );
 
         polyPatchID shadowPatch(shadowPatchName_, sr.boundaryMesh());
 

From 68401f7d4cb4d7b71234ee5333f43b2c71fac19e Mon Sep 17 00:00:00 2001
From: Hrvoje Jasak <h.jasak@wikki.co.uk>
Date: Mon, 5 Sep 2016 16:16:50 +0100
Subject: [PATCH 033/130] Reorganise source code: pEqn.H and UEqn.H

---
 .../MRFSimpleFoam/MRFSimpleFoam.C             | 48 +------------------
 1 file changed, 2 insertions(+), 46 deletions(-)

diff --git a/applications/solvers/incompressible/MRFSimpleFoam/MRFSimpleFoam.C b/applications/solvers/incompressible/MRFSimpleFoam/MRFSimpleFoam.C
index a17092cad..7f5c77ad8 100644
--- a/applications/solvers/incompressible/MRFSimpleFoam/MRFSimpleFoam.C
+++ b/applications/solvers/incompressible/MRFSimpleFoam/MRFSimpleFoam.C
@@ -61,52 +61,8 @@ int main(int argc, char *argv[])
 
         // Pressure-velocity SIMPLE corrector
         {
-            // Momentum predictor
-
-            tmp<fvVectorMatrix> UEqn
-            (
-                fvm::div(phi, U)
-              + turbulence->divDevReff()
-            );
-            mrfZones.addCoriolis(UEqn());
-
-            UEqn().relax();
-
-            solve(UEqn() == -fvc::grad(p));
-
-            volScalarField rAU = 1.0/UEqn().A();
-            U = rAU*UEqn().H();
-            UEqn.clear();
-
-            phi = fvc::interpolate(U, "interpolate(HbyA)") & mesh.Sf();
-            mrfZones.relativeFlux(phi);
-            adjustPhi(phi, U, p);
-
-            // Non-orthogonal pressure corrector loop
-            while (simple.correctNonOrthogonal())
-            {
-                fvScalarMatrix pEqn
-                (
-                    fvm::laplacian(rAU, p) == fvc::div(phi)
-                );
-
-                pEqn.setReference(pRefCell, pRefValue);
-                pEqn.solve();
-
-                if (simple.finalNonOrthogonalIter())
-                {
-                    phi -= pEqn.flux();
-                }
-            }
-
-#           include "continuityErrs.H"
-
-            // Explicitly relax pressure for momentum corrector
-            p.relax();
-
-            // Momentum corrector
-            U -= rAU*fvc::grad(p);
-            U.correctBoundaryConditions();
+#           include "UEqn.H"
+#           include "pEqn.H"
         }
 
         turbulence->correct();

From dfaf6e71da631b091d6aa87a6ef8c280096db359 Mon Sep 17 00:00:00 2001
From: Hrvoje Jasak <h.jasak@wikki.co.uk>
Date: Mon, 5 Sep 2016 16:18:44 +0100
Subject: [PATCH 034/130] Added mvapich2 build options

---
 ThirdParty/AllMake.stage2                   |  11 +-
 ThirdParty/rpmBuild/SPECS/mvapich2-2.2.spec | 268 ++++++++++++++++++++
 etc/prefs.csh-EXAMPLE                       |   3 +
 etc/prefs.sh-EXAMPLE                        |   3 +
 etc/settings.csh                            |  15 +-
 etc/settings.sh                             |  32 +--
 6 files changed, 301 insertions(+), 31 deletions(-)
 create mode 100644 ThirdParty/rpmBuild/SPECS/mvapich2-2.2.spec

diff --git a/ThirdParty/AllMake.stage2 b/ThirdParty/AllMake.stage2
index 44613e46c..b8d04fdeb 100755
--- a/ThirdParty/AllMake.stage2
+++ b/ThirdParty/AllMake.stage2
@@ -40,8 +40,8 @@
 # Requirements:
 #     1: Your foam-extend environment must be properly initialized
 #     2: AllMake.stage1 if you are overriding your system compiler
-#     3: The file etc/prefs.sh should be used for setting the variables enabling
-#        the compilation of the various packages
+#     3: The file etc/prefs.sh should be used for setting the variables
+#        enabling the compilation of the various packages
 #
 # Author:
 #     Martin Beaudoin, Hydro-Quebec, (2015)
@@ -104,6 +104,13 @@ then
 	( rpm_make -p openmpi-1.8.8 -s openmpi-1.8.8.spec  -u http://www.open-mpi.org/software/ompi/v1.8/downloads/openmpi-1.8.8.tar.gz \
           -f --define '_configureAdditionalArgs "$WM_THIRD_PARTY_USE_OPENMPI_188_ConfigureAdditionalArgs"')
     }
+    # mvipich2-2.2
+    #
+    [ ! -z "$WM_THIRD_PARTY_USE_MVAPICH2_22" ] && {
+        echo "Building mvapich2 2.2"
+        ( rpm_make -p mvapich2-2.2 -s mvapich2-2.2.spec -u http://mvapich.cse.ohio-state.edu/download/mvapich/mv2/mvapich2-2.2.tar.gz \
+          -f --define '_configureAdditionalArgs "$WM_THIRD_PARTY_USE_MVAPICH2_22_ConfigureAdditionalArgs"')
+    }
 else
     echo "Using system installed OpenMPI"
     echo ""
diff --git a/ThirdParty/rpmBuild/SPECS/mvapich2-2.2.spec b/ThirdParty/rpmBuild/SPECS/mvapich2-2.2.spec
new file mode 100644
index 000000000..fed9345c4
--- /dev/null
+++ b/ThirdParty/rpmBuild/SPECS/mvapich2-2.2.spec
@@ -0,0 +1,268 @@
+#------------------------------------------------------------------------------
+# =========                 |
+# \\      /  F ield         | foam-extend: Open Source CFD
+#  \\    /   O peration     |
+#   \\  /    A nd           | For copyright notice see file Copyright
+#    \\/     M anipulation  |
+#------------------------------------------------------------------------------
+# License
+#     This file is part of foam-extend.
+#
+#     foam-extend is free software: you can redistribute it and/or modify it
+#     under the terms of the GNU General Public License as published by the
+#     Free Software Foundation, either version 3 of the License, or (at your
+#     option) any later version.
+#
+#     foam-extend is distributed in the hope that it will be useful, but
+#     WITHOUT ANY WARRANTY; without even the implied warranty of
+#     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+#     General Public License for more details.
+#
+#     You should have received a copy of the GNU General Public License
+#     along with foam-extend.  If not, see <http://www.gnu.org/licenses/>.
+#
+# Script
+#     RPM spec file for mvapich2-2.2
+#
+# Description
+#     RPM spec file for creating a relocatable RPM
+#
+# Author:
+#     Martin Beaudoin, Hydro-Quebec, (2015)
+#
+#------------------------------------------------------------------------------
+
+# We grab the value of WM_THIRD_PARTY and WM_OPTIONS from the environment variable
+%{expand:%%define _WM_THIRD_PARTY_DIR %(echo $WM_THIRD_PARTY_DIR)}
+%{expand:%%define _WM_OPTIONS         %(echo $WM_OPTIONS)}
+
+# Disable the generation of debuginfo packages
+%define debug_package %{nil}
+
+# The topdir needs to point to the $WM_THIRD_PARTY/rpmbuild directory
+%define _topdir	 	%{_WM_THIRD_PARTY_DIR}/rpmBuild
+%define _tmppath	%{_topdir}/tmp
+
+# Will install the package directly $WM_THIRD_PARTY_DIR
+#   Some comments about package relocation:
+#   By using this prefix for the Prefix:  parameter in this file, you will make this
+#   package relocatable.
+#
+#   This is fine, as long as your software is itself relocatable.
+#
+#   Simply take note that libraries built with libtool are not relocatable because the
+#   prefix we specify will be hard-coded in the library .la files.
+#   Ref: http://sourceware.org/autobook/autobook/autobook_80.html
+#
+#   In that case, if you ever change the value of the $WM_THIRD_PARTY_DIR, you will
+#   not be able to reutilize this RPM, even though it is relocatable. You will need to
+#   regenerate the RPM.
+#
+%define _prefix         %{_WM_THIRD_PARTY_DIR}
+
+%define name		mvapich2
+%define release		%{_WM_OPTIONS}
+%define version 	2.2
+
+%define buildroot       %{_topdir}/BUILD/%{name}-%{version}-root
+
+BuildRoot:	        %{buildroot}
+Summary: 		mvapich2
+License: 		BSD
+Name: 			%{name}
+Version: 		%{version}
+Release: 		%{release}
+URL:            http://mvapich.cse.ohio-state.edu/download/mvapich/mv2/
+Source: 		%url/%{name}-%{version}.tar.gz
+Prefix: 		%{_prefix}
+Group: 			Development/Tools
+
+
+%define _installPrefix  %{_prefix}/packages/%{name}-%{version}/platforms/%{_WM_OPTIONS}
+
+#--------------------------------------------------------------------------
+#
+# Here, we define default compiling options for mvapich2
+#
+# One can override the option on the commande line : --define='MACRO EXPR'
+#
+%{!?_configureAdditionalArgs: %define _configureAdditionalArgs Undefined}
+
+
+%description
+%{summary}
+
+%prep
+%setup -q
+
+
+%build
+    # export WM settings in a form that GNU configure recognizes
+    [ -n "$WM_CC" ]         &&  export CC="$WM_CC"
+    [ -n "$WM_CXX" ]        &&  export CXX="$WM_CXX"
+    [ -n "$WM_CFLAGS" ]     &&  export CFLAGS="$WM_CFLAGS"
+    [ -n "$WM_CXXFLAGS" ]   &&  export CXXFLAGS="$WM_CXXFLAGS"
+    [ -n "$WM_LDFLAGS" ]    &&  export LDFLAGS="$WM_LDFLAGS"
+
+    set +x
+    echo ""
+    echo "Compilation options:"
+    echo "     _configureAdditionalArgs : %{_configureAdditionalArgs}"
+    echo ""
+    set -x
+
+    unset mpiWith
+    # Enable GridEngine if it appears to be in use
+    # If you don't want any integration with SGE, simply unset the SGE
+    # environment variable
+    if [ -n "$SGE_ROOT" ]
+    then
+        mpiWith="$mpiWith --with-sge"
+    else
+        mpiWith="$mpiWith --without-sge"
+	mpiWith="$mpiWith --enable-mca-no-build=ras-gridengine,pls-gridengine"
+    fi
+
+    # Infiniband support
+    # Adjust according to your local paths
+    # if [ -d /usr/local/ofed -a -d /usr/local/ofed/lib64 ]
+    # then
+    #     mpiWith="$mpiWith --with-openib=/usr/local/ofed"
+    #     mpiWith="$mpiWith --with-openib-libdir=/usr/local/ofed/lib64"
+    # fi
+
+    # NOTE. Option --disable-mcast disables Infiniband when libs are not present
+
+    ./configure \
+        --prefix=%{_installPrefix}  \
+        --exec_prefix=%{_installPrefix}  \
+        --enable-mpirun-prefix-by-default \
+        --enable-orterun-prefix-by-default \
+        --enable-shared \
+        --enable-mpi-cxx \
+        --disable-static \
+        --disable-mpi-f77 \
+        --disable-mpi-f90 \
+        --without-slurm \
+        --enable-mpi-profile $mpiWith \
+        --disable-vt \
+        echo %{?_configureAdditionalArgs}`
+
+    [ -z "$WM_NCOMPPROCS" ] && WM_NCOMPPROCS=1
+    make -j $WM_NCOMPPROCS
+
+%install
+    make install DESTDIR=$RPM_BUILD_ROOT
+
+    # Creation of foam-extend specific .csh and .sh files"
+
+    echo ""
+    echo "Generating foam-extend specific .csh and .sh files for the package %{name}-%{version}"
+    echo ""
+    #
+    # Generate package specific .sh file for foam-extend
+    #
+mkdir -p $RPM_BUILD_ROOT/%{_installPrefix}/etc
+cat << DOT_SH_EOF > $RPM_BUILD_ROOT/%{_installPrefix}/etc/%{name}-%{version}.sh
+# Load %{name}-%{version} libraries and binaries
+# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+export MVAPICH2_DIR=\$WM_THIRD_PARTY_DIR/packages/%{name}-%{version}/platforms/\$WM_OPTIONS
+export MVAPICH2_BIN_DIR=\$MVAPICH2_DIR/bin
+export MVAPICH2_LIB_DIR=\$MVAPICH2_DIR/lib
+
+# Enable access to the package runtime applications and libraries
+[ -d \$MVAPICH2_BIN_DIR ] && _foamAddPath \$MVAPICH2_BIN_DIR
+[ -d \$MVAPICH2_LIB_DIR ] && _foamAddLib  \$MVAPICH2_LIB_DIR
+
+export MPI_HOME=\$MVAPICH2_DIR
+export MPI_ARCH_PATH=\$MPI_HOME
+export OPAL_PREFIX=\$MPI_ARCH_PATH
+
+# We initialize the rest of the environment using mpicc --showme:
+# This does not work with MVAPICH2
+#export MVAPICH2_INCLUDE_DIR="\`mpicc --showme:incdirs\`"
+#export MVAPICH2_COMPILE_FLAGS="\`mpicc --showme:compile\`"
+#export MVAPICH2_LINK_FLAGS="\`mpicc --showme:link\`"
+
+# Set the foam-extend compilation flags
+export PINC=\$MVAPICH2_COMPILE_FLAGS
+export PLIBS=\$MVAPICH2_LINK_FLAGS
+
+
+if [ "\$FOAM_VERBOSE" -a "\$PS1" ]
+then
+    echo "  Environment variables defined for OpenMPI:"
+    echo "    MVAPICH2_BIN_DIR       : \$MVAPICH2_BIN_DIR"
+    echo "    MVAPICH2_LIB_DIR       : \$MVAPICH2_LIB_DIR"
+    echo "    MVAPICH2_INCLUDE_DIR   : \$MVAPICH2_INCLUDE_DIR"
+    echo "    MVAPICH2_COMPILE_FLAGS : \$MVAPICH2_COMPILE_FLAGS"
+    echo "    MVAPICH2_LINK_FLAGS    : \$MVAPICH2_LINK_FLAGS"
+    echo ""
+    echo "    MPI_HOME              : \$MPI_HOME"
+    echo "    MPI_ARCH_PATH         : \$MPI_ARCH_PATH"
+    echo "    OPAL_PREFIX           : \$OPAL_PREFIX"
+    echo "    PINC                  : \$PINC"
+    echo "    PLIBS                 : \$PLIBS"
+fi
+DOT_SH_EOF
+
+    #
+    # Generate package specific .csh file for foam-extend
+    #
+cat << DOT_CSH_EOF > $RPM_BUILD_ROOT/%{_installPrefix}/etc/%{name}-%{version}.csh
+# Load %{name}-%{version} libraries and binaries
+# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+setenv MVAPICH2_DIR \$WM_THIRD_PARTY_DIR/packages/%{name}-%{version}/platforms/\$WM_OPTIONS
+setenv MVAPICH2_BIN_DIR \$MVAPICH2_DIR/bin
+setenv MVAPICH2_LIB_DIR \$MVAPICH2_DIR/lib
+
+# Enable access to the package runtime applications and libraries
+if ( -e \$MVAPICH2_BIN_DIR ) then
+    _foamAddPath \$MVAPICH2_BIN_DIR
+endif
+
+if ( -e \$MVAPICH2_LIB_DIR ) then
+    _foamAddLib \$MVAPICH2_LIB_DIR
+endif
+
+setenv MPI_HOME \$MVAPICH2_DIR
+setenv MPI_ARCH_PATH \$MPI_HOME
+setenv OPAL_PREFIX \$MPI_ARCH_PATH
+
+# We initialize the rest of the environment using mpicc --showme:
+setenv MVAPICH2_INCLUDE_DIR "\`mpicc --showme:incdirs\`"
+setenv MVAPICH2_COMPILE_FLAGS "\`mpicc --showme:compile\`"
+setenv MVAPICH2_LINK_FLAGS "\`mpicc --showme:link\`"
+
+# Set the foam-extend compilation flags
+setenv PINC "\$MVAPICH2_COMPILE_FLAGS"
+setenv PLIBS "\$MVAPICH2_LINK_FLAGS"
+
+
+if (\$?FOAM_VERBOSE && \$?prompt) then
+    echo "  Environment variables defined for OpenMPI:"
+    echo "    MVAPICH2_BIN_DIR       : \$MVAPICH2_BIN_DIR"
+    echo "    MVAPICH2_LIB_DIR       : \$MVAPICH2_LIB_DIR"
+    echo "    MVAPICH2_INCLUDE_DIR   : \$MVAPICH2_INCLUDE_DIR"
+    echo "    MVAPICH2_COMPILE_FLAGS : \$MVAPICH2_COMPILE_FLAGS"
+    echo "    MVAPICH2_LINK_FLAGS    : \$MVAPICH2_LINK_FLAGS"
+    echo ""
+    echo "    MPI_HOME              : \$MPI_HOME"
+    echo "    MPI_ARCH_PATH         : \$MPI_ARCH_PATH"
+    echo "    OPAL_PREFIX           : \$OPAL_PREFIX"
+    echo "    PINC                  : \$PINC"
+    echo "    PLIBS                 : \$PLIBS"
+endif
+DOT_CSH_EOF
+
+    #finally, generate a .tgz file for systems where using rpm for installing packages
+    # as a non-root user might be a problem.
+    (mkdir -p  %{_topdir}/TGZS/%{_target_cpu}; cd $RPM_BUILD_ROOT/%{_prefix}; tar -zcvf %{_topdir}/TGZS/%{_target_cpu}/%{name}-%{version}.tgz  packages/%{name}-%{version})
+
+
+%clean
+rm -rf %{buildroot}
+
+%files
+%defattr(-,root,root)
+%{_installPrefix}
diff --git a/etc/prefs.csh-EXAMPLE b/etc/prefs.csh-EXAMPLE
index f07db0d28..e77bf12f2 100644
--- a/etc/prefs.csh-EXAMPLE
+++ b/etc/prefs.csh-EXAMPLE
@@ -210,6 +210,9 @@ setenv WM_THIRD_PARTY_USE_OPENMPI_188 1
 #setenv WM_THIRD_PARTY_USE_OPENMPI_15 1
 #setenv WM_THIRD_PARTY_USE_OPENMPI_143 1
 #setenv WM_THIRD_PARTY_USE_OPENMPI_141 1
+#setenv WM_THIRD_PARTY_USE_MVAPICH2_22 1
+#setenv WM_THIRD_PARTY_USE_MVAPICH2_22_ConfigureAdditionalArgs '--disable-mcast'
+
 
 #
 # For AllMake.stage3
diff --git a/etc/prefs.sh-EXAMPLE b/etc/prefs.sh-EXAMPLE
index 0ed4194d3..02d6114d8 100644
--- a/etc/prefs.sh-EXAMPLE
+++ b/etc/prefs.sh-EXAMPLE
@@ -220,6 +220,9 @@ export WM_THIRD_PARTY_USE_OPENMPI_188=1
 #export WM_THIRD_PARTY_USE_OPENMPI_15=1
 #export WM_THIRD_PARTY_USE_OPENMPI_143=1
 #export WM_THIRD_PARTY_USE_OPENMPI_141=1
+#export WM_THIRD_PARTY_USE_MVAPICH2_22=1
+#export WM_THIRD_PARTY_USE_MVAPICH2_22_ConfigureAdditionalArgs='--disable-mcast'
+
 
 #
 # For AllMake.stage3
diff --git a/etc/settings.csh b/etc/settings.csh
index 586bb0add..acfd35fe0 100644
--- a/etc/settings.csh
+++ b/etc/settings.csh
@@ -300,17 +300,12 @@ case SYSTEMOPENMPI:
     breaksw
 
 case MVAPICH2:
-    set mpi_version=mvapich2
-
-    if ($?MVAPICH2_BIN_DIR != 0) then
-        if (-d "${MVAPICH2_BIN_DIR}" ) then
-        _foamAddPath $MVAPICH2_BIN_DIR
+    set mpi_version=mvapich2-2.2
+    if ($?WM_THIRD_PARTY_USE_MVAPICH2_22 != 0 && -d $WM_THIRD_PARTY_DIR/packages/mvapich2-2.2/platforms/$WM_OPTIONS ) then
+        if ($?FOAM_VERBOSE && $?prompt) then
+            echo "Using mvapich2-2.2 from the ThirdParty package: $WM_THIRD_PARTY_DIR/packages/$mpi_version"
         endif
-    else
-        set mpicc_cmd=`which mpicc`
-        setenv MVAPICH2_BIN_DIR `dirname $mpicc_cmd`
-        unset mpicc_cmd
-    endif
+        _foamSource  $WM_THIRD_PARTY_DIR/packages/$mpi_version/platforms/$WM_OPTIONS/etc/$mpi_version.csh
 
     setenv MPI_HOME `dirname $MVAPICH2_BIN_DIR`
     setenv MPI_ARCH_PATH $MPI_HOME
diff --git a/etc/settings.sh b/etc/settings.sh
index ca4d7c062..311b8714b 100755
--- a/etc/settings.sh
+++ b/etc/settings.sh
@@ -220,7 +220,7 @@ OPENMPI)
         then
             echo "Using openmpi-1.8.8 from the ThirdParty package: $WM_THIRD_PARTY_DIR/packages/$mpi_version"
         fi
-        _foamSource  $WM_THIRD_PARTY_DIR/packages/$mpi_version/platforms/$WM_OPTIONS/etc/$mpi_version.sh
+        _foamSource $WM_THIRD_PARTY_DIR/packages/$mpi_version/platforms/$WM_OPTIONS/etc/$mpi_version.sh
 
     elif [ ! -z $WM_THIRD_PARTY_USE_OPENMPI_165 ] && [ -e $WM_THIRD_PARTY_DIR/packages/openmpi-1.6.5/platforms/$WM_OPTIONS ]
         then
@@ -229,7 +229,7 @@ OPENMPI)
         then
             echo "Using openmpi-1.6.5 from the ThirdParty package: $WM_THIRD_PARTY_DIR/packages/$mpi_version"
         fi
-        _foamSource  $WM_THIRD_PARTY_DIR/packages/$mpi_version/platforms/$WM_OPTIONS/etc/$mpi_version.sh
+        _foamSource $WM_THIRD_PARTY_DIR/packages/$mpi_version/platforms/$WM_OPTIONS/etc/$mpi_version.sh
 
     elif [ ! -z $WM_THIRD_PARTY_USE_OPENMPI_143 ] && [ -e $WM_THIRD_PARTY_DIR/packages/openmpi-1.4.3/platforms/$WM_OPTIONS ]
         then
@@ -238,7 +238,7 @@ OPENMPI)
         then
             echo "Using openmpi-1.4.3 from the ThirdParty package: $WM_THIRD_PARTY_DIR/packages/$mpi_version"
         fi
-        _foamSource  $WM_THIRD_PARTY_DIR/packages/$mpi_version/platforms/$WM_OPTIONS/etc/$mpi_version.sh
+        _foamSource $WM_THIRD_PARTY_DIR/packages/$mpi_version/platforms/$WM_OPTIONS/etc/$mpi_version.sh
 
     elif [ ! -z $WM_THIRD_PARTY_USE_OPENMPI_15 ] && [ -e $WM_THIRD_PARTY_DIR/packages/openmpi-1.5/platforms/$WM_OPTIONS ]
         then
@@ -247,7 +247,7 @@ OPENMPI)
         then
             echo "Using openmpi-1.5 from the ThirdParty package: $WM_THIRD_PARTY_DIR/packages/$mpi_version"
         fi
-        _foamSource  $WM_THIRD_PARTY_DIR/packages/$mpi_version/platforms/$WM_OPTIONS/etc/$mpi_version.sh
+        _foamSource $WM_THIRD_PARTY_DIR/packages/$mpi_version/platforms/$WM_OPTIONS/etc/$mpi_version.sh
     fi
 
     # On Windows set mpi_version from value defined in bashrc.mingw:
@@ -375,22 +375,16 @@ SYSTEMOPENMPI)
     unset mpi_version
     ;;
 
+
 MVAPICH2)
-    mpi_version=mvapich2
-
-    if [ -n "${MVAPICH2_BIN_DIR}" ] && [ -d "${MVAPICH2_BIN_DIR}" ]
-    then
-        _foamAddPath $MVAPICH2_BIN_DIR
-    else
-        MVAPICH2_BIN_DIR=$(dirname `which mpicc`)
-    fi
-
-    if which mpicc >/dev/null
-    then
-        mpicc -v 2>/dev/null | grep -q "mpicc for MVAPICH2" ||
-            echo "Warning: `which mpicc` does not appear to be for MVAPICH2"
-    else
-        echo "Warning: mpicc not available"
+    mpi_version=mvapich2-2.2
+    if [ ! -z $WM_THIRD_PARTY_USE_MVAPICH2_20 ] && [ -e $WM_THIRD_PARTY_DIR/packages/mvapich2-2.2/platforms/$WM_OPTIONS ]
+        then
+        if [ "$FOAM_VERBOSE" -a "$PS1" ]
+        then
+            echo "Using mvapich2 from the ThirdParty package: $WM_THIRD_PARTY_DIR/packages/$mpi_version"
+        fi
+        _foamSource $WM_THIRD_PARTY_DIR/packages/$mpi_version/platforms/$WM_OPTIONS/etc/$mpi_version.sh
     fi
 
     export MPI_HOME=`dirname $MVAPICH2_BIN_DIR`

From 17129062cd4cea7aff3de2885c74a03dd4c05d53 Mon Sep 17 00:00:00 2001
From: Hrvoje Jasak <h.jasak@wikki.co.uk>
Date: Wed, 7 Sep 2016 13:49:02 +0100
Subject: [PATCH 035/130] Added dat file for forces and moments

Conflicts:
	bin/plotForces.py
---
 bin/plotForces.py | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

diff --git a/bin/plotForces.py b/bin/plotForces.py
index fd8ff938d..a7d779848 100755
--- a/bin/plotForces.py
+++ b/bin/plotForces.py
@@ -35,6 +35,7 @@ for line in lines:
                 mvy.append(float(match.group(12)))
                 mvz.append(float(match.group(13)))
 
+
 # combine pressure and viscous forces and moments
 fx = fpx
 fy = fpy
@@ -53,10 +54,17 @@ for i in range(1,len(t)):
         mz[i] += mvz[i]
 
 # write clean data file
-outfile=open('forces.dat','w')
+outForces=open('forces.dat','w')
 
 for data in zip(t,fx,fy,fz):
-        outfile.write(' '.join([str(d) for d in data])+'\n')
+        outForces.write(' '.join([str(d) for d in data])+'\n')
+outForces.close()
+
+outMoments=open('moments.dat','w')
+
+for data in zip(t,mx,my,mz):
+        outMoments.write(' '.join([str(d) for d in data])+'\n')
+outMoments.close()
 
 # plot forces
 import pylab

From aa518c69314d011549f3dd726725b382d58a2d74 Mon Sep 17 00:00:00 2001
From: Hrvoje Jasak <h.jasak@wikki.co.uk>
Date: Wed, 7 Sep 2016 13:49:21 +0100
Subject: [PATCH 036/130] Added Urel for post-processing and function objects

---
 .../incompressible/MRFSimpleFoam/MRFSimpleFoam.C |  4 ++++
 .../incompressible/MRFSimpleFoam/createFields.H  | 16 ++++++++++++++++
 2 files changed, 20 insertions(+)

diff --git a/applications/solvers/incompressible/MRFSimpleFoam/MRFSimpleFoam.C b/applications/solvers/incompressible/MRFSimpleFoam/MRFSimpleFoam.C
index 7f5c77ad8..996fb9631 100644
--- a/applications/solvers/incompressible/MRFSimpleFoam/MRFSimpleFoam.C
+++ b/applications/solvers/incompressible/MRFSimpleFoam/MRFSimpleFoam.C
@@ -65,6 +65,10 @@ int main(int argc, char *argv[])
 #           include "pEqn.H"
         }
 
+        // Calculate relative velocity
+        Urel == U;
+        mrfZones.relativeVelocity(Urel);
+
         turbulence->correct();
 
         runTime.write();
diff --git a/applications/solvers/incompressible/MRFSimpleFoam/createFields.H b/applications/solvers/incompressible/MRFSimpleFoam/createFields.H
index 8ae490c37..e43f40963 100644
--- a/applications/solvers/incompressible/MRFSimpleFoam/createFields.H
+++ b/applications/solvers/incompressible/MRFSimpleFoam/createFields.H
@@ -44,3 +44,19 @@
 
     MRFZones mrfZones(mesh);
     mrfZones.correctBoundaryVelocity(U);
+
+    // Create Urel as a permanent field to make it available for on-the-fly
+    // post-processing operations
+    volVectorField Urel
+    (
+        IOobject
+        (
+            "Urel",
+            runTime.timeName(),
+            mesh,
+            IOobject::NO_READ,
+            IOobject::AUTO_WRITE
+        ),
+        U
+    );
+    mrfZones.relativeVelocity(Urel);

From 7bab48776c8318e83540821839596cdd68b4c62c Mon Sep 17 00:00:00 2001
From: Hrvoje Jasak <h.jasak@wikki.co.uk>
Date: Wed, 14 Sep 2016 05:24:50 +0100
Subject: [PATCH 037/130] Pstream with a single comm, updated parallel ops

---
 src/finiteArea/faMesh/faMesh.C                |   12 +
 src/finiteArea/faMesh/faMesh.H                |   20 +-
 .../constraint/processor/processorFaPatch.C   |   12 +
 .../constraint/processor/processorFaPatch.H   |   47 +-
 .../processor/processorFvPatchField.H         |   12 +-
 .../fvMatrices/fvMatrix/fvMatrix.C            |    5 +-
 src/finiteVolume/fvMesh/fvMesh.H              |   14 +-
 .../constraint/cyclicGgi/cyclicGgiFvPatch.H   |    7 +-
 .../fvPatches/constraint/ggi/ggiFvPatch.H     |   20 +-
 .../mixingPlane/mixingPlaneFvPatch.H          |   20 +-
 .../constraint/processor/processorFvPatch.H   |   22 +-
 .../regionCouple/regionCoupleFvPatch.H        |   20 +-
 src/foam/Make/files                           |    1 -
 src/foam/db/IOstreams/Pstreams/IPread.C       |  128 +-
 src/foam/db/IOstreams/Pstreams/IPreadToken.C  |  187 ---
 src/foam/db/IOstreams/Pstreams/IPstream.C     |    9 +
 src/foam/db/IOstreams/Pstreams/IPstream.H     |   20 +-
 src/foam/db/IOstreams/Pstreams/OPstream.C     |   14 +-
 src/foam/db/IOstreams/Pstreams/OPstream.H     |   23 +-
 src/foam/db/IOstreams/Pstreams/OPwrite.C      |  129 +-
 src/foam/db/IOstreams/Pstreams/Pstream.C      | 1052 ++++++++++++-----
 src/foam/db/IOstreams/Pstreams/Pstream.H      |  352 +++++-
 .../Pstreams/PstreamCombineReduceOps.H        |   52 +-
 .../IOstreams/Pstreams/PstreamCommsStruct.C   |    4 +-
 .../db/IOstreams/Pstreams/PstreamExchange.C   |  159 +++
 .../db/IOstreams/Pstreams/PstreamGlobals.C    |   48 +-
 .../db/IOstreams/Pstreams/PstreamGlobals.H    |   16 +-
 .../{PstreamsPrint.C => PstreamReduceOps.C}   |   21 +-
 .../db/IOstreams/Pstreams/PstreamReduceOps.H  |  112 +-
 src/foam/db/IOstreams/Pstreams/allReduce.H    |   71 ++
 .../IOstreams/Pstreams/allReduceTemplates.C   |  211 ++++
 .../IOstreams/Pstreams/combineGatherScatter.C |  309 +++--
 .../db/IOstreams/Pstreams/gatherScatter.C     |   98 +-
 .../db/IOstreams/Pstreams/gatherScatterList.C |  122 +-
 src/foam/db/regIOobject/regIOobjectRead.C     |    1 -
 .../BlockMatrixAgglomeration.C                |    4 +-
 .../regionCoupleLduInterfaceField.H           |    2 +-
 .../ggiLduInterface/ggiLduInterface.H         |    9 +
 .../mixingPlaneLduInterface.H                 |    9 +
 .../processorLduInterface.H                   |   14 +-
 .../processorLduInterfaceTemplates.C          |   42 +-
 .../regionCoupleLduInterface.H                |    1 -
 .../GAMGAgglomerateLduAddressing.C            |    4 +-
 .../interfaces/AMGInterface/AMGInterface.H    |   14 +-
 .../interfaces/AMGInterface/newAMGInterface.C |    9 +-
 .../cyclicAMGInterface/cyclicAMGInterface.C   |    3 +-
 .../cyclicAMGInterface/cyclicAMGInterface.H   |    8 +-
 .../cyclicGGIAMGInterface.C                   |    6 +-
 .../cyclicGGIAMGInterface.H                   |    3 +-
 .../ggiAMGInterface/ggiAMGInterface.C         |    5 +-
 .../ggiAMGInterface/ggiAMGInterface.H         |   25 +-
 .../mixingPlaneAMGInterface.C                 |    7 +-
 .../mixingPlaneAMGInterface.H                 |   32 +-
 .../processorAMGInterface.C                   |    7 +-
 .../processorAMGInterface.H                   |   29 +-
 .../regionCoupleAMGInterface.C                |    6 +-
 .../regionCoupleAMGInterface.H                |   10 +-
 src/foam/meshes/lduMesh/lduMesh.H             |   11 +-
 src/foam/meshes/lduMesh/lduPrimitiveMesh.H    |   19 +-
 .../constraint/ggi/ggiPointPatch.H            |   20 +-
 .../processor/processorPointPatch.H           |   20 +-
 .../mapPolyMesh/mapDistribute/mapDistribute.C |   35 +-
 .../mapPolyMesh/mapDistribute/mapDistribute.H |   22 +-
 .../mapDistribute/mapDistributeTemplates.C    |  712 ++++++-----
 src/foam/meshes/polyMesh/polyMesh.C           |   15 +
 src/foam/meshes/polyMesh/polyMesh.H           |   51 +-
 .../polyPatches/constraint/ggi/ggiPolyPatch.C |   15 +-
 .../polyPatches/constraint/ggi/ggiPolyPatch.H |    9 +
 .../mixingPlane/mixingPlanePolyPatch.C        |   12 +
 .../mixingPlane/mixingPlanePolyPatch.H        |   14 +-
 .../constraint/processor/processorPolyPatch.C |   15 +-
 .../constraint/processor/processorPolyPatch.H |   15 +-
 .../regionCouple/regionCouplePolyPatch.C      |   12 +
 .../regionCouple/regionCouplePolyPatch.H      |   11 +-
 src/lduSolvers/amg/amgPolicy/pamgPolicy.C     |    3 +-
 .../tetPolyMesh/tetPolyMesh.C                 |   14 +
 .../tetPolyMesh/tetPolyMesh.H                 |   18 +-
 77 files changed, 3288 insertions(+), 1354 deletions(-)
 delete mode 100644 src/foam/db/IOstreams/Pstreams/IPreadToken.C
 create mode 100644 src/foam/db/IOstreams/Pstreams/PstreamExchange.C
 rename src/foam/db/IOstreams/Pstreams/{PstreamsPrint.C => PstreamReduceOps.C} (76%)
 create mode 100644 src/foam/db/IOstreams/Pstreams/allReduce.H
 create mode 100644 src/foam/db/IOstreams/Pstreams/allReduceTemplates.C

diff --git a/src/finiteArea/faMesh/faMesh.C b/src/finiteArea/faMesh/faMesh.C
index 481ccaf35..1c1ab187b 100644
--- a/src/finiteArea/faMesh/faMesh.C
+++ b/src/finiteArea/faMesh/faMesh.C
@@ -1047,6 +1047,18 @@ void Foam::faMesh::addFaPatches(const List<faPatch*>& p)
 }
 
 
+Foam::label Foam::faMesh::comm() const
+{
+    return comm_;
+}
+
+
+Foam::label& Foam::faMesh::comm()
+{
+    return comm_;
+}
+
+
 const Foam::objectRegistry& Foam::faMesh::thisDb() const
 {
     return mesh().thisDb();
diff --git a/src/finiteArea/faMesh/faMesh.H b/src/finiteArea/faMesh/faMesh.H
index 8f340701b..89f81abec 100644
--- a/src/finiteArea/faMesh/faMesh.H
+++ b/src/finiteArea/faMesh/faMesh.H
@@ -110,6 +110,12 @@ class faMesh
             mutable label nFaces_;
 
 
+        // Communication support
+
+            //- Communicator used for parallel communication
+            label comm_;
+
+
     // Demand-driven data
 
         //- Primitive patch
@@ -287,9 +293,8 @@ public:
         );
 
 
-    // Destructor
-
-        virtual ~faMesh();
+    //- Destructor
+    virtual ~faMesh();
 
 
     // Member Functions
@@ -369,6 +374,15 @@ public:
                 }
 
 
+            // Communication support
+
+                //- Return communicator used for parallel communication
+                label comm() const;
+
+                //- Return communicator used for parallel communication
+                label& comm();
+
+
         // Access
 
             //- Return reference to the mesh database
diff --git a/src/finiteArea/faMesh/faPatches/constraint/processor/processorFaPatch.C b/src/finiteArea/faMesh/faPatches/constraint/processor/processorFaPatch.C
index 050f763d1..08c4809bb 100644
--- a/src/finiteArea/faMesh/faPatches/constraint/processor/processorFaPatch.C
+++ b/src/finiteArea/faMesh/faPatches/constraint/processor/processorFaPatch.C
@@ -55,6 +55,18 @@ processorFaPatch::~processorFaPatch()
 
 // * * * * * * * * * * * * * * Member Functions  * * * * * * * * * * * * * * //
 
+Foam::label Foam::processorFaPatch::comm() const
+{
+    return boundaryMesh().mesh().comm();
+}
+
+
+int Foam::processorFaPatch::tag() const
+{
+    return Pstream::msgType();
+}
+
+
 void processorFaPatch::makeNonGlobalPatchPoints() const
 {
     // If it is not runing parallel or there are no global points
diff --git a/src/finiteArea/faMesh/faPatches/constraint/processor/processorFaPatch.H b/src/finiteArea/faMesh/faPatches/constraint/processor/processorFaPatch.H
index 6939342aa..0f0c96d81 100644
--- a/src/finiteArea/faMesh/faPatches/constraint/processor/processorFaPatch.H
+++ b/src/finiteArea/faMesh/faPatches/constraint/processor/processorFaPatch.H
@@ -37,7 +37,6 @@ SourceFiles
 
 #include "coupledFaPatch.H"
 #include "processorLduInterface.H"
-// #include "processorPolyPatch.H"
 
 // * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
 
@@ -55,7 +54,10 @@ class processorFaPatch
 {
     // Private data
 
+        //- My processro number
         int myProcNo_;
+
+        //- Neighbour processor number
         int neighbProcNo_;
 
         //- Processor-neighbbour patch edge centres
@@ -75,6 +77,7 @@ class processorFaPatch
         //  non-global, i.e. present in this processor patch
         mutable labelList* nonGlobalPatchPointsPtr_;
 
+
 protected:
 
     // Protected Member functions
@@ -88,27 +91,27 @@ protected:
         //- Find non-globa patch points
         void makeNonGlobalPatchPoints() const;
 
-protected:
 
-    // Protected Member functions
+        // Geometry functions
 
-        //- Initialise the calculation of the patch geometry
-        void initGeometry();
+            //- Initialise the calculation of the patch geometry
+            void initGeometry();
 
-        //- Calculate the patch geometry
-        void calcGeometry();
+            //- Calculate the patch geometry
+            void calcGeometry();
 
-        //- Initialise the patches for moving points
-        void initMovePoints(const pointField&);
+            //- Initialise the patches for moving points
+            void initMovePoints(const pointField&);
 
-        //- Correct patches after moving points
-        void movePoints(const pointField&);
+            //- Correct patches after moving points
+            void movePoints(const pointField&);
 
-        //- Initialise the update of the patch topology
-        virtual void initUpdateMesh();
+            //- Initialise the update of the patch topology
+            virtual void initUpdateMesh();
+
+            //- Update of the patch topology
+            virtual void updateMesh();
 
-        //- Update of the patch topology
-        virtual void updateMesh();
 
 public:
 
@@ -160,9 +163,9 @@ public:
             nonGlobalPatchPointsPtr_(NULL)
         {}
 
-    // Destructor
 
-        virtual ~processorFaPatch();
+    //- Destructor
+    virtual ~processorFaPatch();
 
 
     // Member functions
@@ -192,6 +195,16 @@ public:
             }
         }
 
+
+        // Communications support
+
+            //- Return communicator used for communication
+            virtual label comm() const;
+
+            //- Return message tag to use for communication
+            virtual int tag() const;
+
+
         //- Return face transformation tensor
         virtual const tensorField& forwardT() const
         {
diff --git a/src/finiteVolume/fields/fvPatchFields/constraint/processor/processorFvPatchField.H b/src/finiteVolume/fields/fvPatchFields/constraint/processor/processorFvPatchField.H
index 4f23bcc25..b3711d6ab 100644
--- a/src/finiteVolume/fields/fvPatchFields/constraint/processor/processorFvPatchField.H
+++ b/src/finiteVolume/fields/fvPatchFields/constraint/processor/processorFvPatchField.H
@@ -249,7 +249,7 @@ public:
             ) const;
 
 
-        //- Processor coupled interface functions
+        // Processor coupled interface functions
 
             //- Return processor number
             virtual int myProcNo() const
@@ -263,6 +263,16 @@ public:
                 return procPatch_.neighbProcNo();
             }
 
+
+            // Communication support
+
+                //- Return communicator used for parallel communication
+                virtual int comm() const
+                {
+                    return procPatch_.comm();
+                }
+
+
             //- Does the patch field perform the transfromation
             virtual bool doTransform() const
             {
diff --git a/src/finiteVolume/fvMatrices/fvMatrix/fvMatrix.C b/src/finiteVolume/fvMatrices/fvMatrix/fvMatrix.C
index a17e1ff44..793a55d90 100644
--- a/src/finiteVolume/fvMatrices/fvMatrix/fvMatrix.C
+++ b/src/finiteVolume/fvMatrices/fvMatrix/fvMatrix.C
@@ -698,7 +698,10 @@ void Foam::fvMatrix<Type>::relax()
 {
     if (psi_.mesh().solutionDict().relaxEquation(psi_.name()))
     {
-        relax(psi_.mesh().solutionDict().equationRelaxationFactor(psi_.name()));
+        relax
+        (
+            psi_.mesh().solutionDict().equationRelaxationFactor(psi_.name())
+        );
     }
     else
     {
diff --git a/src/finiteVolume/fvMesh/fvMesh.H b/src/finiteVolume/fvMesh/fvMesh.H
index 68e9bef2d..44e7dc78c 100644
--- a/src/finiteVolume/fvMesh/fvMesh.H
+++ b/src/finiteVolume/fvMesh/fvMesh.H
@@ -216,9 +216,8 @@ public:
         );
 
 
-    // Destructor
-
-        virtual ~fvMesh();
+    //- Destructor
+    virtual ~fvMesh();
 
 
     // Member Functions
@@ -285,6 +284,15 @@ public:
             }
 
 
+            // Communication support
+
+                //- Return communicator used for parallel communication
+                label comm() const
+                {
+                    return polyMesh::comm();
+                }
+
+
             //- Return cell volumes
             const DimensionedField<scalar, volMesh>& V() const;
 
diff --git a/src/finiteVolume/fvMesh/fvPatches/constraint/cyclicGgi/cyclicGgiFvPatch.H b/src/finiteVolume/fvMesh/fvPatches/constraint/cyclicGgi/cyclicGgiFvPatch.H
index d47c4079d..35f550a39 100644
--- a/src/finiteVolume/fvMesh/fvPatches/constraint/cyclicGgi/cyclicGgiFvPatch.H
+++ b/src/finiteVolume/fvMesh/fvPatches/constraint/cyclicGgi/cyclicGgiFvPatch.H
@@ -90,10 +90,9 @@ public:
         {}
 
 
-    // Destructor
-
-        virtual ~cyclicGgiFvPatch()
-        {}
+    //- Destructor
+    virtual ~cyclicGgiFvPatch()
+    {}
 
 
     // Member functions
diff --git a/src/finiteVolume/fvMesh/fvPatches/constraint/ggi/ggiFvPatch.H b/src/finiteVolume/fvMesh/fvPatches/constraint/ggi/ggiFvPatch.H
index 3006723c8..c063d185d 100644
--- a/src/finiteVolume/fvMesh/fvPatches/constraint/ggi/ggiFvPatch.H
+++ b/src/finiteVolume/fvMesh/fvPatches/constraint/ggi/ggiFvPatch.H
@@ -92,13 +92,27 @@ public:
         {}
 
 
-    // Destructor
-
-        virtual ~ggiFvPatch();
+    //- Destructor
+    virtual ~ggiFvPatch();
 
 
     // Member functions
 
+        // Communication support
+
+            //- Return communicator used for parallel communication
+            virtual int comm() const
+            {
+                return ggiPolyPatch_.comm();
+            }
+
+            //- Return message tag used for sending
+            virtual int tag() const
+            {
+                return ggiPolyPatch_.tag();
+            }
+
+
         // Access
 
             //- Return shadow patch
diff --git a/src/finiteVolume/fvMesh/fvPatches/constraint/mixingPlane/mixingPlaneFvPatch.H b/src/finiteVolume/fvMesh/fvPatches/constraint/mixingPlane/mixingPlaneFvPatch.H
index c03fcab6f..5f4348b4a 100644
--- a/src/finiteVolume/fvMesh/fvPatches/constraint/mixingPlane/mixingPlaneFvPatch.H
+++ b/src/finiteVolume/fvMesh/fvPatches/constraint/mixingPlane/mixingPlaneFvPatch.H
@@ -95,13 +95,27 @@ public:
         {}
 
 
-    // Destructor
-
-        virtual ~mixingPlaneFvPatch();
+    //- Destructor
+    virtual ~mixingPlaneFvPatch();
 
 
     // Member functions
 
+        // Communication support
+
+            //- Return communicator used for parallel communication
+            virtual int comm() const
+            {
+                return mixingPlanePolyPatch_.comm();
+            }
+
+            //- Return message tag used for sending
+            virtual int tag() const
+            {
+                return mixingPlanePolyPatch_.tag();
+            }
+
+
         // Access
 
             //- Return shadow patch
diff --git a/src/finiteVolume/fvMesh/fvPatches/constraint/processor/processorFvPatch.H b/src/finiteVolume/fvMesh/fvPatches/constraint/processor/processorFvPatch.H
index 4e4cbfa6e..77fc4809b 100644
--- a/src/finiteVolume/fvMesh/fvPatches/constraint/processor/processorFvPatch.H
+++ b/src/finiteVolume/fvMesh/fvPatches/constraint/processor/processorFvPatch.H
@@ -86,10 +86,9 @@ public:
         {}
 
 
-    // Destructor
-
-        virtual ~processorFvPatch()
-        {}
+    //- Destructor
+    virtual ~processorFvPatch()
+    {}
 
 
     // Member functions
@@ -119,6 +118,21 @@ public:
             }
         }
 
+        // Communication support
+
+            //- Return communicator used for parallel communication
+            virtual int comm() const
+            {
+                return procPolyPatch_.comm();
+            }
+
+            //- Return message tag used for sending
+            virtual int tag() const
+            {
+                return procPolyPatch_.tag();
+            }
+
+
         //- Return face transformation tensor
         virtual const tensorField& forwardT() const
         {
diff --git a/src/finiteVolume/fvMesh/fvPatches/constraint/regionCouple/regionCoupleFvPatch.H b/src/finiteVolume/fvMesh/fvPatches/constraint/regionCouple/regionCoupleFvPatch.H
index 9603b54a1..50b529962 100644
--- a/src/finiteVolume/fvMesh/fvPatches/constraint/regionCouple/regionCoupleFvPatch.H
+++ b/src/finiteVolume/fvMesh/fvPatches/constraint/regionCouple/regionCoupleFvPatch.H
@@ -95,9 +95,8 @@ public:
         {}
 
 
-    // Destructor
-
-        virtual ~regionCoupleFvPatch();
+    //- Destructor
+    virtual ~regionCoupleFvPatch();
 
 
     // Member functions
@@ -123,6 +122,21 @@ public:
             virtual tmp<vectorField> delta() const;
 
 
+        // Communication support
+
+            //- Return communicator used for parallel communication
+            virtual int comm() const
+            {
+                return rcPolyPatch_.comm();
+            }
+
+            //- Return message tag used for sending
+            virtual int tag() const
+            {
+                return rcPolyPatch_.tag();
+            }
+
+
         // Interpolation
 
             //- Interpolate face field
diff --git a/src/foam/Make/files b/src/foam/Make/files
index 4218eac70..d506e7a3f 100644
--- a/src/foam/Make/files
+++ b/src/foam/Make/files
@@ -147,7 +147,6 @@ $(Pstreams)/IPstream.C
 $(Pstreams)/OPstream.C
 $(Pstreams)/IPread.C
 $(Pstreams)/OPwrite.C
-$(Pstreams)/PstreamsPrint.C
 
 dictionary = db/dictionary
 $(dictionary)/dictionary.C
diff --git a/src/foam/db/IOstreams/Pstreams/IPread.C b/src/foam/db/IOstreams/Pstreams/IPread.C
index d493491f2..69142cc4a 100644
--- a/src/foam/db/IOstreams/Pstreams/IPread.C
+++ b/src/foam/db/IOstreams/Pstreams/IPread.C
@@ -38,6 +38,8 @@ Foam::IPstream::IPstream
     const commsTypes commsType,
     const int fromProcNo,
     const label bufSize,
+    const int tag,
+    const label comm,
     streamFormat format,
     versionNumber version
 )
@@ -45,6 +47,8 @@ Foam::IPstream::IPstream
     Pstream(commsType, bufSize),
     Istream(format, version),
     fromProcNo_(fromProcNo),
+    tag_(tag),
+    comm_(comm),
     messageSize_(0)
 {
     setOpened();
@@ -52,17 +56,31 @@ Foam::IPstream::IPstream
 
     MPI_Status status;
 
-    // If the buffer size is not specified, probe the incomming message
+    // If the buffer size is not specified, probe the incoming message
     // and set it
     if (!bufSize)
     {
-        MPI_Probe(procID(fromProcNo_), msgType(), MPI_COMM_WORLD, &status);
+        MPI_Probe
+        (
+            fromProcNo_,
+            tag_,
+            PstreamGlobals::MPICommunicators_[comm_],
+            &status
+        );
         MPI_Get_count(&status, MPI_BYTE, &messageSize_);
 
         buf_.setSize(messageSize_);
     }
 
-    messageSize_ = read(commsType, fromProcNo_, buf_.begin(), buf_.size());
+    messageSize_ = IPstream::read
+    (
+        commsType,
+        fromProcNo_,
+        buf_.begin(),
+        buf_.size(),
+        tag_,
+        comm_
+    );
 
     if (!messageSize_)
     {
@@ -83,9 +101,31 @@ Foam::label Foam::IPstream::read
     const commsTypes commsType,
     const int fromProcNo,
     char* buf,
-    const std::streamsize bufSize
+    const std::streamsize bufSize,
+    const int tag,
+    const label comm
 )
 {
+    if (debug)
+    {
+        Pout<< "UIPstream::read : starting read from:" << fromProcNo
+            << " tag:" << tag << " comm:" << comm
+            << " wanted size:" << label(bufSize)
+            << " commsType:" << Pstream::commsTypeNames[commsType]
+            << Foam::endl;
+    }
+
+    if (Pstream::warnComm != -1 && comm != Pstream::warnComm)
+    {
+        Pout<< "UIPstream::read : starting read from:" << fromProcNo
+            << " tag:" << tag << " comm:" << comm
+            << " wanted size:" << label(bufSize)
+            << " commsType:" << Pstream::commsTypeNames[commsType]
+            << " warnComm:" << Pstream::warnComm
+            << Foam::endl;
+        error::printStack(Pout);
+    }
+
     if (commsType == blocking || commsType == scheduled)
     {
         MPI_Status status;
@@ -96,10 +136,10 @@ Foam::label Foam::IPstream::read
             (
                 buf,
                 bufSize,
-                MPI_PACKED,
-                procID(fromProcNo),
-                msgType(),
-                MPI_COMM_WORLD,
+                MPI_BYTE,
+                fromProcNo,
+                tag,
+                PstreamGlobals::MPICommunicators_[comm],
                 &status
             )
         )
@@ -144,10 +184,10 @@ Foam::label Foam::IPstream::read
             (
                 buf,
                 bufSize,
-                MPI_PACKED,
-                procID(fromProcNo),
-                msgType(),
-                MPI_COMM_WORLD,
+                MPI_BYTE,
+                fromProcNo,
+                tag,
+                PstreamGlobals::MPICommunicators_[comm],
                 &request
             )
         )
@@ -162,8 +202,18 @@ Foam::label Foam::IPstream::read
             return 0;
         }
 
-        PstreamGlobals::IPstream_outstandingRequests_.append(request);
+        if (debug)
+        {
+            Pout<< "UIPstream::read : started read from:" << fromProcNo
+                << " tag:" << tag << " read size:" << label(bufSize)
+                << " commsType:" << Pstream::commsTypeNames[commsType]
+                << " request:" << PstreamGlobals::outstandingRequests_.size()
+                << Foam::endl;
+        }
 
+        PstreamGlobals::outstandingRequests_.append(request);
+
+        // Assume the message is completely received.
         return 1;
     }
     else
@@ -180,56 +230,4 @@ Foam::label Foam::IPstream::read
 }
 
 
-void Foam::IPstream::waitRequests()
-{
-    if (PstreamGlobals::IPstream_outstandingRequests_.size())
-    {
-        if
-        (
-            MPI_Waitall
-            (
-                PstreamGlobals::IPstream_outstandingRequests_.size(),
-                PstreamGlobals::IPstream_outstandingRequests_.begin(),
-                MPI_STATUSES_IGNORE
-            )
-        )
-        {
-            FatalErrorIn
-            (
-                "IPstream::waitRequests()"
-            )   << "MPI_Waitall returned with error" << endl;
-        }
-
-        PstreamGlobals::IPstream_outstandingRequests_.clear();
-    }
-}
-
-
-bool Foam::IPstream::finishedRequest(const label i)
-{
-    if (i >= PstreamGlobals::IPstream_outstandingRequests_.size())
-    {
-        FatalErrorIn
-        (
-            "IPstream::finishedRequest(const label)"
-        )   << "There are "
-            << PstreamGlobals::IPstream_outstandingRequests_.size()
-            << " outstanding send requests and you are asking for i=" << i
-            << nl
-            << "Maybe you are mixing blocking/non-blocking comms?"
-            << Foam::abort(FatalError);
-    }
-
-    int flag;
-    MPI_Test
-    (
-        &PstreamGlobals::IPstream_outstandingRequests_[i],
-        &flag,
-        MPI_STATUS_IGNORE
-    );
-
-    return flag != 0;
-}
-
-
 // ************************************************************************* //
diff --git a/src/foam/db/IOstreams/Pstreams/IPreadToken.C b/src/foam/db/IOstreams/Pstreams/IPreadToken.C
deleted file mode 100644
index d4b8d228c..000000000
--- a/src/foam/db/IOstreams/Pstreams/IPreadToken.C
+++ /dev/null
@@ -1,187 +0,0 @@
-/*---------------------------------------------------------------------------*\
-  =========                 |
-  \\      /  F ield         | foam-extend: Open Source CFD
-   \\    /   O peration     | Version:     4.0
-    \\  /    A nd           | Web:         http://www.foam-extend.org
-     \\/     M anipulation  | For copyright notice see file Copyright
--------------------------------------------------------------------------------
-License
-    This file is part of foam-extend.
-
-    foam-extend is free software: you can redistribute it and/or modify it
-    under the terms of the GNU General Public License as published by the
-    Free Software Foundation, either version 3 of the License, or (at your
-    option) any later version.
-
-    foam-extend is distributed in the hope that it will be useful, but
-    WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-    General Public License for more details.
-
-    You should have received a copy of the GNU General Public License
-    along with foam-extend.  If not, see <http://www.gnu.org/licenses/>.
-
-\*---------------------------------------------------------------------------*/
-
-#include "IPstream.H"
-#include "token.H"
-#include <cctype>
-
-// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
-
-namespace Foam
-{
-
-// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
-
-Istream& IPstream::read(token& t)
-{
-    // Return the put back token if it exists
-    if (Istream::getBack(t))
-    {
-        return *this;
-    }
-
-    char c;
-
-    // return on error
-    if (!read(c))
-    {
-        t.setBad();
-        return *this;
-    }
-
-    // Set the line number of this token to the current stream line number
-    t.lineNumber() = lineNumber();
-
-    // Analyse input starting with this character.
-    switch (c)
-    {
-        // Punctuation
-        case token::END_STATEMENT :
-        case token::BEGIN_LIST :
-        case token::END_LIST :
-        case token::BEGIN_SQR :
-        case token::END_SQR :
-        case token::BEGIN_BLOCK :
-        case token::END_BLOCK :
-        case token::COLON :
-        case token::COMMA :
-        case token::ASSIGN :
-        case token::ADD :
-        case token::SUBTRACT :
-        case token::MULTIPLY :
-        case token::DIVIDE :
-        {
-            t = token::punctuationToken(c);
-            return *this;
-        }
-
-        // Word
-        case token::WORD :
-        {
-            word* wPtr = new word;
-            if (read(*wPtr))
-            {
-                if (token::compound::isCompound(*wPtr))
-                {
-                    t = token::compound::New(*wPtr, *this).ptr();
-                    delete wPtr;
-                }
-                else
-                {
-                    t = wPtr;
-                }
-            }
-            else
-            {
-                delete wPtr;
-                t.setBad();
-            }
-            return *this;
-        }
-
-        // String
-        case token::STRING :
-        {
-            string* sPtr = new string;
-            if (read(*sPtr))
-            {
-                t = sPtr;
-            }
-            else
-            {
-                delete sPtr;
-                t.setBad();
-            }
-            return *this;
-        }
-
-        // Label
-        case token::LABEL :
-        {
-            label l;
-            if (read(l))
-            {
-                t = l;
-            }
-            else
-            {
-                t.setBad();
-            }
-            return *this;
-        }
-
-        // floatScalar
-        case token::FLOAT_SCALAR :
-        {
-            floatScalar s;
-            if (read(s))
-            {
-                t = s;
-            }
-            else
-            {
-                t.setBad();
-            }
-            return *this;
-        }
-
-        // doubleScalar
-        case token::DOUBLE_SCALAR :
-        {
-            doubleScalar s;
-            if (read(s))
-            {
-                t = s;
-            }
-            else
-            {
-                t.setBad();
-            }
-            return *this;
-        }
-
-        // Character (returned as a single character word) or error
-        default:
-        {
-            if (isalpha(c))
-            {
-                t = word(c);
-                return *this;
-            }
-
-            setBad();
-            t.setBad();
-
-            return *this;
-        }
-    }
-}
-
-
-// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
-
-} // End namespace Foam
-
-// ************************************************************************* //
diff --git a/src/foam/db/IOstreams/Pstreams/IPstream.C b/src/foam/db/IOstreams/Pstreams/IPstream.C
index b3b31946d..090c1ceb6 100644
--- a/src/foam/db/IOstreams/Pstreams/IPstream.C
+++ b/src/foam/db/IOstreams/Pstreams/IPstream.C
@@ -324,4 +324,13 @@ Foam::Istream& Foam::IPstream::rewind()
 }
 
 
+void Foam::IPstream::print(Ostream& os) const
+{
+    os  << "Reading from processor " << fromProcNo_
+        << " using communicator " << comm_
+        <<  " and tag " << tag_
+        << Foam::endl;
+}
+
+
 // ************************************************************************* //
diff --git a/src/foam/db/IOstreams/Pstreams/IPstream.H b/src/foam/db/IOstreams/Pstreams/IPstream.H
index c69b024bc..bac005a20 100644
--- a/src/foam/db/IOstreams/Pstreams/IPstream.H
+++ b/src/foam/db/IOstreams/Pstreams/IPstream.H
@@ -58,6 +58,12 @@ class IPstream
         //- ID of sending processor
         int fromProcNo_;
 
+        //- Message tag
+        const int tag_;
+
+        //- Communicator
+        const label comm_;
+
         //- Message size
         label messageSize_;
 
@@ -86,12 +92,14 @@ public:
             const commsTypes commsType,
             const int fromProcNo,
             const label bufSize = 0,
+            const int tag = Pstream::msgType(),
+            const label comm = Pstream::worldComm,
             streamFormat format = BINARY,
             versionNumber version = currentVersion
         );
 
 
-    // Destructor
+    //- Destructor
     ~IPstream();
 
 
@@ -115,15 +123,11 @@ public:
                 const commsTypes commsType,
                 const int fromProcNo,
                 char* buf,
-                const std::streamsize bufSize
+                const std::streamsize bufSize,
+                const int tag = Pstream::msgType(),
+                const label communicator = 0
             );
 
-            //- Non-blocking receives: wait until all have finished.
-            static void waitRequests();
-
-            //- Non-blocking receives: has request i finished?
-            static bool finishedRequest(const label i);
-
             //- Return next token from stream
             Istream& read(token&);
 
diff --git a/src/foam/db/IOstreams/Pstreams/OPstream.C b/src/foam/db/IOstreams/Pstreams/OPstream.C
index 204330e6b..d643f27f1 100644
--- a/src/foam/db/IOstreams/Pstreams/OPstream.C
+++ b/src/foam/db/IOstreams/Pstreams/OPstream.C
@@ -92,13 +92,17 @@ Foam::OPstream::OPstream
     const commsTypes commsType,
     const int toProcNo,
     const label bufSize,
+    const int tag,
+    const label comm,
     streamFormat format,
     versionNumber version
 )
 :
     Pstream(commsType, bufSize),
     Ostream(format, version),
-    toProcNo_(toProcNo)
+    toProcNo_(toProcNo),
+    tag_(tag),
+    comm_(comm)
 {
     setOpened();
     setGood();
@@ -233,4 +237,12 @@ Foam::Ostream& Foam::OPstream::write(const char* data, std::streamsize count)
 }
 
 
+void Foam::OPstream::print(Ostream& os) const
+{
+    os  << "Writing from processor " << toProcNo_
+        << " to processor " << myProcNo() << " in communicator " << comm_
+        << " and tag " << tag_ << Foam::endl;
+}
+
+
 // ************************************************************************* //
diff --git a/src/foam/db/IOstreams/Pstreams/OPstream.H b/src/foam/db/IOstreams/Pstreams/OPstream.H
index 623637618..c68e563f8 100644
--- a/src/foam/db/IOstreams/Pstreams/OPstream.H
+++ b/src/foam/db/IOstreams/Pstreams/OPstream.H
@@ -58,6 +58,12 @@ class OPstream
          // ID of receiving processor
         int toProcNo_;
 
+        //- Message tag
+        const int tag_;
+
+        //- Communicator
+        const label comm_;
+
 
     // Private member functions
 
@@ -88,14 +94,15 @@ public:
             const commsTypes commsType,
             const int toProcNo,
             const label bufSize = 0,
+            const int tag = Pstream::msgType(),
+            const label comm = Pstream::worldComm,
             streamFormat format = BINARY,
             versionNumber version = currentVersion
         );
 
 
-    // Destructor
-
-        ~OPstream();
+    //- Destructor
+    ~OPstream();
 
 
     // Member functions
@@ -117,15 +124,11 @@ public:
                 const commsTypes commsType,
                 const int toProcNo,
                 const char* buf,
-                const std::streamsize bufSize
+                const std::streamsize bufSize,
+                const int tag = Pstream::msgType(),
+                const label comm = Pstream::worldComm
             );
 
-            //- Non-blocking writes: wait until all have finished.
-            static void waitRequests();
-
-            //- Non-blocking writes: has request i finished?
-            static bool finishedRequest(const label i);
-
             //- Write next token to stream
             Ostream& write(const token&);
 
diff --git a/src/foam/db/IOstreams/Pstreams/OPwrite.C b/src/foam/db/IOstreams/Pstreams/OPwrite.C
index 644a83b00..8fee41230 100644
--- a/src/foam/db/IOstreams/Pstreams/OPwrite.C
+++ b/src/foam/db/IOstreams/Pstreams/OPwrite.C
@@ -60,9 +60,33 @@ bool Foam::OPstream::write
     const commsTypes commsType,
     const int toProcNo,
     const char* buf,
-    const std::streamsize bufSize
+    const std::streamsize bufSize,
+    const int tag,
+    const label comm
 )
 {
+    if (debug)
+    {
+        Pout<< "OPstream::write : starting write to:" << toProcNo
+            << " tag:" << tag
+            << " comm:" << comm << " size:" << label(bufSize)
+            << " commsType:" << Pstream::commsTypeNames[commsType]
+            << Foam::endl;
+    }
+    if (Pstream::warnComm != -1 && comm != Pstream::warnComm)
+    {
+        Pout<< "OPstream::write : starting write to:" << toProcNo
+            << " tag:" << tag
+            << " comm:" << comm << " size:" << label(bufSize)
+            << " commsType:" << Pstream::commsTypeNames[commsType]
+            << " warnComm:" << Pstream::warnComm
+            << Foam::endl;
+        error::printStack(Pout);
+    }
+
+
+    PstreamGlobals::checkCommunicator(comm, toProcNo);
+
     bool transferFailed = true;
 
     if (commsType == blocking)
@@ -71,11 +95,19 @@ bool Foam::OPstream::write
         (
             const_cast<char*>(buf),
             bufSize,
-            MPI_PACKED,
-            procID(toProcNo),
-            msgType(),
-            MPI_COMM_WORLD
+            MPI_BYTE,
+            toProcNo,   //procID(toProcNo),
+            tag,
+            PstreamGlobals::MPICommunicators_[comm] // MPI_COMM_WORLD
         );
+
+        if (debug)
+        {
+            Pout<< "OPstream::write : finished write to:" << toProcNo
+                << " tag:" << tag << " size:" << label(bufSize)
+                << " commsType:" << Pstream::commsTypeNames[commsType]
+                << Foam::endl;
+        }
     }
     else if (commsType == scheduled)
     {
@@ -83,11 +115,19 @@ bool Foam::OPstream::write
         (
             const_cast<char*>(buf),
             bufSize,
-            MPI_PACKED,
-            procID(toProcNo),
-            msgType(),
-            MPI_COMM_WORLD
+            MPI_BYTE,
+            toProcNo,   //procID(toProcNo),
+            tag,
+            PstreamGlobals::MPICommunicators_[comm] // MPI_COMM_WORLD
         );
+
+        if (debug)
+        {
+            Pout<< "OPstream::write : finished write to:" << toProcNo
+                << " tag:" << tag << " size:" << label(bufSize)
+                << " commsType:" << Pstream::commsTypeNames[commsType]
+                << Foam::endl;
+        }
     }
     else if (commsType == nonBlocking)
     {
@@ -97,14 +137,23 @@ bool Foam::OPstream::write
         (
             const_cast<char*>(buf),
             bufSize,
-            MPI_PACKED,
-            procID(toProcNo),
-            msgType(),
-            MPI_COMM_WORLD,
+            MPI_BYTE,
+            toProcNo,   //procID(toProcNo),
+            tag,
+            PstreamGlobals::MPICommunicators_[comm],// MPI_COMM_WORLD,
             &request
         );
 
-        PstreamGlobals::OPstream_outstandingRequests_.append(request);
+        if (debug)
+        {
+            Pout<< "OPstream::write : started write to:" << toProcNo
+                << " tag:" << tag << " size:" << label(bufSize)
+                << " commsType:" << Pstream::commsTypeNames[commsType]
+                << " request:" << PstreamGlobals::outstandingRequests_.size()
+                << Foam::endl;
+        }
+
+        PstreamGlobals::outstandingRequests_.append(request);
     }
     else
     {
@@ -120,56 +169,4 @@ bool Foam::OPstream::write
 }
 
 
-void Foam::OPstream::waitRequests()
-{
-    if (PstreamGlobals::OPstream_outstandingRequests_.size())
-    {
-        if
-        (
-            MPI_Waitall
-            (
-                PstreamGlobals::OPstream_outstandingRequests_.size(),
-                PstreamGlobals::OPstream_outstandingRequests_.begin(),
-                MPI_STATUSES_IGNORE
-            )
-        )
-        {
-            FatalErrorIn
-            (
-                "OPstream::waitRequests()"
-            )   << "MPI_Waitall returned with error" << Foam::endl;
-        }
-
-        PstreamGlobals::OPstream_outstandingRequests_.clear();
-    }
-}
-
-
-bool Foam::OPstream::finishedRequest(const label i)
-{
-    if (i >= PstreamGlobals::OPstream_outstandingRequests_.size())
-    {
-        FatalErrorIn
-        (
-            "OPstream::finishedRequest(const label)"
-        )   << "There are "
-            << PstreamGlobals::OPstream_outstandingRequests_.size()
-            << " outstanding send requests and you are asking for i=" << i
-            << nl
-            << "Maybe you are mixing blocking/non-blocking comms?"
-            << Foam::abort(FatalError);
-    }
-
-    int flag;
-    MPI_Test
-    (
-        &PstreamGlobals::OPstream_outstandingRequests_[i],
-        &flag,
-        MPI_STATUS_IGNORE
-    );
-
-    return flag != 0;
-}
-
-
 // ************************************************************************* //
diff --git a/src/foam/db/IOstreams/Pstreams/Pstream.C b/src/foam/db/IOstreams/Pstreams/Pstream.C
index ad89c4591..14fc149ac 100644
--- a/src/foam/db/IOstreams/Pstreams/Pstream.C
+++ b/src/foam/db/IOstreams/Pstreams/Pstream.C
@@ -23,11 +23,6 @@ License
 
 \*---------------------------------------------------------------------------*/
 
-
-#include <cstring>
-#include <cstdlib>
-#include <csignal>
-
 #include "mpi.h"
 
 #include "Pstream.H"
@@ -35,6 +30,13 @@ License
 #include "debug.H"
 #include "dictionary.H"
 #include "OSspecific.H"
+#include "PstreamGlobals.H"
+#include "SubList.H"
+#include "allReduce.H"
+
+#include <cstring>
+#include <cstdlib>
+#include <csignal>
 
 #if defined(WM_SP)
 #   define MPI_SCALAR MPI_FLOAT
@@ -63,27 +65,42 @@ const Foam::NamedEnum<Foam::Pstream::commsTypes, 3>
 
 // * * * * * * * * * * * * * Private Member Functions  * * * * * * * * * * * //
 
-void Foam::Pstream::setParRun()
+void Foam::Pstream::setParRun(const label nProcs)
 {
     parRun_ = true;
 
+    // Redo worldComm communicator (created at static initialisation)
+    freeCommunicator(Pstream::worldComm);
+    label comm = allocateCommunicator(-1, identity(nProcs), true);
+
+    if (comm != Pstream::worldComm)
+    {
+        FatalErrorIn("Pstream::setParRun(const label)")
+            << "problem : comm:" << comm
+            << "  Pstream::worldComm:" << Pstream::worldComm
+            << Foam::exit(FatalError);
+    }
+
     Pout.prefix() = '[' +  name(myProcNo()) + "] ";
     Perr.prefix() = '[' +  name(myProcNo()) + "] ";
 }
 
 
-void Foam::Pstream::calcLinearComm(const label nProcs)
+Foam::List<Foam::Pstream::commsStruct> Foam::Pstream::calcLinearComm
+(
+    const label nProcs
+)
 {
-    linearCommunication_.setSize(nProcs);
+    List<commsStruct> linearCommunication(nProcs);
 
     // Master
     labelList belowIDs(nProcs - 1);
-    forAll(belowIDs, i)
+    forAll (belowIDs, i)
     {
         belowIDs[i] = i + 1;
     }
 
-    linearCommunication_[0] = commsStruct
+    linearCommunication[0] = commsStruct
     (
         nProcs,
         0,
@@ -91,11 +108,11 @@ void Foam::Pstream::calcLinearComm(const label nProcs)
         belowIDs,
         labelList(0)
     );
-
+    Pout<< "SIZE: " << linearCommunication.size() << " nProcs: " << nProcs << endl;
     // Slaves. Have no below processors, only communicate up to master
     for (label procID = 1; procID < nProcs; procID++)
     {
-        linearCommunication_[procID] = commsStruct
+        linearCommunication[procID] = commsStruct
         (
             nProcs,
             procID,
@@ -104,24 +121,8 @@ void Foam::Pstream::calcLinearComm(const label nProcs)
             labelList(0)
         );
     }
-}
 
-
-// Append my children (and my children children etc.) to allReceives.
-void Foam::Pstream::collectReceives
-(
-    const label procID,
-    const List<dynamicLabelList >& receives,
-    dynamicLabelList& allReceives
-)
-{
-    const dynamicLabelList& myChildren = receives[procID];
-
-    forAll(myChildren, childI)
-    {
-        allReceives.append(myChildren[childI]);
-        collectReceives(myChildren[childI], receives, allReceives);
-    }
+    return linearCommunication;
 }
 
 
@@ -151,7 +152,10 @@ void Foam::Pstream::collectReceives
 //  5       -               4
 //  6       7               4
 //  7       -               6
-void Foam::Pstream::calcTreeComm(label nProcs)
+Foam::List<Foam::Pstream::commsStruct> Foam::Pstream::calcTreeComm
+(
+    const label nProcs
+)
 {
     label nLevels = 1;
     while ((1 << nLevels) < nProcs)
@@ -159,7 +163,7 @@ void Foam::Pstream::calcTreeComm(label nProcs)
         nLevels++;
     }
 
-    List<dynamicLabelList > receives(nProcs);
+    List<dynamicLabelList> receives(nProcs);
     labelList sends(nProcs, -1);
 
     // Info<< "Using " << nLevels << " communication levels" << endl;
@@ -190,18 +194,18 @@ void Foam::Pstream::calcTreeComm(label nProcs)
 
     // For all processors find the processors it receives data from
     // (and the processors they receive data from etc.)
-    List<dynamicLabelList > allReceives(nProcs);
+    List<dynamicLabelList> allReceives(nProcs);
     for (label procID = 0; procID < nProcs; procID++)
     {
         collectReceives(procID, receives, allReceives[procID]);
     }
 
 
-    treeCommunication_.setSize(nProcs);
+    List<commsStruct> treeCommunication(nProcs);
 
     for (label procID = 0; procID < nProcs; procID++)
     {
-        treeCommunication_[procID] = commsStruct
+        treeCommunication[procID] = commsStruct
         (
             nProcs,
             procID,
@@ -210,6 +214,26 @@ void Foam::Pstream::calcTreeComm(label nProcs)
             allReceives[procID].shrink()
         );
     }
+
+    return treeCommunication;
+}
+
+
+// Append my children (and my children;s children etc.) to allReceives.
+void Foam::Pstream::collectReceives
+(
+    const label procID,
+    const List<dynamicLabelList>& receives,
+    dynamicLabelList& allReceives
+)
+{
+    const dynamicLabelList& myChildren = receives[procID];
+
+    forAll (myChildren, childI)
+    {
+        allReceives.append(myChildren[childI]);
+        collectReceives(myChildren[childI], receives, allReceives);
+    }
 }
 
 
@@ -222,6 +246,294 @@ void Foam::Pstream::initCommunicationSchedule()
 }
 
 
+void Foam::Pstream::allocatePstreamCommunicator
+(
+    const label parentIndex,
+    const label index
+)
+{
+    if (index == PstreamGlobals::MPIGroups_.size())
+    {
+        // Extend storage with dummy values
+        MPI_Group newGroup;
+        PstreamGlobals::MPIGroups_.append(newGroup);
+        MPI_Comm newComm;
+        PstreamGlobals::MPICommunicators_.append(newComm);
+    }
+    else if (index > PstreamGlobals::MPIGroups_.size())
+    {
+        FatalErrorIn
+        (
+            "Pstream::allocatePstreamCommunicator\n"
+            "(\n"
+            "    const label parentIndex,\n"
+            "    const labelList& subRanks\n"
+            ")\n"
+        )   << "PstreamGlobals out of sync with Pstream data. Problem."
+            << Foam::exit(FatalError);
+    }
+
+
+    if (parentIndex == -1)
+    {
+        // Allocate world communicator
+
+        if (index != Pstream::worldComm)
+        {
+            FatalErrorIn
+            (
+                "Pstream::allocateCommunicator\n"
+                "(\n"
+                "    const label parentIndex,\n"
+                "    const labelList& subRanks\n"
+                ")\n"
+            )   << "world communicator should always be index "
+                << Pstream::worldComm << Foam::exit(FatalError);
+        }
+
+        PstreamGlobals::MPICommunicators_[index] = MPI_COMM_WORLD;
+        MPI_Comm_group(MPI_COMM_WORLD, &PstreamGlobals::MPIGroups_[index]);
+        MPI_Comm_rank
+        (
+            PstreamGlobals::MPICommunicators_[index],
+           &myProcNo_[index]
+        );
+
+        // Set the number of processes to the actual number
+        int numProcs;
+        MPI_Comm_size(PstreamGlobals::MPICommunicators_[index], &numProcs);
+        procIDs_[index] = identity(numProcs);
+    }
+    else
+    {
+        // Create new group
+        MPI_Group_incl
+        (
+            PstreamGlobals::MPIGroups_[parentIndex],
+            procIDs_[index].size(),
+            procIDs_[index].begin(),
+            &PstreamGlobals::MPIGroups_[index]
+        );
+
+        // Create new communicator
+        MPI_Comm_create
+        (
+            PstreamGlobals::MPICommunicators_[parentIndex],
+            PstreamGlobals::MPIGroups_[index],
+            &PstreamGlobals::MPICommunicators_[index]
+        );
+
+        if (PstreamGlobals::MPICommunicators_[index] == MPI_COMM_NULL)
+        {
+            myProcNo_[index] = -1;
+        }
+        else
+        {
+            if
+            (
+                MPI_Comm_rank
+                (
+                    PstreamGlobals::MPICommunicators_[index],
+                   &myProcNo_[index]
+                )
+            )
+            {
+                FatalErrorIn
+                (
+                    "Pstream::allocatePstreamCommunicator\n"
+                    "(\n"
+                    "    const label,\n"
+                    "    const labelList&\n"
+                    ")\n"
+                )   << "Problem :"
+                    << " when allocating communicator at " << index
+                    << " from ranks " << procIDs_[index]
+                    << " of parent " << parentIndex
+                    << " cannot find my own rank"
+                    << Foam::exit(FatalError);
+            }
+        }
+    }
+}
+
+
+void Foam::Pstream::freePstreamCommunicator(const label communicator)
+{
+    if (communicator != Pstream::worldComm)
+    {
+        if (PstreamGlobals::MPICommunicators_[communicator] != MPI_COMM_NULL)
+        {
+            // Free communicator. Sets communicator to MPI_COMM_NULL
+            MPI_Comm_free(&PstreamGlobals::MPICommunicators_[communicator]);
+        }
+        if (PstreamGlobals::MPIGroups_[communicator] != MPI_GROUP_NULL)
+        {
+            // Free greoup. Sets group to MPI_GROUP_NULL
+            MPI_Group_free(&PstreamGlobals::MPIGroups_[communicator]);
+        }
+    }
+}
+
+
+Foam::label Foam::Pstream::allocateCommunicator
+(
+    const label parentIndex,
+    const labelList& subRanks,
+    const bool doPstream
+)
+{
+    label index;
+    if (!freeComms_.empty())
+    {
+        index = freeComms_.pop();
+    }
+    else
+    {
+        // Extend storage
+        index = parentCommunicator_.size();
+
+        myProcNo_.append(-1);
+        procIDs_.append(List<int>(0));
+        parentCommunicator_.append(-1);
+        linearCommunication_.append(List<commsStruct>(0));
+        treeCommunication_.append(List<commsStruct>(0));
+    }
+
+    if (debug)
+    {
+        Pout<< "Communicators : Allocating communicator " << index << endl
+            << "    parent : " << parentIndex << endl
+            << "    procs  : " << subRanks << endl
+            << endl;
+    }
+
+    // Initialise; overwritten by allocatePstreamCommunicator
+    myProcNo_[index] = 0;
+
+    // Convert from label to int
+    procIDs_[index].setSize(subRanks.size());
+    forAll (procIDs_[index], i)
+    {
+        procIDs_[index][i] = subRanks[i];
+
+        // Enforce incremental order (so index is rank in next communicator)
+        if (i >= 1 && subRanks[i] <= subRanks[i-1])
+        {
+            FatalErrorIn
+            (
+                "Pstream::allocateCommunicator"
+                "(const label, const labelList&, const bool)"
+            )   << "subranks not sorted : " << subRanks
+                << " when allocating subcommunicator from parent "
+                << parentIndex
+                << Foam::abort(FatalError);
+        }
+    }
+    parentCommunicator_[index] = parentIndex;
+
+    linearCommunication_[index] = calcLinearComm(procIDs_[index].size());
+    treeCommunication_[index] = calcTreeComm(procIDs_[index].size());
+
+
+    if (doPstream && parRun())
+    {
+        allocatePstreamCommunicator(parentIndex, index);
+    }
+
+    return index;
+}
+
+
+void Foam::Pstream::freeCommunicator
+(
+    const label communicator,
+    const bool doPstream
+)
+{
+    if (debug)
+    {
+        Pout<< "Communicators : Freeing communicator " << communicator << endl
+            << "    parent   : " << parentCommunicator_[communicator] << endl
+            << "    myProcNo : " << myProcNo_[communicator] << endl
+            << endl;
+    }
+
+    if (doPstream && parRun())
+    {
+        freePstreamCommunicator(communicator);
+    }
+    myProcNo_[communicator] = -1;
+    //procIDs_[communicator].clear();
+    parentCommunicator_[communicator] = -1;
+    linearCommunication_[communicator].clear();
+    treeCommunication_[communicator].clear();
+
+    freeComms_.push(communicator);
+}
+
+
+void Foam::Pstream::freeCommunicators(const bool doPstream)
+{
+    forAll (myProcNo_, communicator)
+    {
+        if (myProcNo_[communicator] != -1)
+        {
+            freeCommunicator(communicator, doPstream);
+        }
+    }
+}
+
+
+int Foam::Pstream::baseProcNo(const label myComm, const int myProcID)
+{
+    int procID = myProcID;
+    label comm = myComm;
+
+    while (parent(comm) != -1)
+    {
+        const List<int>& parentRanks = Pstream::procID(comm);
+        procID = parentRanks[procID];
+        comm = Pstream::parent(comm);
+    }
+
+    return procID;
+}
+
+
+Foam::label Foam::Pstream::procNo(const label myComm, const int baseProcID)
+{
+    const List<int>& parentRanks = procID(myComm);
+    label parentComm = parent(myComm);
+
+    if (parentComm == -1)
+    {
+        return findIndex(parentRanks, baseProcID);
+    }
+    else
+    {
+        label parentRank = procNo(parentComm, baseProcID);
+        return findIndex(parentRanks, parentRank);
+    }
+}
+
+
+Foam::label Foam::Pstream::procNo
+(
+    const label myComm,
+    const label currentComm,
+    const int currentProcID
+)
+{
+    label physProcID = Pstream::baseProcNo(currentComm, currentProcID);
+    return procNo(myComm, physProcID);
+}
+
+
+
+
+// * * * * * * * * * * * * * * * Member Functions  * * * * * * * * * * * * * //
+
+
 // NOTE:
 // valid parallel options vary between implementations, but flag common ones.
 // if they are not removed by MPI_Init(), the subsequent argument processing
@@ -233,7 +545,6 @@ void Foam::Pstream::addValidParOptions(HashTable<string>& validParOptions)
     validParOptions.insert("p4wd", "directory");
     validParOptions.insert("p4amslave", "");
     validParOptions.insert("p4yourname", "hostname");
-    validParOptions.insert("GAMMANP", "number of instances");
     validParOptions.insert("machinefile", "machine file");
 }
 
@@ -244,24 +555,25 @@ bool Foam::Pstream::init(int& argc, char**& argv)
 
     int numprocs;
     MPI_Comm_size(MPI_COMM_WORLD, &numprocs);
-    MPI_Comm_rank(MPI_COMM_WORLD, &myProcNo_);
+    int myRank;
+    MPI_Comm_rank(MPI_COMM_WORLD, &myRank);
+
+    if (debug)
+    {
+        Pout<< "Pstream::init : initialised with numProcs:" << numprocs
+            << " myRank:" << myRank << endl;
+    }
 
     if (numprocs <= 1)
     {
         FatalErrorIn("Pstream::init(int& argc, char**& argv)")
-            << "bool Pstream::init(int& argc, char**& argv) : "
+            << "bool IPstream::init(int& argc, char**& argv) : "
                "attempt to run parallel on 1 processor"
             << Foam::abort(FatalError);
     }
 
-    procIDs_.setSize(numprocs);
-
-    forAll(procIDs_, procNo)
-    {
-        procIDs_[procNo] = procNo;
-    }
-
-    setParRun();
+    // Initialise parallel structure
+    setParRun(numprocs);
 
 #   ifndef SGIMPI
     string bufferSizeName = getEnv("MPI_BUFFER_SIZE");
@@ -284,15 +596,12 @@ bool Foam::Pstream::init(int& argc, char**& argv)
     }
 #   endif
 
-    int processorNameLen;
-    char processorName[MPI_MAX_PROCESSOR_NAME];
-
-    MPI_Get_processor_name(processorName, &processorNameLen);
-
-    //signal(SIGABRT, stop);
-
-    // Now that nprocs is known construct communication tables.
-    initCommunicationSchedule();
+    //int processorNameLen;
+    //char processorName[MPI_MAX_PROCESSOR_NAME];
+    //
+    //MPI_Get_processor_name(processorName, &processorNameLen);
+    //processorName[processorNameLen] = '\0';
+    //Pout<< "Processor name:" << processorName << endl;
 
     return true;
 }
@@ -300,6 +609,11 @@ bool Foam::Pstream::init(int& argc, char**& argv)
 
 void Foam::Pstream::exit(int errnum)
 {
+    if (debug)
+    {
+        Pout<< "Pstream::exit." << endl;
+    }
+
 #   ifndef SGIMPI
     int size;
     char* buff;
@@ -307,6 +621,28 @@ void Foam::Pstream::exit(int errnum)
     delete[] buff;
 #   endif
 
+    if (PstreamGlobals::outstandingRequests_.size())
+    {
+        label n = PstreamGlobals::outstandingRequests_.size();
+        PstreamGlobals::outstandingRequests_.clear();
+
+        WarningIn("Pstream::exit(int)")
+            << "There are still " << n << " outstanding MPI_Requests." << endl
+            << "This means that your code exited before doing a"
+            << " Pstream::waitRequests()." << endl
+            << "This should not happen for a normal code exit."
+            << endl;
+    }
+
+    // Clean mpi communicators
+    forAll (myProcNo_, communicator)
+    {
+        if (myProcNo_[communicator] != -1)
+        {
+            freePstreamCommunicator(communicator);
+        }
+    }
+
     if (errnum == 0)
     {
         MPI_Finalize();
@@ -325,306 +661,369 @@ void Foam::Pstream::abort()
 }
 
 
-void Foam::reduce(scalar& Value, const sumOp<scalar>& bop)
+void Foam::reduce
+(
+    scalar& Value,
+    const sumOp<scalar>& bop,
+    const int tag,
+    const label comm
+)
 {
-    if (!Pstream::parRun())
+    if (Pstream::warnComm != -1 && comm != Pstream::warnComm)
     {
-        return;
+        Pout<< "** reducing:" << Value << " with comm:" << comm
+            << " warnComm:" << Pstream::warnComm
+            << endl;
+        error::printStack(Pout);
+    }
+    allReduce(Value, 1, MPI_SCALAR, MPI_SUM, bop, tag, comm);
+}
+
+
+void Foam::reduce
+(
+    scalar& Value,
+    const minOp<scalar>& bop,
+    const int tag,
+    const label comm
+)
+{
+    if (Pstream::warnComm != -1 && comm != Pstream::warnComm)
+    {
+        Pout<< "** reducing:" << Value << " with comm:" << comm
+            << " warnComm:" << Pstream::warnComm
+            << endl;
+        error::printStack(Pout);
+    }
+    allReduce(Value, 1, MPI_SCALAR, MPI_MIN, bop, tag, comm);
+}
+
+
+void Foam::reduce
+(
+    vector2D& Value,
+    const sumOp<vector2D>& bop,
+    const int tag,
+    const label comm
+)
+{
+    if (Pstream::warnComm != -1 && comm != Pstream::warnComm)
+    {
+        Pout<< "** reducing:" << Value << " with comm:" << comm
+            << " warnComm:" << Pstream::warnComm
+            << endl;
+        error::printStack(Pout);
+    }
+    allReduce(Value, 2, MPI_SCALAR, MPI_SUM, bop, tag, comm);
+}
+
+
+void Foam::sumReduce
+(
+    scalar& Value,
+    label& Count,
+    const int tag,
+    const label comm
+)
+{
+    if (Pstream::warnComm != -1 && comm != Pstream::warnComm)
+    {
+        Pout<< "** reducing:" << Value << " with comm:" << comm
+            << " warnComm:" << Pstream::warnComm
+            << endl;
+        error::printStack(Pout);
+    }
+    vector2D twoScalars(Value, scalar(Count));
+    reduce(twoScalars, sumOp<vector2D>(), tag, comm);
+
+    Value = twoScalars.x();
+    Count = twoScalars.y();
+}
+
+
+void Foam::reduce
+(
+    scalar& Value,
+    const sumOp<scalar>& bop,
+    const int tag,
+    const label comm,
+    label& requestID
+)
+{
+#ifdef MPIX_COMM_TYPE_SHARED
+    // Assume mpich2 with non-blocking collectives extensions. Once mpi3
+    // is available this will change.
+    MPI_Request request;
+    scalar v = Value;
+    MPIX_Ireduce
+    (
+        &v,
+        &Value,
+        1,
+        MPI_SCALAR,
+        MPI_SUM,
+        0,              // root
+        PstreamGlobals::MPICommunicators_[comm],
+        &request
+    );
+
+    requestID = PstreamGlobals::outstandingRequests_.size();
+    PstreamGlobals::outstandingRequests_.append(request);
+
+    if (debug)
+    {
+        Pout<< "Pstream::allocateRequest for non-blocking reduce"
+            << " : request:" << requestID
+            << endl;
+    }
+#else
+    // Non-blocking not yet implemented in mpi
+    reduce(Value, bop, tag, comm);
+
+    requestID = -1;
+#endif
+}
+
+
+Foam::label Foam::Pstream::nRequests()
+{
+    return PstreamGlobals::outstandingRequests_.size();
+}
+
+
+void Foam::Pstream::resetRequests(const label i)
+{
+    if (i < PstreamGlobals::outstandingRequests_.size())
+    {
+        PstreamGlobals::outstandingRequests_.setSize(i);
+    }
+}
+
+
+void Foam::Pstream::waitRequests(const label start)
+{
+    if (debug)
+    {
+        Pout<< "Pstream::waitRequests : starting wait for "
+            << PstreamGlobals::outstandingRequests_.size()-start
+            << " outstanding requests starting at " << start << endl;
     }
 
-    if (Pstream::nProcs() <= Pstream::nProcsSimpleSum())
+    if (PstreamGlobals::outstandingRequests_.size())
     {
-        if (Pstream::master())
-        {
-            for
-            (
-                int slave=Pstream::firstSlave();
-                slave<=Pstream::lastSlave();
-                slave++
-            )
-            {
-                scalar value;
+        SubList<MPI_Request> waitRequests
+        (
+            PstreamGlobals::outstandingRequests_,
+            PstreamGlobals::outstandingRequests_.size() - start,
+            start
+        );
 
-                if
-                (
-                    MPI_Recv
-                    (
-                        &value,
-                        1,
-                        MPI_SCALAR,
-                        Pstream::procID(slave),
-                        Pstream::msgType(),
-                        MPI_COMM_WORLD,
-                        MPI_STATUS_IGNORE
-                    )
-                )
-                {
-                    FatalErrorIn
-                    (
-                        "reduce(scalar& Value, const sumOp<scalar>& sumOp)"
-                    )   << "MPI_Recv failed"
-                        << Foam::abort(FatalError);
-                }
-
-                Value = bop(Value, value);
-            }
-        }
-        else
-        {
-            if
+        if
+        (
+            MPI_Waitall
             (
-                MPI_Send
-                (
-                    &Value,
-                    1,
-                    MPI_SCALAR,
-                    Pstream::procID(Pstream::masterNo()),
-                    Pstream::msgType(),
-                    MPI_COMM_WORLD
-                )
+                waitRequests.size(),
+                waitRequests.begin(),
+                MPI_STATUSES_IGNORE
             )
-            {
-                FatalErrorIn
-                (
-                    "reduce(scalar& Value, const sumOp<scalar>& sumOp)"
-                )   << "MPI_Send failed"
-                    << Foam::abort(FatalError);
-            }
+        )
+        {
+            FatalErrorIn
+            (
+                "Pstream::waitRequests()"
+            )   << "MPI_Waitall returned with error" << Foam::endl;
         }
 
+        resetRequests(start);
+    }
 
-        if (Pstream::master())
-        {
-            for
-            (
-                int slave=Pstream::firstSlave();
-                slave<=Pstream::lastSlave();
-                slave++
-            )
-            {
-                if
-                (
-                    MPI_Send
-                    (
-                        &Value,
-                        1,
-                        MPI_SCALAR,
-                        Pstream::procID(slave),
-                        Pstream::msgType(),
-                        MPI_COMM_WORLD
-                    )
-                )
-                {
-                    FatalErrorIn
-                    (
-                        "reduce(scalar& Value, const sumOp<scalar>& sumOp)"
-                    )   << "MPI_Send failed"
-                        << Foam::abort(FatalError);
-                }
-            }
-        }
-        else
-        {
-            if
-            (
-                MPI_Recv
-                (
-                    &Value,
-                    1,
-                    MPI_SCALAR,
-                    Pstream::procID(Pstream::masterNo()),
-                    Pstream::msgType(),
-                    MPI_COMM_WORLD,
-                    MPI_STATUS_IGNORE
-                )
-            )
-            {
-                FatalErrorIn
-                (
-                    "reduce(scalar& Value, const sumOp<scalar>& sumOp)"
-                )   << "MPI_Recv failed"
-                    << Foam::abort(FatalError);
-            }
-        }
+    if (debug)
+    {
+        Pout<< "Pstream::waitRequests : finished wait." << endl;
+    }
+}
+
+
+void Foam::Pstream::waitRequest(const label i)
+{
+    if (debug)
+    {
+        Pout<< "Pstream::waitRequest : starting wait for request:" << i
+            << endl;
+    }
+
+    if (i >= PstreamGlobals::outstandingRequests_.size())
+    {
+        FatalErrorIn
+        (
+            "Pstream::waitRequest(const label)"
+        )   << "There are " << PstreamGlobals::outstandingRequests_.size()
+            << " outstanding send requests and you are asking for i=" << i
+            << nl
+            << "Maybe you are mixing blocking/non-blocking comms?"
+            << Foam::abort(FatalError);
+    }
+
+    if
+    (
+        MPI_Wait
+        (
+           &PstreamGlobals::outstandingRequests_[i],
+            MPI_STATUS_IGNORE
+        )
+    )
+    {
+        FatalErrorIn
+        (
+            "Pstream::waitRequest()"
+        )   << "MPI_Wait returned with error" << Foam::endl;
+    }
+
+    if (debug)
+    {
+        Pout<< "Pstream::waitRequest : finished wait for request:" << i
+            << endl;
+    }
+}
+
+
+bool Foam::Pstream::finishedRequest(const label i)
+{
+    if (debug)
+    {
+        Pout<< "Pstream::finishedRequest : checking request:" << i
+            << endl;
+    }
+
+    if (i >= PstreamGlobals::outstandingRequests_.size())
+    {
+        FatalErrorIn
+        (
+            "Pstream::finishedRequest(const label)"
+        )   << "There are " << PstreamGlobals::outstandingRequests_.size()
+            << " outstanding send requests and you are asking for i=" << i
+            << nl
+            << "Maybe you are mixing blocking/non-blocking comms?"
+            << Foam::abort(FatalError);
+    }
+
+    int flag;
+    MPI_Test
+    (
+       &PstreamGlobals::outstandingRequests_[i],
+       &flag,
+        MPI_STATUS_IGNORE
+    );
+
+    if (debug)
+    {
+        Pout<< "Pstream::finishedRequest : finished request:" << i
+            << endl;
+    }
+
+    return flag != 0;
+}
+
+
+int Foam::Pstream::allocateTag(const char* s)
+{
+    int tag;
+    if (PstreamGlobals::freedTags_.size())
+    {
+        tag = PstreamGlobals::freedTags_.remove();
     }
     else
     {
-        scalar sum;
-        MPI_Allreduce(&Value, &sum, 1, MPI_SCALAR, MPI_SUM, MPI_COMM_WORLD);
-        Value = sum;
-
-        /*
-        int myProcNo = Pstream::myProcNo();
-        int nProcs = Pstream::nProcs();
-
-        //
-        // receive from children
-        //
-        int level = 1;
-        int thisLevelOffset = 2;
-        int childLevelOffset = thisLevelOffset/2;
-        int childProcId = 0;
-
-        while
-        (
-            (childLevelOffset < nProcs)
-         && (myProcNo % thisLevelOffset) == 0
-        )
-        {
-            childProcId = myProcNo + childLevelOffset;
-
-            scalar value;
-
-            if (childProcId < nProcs)
-            {
-                if
-                (
-                    MPI_Recv
-                    (
-                        &value,
-                        1,
-                        MPI_SCALAR,
-                        Pstream::procID(childProcId),
-                        Pstream::msgType(),
-                        MPI_COMM_WORLD,
-                        MPI_STATUS_IGNORE
-                    )
-                )
-                {
-                    FatalErrorIn
-                    (
-                        "reduce(scalar& Value, const sumOp<scalar>& sumOp)"
-                    )   << "MPI_Recv failed"
-                        << Foam::abort(FatalError);
-                }
-
-                Value = bop(Value, value);
-            }
-
-            level++;
-            thisLevelOffset <<= 1;
-            childLevelOffset = thisLevelOffset/2;
-        }
-
-        //
-        // send and receive from parent
-        //
-        if (!Pstream::master())
-        {
-            int parentId = myProcNo - (myProcNo % thisLevelOffset);
-
-            if
-            (
-                MPI_Send
-                (
-                    &Value,
-                    1,
-                    MPI_SCALAR,
-                    Pstream::procID(parentId),
-                    Pstream::msgType(),
-                    MPI_COMM_WORLD
-                )
-            )
-            {
-                FatalErrorIn
-                (
-                    "reduce(scalar& Value, const sumOp<scalar>& sumOp)"
-                )   << "MPI_Send failed"
-                    << Foam::abort(FatalError);
-            }
-
-            if
-            (
-                MPI_Recv
-                (
-                    &Value,
-                    1,
-                    MPI_SCALAR,
-                    Pstream::procID(parentId),
-                    Pstream::msgType(),
-                    MPI_COMM_WORLD,
-                    MPI_STATUS_IGNORE
-                )
-            )
-            {
-                FatalErrorIn
-                (
-                    "reduce(scalar& Value, const sumOp<scalar>& sumOp)"
-                )   << "MPI_Recv failed"
-                    << Foam::abort(FatalError);
-            }
-        }
-
-
-        //
-        // distribute to my children
-        //
-        level--;
-        thisLevelOffset >>= 1;
-        childLevelOffset = thisLevelOffset/2;
-
-        while (level > 0)
-        {
-            childProcId = myProcNo + childLevelOffset;
-
-            if (childProcId < nProcs)
-            {
-                if
-                (
-                    MPI_Send
-                    (
-                        &Value,
-                        1,
-                        MPI_SCALAR,
-                        Pstream::procID(childProcId),
-                        Pstream::msgType(),
-                        MPI_COMM_WORLD
-                    )
-                )
-                {
-                    FatalErrorIn
-                    (
-                        "reduce(scalar& Value, const sumOp<scalar>& sumOp)"
-                    )   << "MPI_Send failed"
-                        << Foam::abort(FatalError);
-                }
-            }
-
-            level--;
-            thisLevelOffset >>= 1;
-            childLevelOffset = thisLevelOffset/2;
-        }
-        */
+        tag = PstreamGlobals::nTags_++;
     }
+
+    if (debug)
+    {
+        Pout<< "Pstream::allocateTag " << s
+            << " : tag:" << tag
+            << endl;
+    }
+
+    return tag;
+}
+
+
+int Foam::Pstream::allocateTag(const word& s)
+{
+    int tag;
+    if (PstreamGlobals::freedTags_.size())
+    {
+        tag = PstreamGlobals::freedTags_.remove();
+    }
+    else
+    {
+        tag = PstreamGlobals::nTags_++;
+    }
+
+    if (debug)
+    {
+        Pout<< "Pstream::allocateTag " << s
+            << " : tag:" << tag
+            << endl;
+    }
+
+    return tag;
+}
+
+
+void Foam::Pstream::freeTag(const char* s, const int tag)
+{
+    if (debug)
+    {
+        Pout<< "Pstream::freeTag " << s << " tag:" << tag << endl;
+    }
+    PstreamGlobals::freedTags_.append(tag);
+}
+
+
+void Foam::Pstream::freeTag(const word& s, const int tag)
+{
+    if (debug)
+    {
+        Pout<< "Pstream::freeTag " << s << " tag:" << tag << endl;
+    }
+    PstreamGlobals::freedTags_.append(tag);
 }
 
 
 // * * * * * * * * * * * * * * Static Data Members * * * * * * * * * * * * * //
 
-// Initialise my process number to 0 (the master)
-int Foam::Pstream::myProcNo_(0);
-
-
 // By default this is not a parallel run
 bool Foam::Pstream::parRun_(false);
 
+// Free communicators
+Foam::LIFOStack<Foam::label> Foam::Pstream::freeComms_;
+
+// My processor number
+Foam::DynamicList<int> Foam::Pstream::myProcNo_(10);
 
 // List of process IDs
-Foam::List<int> Foam::Pstream::procIDs_(1, 0);
+Foam::DynamicList<Foam::List<int> > Foam::Pstream::procIDs_(10);
 
+// Parent communicator
+Foam::DynamicList<Foam::label> Foam::Pstream::parentCommunicator_(10);
 
 // Standard transfer message type
 const int Foam::Pstream::msgType_(1);
 
-
 // Linear communication schedule
-Foam::List<Foam::Pstream::commsStruct> Foam::Pstream::linearCommunication_(0);
-
+Foam::DynamicList<Foam::List<Foam::Pstream::commsStruct> >
+Foam::Pstream::linearCommunication_(10);
 
 // Multi level communication schedule
-Foam::List<Foam::Pstream::commsStruct> Foam::Pstream::treeCommunication_(0);
+Foam::DynamicList<Foam::List<Foam::Pstream::commsStruct> >
+Foam::Pstream::treeCommunication_(10);
 
 
+// Allocate a serial communicator. This gets overwritten in parallel mode
+// (by Pstream::setParRun())
+Foam::Pstream::communicator serialComm(-1, Foam::labelList(1, 0), false);
+
 // Number of processors at which the reduce algorithm changes from linear to
 // tree
 const Foam::debug::optimisationSwitch
@@ -646,4 +1045,19 @@ Foam::Pstream::defaultCommsType
 );
 
 
+const Foam::debug::optimisationSwitch
+Foam::Pstream::nPollProcInterfaces
+(
+    "nPollProcInterfaces",
+    0
+);
+
+// Default communicator
+Foam::label Foam::Pstream::worldComm(0);
+
+
+// Warn for use of any communicator
+Foam::label Foam::Pstream::warnComm(-1);
+
+
 // ************************************************************************* //
diff --git a/src/foam/db/IOstreams/Pstreams/Pstream.H b/src/foam/db/IOstreams/Pstreams/Pstream.H
index 4d72af95e..5d0110e24 100644
--- a/src/foam/db/IOstreams/Pstreams/Pstream.H
+++ b/src/foam/db/IOstreams/Pstreams/Pstream.H
@@ -29,11 +29,12 @@ Description
 
 SourceFiles
     Pstream.C
-    PstreamsPrint.C
+    PstreamReduceOps.C
     PstreamCommsStruct.C
     gatherScatter.C
     combineGatherScatter.C
     gatherScatterList.C
+    PstreamExchange.C
 
 \*---------------------------------------------------------------------------*/
 
@@ -47,6 +48,8 @@ SourceFiles
 #include "NamedEnum.H"
 #include "dynamicLabelList.H"
 #include "optimisationSwitch.H"
+#include "ListOps.H"
+#include "LIFOStack.H"
 
 // * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
 
@@ -59,7 +62,6 @@ namespace Foam
 
 class Pstream
 {
-
 public:
 
     //- Types of communications
@@ -72,6 +74,7 @@ public:
 
     static const NamedEnum<commsTypes, 3> commsTypeNames;
 
+
     // Public classes
 
         //- Structure for communicating between processors
@@ -85,7 +88,7 @@ public:
                 //- procIDs of processors directly below me
                 labelList below_;
 
-                //- procIDs of all processors below (so not just directly below)
+                //- procIDs of all processors below (not just directly below)
                 labelList allBelow_;
 
                 //- procIDs of all processors not below. (inverse set of
@@ -157,7 +160,7 @@ public:
                 friend Ostream& operator<<(Ostream&, const commsStruct&);
         };
 
-        //- combineReduce operator for lists. Used for counting.
+        //- combineReduce operator for lists.  Used for counting.
         class listEq
         {
         public:
@@ -165,7 +168,7 @@ public:
             template<class T>
             void operator()(T& x, const T& y) const
             {
-                forAll(y, i)
+                forAll (y, i)
                 {
                     if (y[i].size())
                     {
@@ -183,32 +186,38 @@ private:
         //- Is this a parallel run?
         static bool parRun_;
 
-        //- My processor index
-        static int myProcNo_;
-
-        //- Process IDs
-        static List<int> procIDs_;
-
-        //- Default message type
+        //- Default message type info
         static const int msgType_;
 
+        //- Stack of free comms
+        static LIFOStack<label> freeComms_;
+
+        //- My processor index
+        static DynamicList<int> myProcNo_;
+
+        //- Process IDs
+        static DynamicList<List<int> > procIDs_;
+
+        //- List of parent communicators
+        static DynamicList<label> parentCommunicator_;
+
         //- Structure for linear communications
-        static List<commsStruct> linearCommunication_;
+        static DynamicList<List<commsStruct> > linearCommunication_;
 
         //- Structure for tree communications
-        static List<commsStruct> treeCommunication_;
+        static DynamicList<List<commsStruct> > treeCommunication_;
 
 
     // Private member functions
 
         //- Set data for parallel running
-        static void setParRun();
+        static void setParRun(const label nProcs);
 
         //- Calculate linear communication schedule
-        static void calcLinearComm(const label nProcs);
+        static List<commsStruct> calcLinearComm(const label nProcs);
 
         //- Calculate tree communication schedule
-        static void calcTreeComm(const label nProcs);
+        static List<commsStruct> calcTreeComm(const label nProcs);
 
         //- Helper function for tree communication schedule determination
         //  Collects all processorIDs below a processor
@@ -223,6 +232,19 @@ private:
         //  Pstream::init()
         static void initCommunicationSchedule();
 
+        //- Allocate a communicator with index
+        static void allocatePstreamCommunicator
+        (
+            const label parentIndex,
+            const label index
+        );
+
+        //- Free a communicator
+        static void freePstreamCommunicator
+        (
+            const label index
+        );
+
 
 protected:
 
@@ -268,6 +290,15 @@ public:
             );
         }
 
+        //- Number of polling cycles in processor updates
+        static const debug::optimisationSwitch nPollProcInterfaces;
+
+        //- Default communicator (all processors)
+        static label worldComm;
+
+        //- Debugging: warn for use of any communicator differing from warnComm
+        static label warnComm;
+
 
     // Constructors
 
@@ -290,6 +321,79 @@ public:
 
     // Member functions
 
+        //- Allocate a new communicator
+        static label allocateCommunicator
+        (
+            const label parent,
+            const labelList& subRanks,
+            const bool doPstream = true
+        );
+
+        //- Free a previously allocated communicator
+        static void freeCommunicator
+        (
+            const label communicator,
+            const bool doPstream = true
+        );
+
+        //- Free all communicators
+        static void freeCommunicators(const bool doPstream);
+
+
+        //- Helper class for allocating/freeing communicators
+        class communicator
+        {
+            //- Communicator identifier
+            label comm_;
+
+            //- Disallow copy and assignment
+            communicator(const communicator&);
+            void operator=(const communicator&);
+
+        public:
+
+            //- Constructo from components
+            communicator
+            (
+                const label parent,
+                const labelList& subRanks,
+                const bool doPstream
+            )
+            :
+                comm_(allocateCommunicator(parent, subRanks, doPstream))
+            {}
+
+            //- Destructor
+            ~communicator()
+            {
+                freeCommunicator(comm_);
+            }
+
+            //- Cast to label
+            operator label() const
+            {
+                return comm_;
+            }
+        };
+
+
+        //- Return physical processor number (i.e. processor number in
+        //  worldComm) given communicator and processor
+        static int baseProcNo(const label myComm, const int procID);
+
+        //- Return processor number in communicator (given physical processor
+        //  number) (= reverse of baseProcNo)
+        static label procNo(const label comm, const int baseProcID);
+
+        //- Return processor number in communicator (given processor number
+        //  and communicator)
+        static label procNo
+        (
+            const label myComm,
+            const label currentComm,
+            const int currentProcID
+        );
+
         //- Add the valid option this type of communications library
         //  adds/requires on the command line
         static void addValidParOptions(HashTable<string>& validParOptions);
@@ -298,44 +402,78 @@ public:
         //  Spawns slave processes and initialises inter-communication
         static bool init(int& argc, char**& argv);
 
+
+        // Non-blocking comms
+
+            //- Get number of outstanding requests
+            static label nRequests();
+
+            //- Truncate number of outstanding requests
+            static void resetRequests(const label sz);
+
+            //- Wait until all requests (from start onwards) have finished.
+            static void waitRequests(const label start = 0);
+
+            //- Wait until request i has finished.
+            static void waitRequest(const label i);
+
+            //- Non-blocking comms: has request i finished?
+            static bool finishedRequest(const label i);
+
+            static int allocateTag(const char*);
+
+            static int allocateTag(const word&);
+
+            static void freeTag(const char*, const int tag);
+
+            static void freeTag(const word&, const int tag);
+
+
         //- Is this a parallel run?
         static bool& parRun()
         {
             return parRun_;
         }
 
-        //- Number of processes in parallel run
-        static label nProcs()
+        //- Number of processes in parallel run for a given communicator
+        static label nProcs(const label communicator = 0)
         {
-            return procIDs_.size();
+            return procIDs_[communicator].size();
         }
 
-        //- Am I the master process
-        static bool master()
-        {
-            return myProcNo_ == masterNo();
-        }
-
-        //- Process index of the master
+        //- Process index of the master for the global communicator
         static int masterNo()
         {
             return 0;
         }
 
-        //- Number of this process (starting from masterNo() = 0)
-        static int myProcNo()
+        //- Am I the master process
+        static bool master(const label communicator = 0)
         {
-            return myProcNo_;
+            return myProcNo_[communicator] == masterNo();
+        }
+
+
+        //- Number of this process (starting from masterNo() = 0)
+        static int myProcNo(const label communicator = 0)
+        {
+            return myProcNo_[communicator];
+        }
+
+        //- Return parent communicator
+        static label parent(const label communicator)
+        {
+            return parentCommunicator_(communicator);
         }
 
         //- Process IDs
-        static const List<int>& procIDs()
+        static const List<int>& procIDs(const int communicator)
         {
-            return procIDs_;
+            return procIDs_[communicator];
         }
 
         //- Process ID of given process index
-        static int procID(int procNo)
+        static List<int>& procID(const int procNo)
         {
             return procIDs_[procNo];
         }
@@ -347,21 +485,27 @@ public:
         }
 
         //- Process index of last slave
-        static int lastSlave()
+        static int lastSlave(const label communicator = 0)
         {
-            return nProcs() - 1;
+            return nProcs(communicator) - 1;
         }
 
         //- Communication schedule for linear all-to-master (proc 0)
-        static const List<commsStruct>& linearCommunication()
+        static const List<commsStruct>& linearCommunication
+        (
+            const label communicator = 0
+        )
         {
-            return linearCommunication_;
+            return linearCommunication_[communicator];
         }
 
         //- Communication schedule for tree all-to-master (proc 0)
-        static const List<commsStruct>& treeCommunication()
+        static const List<commsStruct>& treeCommunication
+        (
+            const label communicator = 0
+        )
         {
-            return treeCommunication_;
+            return treeCommunication_[communicator];
         }
 
         //- Message tag of standard messages
@@ -400,21 +544,40 @@ public:
             (
                 const List<commsStruct>& comms,
                 T& Value,
-                const BinaryOp& bop
+                const BinaryOp& bop,
+                const int tag,
+                const label comm
             );
 
             //- Like above but switches between linear/tree communication
             template <class T, class BinaryOp>
-            static void gather(T& Value, const BinaryOp& bop);
+            static void gather
+            (
+                T& Value,
+                const BinaryOp& bop,
+                const int tag = Pstream::msgType(),
+                const label comm = Pstream::worldComm
+            );
 
             //- Scatter data. Distribute without modification.
             //  Reverse of gather
             template <class T>
-            static void scatter(const List<commsStruct>& comms, T& Value);
+            static void scatter
+            (
+                const List<commsStruct>& comms,
+                T& Value,
+                const int tag,
+                const label comm
+            );
 
             //- Like above but switches between linear/tree communication
             template <class T>
-            static void scatter(T& Value);
+            static void scatter
+            (
+                T& Value,
+                const int tag = Pstream::msgType(),
+                const label comm = Pstream::worldComm
+            );
 
 
         // Combine variants. Inplace combine values from processors.
@@ -425,24 +588,39 @@ public:
             (
                 const List<commsStruct>& comms,
                 T& Value,
-                const CombineOp& cop
+                const CombineOp& cop,
+                const int tag,
+                const label comm
             );
 
             //- Like above but switches between linear/tree communication
             template <class T, class CombineOp>
-            static void combineGather(T& Value, const CombineOp& cop);
+            static void combineGather
+            (
+                T& Value,
+                const CombineOp& cop,
+                const int tag = Pstream::msgType(),
+                const label comm = Pstream::worldComm
+            );
 
             //- Scatter data. Reverse of combineGather
             template <class T>
             static void combineScatter
             (
                 const List<commsStruct>& comms,
-                T& Value
+                T& Value,
+                const int tag,
+                const label comm
             );
 
             //- Like above but switches between linear/tree communication
             template <class T>
-            static void combineScatter(T& Value);
+            static void combineScatter
+            (
+                T& Value,
+                const int tag = Pstream::msgType(),
+                const label comm = Pstream::worldComm
+            );
 
         // Combine variants working on whole List at a time.
 
@@ -451,7 +629,9 @@ public:
             (
                 const List<commsStruct>& comms,
                 List<T>& Value,
-                const CombineOp& cop
+                const CombineOp& cop,
+                const int tag,
+                const label comm
             );
 
             //- Like above but switches between linear/tree communication
@@ -459,7 +639,9 @@ public:
             static void listCombineGather
             (
                 List<T>& Value,
-                const CombineOp& cop
+                const CombineOp& cop,
+                const int tag = Pstream::msgType(),
+                const label comm = Pstream::worldComm
             );
 
             //- Scatter data. Reverse of combineGather
@@ -467,12 +649,19 @@ public:
             static void listCombineScatter
             (
                 const List<commsStruct>& comms,
-                List<T>& Value
+                List<T>& Value,
+                const int tag,
+                const label comm
             );
 
             //- Like above but switches between linear/tree communication
             template <class T>
-            static void listCombineScatter(List<T>& Value);
+            static void listCombineScatter
+            (
+                List<T>& Value,
+                const int tag = Pstream::msgType(),
+                const label comm = Pstream::worldComm
+            );
 
         // Combine variants working on whole map at a time. Container needs to
         // have iterators and find() defined.
@@ -482,7 +671,9 @@ public:
             (
                 const List<commsStruct>& comms,
                 Container& Values,
-                const CombineOp& cop
+                const CombineOp& cop,
+                const int tag,
+                const label comm
             );
 
             //- Like above but switches between linear/tree communication
@@ -490,7 +681,9 @@ public:
             static void mapCombineGather
             (
                 Container& Values,
-                const CombineOp& cop
+                const CombineOp& cop,
+                const int tag = Pstream::msgType(),
+                const label comm = Pstream::worldComm
             );
 
             //- Scatter data. Reverse of combineGather
@@ -498,13 +691,19 @@ public:
             static void mapCombineScatter
             (
                 const List<commsStruct>& comms,
-                Container& Values
+                Container& Values,
+                const int tag,
+                const label comm
             );
 
             //- Like above but switches between linear/tree communication
             template <class Container>
-            static void mapCombineScatter(Container& Values);
-
+            static void mapCombineScatter
+            (
+                Container& Values,
+                const int tag = Pstream::msgType(),
+                const label comm = Pstream::worldComm
+            );
 
 
         // Gather/scatter keeping the individual processor data separate.
@@ -516,24 +715,56 @@ public:
             static void gatherList
             (
                 const List<commsStruct>& comms,
-                List<T>& Values
+                List<T>& Values,
+                const int tag,
+                const label comm
             );
 
             //- Like above but switches between linear/tree communication
             template <class T>
-            static void gatherList(List<T>& Values);
+            static void gatherList
+            (
+                List<T>& Values,
+                const int tag = Pstream::msgType(),
+                const label comm = Pstream::worldComm
+            );
 
             //- Scatter data. Reverse of gatherList
             template <class T>
             static void scatterList
             (
                 const List<commsStruct>& comms,
-                List<T>& Values
+                List<T>& Values,
+                const int tag,
+                const label comm
             );
 
             //- Like above but switches between linear/tree communication
             template <class T>
-            static void scatterList(List<T>& Values);
+            static void scatterList
+            (
+                List<T>& Values,
+                const int tag = Pstream::msgType(),
+                const label comm = Pstream::worldComm
+            );
+
+
+        // Exchange
+
+            //- Exchange data. Sends sendData, receives into recvData, sets
+            //  sizes (not bytes). sizes[p0][p1] is what processor p0 has
+            //  sent to p1. Continuous data only.
+            //  If block=true will wait for all transfers to finish.
+            template<class Container, class T>
+            static void exchange
+            (
+                const List<Container>&,
+                List<Container>&,
+                labelListList& sizes,
+                const int tag = Pstream::msgType(),
+                const label comm = Pstream::worldComm,
+                const bool block = true
+            );
 };
 
 
@@ -556,6 +787,7 @@ Ostream& operator<<(Ostream&, const Pstream::commsStruct&);
 #   include "gatherScatter.C"
 #   include "combineGatherScatter.C"
 #   include "gatherScatterList.C"
+#   include "PstreamExchange.C"
 #endif
 
 
diff --git a/src/foam/db/IOstreams/Pstreams/PstreamCombineReduceOps.H b/src/foam/db/IOstreams/Pstreams/PstreamCombineReduceOps.H
index 2e30e0962..48a99274e 100644
--- a/src/foam/db/IOstreams/Pstreams/PstreamCombineReduceOps.H
+++ b/src/foam/db/IOstreams/Pstreams/PstreamCombineReduceOps.H
@@ -50,26 +50,62 @@ void combineReduce
 (
     const List<Pstream::commsStruct>& comms,
     T& Value,
-    const CombineOp& cop
+    const CombineOp& cop,
+    const int tag,
+    const label comm
 )
 {
-    Pstream::combineGather(comms, Value, cop);
-    Pstream::combineScatter(comms, Value);
+    Pstream::combineGather(comms, Value, cop, tag, comm);
+    Pstream::combineScatter(comms, Value, tag, comm);
 }
 
 
 template <class T, class CombineOp>
-void combineReduce(T& Value, const CombineOp& cop)
+void combineReduce
+(
+    T& Value,
+    const CombineOp& cop,
+    const int tag = Pstream::msgType(),
+    const label comm = Pstream::worldComm
+)
 {
     if (Pstream::nProcs() < Pstream::nProcsSimpleSum())
     {
-        Pstream::combineGather(Pstream::linearCommunication(), Value, cop);
-        Pstream::combineScatter(Pstream::linearCommunication(), Value);
+        Pstream::combineGather
+        (
+            Pstream::linearCommunication(comm),
+            Value,
+            cop,
+            tag,
+            comm
+        );
+
+        Pstream::combineScatter
+        (
+            Pstream::linearCommunication(comm),
+            Value,
+            tag,
+            comm
+        );
     }
     else
     {
-        Pstream::combineGather(Pstream::treeCommunication(), Value, cop);
-        Pstream::combineScatter(Pstream::treeCommunication(), Value);
+        Pstream::combineGather
+        (
+            Pstream::treeCommunication(comm),
+            Value,
+            cop,
+            tag,
+            comm
+        );
+
+        Pstream::combineScatter
+        (
+            Pstream::treeCommunication(comm),
+            Value,
+            tag,
+            comm
+        );
     }
 }
 
diff --git a/src/foam/db/IOstreams/Pstreams/PstreamCommsStruct.C b/src/foam/db/IOstreams/Pstreams/PstreamCommsStruct.C
index af02e2504..05312245f 100644
--- a/src/foam/db/IOstreams/Pstreams/PstreamCommsStruct.C
+++ b/src/foam/db/IOstreams/Pstreams/PstreamCommsStruct.C
@@ -68,13 +68,13 @@ Foam::Pstream::commsStruct::commsStruct
 {
     boolList inBelow(nProcs, false);
 
-    forAll(allBelow, belowI)
+    forAll (allBelow, belowI)
     {
         inBelow[allBelow[belowI]] = true;
     }
 
     label notI = 0;
-    forAll(inBelow, procI)
+    forAll (inBelow, procI)
     {
         if ((procI != myProcID) && !inBelow[procI])
         {
diff --git a/src/foam/db/IOstreams/Pstreams/PstreamExchange.C b/src/foam/db/IOstreams/Pstreams/PstreamExchange.C
new file mode 100644
index 000000000..d846c445e
--- /dev/null
+++ b/src/foam/db/IOstreams/Pstreams/PstreamExchange.C
@@ -0,0 +1,159 @@
+/*---------------------------------------------------------------------------*\
+  =========                 |
+  \\      /  F ield         | foam-extend: Open Source CFD
+   \\    /   O peration     | Version:     4.0
+    \\  /    A nd           | Web:         http://www.foam-extend.org
+     \\/     M anipulation  | For copyright notice see file Copyright
+-------------------------------------------------------------------------------
+License
+    This file is part of foam-extend.
+
+    foam-extend is free software: you can redistribute it and/or modify it
+    under the terms of the GNU General Public License as published by the
+    Free Software Foundation, either version 3 of the License, or (at your
+    option) any later version.
+
+    foam-extend is distributed in the hope that it will be useful, but
+    WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+    General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with foam-extend.  If not, see <http://www.gnu.org/licenses/>.
+
+Description
+    Pstream exchange data.
+
+\*---------------------------------------------------------------------------*/
+
+#include "Pstream.H"
+#include "contiguous.H"
+#include "PstreamCombineReduceOps.H"
+
+// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
+
+namespace Foam
+{
+
+// * * * * * * * * * * * * * * * Member Functions  * * * * * * * * * * * * * //
+
+//template<template<class> class ListType, class T>
+template<class Container, class T>
+void Pstream::exchange
+(
+    const List<Container>& sendBufs,
+    List<Container>& recvBufs,
+    labelListList& sizes,
+    const int tag,
+    const label comm,
+    const bool block
+)
+{
+    if (!contiguous<T>())
+    {
+        FatalErrorIn
+        (
+            "Pstream::exchange(..)"
+        )   << "Continuous data only." << Foam::abort(FatalError);
+    }
+
+    if (sendBufs.size() != Pstream::nProcs(comm))
+    {
+        FatalErrorIn
+        (
+            "Pstream::exchange(..)"
+        )   << "Size of list:" << sendBufs.size()
+            << " does not equal the number of processors:"
+            << Pstream::nProcs(comm)
+            << Foam::abort(FatalError);
+    }
+
+    sizes.setSize(Pstream::nProcs(comm));
+    labelList& nsTransPs = sizes[Pstream::myProcNo(comm)];
+    nsTransPs.setSize(Pstream::nProcs(comm));
+
+    forAll (sendBufs, procI)
+    {
+        nsTransPs[procI] = sendBufs[procI].size();
+    }
+
+    // Send sizes across. Note: blocks.
+    combineReduce(sizes, Pstream::listEq(), tag, comm);
+
+    if (Pstream::nProcs(comm) > 1)
+    {
+        label startOfRequests = Pstream::nRequests();
+
+        // Set up receives
+        // ~~~~~~~~~~~~~~~
+
+        recvBufs.setSize(sendBufs.size());
+        forAll (sizes, procI)
+        {
+            label nRecv = sizes[procI][Pstream::myProcNo(comm)];
+
+            if (procI != Pstream::myProcNo(comm) && nRecv > 0)
+            {
+                recvBufs[procI].setSize(nRecv);
+                IPstream::read
+                (
+                    Pstream::nonBlocking,
+                    procI,
+                    reinterpret_cast<char*>(recvBufs[procI].begin()),
+                    nRecv*sizeof(T),
+                    tag,
+                    comm
+                );
+            }
+        }
+
+
+        // Set up sends
+        // ~~~~~~~~~~~~
+
+        forAll (sendBufs, procI)
+        {
+            if (procI != Pstream::myProcNo(comm) && sendBufs[procI].size() > 0)
+            {
+                if
+                (
+                   !OPstream::write
+                    (
+                        Pstream::nonBlocking,
+                        procI,
+                        reinterpret_cast<const char*>(sendBufs[procI].begin()),
+                        sendBufs[procI].size()*sizeof(T),
+                        tag,
+                        comm
+                    )
+                )
+                {
+                    FatalErrorIn("Pstream::exchange(..)")
+                        << "Cannot send outgoing message. "
+                        << "to:" << procI << " nBytes:"
+                        << label(sendBufs[procI].size()*sizeof(T))
+                        << Foam::abort(FatalError);
+                }
+            }
+        }
+
+
+        // Wait for all to finish
+        // ~~~~~~~~~~~~~~~~~~~~~~
+
+        if (block)
+        {
+            Pstream::waitRequests(startOfRequests);
+        }
+    }
+
+    // Do myself
+    recvBufs[Pstream::myProcNo(comm)] = sendBufs[Pstream::myProcNo(comm)];
+}
+
+
+// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
+
+} // End namespace Foam
+
+// ************************************************************************* //
diff --git a/src/foam/db/IOstreams/Pstreams/PstreamGlobals.C b/src/foam/db/IOstreams/Pstreams/PstreamGlobals.C
index 915d525aa..e3fd17c71 100644
--- a/src/foam/db/IOstreams/Pstreams/PstreamGlobals.C
+++ b/src/foam/db/IOstreams/Pstreams/PstreamGlobals.C
@@ -33,10 +33,50 @@ namespace Foam
 // * * * * * * * * * * * * * * Static Data Members * * * * * * * * * * * * * //
 
 // Outstanding non-blocking operations.
-//! @cond fileScope
-DynamicList<MPI_Request> PstreamGlobals::IPstream_outstandingRequests_;
-DynamicList<MPI_Request> PstreamGlobals::OPstream_outstandingRequests_;
-//! @endcond
+//! \cond fileScope
+DynamicList<MPI_Request> PstreamGlobals::outstandingRequests_;
+//! \endcond
+
+// Max outstanding message tag operations.
+//! \cond fileScope
+int PstreamGlobals::nTags_ = 0;
+//! \endcond
+
+// Free'd message tags
+//! \cond fileScope
+DynamicList<int> PstreamGlobals::freedTags_;
+//! \endcond
+
+
+// Allocated communicators.
+//! \cond fileScope
+DynamicList<MPI_Comm> PstreamGlobals::MPICommunicators_;
+DynamicList<MPI_Group> PstreamGlobals::MPIGroups_;
+//! \endcond
+
+void PstreamGlobals::checkCommunicator
+(
+    const label comm,
+    const label otherProcNo
+)
+{
+    if
+    (
+        comm < 0
+     || comm >= PstreamGlobals::MPICommunicators_.size()
+    )
+    {
+        FatalErrorIn
+        (
+            "PstreamGlobals::checkCommunicator(const label, const label)"
+        )   << "otherProcNo:" << otherProcNo << " : illegal communicator "
+            << comm << endl
+            << "Communicator should be within range 0.."
+            << PstreamGlobals::MPICommunicators_.size() - 1
+            << abort(FatalError);
+    }
+}
+
 
 // * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
 
diff --git a/src/foam/db/IOstreams/Pstreams/PstreamGlobals.H b/src/foam/db/IOstreams/Pstreams/PstreamGlobals.H
index c68ea965e..1fc154548 100644
--- a/src/foam/db/IOstreams/Pstreams/PstreamGlobals.H
+++ b/src/foam/db/IOstreams/Pstreams/PstreamGlobals.H
@@ -46,14 +46,24 @@ namespace Foam
 {
 
 /*---------------------------------------------------------------------------*\
-                     Class PstreamGlobals Declaration
+                        Class PstreamGlobals Declaration
 \*---------------------------------------------------------------------------*/
 
 namespace PstreamGlobals
 {
 
-extern DynamicList<MPI_Request> IPstream_outstandingRequests_;
-extern DynamicList<MPI_Request> OPstream_outstandingRequests_;
+extern DynamicList<MPI_Request> outstandingRequests_;
+
+extern int nTags_;
+
+extern DynamicList<int> freedTags_;
+
+
+// Current communicators. First element will be MPI_COMM_WORLD
+extern DynamicList<MPI_Comm> MPICommunicators_;
+extern DynamicList<MPI_Group> MPIGroups_;
+
+void checkCommunicator(const label, const label procNo);
 
 };
 
diff --git a/src/foam/db/IOstreams/Pstreams/PstreamsPrint.C b/src/foam/db/IOstreams/Pstreams/PstreamReduceOps.C
similarity index 76%
rename from src/foam/db/IOstreams/Pstreams/PstreamsPrint.C
rename to src/foam/db/IOstreams/Pstreams/PstreamReduceOps.C
index ac0fdf58c..be693beb8 100644
--- a/src/foam/db/IOstreams/Pstreams/PstreamsPrint.C
+++ b/src/foam/db/IOstreams/Pstreams/PstreamReduceOps.C
@@ -21,28 +21,11 @@ License
     You should have received a copy of the GNU General Public License
     along with foam-extend.  If not, see <http://www.gnu.org/licenses/>.
 
-Description
-    Prints out a description of the streams
-
 \*---------------------------------------------------------------------------*/
 
-#include "IPstream.H"
-#include "OPstream.H"
+#include "PstreamReduceOps.H"
+#include "allReduce.H"
 
 // * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
 
-void Foam::IPstream::print(Ostream& os) const
-{
-    os  << "Reading from processor " << fromProcNo_
-        << " to processor " << myProcNo() << Foam::endl;
-}
-
-
-void Foam::OPstream::print(Ostream& os) const
-{
-    os  << "Writing from processor " << toProcNo_
-        << " to processor " << myProcNo() << Foam::endl;
-}
-
-
 // ************************************************************************* //
diff --git a/src/foam/db/IOstreams/Pstreams/PstreamReduceOps.H b/src/foam/db/IOstreams/Pstreams/PstreamReduceOps.H
index 8c14b0f84..2b7f45263 100644
--- a/src/foam/db/IOstreams/Pstreams/PstreamReduceOps.H
+++ b/src/foam/db/IOstreams/Pstreams/PstreamReduceOps.H
@@ -28,6 +28,7 @@ License
 
 #include "Pstream.H"
 #include "ops.H"
+#include "vector2D.H"
 
 // * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
 
@@ -42,11 +43,20 @@ void reduce
 (
     const List<Pstream::commsStruct>& comms,
     T& Value,
-    const BinaryOp& bop
+    const BinaryOp& bop,
+    const int tag,
+    const label comm
 )
 {
-    Pstream::gather(comms, Value, bop);
-    Pstream::scatter(comms, Value);
+    if (Pstream::warnComm != -1 && comm != Pstream::warnComm)
+    {
+        Pout<< "** reducing:" << Value << " with comm:" << comm
+            << endl;
+        error::printStack(Pout);
+    }
+
+    Pstream::gather(comms, Value, bop, tag, comm);
+    Pstream::scatter(comms, Value, tag, comm);
 }
 
 
@@ -55,16 +65,18 @@ template <class T, class BinaryOp>
 void reduce
 (
     T& Value,
-    const BinaryOp& bop
+    const BinaryOp& bop,
+    const int tag = Pstream::msgType(),
+    const label comm = Pstream::worldComm
 )
 {
-    if (Pstream::nProcs() < Pstream::nProcsSimpleSum())
+    if (Pstream::nProcs(comm) < Pstream::nProcsSimpleSum())
     {
-        reduce(Pstream::linearCommunication(), Value, bop);
+        reduce(Pstream::linearCommunication(comm), Value, bop, tag, comm);
     }
     else
     {
-        reduce(Pstream::treeCommunication(), Value, bop);
+        reduce(Pstream::treeCommunication(comm), Value, bop, tag, comm);
     }
 }
 
@@ -74,26 +86,100 @@ template <class T, class BinaryOp>
 T returnReduce
 (
     const T& Value,
-    const BinaryOp& bop
+    const BinaryOp& bop,
+    const int tag = Pstream::msgType(),
+    const label comm = Pstream::worldComm
 )
 {
     T WorkValue(Value);
 
-    if (Pstream::nProcs() < Pstream::nProcsSimpleSum())
+    if (Pstream::nProcs(comm) < Pstream::nProcsSimpleSum())
     {
-        reduce(Pstream::linearCommunication(), WorkValue, bop);
+        reduce(Pstream::linearCommunication(comm), WorkValue, bop, tag, comm);
     }
     else
     {
-        reduce(Pstream::treeCommunication(), WorkValue, bop);
+        reduce(Pstream::treeCommunication(comm), WorkValue, bop, tag, comm);
     }
 
     return WorkValue;
 }
 
 
-// Insist there is a specialisation for the reduction of a scalar
-void reduce(scalar& Value, const sumOp<scalar>& bop);
+// Reduce with sum of both value and count (for averaging)
+template<class T>
+void sumReduce
+(
+    T& Value,
+    label& Count,
+    const int tag = Pstream::msgType(),
+    const label comm = Pstream::worldComm
+)
+{
+    reduce(Value, sumOp<T>(), tag, comm);
+    reduce(Count, sumOp<label>(), tag, comm);
+}
+
+
+// Non-blocking version of reduce. Sets request.
+template<class T, class BinaryOp>
+void reduce
+(
+    T& Value,
+    const BinaryOp& bop,
+    const int tag,
+    const label comm,
+    label& request
+)
+{
+    notImplemented
+    (
+        "reduce(T&, const BinaryOp&, const int, const label, label&"
+    );
+}
+
+
+// Insist there are specialisations for the common reductions of scalar(s)
+void reduce
+(
+    scalar& Value,
+    const sumOp<scalar>& bop,
+    const int tag = Pstream::msgType(),
+    const label comm = Pstream::worldComm
+);
+
+void reduce
+(
+    scalar& Value,
+    const minOp<scalar>& bop,
+    const int tag = Pstream::msgType(),
+    const label comm = Pstream::worldComm
+);
+
+void reduce
+(
+    vector2D& Value,
+    const sumOp<vector2D>& bop,
+    const int tag = Pstream::msgType(),
+    const label comm = Pstream::worldComm
+);
+
+void sumReduce
+(
+    scalar& Value,
+    label& Count,
+    const int tag = Pstream::msgType(),
+    const label comm = Pstream::worldComm
+);
+
+void reduce
+(
+    scalar& Value,
+    const sumOp<scalar>& bop,
+    const int tag,
+    const label comm,
+    label& request
+);
 
 
 // * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
diff --git a/src/foam/db/IOstreams/Pstreams/allReduce.H b/src/foam/db/IOstreams/Pstreams/allReduce.H
new file mode 100644
index 000000000..792dcea54
--- /dev/null
+++ b/src/foam/db/IOstreams/Pstreams/allReduce.H
@@ -0,0 +1,71 @@
+/*---------------------------------------------------------------------------*\
+  =========                 |
+  \\      /  F ield         | foam-extend: Open Source CFD
+   \\    /   O peration     | Version:     4.0
+    \\  /    A nd           | Web:         http://www.foam-extend.org
+     \\/     M anipulation  | For copyright notice see file Copyright
+-------------------------------------------------------------------------------
+License
+    This file is part of foam-extend.
+
+    foam-extend is free software: you can redistribute it and/or modify it
+    under the terms of the GNU General Public License as published by the
+    Free Software Foundation, either version 3 of the License, or (at your
+    option) any later version.
+
+    foam-extend is distributed in the hope that it will be useful, but
+    WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+    General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with foam-extend.  If not, see <http://www.gnu.org/licenses/>.
+
+Description
+    Various functions to wrap MPI_Allreduce
+
+SourceFiles
+    allReduceTemplates.C
+
+\*---------------------------------------------------------------------------*/
+
+#ifndef allReduce_H
+#define allReduce_H
+
+#include "mpi.h"
+
+#include "Pstream.H"
+#include "PstreamGlobals.H"
+
+// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
+
+namespace Foam
+{
+
+template<class Type, class BinaryOp>
+void allReduce
+(
+    Type& Value,
+    int count,
+    MPI_Datatype MPIType,
+    MPI_Op op,
+    const BinaryOp& bop,
+    const int tag,
+    const int communicator
+);
+
+// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
+
+} // End namespace Foam
+
+// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
+
+#ifdef NoRepository
+#   include "allReduceTemplates.C"
+#endif
+
+// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
+
+#endif
+
+// ************************************************************************* //
diff --git a/src/foam/db/IOstreams/Pstreams/allReduceTemplates.C b/src/foam/db/IOstreams/Pstreams/allReduceTemplates.C
new file mode 100644
index 000000000..ab16055a6
--- /dev/null
+++ b/src/foam/db/IOstreams/Pstreams/allReduceTemplates.C
@@ -0,0 +1,211 @@
+/*---------------------------------------------------------------------------*\
+  =========                 |
+  \\      /  F ield         | foam-extend: Open Source CFD
+   \\    /   O peration     | Version:     4.0
+    \\  /    A nd           | Web:         http://www.foam-extend.org
+     \\/     M anipulation  | For copyright notice see file Copyright
+-------------------------------------------------------------------------------
+License
+    This file is part of foam-extend.
+
+    foam-extend is free software: you can redistribute it and/or modify it
+    under the terms of the GNU General Public License as published by the
+    Free Software Foundation, either version 3 of the License, or (at your
+    option) any later version.
+
+    foam-extend is distributed in the hope that it will be useful, but
+    WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+    General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with foam-extend.  If not, see <http://www.gnu.org/licenses/>.
+
+\*---------------------------------------------------------------------------*/
+
+#include "allReduce.H"
+
+// * * * * * * * * * * * * * * * Global Functions  * * * * * * * * * * * * * //
+
+template<class Type, class BinaryOp>
+void Foam::allReduce
+(
+    Type& Value,
+    int MPICount,
+    MPI_Datatype MPIType,
+    MPI_Op MPIOp,
+    const BinaryOp& bop,
+    const int tag,
+    const label comm
+)
+{
+    if (!Pstream::parRun())
+    {
+        return;
+    }
+
+    if (Pstream::nProcs(comm) <= Pstream::nProcsSimpleSum)
+    {
+        if (Pstream::master(comm))
+        {
+            for
+            (
+                int slave = Pstream::firstSlave();
+                slave <= Pstream::lastSlave(comm);
+                slave++
+            )
+            {
+                Type value;
+
+                if
+                (
+                    MPI_Recv
+                    (
+                        &value,
+                        MPICount,
+                        MPIType,
+                        slave,  //Pstream::procID(slave),
+                        tag,
+                        PstreamGlobals::MPICommunicators_[comm],
+                        MPI_STATUS_IGNORE
+                    )
+                )
+                {
+                    FatalErrorIn
+                    (
+                        "void Foam::allReduce\n"
+                        "(\n"
+                        "    Type&,\n"
+                        "    int,\n"
+                        "    MPI_Datatype,\n"
+                        "    MPI_Op,\n"
+                        "    const BinaryOp&,\n"
+                        "    const int\n"
+                        ")\n"
+                    )   << "MPI_Recv failed"
+                        << Foam::abort(FatalError);
+                }
+
+                Value = bop(Value, value);
+            }
+        }
+        else
+        {
+            if
+            (
+                MPI_Send
+                (
+                    &Value,
+                    MPICount,
+                    MPIType,
+                    Pstream::masterNo(),//Pstream::procID(masterNo()),
+                    tag,
+                    PstreamGlobals::MPICommunicators_[comm]
+                )
+            )
+            {
+                FatalErrorIn
+                (
+                    "void Foam::allReduce\n"
+                    "(\n"
+                    "    Type&,\n"
+                    "    int,\n"
+                    "    MPI_Datatype,\n"
+                    "    MPI_Op,\n"
+                    "    const BinaryOp&,\n"
+                    "    const int\n"
+                    ")\n"
+                )   << "MPI_Send failed"
+                    << Foam::abort(FatalError);
+            }
+        }
+
+
+        if (Pstream::master(comm))
+        {
+            for
+            (
+                int slave = Pstream::firstSlave();
+                slave <= Pstream::lastSlave(comm);
+                slave++
+            )
+            {
+                if
+                (
+                    MPI_Send
+                    (
+                        &Value,
+                        MPICount,
+                        MPIType,
+                        slave,      //Pstream::procID(slave),
+                        tag,
+                        PstreamGlobals::MPICommunicators_[comm]
+                    )
+                )
+                {
+                    FatalErrorIn
+                    (
+                        "void Foam::allReduce\n"
+                        "(\n"
+                        "    Type&,\n"
+                        "    int,\n"
+                        "    MPI_Datatype,\n"
+                        "    MPI_Op,\n"
+                        "    const BinaryOp&,\n"
+                        "    const int\n"
+                        ")\n"
+                    )   << "MPI_Send failed"
+                        << Foam::abort(FatalError);
+                }
+            }
+        }
+        else
+        {
+            if
+            (
+                MPI_Recv
+                (
+                    &Value,
+                    MPICount,
+                    MPIType,
+                    Pstream::masterNo(),//Pstream::procID(masterNo()),
+                    tag,
+                    PstreamGlobals::MPICommunicators_[comm],
+                    MPI_STATUS_IGNORE
+                )
+            )
+            {
+                FatalErrorIn
+                (
+                    "void Foam::allReduce\n"
+                    "(\n"
+                    "    Type&,\n"
+                    "    int,\n"
+                    "    MPI_Datatype,\n"
+                    "    MPI_Op,\n"
+                    "    const BinaryOp&,\n"
+                    "    const int\n"
+                    ")\n"
+                )   << "MPI_Recv failed"
+                    << Foam::abort(FatalError);
+            }
+        }
+    }
+    else
+    {
+        Type sum;
+        MPI_Allreduce
+        (
+            &Value,
+            &sum,
+            MPICount,
+            MPIType,
+            MPIOp,
+            PstreamGlobals::MPICommunicators_[comm]
+        );
+        Value = sum;
+    }
+}
+
+
+// ************************************************************************* //
diff --git a/src/foam/db/IOstreams/Pstreams/combineGatherScatter.C b/src/foam/db/IOstreams/Pstreams/combineGatherScatter.C
index 4f4d40e71..0c9c59b41 100644
--- a/src/foam/db/IOstreams/Pstreams/combineGatherScatter.C
+++ b/src/foam/db/IOstreams/Pstreams/combineGatherScatter.C
@@ -50,16 +50,18 @@ void Pstream::combineGather
 (
     const List<Pstream::commsStruct>& comms,
     T& Value,
-    const CombineOp& cop
+    const CombineOp& cop,
+    const int tag,
+    const label comm
 )
 {
-    if (Pstream::parRun())
+    if (Pstream::nProcs(comm) > 1)
     {
         // Get my communication order
-        const commsStruct& myComm = comms[Pstream::myProcNo()];
+        const commsStruct& myComm = comms[Pstream::myProcNo(comm)];
 
         // Receive from my downstairs neighbours
-        forAll(myComm.below(), belowI)
+        forAll (myComm.below(), belowI)
         {
             label belowID = myComm.below()[belowI];
 
@@ -71,7 +73,9 @@ void Pstream::combineGather
                     Pstream::scheduled,
                     belowID,
                     reinterpret_cast<char*>(&value),
-                    sizeof(T)
+                    sizeof(T),
+                    tag,
+                    comm
                 );
 
                 if (debug > 1)
@@ -84,7 +88,7 @@ void Pstream::combineGather
             }
             else
             {
-                IPstream fromBelow(Pstream::scheduled, belowID);
+                IPstream fromBelow(Pstream::scheduled, belowID, 0, tag, comm);
                 T value(fromBelow);
 
                 if (debug > 1)
@@ -113,12 +117,21 @@ void Pstream::combineGather
                     Pstream::scheduled,
                     myComm.above(),
                     reinterpret_cast<const char*>(&Value),
-                    sizeof(T)
+                    sizeof(T),
+                    tag,
+                    comm
                 );
             }
             else
             {
-                OPstream toAbove(Pstream::scheduled, myComm.above());
+                OPstream toAbove
+                (
+                    Pstream::scheduled,
+                    myComm.above(),
+                    0,
+                    tag,
+                    comm
+                );
                 toAbove << Value;
             }
         }
@@ -127,26 +140,52 @@ void Pstream::combineGather
 
 
 template <class T, class CombineOp>
-void Pstream::combineGather(T& Value, const CombineOp& cop)
+void Pstream::combineGather
+(
+    T& Value,
+    const CombineOp& cop,
+    const int tag,
+    const label comm
+)
 {
-    if (Pstream::nProcs() < Pstream::nProcsSimpleSum())
+    if (Pstream::nProcs(comm) < Pstream::nProcsSimpleSum())
     {
-        combineGather(Pstream::linearCommunication(), Value, cop);
+        combineGather
+        (
+            Pstream::linearCommunication(comm),
+            Value,
+            cop,
+            tag,
+            comm
+        );
     }
     else
     {
-        combineGather(Pstream::treeCommunication(), Value, cop);
+        combineGather
+        (
+            Pstream::treeCommunication(comm),
+            Value,
+            cop,
+            tag,
+            comm
+        );
     }
 }
 
 
 template <class T>
-void Pstream::combineScatter(const List<Pstream::commsStruct>& comms, T& Value)
+void Pstream::combineScatter
+(
+    const List<Pstream::commsStruct>& comms,
+    T& Value,
+    const int tag,
+    const label comm
+)
 {
-    if (Pstream::parRun())
+    if (Pstream::nProcs(comm) > 1)
     {
         // Get my communication order
-        const Pstream::commsStruct& myComm = comms[Pstream::myProcNo()];
+        const Pstream::commsStruct& myComm = comms[Pstream::myProcNo(comm)];
 
         // Receive from up
         if (myComm.above() != -1)
@@ -158,12 +197,21 @@ void Pstream::combineScatter(const List<Pstream::commsStruct>& comms, T& Value)
                     Pstream::scheduled,
                     myComm.above(),
                     reinterpret_cast<char*>(&Value),
-                    sizeof(T)
+                    sizeof(T),
+                    tag,
+                    comm
                 );
             }
             else
             {
-                IPstream fromAbove(Pstream::scheduled, myComm.above());
+                IPstream fromAbove
+                (
+                    Pstream::scheduled,
+                    myComm.above(),
+                    0,
+                    tag,
+                    comm
+                );
                 Value = T(fromAbove);
             }
 
@@ -175,7 +223,7 @@ void Pstream::combineScatter(const List<Pstream::commsStruct>& comms, T& Value)
         }
 
         // Send to my downstairs neighbours
-        forAll(myComm.below(), belowI)
+        forAll (myComm.below(), belowI)
         {
             label belowID = myComm.below()[belowI];
 
@@ -191,12 +239,14 @@ void Pstream::combineScatter(const List<Pstream::commsStruct>& comms, T& Value)
                     Pstream::scheduled,
                     belowID,
                     reinterpret_cast<const char*>(&Value),
-                    sizeof(T)
+                    sizeof(T),
+                    tag,
+                    comm
                 );
             }
             else
             {
-                OPstream toBelow(Pstream::scheduled, belowID);
+                OPstream toBelow(Pstream::scheduled, belowID, 0, tag, comm);
                 toBelow << Value;
             }
         }
@@ -205,15 +255,20 @@ void Pstream::combineScatter(const List<Pstream::commsStruct>& comms, T& Value)
 
 
 template <class T>
-void Pstream::combineScatter(T& Value)
+void Pstream::combineScatter
+(
+    T& Value,
+    const int tag,
+    const label comm
+)
 {
-    if (Pstream::nProcs() < Pstream::nProcsSimpleSum())
+    if (Pstream::nProcs(comm) < Pstream::nProcsSimpleSum())
     {
-        combineScatter(Pstream::linearCommunication(), Value);
+        combineScatter(Pstream::linearCommunication(comm), Value, tag, comm);
     }
     else
     {
-        combineScatter(Pstream::treeCommunication(), Value);
+        combineScatter(Pstream::treeCommunication(comm), Value, tag, comm);
     }
 }
 
@@ -227,16 +282,18 @@ void Pstream::listCombineGather
 (
     const List<Pstream::commsStruct>& comms,
     List<T>& Values,
-    const CombineOp& cop
+    const CombineOp& cop,
+    const int tag,
+    const label comm
 )
 {
-    if (Pstream::parRun())
+    if (Pstream::nProcs(comm) > 1)
     {
         // Get my communication order
-        const commsStruct& myComm = comms[Pstream::myProcNo()];
+        const commsStruct& myComm = comms[Pstream::myProcNo(comm)];
 
         // Receive from my downstairs neighbours
-        forAll(myComm.below(), belowI)
+        forAll (myComm.below(), belowI)
         {
             label belowID = myComm.below()[belowI];
 
@@ -249,7 +306,9 @@ void Pstream::listCombineGather
                     Pstream::scheduled,
                     belowID,
                     reinterpret_cast<char*>(receivedValues.begin()),
-                    receivedValues.byteSize()
+                    receivedValues.byteSize(),
+                    tag,
+                    comm
                 );
 
                 if (debug > 1)
@@ -258,14 +317,14 @@ void Pstream::listCombineGather
                         << belowID << " data:" << receivedValues << endl;
                 }
 
-                forAll(Values, i)
+                forAll (Values, i)
                 {
                     cop(Values[i], receivedValues[i]);
                 }
             }
             else
             {
-                IPstream fromBelow(Pstream::scheduled, belowID);
+                IPstream fromBelow(Pstream::scheduled, belowID, 0, tag, comm);
                 List<T> receivedValues(fromBelow);
 
                 if (debug > 1)
@@ -274,7 +333,7 @@ void Pstream::listCombineGather
                         << belowID << " data:" << receivedValues << endl;
                 }
 
-                forAll(Values, i)
+                forAll (Values, i)
                 {
                     cop(Values[i], receivedValues[i]);
                 }
@@ -297,12 +356,21 @@ void Pstream::listCombineGather
                     Pstream::scheduled,
                     myComm.above(),
                     reinterpret_cast<const char*>(Values.begin()),
-                    Values.byteSize()
+                    Values.byteSize(),
+                    tag,
+                    comm
                 );
             }
             else
             {
-                OPstream toAbove(Pstream::scheduled, myComm.above());
+                OPstream toAbove
+                (
+                    Pstream::scheduled,
+                    myComm.above(),
+                    0,
+                    tag,
+                    comm
+                );
                 toAbove << Values;
             }
         }
@@ -311,15 +379,35 @@ void Pstream::listCombineGather
 
 
 template <class T, class CombineOp>
-void Pstream::listCombineGather(List<T>& Values, const CombineOp& cop)
+void Pstream::listCombineGather
+(
+    List<T>& Values,
+    const CombineOp& cop,
+    const int tag,
+    const label comm
+)
 {
-    if (Pstream::nProcs() < Pstream::nProcsSimpleSum())
+    if (Pstream::nProcs(comm) < Pstream::nProcsSimpleSum())
     {
-        listCombineGather(Pstream::linearCommunication(), Values, cop);
+        listCombineGather
+        (
+            Pstream::linearCommunication(comm),
+            Values,
+            cop,
+            tag,
+            comm
+        );
     }
     else
     {
-        listCombineGather(Pstream::treeCommunication(), Values, cop);
+        listCombineGather
+        (
+            Pstream::treeCommunication(comm),
+            Values,
+            cop,
+            tag,
+            comm
+        );
     }
 }
 
@@ -328,13 +416,15 @@ template <class T>
 void Pstream::listCombineScatter
 (
     const List<Pstream::commsStruct>& comms,
-    List<T>& Values
+    List<T>& Values,
+    const int tag,
+    const label comm
 )
 {
-    if (Pstream::parRun())
+    if (Pstream::nProcs(comm) > 1)
     {
         // Get my communication order
-        const Pstream::commsStruct& myComm = comms[Pstream::myProcNo()];
+        const Pstream::commsStruct& myComm = comms[Pstream::myProcNo(comm)];
 
         // Receive from up
         if (myComm.above() != -1)
@@ -346,12 +436,21 @@ void Pstream::listCombineScatter
                     Pstream::scheduled,
                     myComm.above(),
                     reinterpret_cast<char*>(Values.begin()),
-                    Values.byteSize()
+                    Values.byteSize(),
+                    tag,
+                    comm
                 );
             }
             else
             {
-                IPstream fromAbove(Pstream::scheduled, myComm.above());
+                IPstream fromAbove
+                (
+                    Pstream::scheduled,
+                    myComm.above(),
+                    0,
+                    tag,
+                    comm
+                );
                 fromAbove >> Values;
             }
 
@@ -363,7 +462,7 @@ void Pstream::listCombineScatter
         }
 
         // Send to my downstairs neighbours
-        forAll(myComm.below(), belowI)
+        forAll (myComm.below(), belowI)
         {
             label belowID = myComm.below()[belowI];
 
@@ -379,12 +478,14 @@ void Pstream::listCombineScatter
                     Pstream::scheduled,
                     belowID,
                     reinterpret_cast<const char*>(Values.begin()),
-                    Values.byteSize()
+                    Values.byteSize(),
+                    tag,
+                    comm
                 );
             }
             else
             {
-                OPstream toBelow(Pstream::scheduled, belowID);
+                OPstream toBelow(Pstream::scheduled, belowID, 0, tag, comm);
                 toBelow << Values;
             }
         }
@@ -393,44 +494,60 @@ void Pstream::listCombineScatter
 
 
 template <class T>
-void Pstream::listCombineScatter(List<T>& Values)
+void Pstream::listCombineScatter
+(
+    List<T>& Values,
+    const int tag,
+    const label comm
+)
 {
-    if (Pstream::nProcs() < Pstream::nProcsSimpleSum())
+    if (Pstream::nProcs(comm) < Pstream::nProcsSimpleSum())
     {
-        listCombineScatter(Pstream::linearCommunication(), Values);
+        listCombineScatter
+        (
+            Pstream::linearCommunication(),
+            Values,
+            tag,
+            comm
+        );
     }
     else
     {
-        listCombineScatter(Pstream::treeCommunication(), Values);
+        listCombineScatter
+        (
+            Pstream::treeCommunication(),
+            Values,
+            tag,
+            comm
+        );
     }
 }
 
 
-
-
 // Same thing but for sparse list (map)
 // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-
 template <class Container, class CombineOp>
 void Pstream::mapCombineGather
 (
     const List<Pstream::commsStruct>& comms,
     Container& Values,
-    const CombineOp& cop
+    const CombineOp& cop,
+    const int tag,
+    const label comm
 )
 {
-    if (Pstream::parRun())
+    if (Pstream::nProcs(comm) > 1)
     {
         // Get my communication order
-        const commsStruct& myComm = comms[Pstream::myProcNo()];
+        const commsStruct& myComm = comms[Pstream::myProcNo(comm)];
 
         // Receive from my downstairs neighbours
-        forAll(myComm.below(), belowI)
+        forAll (myComm.below(), belowI)
         {
             label belowID = myComm.below()[belowI];
 
-            IPstream fromBelow(Pstream::scheduled, belowID);
+            IPstream fromBelow(Pstream::scheduled, belowID, 0, tag, comm);
             Container receivedValues(fromBelow);
 
             if (debug > 1)
@@ -470,7 +587,7 @@ void Pstream::mapCombineGather
                     << " data:" << Values << endl;
             }
 
-            OPstream toAbove(Pstream::scheduled, myComm.above());
+            OPstream toAbove(Pstream::scheduled, myComm.above(), 0, tag, comm);
             toAbove << Values;
         }
     }
@@ -478,15 +595,35 @@ void Pstream::mapCombineGather
 
 
 template <class Container, class CombineOp>
-void Pstream::mapCombineGather(Container& Values, const CombineOp& cop)
+void Pstream::mapCombineGather
+(
+    Container& Values,
+    const CombineOp& cop,
+    const int tag,
+    const label comm
+)
 {
-    if (Pstream::nProcs() < Pstream::nProcsSimpleSum())
+    if (Pstream::nProcs(comm) < Pstream::nProcsSimpleSum())
     {
-        mapCombineGather(Pstream::linearCommunication(), Values, cop);
+        mapCombineGather
+        (
+            Pstream::linearCommunication(),
+            Values,
+            cop,
+            tag,
+            comm
+        );
     }
     else
     {
-        mapCombineGather(Pstream::treeCommunication(), Values, cop);
+        mapCombineGather
+        (
+            Pstream::treeCommunication(),
+            Values,
+            cop,
+            tag,
+            comm
+        );
     }
 }
 
@@ -495,18 +632,27 @@ template <class Container>
 void Pstream::mapCombineScatter
 (
     const List<Pstream::commsStruct>& comms,
-    Container& Values
+    Container& Values,
+    const int tag,
+    const label comm
 )
 {
-    if (Pstream::parRun())
+    if (Pstream::nProcs(comm) > 1)
     {
         // Get my communication order
-        const Pstream::commsStruct& myComm = comms[Pstream::myProcNo()];
+        const Pstream::commsStruct& myComm = comms[Pstream::myProcNo(comm)];
 
         // Receive from up
         if (myComm.above() != -1)
         {
-            IPstream fromAbove(Pstream::scheduled, myComm.above());
+            IPstream fromAbove
+            (
+                Pstream::scheduled,
+                myComm.above(),
+                0,
+                tag,
+                comm
+            );
             fromAbove >> Values;
 
             if (debug > 1)
@@ -517,7 +663,7 @@ void Pstream::mapCombineScatter
         }
 
         // Send to my downstairs neighbours
-        forAll(myComm.below(), belowI)
+        forAll (myComm.below(), belowI)
         {
             label belowID = myComm.below()[belowI];
 
@@ -526,7 +672,7 @@ void Pstream::mapCombineScatter
                 Pout<< " sending to " << belowID << " data:" << Values << endl;
             }
 
-            OPstream toBelow(Pstream::scheduled, belowID);
+            OPstream toBelow(Pstream::scheduled, belowID, 0, tag, comm);
             toBelow << Values;
         }
     }
@@ -534,15 +680,32 @@ void Pstream::mapCombineScatter
 
 
 template <class Container>
-void Pstream::mapCombineScatter(Container& Values)
+void Pstream::mapCombineScatter
+(
+    Container& Values,
+    const int tag,
+    const label comm
+)
 {
-    if (Pstream::nProcs() < Pstream::nProcsSimpleSum())
+    if (Pstream::nProcs(comm) < Pstream::nProcsSimpleSum())
     {
-        mapCombineScatter(Pstream::linearCommunication(), Values);
+        mapCombineScatter
+        (
+            Pstream::linearCommunication(),
+            Values,
+            tag,
+            comm
+        );
     }
     else
     {
-        mapCombineScatter(Pstream::treeCommunication(), Values);
+        mapCombineScatter
+        (
+            Pstream::treeCommunication(),
+            Values,
+            tag,
+            comm
+        );
     }
 }
 
diff --git a/src/foam/db/IOstreams/Pstreams/gatherScatter.C b/src/foam/db/IOstreams/Pstreams/gatherScatter.C
index f0382ef90..d182b830f 100644
--- a/src/foam/db/IOstreams/Pstreams/gatherScatter.C
+++ b/src/foam/db/IOstreams/Pstreams/gatherScatter.C
@@ -45,16 +45,18 @@ void Pstream::gather
 (
     const List<Pstream::commsStruct>& comms,
     T& Value,
-    const BinaryOp& bop
+    const BinaryOp& bop,
+    const int tag,
+    const label comm
 )
 {
-    if (Pstream::parRun())
+    if (Pstream::nProcs(comm) > 1)
     {
         // Get my communication order
-        const commsStruct& myComm = comms[Pstream::myProcNo()];
+        const commsStruct& myComm = comms[Pstream::myProcNo(comm)];
 
         // Receive from my downstairs neighbours
-        forAll(myComm.below(), belowI)
+        forAll (myComm.below(), belowI)
         {
             T value;
 
@@ -65,12 +67,21 @@ void Pstream::gather
                     Pstream::scheduled,
                     myComm.below()[belowI],
                     reinterpret_cast<char*>(&value),
-                    sizeof(T)
+                    sizeof(T),
+                    tag,
+                    comm
                 );
             }
             else
             {
-                IPstream fromBelow(Pstream::scheduled, myComm.below()[belowI]);
+                IPstream fromBelow
+                (
+                    Pstream::scheduled,
+                    myComm.below()[belowI],
+                    0,
+                    tag,
+                    comm
+                );
                 fromBelow >> value;
             }
 
@@ -87,12 +98,21 @@ void Pstream::gather
                     Pstream::scheduled,
                     myComm.above(),
                     reinterpret_cast<const char*>(&Value),
-                    sizeof(T)
+                    sizeof(T),
+                    tag,
+                    comm
                 );
             }
             else
             {
-                OPstream toAbove(Pstream::scheduled, myComm.above());
+                OPstream toAbove
+                (
+                    Pstream::scheduled,
+                    myComm.above(),
+                    0,
+                    tag,
+                    comm
+                );
                 toAbove << Value;
             }
         }
@@ -101,26 +121,38 @@ void Pstream::gather
 
 
 template <class T, class BinaryOp>
-void Pstream::gather(T& Value, const BinaryOp& bop)
+void Pstream::gather
+(
+    T& Value,
+    const BinaryOp& bop,
+    const int tag,
+    const label comm
+)
 {
-    if (Pstream::nProcs() < Pstream::nProcsSimpleSum())
+    if (Pstream::nProcs(comm) < Pstream::nProcsSimpleSum())
     {
-        gather(Pstream::linearCommunication(), Value, bop);
+        gather(Pstream::linearCommunication(comm), Value, bop, tag, comm);
     }
     else
     {
-        gather(Pstream::treeCommunication(), Value, bop);
+        gather(Pstream::treeCommunication(comm), Value, bop, tag, comm);
     }
 }
 
 
 template <class T>
-void Pstream::scatter(const List<Pstream::commsStruct>& comms, T& Value)
+void Pstream::scatter
+(
+    const List<Pstream::commsStruct>& comms,
+    T& Value,
+    const int tag,
+    const label comm
+)
 {
-    if (Pstream::parRun())
+    if (Pstream::nProcs(comm) > 1)
     {
         // Get my communication order
-        const commsStruct& myComm = comms[Pstream::myProcNo()];
+        const commsStruct& myComm = comms[Pstream::myProcNo(comm)];
 
         // Receive from up
         if (myComm.above() != -1)
@@ -132,18 +164,27 @@ void Pstream::scatter(const List<Pstream::commsStruct>& comms, T& Value)
                     Pstream::scheduled,
                     myComm.above(),
                     reinterpret_cast<char*>(&Value),
-                    sizeof(T)
+                    sizeof(T),
+                    tag,
+                    comm
                 );
             }
             else
             {
-                IPstream fromAbove(Pstream::scheduled, myComm.above());
+                IPstream fromAbove
+                (
+                    Pstream::scheduled,
+                    myComm.above(),
+                    0,
+                    tag,
+                    comm
+                );
                 fromAbove >> Value;
             }
         }
 
         // Send to my downstairs neighbours
-        forAll(myComm.below(), belowI)
+        forAll (myComm.below(), belowI)
         {
             if (contiguous<T>())
             {
@@ -152,12 +193,21 @@ void Pstream::scatter(const List<Pstream::commsStruct>& comms, T& Value)
                     Pstream::scheduled,
                     myComm.below()[belowI],
                     reinterpret_cast<const char*>(&Value),
-                    sizeof(T)
+                    sizeof(T),
+                    tag,
+                    comm
                 );
             }
             else
             {
-                OPstream toBelow(Pstream::scheduled,myComm.below()[belowI]);
+                OPstream toBelow
+                (
+                    Pstream::scheduled,
+                    myComm.below()[belowI],
+                    0,
+                    tag,
+                    comm
+                );
                 toBelow << Value;
             }
         }
@@ -166,15 +216,15 @@ void Pstream::scatter(const List<Pstream::commsStruct>& comms, T& Value)
 
 
 template <class T>
-void Pstream::scatter(T& Value)
+void Pstream::scatter(T& Value, const int tag, const label comm)
 {
-    if (Pstream::nProcs() < Pstream::nProcsSimpleSum())
+    if (Pstream::nProcs(comm) < Pstream::nProcsSimpleSum())
     {
-        scatter(Pstream::linearCommunication(), Value);
+        scatter(Pstream::linearCommunication(comm), Value, tag, comm);
     }
     else
     {
-        scatter(Pstream::treeCommunication(), Value);
+        scatter(Pstream::treeCommunication(comm), Value, tag, comm);
     }
 }
 
diff --git a/src/foam/db/IOstreams/Pstreams/gatherScatterList.C b/src/foam/db/IOstreams/Pstreams/gatherScatterList.C
index ed9b046d3..b056256b1 100644
--- a/src/foam/db/IOstreams/Pstreams/gatherScatterList.C
+++ b/src/foam/db/IOstreams/Pstreams/gatherScatterList.C
@@ -49,28 +49,35 @@ template <class T>
 void Pstream::gatherList
 (
     const List<Pstream::commsStruct>& comms,
-    List<T>& Values
+    List<T>& Values,
+    const int tag,
+    const label comm
 )
 {
-    if (Pstream::parRun())
+    if (Pstream::nProcs(comm) > 1)
     {
-        if (Values.size() != Pstream::nProcs())
+        if (Values.size() != Pstream::nProcs(comm))
         {
             FatalErrorIn
             (
-                "Pstream::gatherList(const List<Pstream::commsStruct>&"
-                ", List<T>)"
+                "void Pstream::gatherList\n"
+                "(\n"
+                "    const List<Pstream::commsStruct>& comms,\n"
+                "    List<T>& Values,\n"
+                "    const int tag,\n"
+                "    const label comm\n"
+                ")"
             )   << "Size of list:" << Values.size()
                 << " does not equal the number of processors:"
-                << Pstream::nProcs()
+                << Pstream::nProcs(comm)
                 << Foam::abort(FatalError);
         }
 
         // Get my communication order
-        const commsStruct& myComm = comms[Pstream::myProcNo()];
+        const commsStruct& myComm = comms[Pstream::myProcNo(comm)];
 
         // Receive from my downstairs neighbours
-        forAll(myComm.below(), belowI)
+        forAll (myComm.below(), belowI)
         {
             label belowID = myComm.below()[belowI];
             const labelList& belowLeaves = comms[belowID].allBelow();
@@ -84,19 +91,21 @@ void Pstream::gatherList
                     Pstream::scheduled,
                     belowID,
                     reinterpret_cast<char*>(receivedValues.begin()),
-                    receivedValues.byteSize()
+                    receivedValues.byteSize(),
+                    tag,
+                    comm
                 );
 
                 Values[belowID] = receivedValues[0];
 
-                forAll(belowLeaves, leafI)
+                forAll (belowLeaves, leafI)
                 {
                     Values[belowLeaves[leafI]] = receivedValues[leafI + 1];
                 }
             }
             else
             {
-                IPstream fromBelow(Pstream::scheduled, belowID);
+                IPstream fromBelow(Pstream::scheduled, belowID, 0, tag, comm);
                 fromBelow >> Values[belowID];
 
                 if (debug > 1)
@@ -107,7 +116,7 @@ void Pstream::gatherList
                 }
 
                 // Receive from all other processors below belowID
-                forAll(belowLeaves, leafI)
+                forAll (belowLeaves, leafI)
                 {
                     label leafID = belowLeaves[leafI];
                     fromBelow >> Values[leafID];
@@ -132,16 +141,16 @@ void Pstream::gatherList
             if (debug > 1)
             {
                 Pout<< " sending to " << myComm.above()
-                    << " data from: " << Pstream::myProcNo()
-                    << " data: " << Values[Pstream::myProcNo()] << endl;
+                    << " data from: " << Pstream::myProcNo(comm)
+                    << " data: " << Values[Pstream::myProcNo(comm)] << endl;
             }
 
             if (contiguous<T>())
             {
                 List<T> sendingValues(belowLeaves.size() + 1);
-                sendingValues[0] = Values[Pstream::myProcNo()];
+                sendingValues[0] = Values[Pstream::myProcNo(comm)];
 
-                forAll(belowLeaves, leafI)
+                forAll (belowLeaves, leafI)
                 {
                     sendingValues[leafI + 1] = Values[belowLeaves[leafI]];
                 }
@@ -151,15 +160,24 @@ void Pstream::gatherList
                     Pstream::scheduled,
                     myComm.above(),
                     reinterpret_cast<const char*>(sendingValues.begin()),
-                    sendingValues.byteSize()
+                    sendingValues.byteSize(),
+                    tag,
+                    comm
                 );
             }
             else
             {
-                OPstream toAbove(Pstream::scheduled, myComm.above());
-                toAbove << Values[Pstream::myProcNo()];
+                OPstream toAbove
+                (
+                    Pstream::scheduled,
+                    myComm.above(),
+                    0,
+                    tag,
+                    comm
+                );
+                toAbove << Values[Pstream::myProcNo(comm)];
 
-                forAll(belowLeaves, leafI)
+                forAll (belowLeaves, leafI)
                 {
                     label leafID = belowLeaves[leafI];
 
@@ -178,15 +196,15 @@ void Pstream::gatherList
 
 
 template <class T>
-void Pstream::gatherList(List<T>& Values)
+void Pstream::gatherList(List<T>& Values, const int tag, const label comm)
 {
-    if (Pstream::nProcs() < Pstream::nProcsSimpleSum())
+    if (Pstream::nProcs(comm) < Pstream::nProcsSimpleSum())
     {
-        gatherList(Pstream::linearCommunication(), Values);
+        gatherList(Pstream::linearCommunication(comm), Values, tag, comm);
     }
     else
     {
-        gatherList(Pstream::treeCommunication(), Values);
+        gatherList(Pstream::treeCommunication(comm), Values, tag, comm);
     }
 }
 
@@ -195,17 +213,24 @@ template <class T>
 void Pstream::scatterList
 (
     const List<Pstream::commsStruct>& comms,
-    List<T>& Values
+    List<T>& Values,
+    const int tag,
+    const label comm
 )
 {
-    if (Pstream::parRun())
+    if (Pstream::nProcs(comm) > 1)
     {
-        if (Values.size() != Pstream::nProcs())
+        if (Values.size() != Pstream::nProcs(comm))
         {
             FatalErrorIn
             (
-                "Pstream::scatterList(const List<Pstream::commsStruct>&"
-                ", List<T>)"
+                "void Pstream::scatterList\n"
+                "(\n"
+                "    const List<Pstream::commsStruct>& comms,\n"
+                "    List<T>& Values,\n"
+                "    const int tag,\n"
+                "    const label comm\n"
+                ")"
             )   << "Size of list:" << Values.size()
                 << " does not equal the number of processors:"
                 << Pstream::nProcs()
@@ -213,7 +238,7 @@ void Pstream::scatterList
         }
 
         // Get my communication order
-        const commsStruct& myComm = comms[Pstream::myProcNo()];
+        const commsStruct& myComm = comms[Pstream::myProcNo(comm)];
 
         // Receive from up
         if (myComm.above() != -1)
@@ -229,19 +254,28 @@ void Pstream::scatterList
                     Pstream::scheduled,
                     myComm.above(),
                     reinterpret_cast<char*>(receivedValues.begin()),
-                    receivedValues.byteSize()
+                    receivedValues.byteSize(),
+                    tag,
+                    comm
                 );
 
-                forAll(notBelowLeaves, leafI)
+                forAll (notBelowLeaves, leafI)
                 {
                     Values[notBelowLeaves[leafI]] = receivedValues[leafI];
                 }
             }
             else
             {
-                IPstream fromAbove(Pstream::scheduled, myComm.above());
+                IPstream fromAbove
+                (
+                    Pstream::scheduled,
+                    myComm.above(),
+                    0,
+                    tag,
+                    comm
+                );
 
-                forAll(notBelowLeaves, leafI)
+                forAll (notBelowLeaves, leafI)
                 {
                     label leafID = notBelowLeaves[leafI];
                     fromAbove >> Values[leafID];
@@ -257,7 +291,7 @@ void Pstream::scatterList
         }
 
         // Send to my downstairs neighbours
-        forAll(myComm.below(), belowI)
+        forAll (myComm.below(), belowI)
         {
             label belowID = myComm.below()[belowI];
             const labelList& notBelowLeaves = comms[belowID].allNotBelow();
@@ -266,7 +300,7 @@ void Pstream::scatterList
             {
                 List<T> sendingValues(notBelowLeaves.size());
 
-                forAll(notBelowLeaves, leafI)
+                forAll (notBelowLeaves, leafI)
                 {
                     sendingValues[leafI] = Values[notBelowLeaves[leafI]];
                 }
@@ -276,15 +310,17 @@ void Pstream::scatterList
                     Pstream::scheduled,
                     belowID,
                     reinterpret_cast<const char*>(sendingValues.begin()),
-                    sendingValues.byteSize()
+                    sendingValues.byteSize(),
+                    tag,
+                    comm
                 );
             }
             else
             {
-                OPstream toBelow(Pstream::scheduled, belowID);
+                OPstream toBelow(Pstream::scheduled, belowID, 0, tag, comm);
 
                 // Send data destined for all other processors below belowID
-                forAll(notBelowLeaves, leafI)
+                forAll (notBelowLeaves, leafI)
                 {
                     label leafID = notBelowLeaves[leafI];
                     toBelow << Values[leafID];
@@ -303,15 +339,15 @@ void Pstream::scatterList
 
 
 template <class T>
-void Pstream::scatterList(List<T>& Values)
+void Pstream::scatterList(List<T>& Values, const int tag, const label comm)
 {
-    if (Pstream::nProcs() < Pstream::nProcsSimpleSum())
+    if (Pstream::nProcs(comm) < Pstream::nProcsSimpleSum())
     {
-        scatterList(Pstream::linearCommunication(), Values);
+        scatterList(Pstream::linearCommunication(comm), Values, tag, comm);
     }
     else
     {
-        scatterList(Pstream::treeCommunication(), Values);
+        scatterList(Pstream::treeCommunication(comm), Values, tag, comm);
     }
 }
 
diff --git a/src/foam/db/regIOobject/regIOobjectRead.C b/src/foam/db/regIOobject/regIOobjectRead.C
index 377d55ed0..474bc02e9 100644
--- a/src/foam/db/regIOobject/regIOobjectRead.C
+++ b/src/foam/db/regIOobject/regIOobjectRead.C
@@ -28,7 +28,6 @@ License
 #include "objectRegistry.H"
 #include "PstreamReduceOps.H"
 
-
 // * * * * * * * * * * * * * * * Member Functions  * * * * * * * * * * * * * //
 
 Foam::Istream& Foam::regIOobject::readStream()
diff --git a/src/foam/matrices/blockLduMatrix/BlockAMG/BlockMatrixCoarsening/BlockMatrixAgglomeration/BlockMatrixAgglomeration.C b/src/foam/matrices/blockLduMatrix/BlockAMG/BlockMatrixCoarsening/BlockMatrixAgglomeration/BlockMatrixAgglomeration.C
index 792a39c0f..bfcef67c0 100644
--- a/src/foam/matrices/blockLduMatrix/BlockAMG/BlockMatrixCoarsening/BlockMatrixAgglomeration/BlockMatrixAgglomeration.C
+++ b/src/foam/matrices/blockLduMatrix/BlockAMG/BlockMatrixCoarsening/BlockMatrixAgglomeration/BlockMatrixAgglomeration.C
@@ -892,6 +892,7 @@ Foam::BlockMatrixAgglomeration<Type>::restrictMatrix() const
             nCoarseEqns_,
             coarseOwner,
             coarseNeighbour,
+            Pstream::worldComm, //HJ, AMG Comm fineMesh.comm(),
             true
         )
     );
@@ -953,7 +954,8 @@ Foam::BlockMatrixAgglomeration<Type>::restrictMatrix() const
                     coarseInterfaces,
                     fineInterface,
                     fineInterface.interfaceInternalField(agglomIndex_),
-                    fineInterfaceAddr[intI]
+                    fineInterfaceAddr[intI],
+                    Pstream::worldComm //HJ, AMG Comm fineMesh.comm(),
                 ).ptr()
             );
         }
diff --git a/src/foam/matrices/lduMatrix/lduAddressing/lduInterfaceFields/regionCoupleLduInterfaceField/regionCoupleLduInterfaceField.H b/src/foam/matrices/lduMatrix/lduAddressing/lduInterfaceFields/regionCoupleLduInterfaceField/regionCoupleLduInterfaceField.H
index 245039979..9760d17e0 100644
--- a/src/foam/matrices/lduMatrix/lduAddressing/lduInterfaceFields/regionCoupleLduInterfaceField/regionCoupleLduInterfaceField.H
+++ b/src/foam/matrices/lduMatrix/lduAddressing/lduInterfaceFields/regionCoupleLduInterfaceField/regionCoupleLduInterfaceField.H
@@ -66,7 +66,7 @@ public:
         {}
 
 
-    //-Destructor
+    //- Destructor
     virtual ~regionCoupleLduInterfaceField();
 };
 
diff --git a/src/foam/matrices/lduMatrix/lduAddressing/lduInterfaces/ggiLduInterface/ggiLduInterface.H b/src/foam/matrices/lduMatrix/lduAddressing/lduInterfaces/ggiLduInterface/ggiLduInterface.H
index 3e5b815f7..3d6bf2261 100644
--- a/src/foam/matrices/lduMatrix/lduAddressing/lduInterfaces/ggiLduInterface/ggiLduInterface.H
+++ b/src/foam/matrices/lduMatrix/lduAddressing/lduInterfaces/ggiLduInterface/ggiLduInterface.H
@@ -132,6 +132,15 @@ public:
         virtual void expandAddrToZone(labelField&) const = 0;
 
 
+        // Communications support
+
+            //- Return communicator used for parallel communication
+            virtual int comm() const = 0;
+
+            //- Return message tag used for sending
+            virtual int tag() const = 0;
+
+
         // Transfer buffer access
 
             //- Return contents of the label transfer buffer
diff --git a/src/foam/matrices/lduMatrix/lduAddressing/lduInterfaces/mixingPlaneLduInterface/mixingPlaneLduInterface.H b/src/foam/matrices/lduMatrix/lduAddressing/lduInterfaces/mixingPlaneLduInterface/mixingPlaneLduInterface.H
index 701845b43..ed66070f1 100644
--- a/src/foam/matrices/lduMatrix/lduAddressing/lduInterfaces/mixingPlaneLduInterface/mixingPlaneLduInterface.H
+++ b/src/foam/matrices/lduMatrix/lduAddressing/lduInterfaces/mixingPlaneLduInterface/mixingPlaneLduInterface.H
@@ -110,6 +110,15 @@ public:
         virtual const tensorField& reverseT() const = 0;
 
 
+        // Communications support
+
+            //- Return communicator used for parallel communication
+            virtual int comm() const = 0;
+
+            //- Return message tag used for sending
+            virtual int tag() const = 0;
+
+
         // Transfer buffer access
 
             //- Return contents of the label transfer buffer
diff --git a/src/foam/matrices/lduMatrix/lduAddressing/lduInterfaces/processorLduInterface/processorLduInterface.H b/src/foam/matrices/lduMatrix/lduAddressing/lduInterfaces/processorLduInterface/processorLduInterface.H
index 1a7a622ec..a600d27a8 100644
--- a/src/foam/matrices/lduMatrix/lduAddressing/lduInterfaces/processorLduInterface/processorLduInterface.H
+++ b/src/foam/matrices/lduMatrix/lduAddressing/lduInterfaces/processorLduInterface/processorLduInterface.H
@@ -76,9 +76,8 @@ public:
         processorLduInterface();
 
 
-    // Destructor
-
-        virtual ~processorLduInterface();
+    //- Destructor
+    virtual ~processorLduInterface();
 
 
     // Member Functions
@@ -95,6 +94,15 @@ public:
             virtual const tensorField& forwardT() const = 0;
 
 
+        // Communications support
+
+            //- Return communicator used for parallel communication
+            virtual int comm() const = 0;
+
+            //- Return message tag used for sending
+            virtual int tag() const = 0;
+
+
         // Transfer functions
 
             //- Raw send function
diff --git a/src/foam/matrices/lduMatrix/lduAddressing/lduInterfaces/processorLduInterface/processorLduInterfaceTemplates.C b/src/foam/matrices/lduMatrix/lduAddressing/lduInterfaces/processorLduInterface/processorLduInterfaceTemplates.C
index cdf0db1a6..1645f505c 100644
--- a/src/foam/matrices/lduMatrix/lduAddressing/lduInterfaces/processorLduInterface/processorLduInterfaceTemplates.C
+++ b/src/foam/matrices/lduMatrix/lduAddressing/lduInterfaces/processorLduInterface/processorLduInterfaceTemplates.C
@@ -36,6 +36,8 @@ void Foam::processorLduInterface::send
     const UList<Type>& f
 ) const
 {
+    label nBytes = f.byteSize();
+
     if (commsType == Pstream::blocking || commsType == Pstream::scheduled)
     {
         OPstream::write
@@ -43,7 +45,9 @@ void Foam::processorLduInterface::send
             commsType,
             neighbProcNo(),
             reinterpret_cast<const char*>(f.begin()),
-            f.byteSize()
+            nBytes,
+            tag(),
+            comm()
         );
     }
     else if (commsType == Pstream::nonBlocking)
@@ -55,18 +59,22 @@ void Foam::processorLduInterface::send
             commsType,
             neighbProcNo(),
             receiveBuf_.begin(),
-            receiveBuf_.size()
+            receiveBuf_.size(),
+            tag(),
+            comm()
         );
 
-        resizeBuf(sendBuf_, f.byteSize());
-        memcpy(sendBuf_.begin(), f.begin(), f.byteSize());
+        resizeBuf(sendBuf_, nBytes);
+        memcpy(sendBuf_.begin(), f.begin(), nBytes);
 
         OPstream::write
         (
             commsType,
             neighbProcNo(),
             sendBuf_.begin(),
-            f.byteSize()
+            nBytes,
+            tag(),
+            comm()
         );
     }
     else
@@ -92,7 +100,9 @@ void Foam::processorLduInterface::receive
             commsType,
             neighbProcNo(),
             reinterpret_cast<char*>(f.begin()),
-            f.byteSize()
+            f.byteSize(),
+            tag(),
+            comm()
         );
     }
     else if (commsType == Pstream::nonBlocking)
@@ -141,7 +151,7 @@ void Foam::processorLduInterface::compressedSend
         resizeBuf(sendBuf_, nBytes);
         float *fArray = reinterpret_cast<float*>(sendBuf_.begin());
 
-        for (register label i=0; i<nm1; i++)
+        for (register label i = 0; i < nm1; i++)
         {
             fArray[i] = sArray[i] - slast[i%nCmpts];
         }
@@ -155,7 +165,9 @@ void Foam::processorLduInterface::compressedSend
                 commsType,
                 neighbProcNo(),
                 sendBuf_.begin(),
-                nBytes
+                nBytes,
+                tag(),
+                comm()
             );
         }
         else if (commsType == Pstream::nonBlocking)
@@ -167,7 +179,9 @@ void Foam::processorLduInterface::compressedSend
                 commsType,
                 neighbProcNo(),
                 receiveBuf_.begin(),
-                receiveBuf_.size()
+                receiveBuf_.size(),
+                tag(),
+                comm()
             );
 
             OPstream::write
@@ -175,7 +189,9 @@ void Foam::processorLduInterface::compressedSend
                 commsType,
                 neighbProcNo(),
                 sendBuf_.begin(),
-                nBytes
+                nBytes,
+                tag(),
+                comm()
             );
         }
         else
@@ -215,7 +231,9 @@ void Foam::processorLduInterface::compressedReceive
                 commsType,
                 neighbProcNo(),
                 receiveBuf_.begin(),
-                nBytes
+                nBytes,
+                tag(),
+                comm()
             );
         }
         else if (commsType != Pstream::nonBlocking)
@@ -231,7 +249,7 @@ void Foam::processorLduInterface::compressedReceive
         scalar *sArray = reinterpret_cast<scalar*>(f.begin());
         const scalar *slast = &sArray[nm1];
 
-        for (register label i=0; i<nm1; i++)
+        for (register label i = 0; i < nm1; i++)
         {
             sArray[i] = fArray[i] + slast[i%nCmpts];
         }
diff --git a/src/foam/matrices/lduMatrix/lduAddressing/lduInterfaces/regionCoupleLduInterface/regionCoupleLduInterface.H b/src/foam/matrices/lduMatrix/lduAddressing/lduInterfaces/regionCoupleLduInterface/regionCoupleLduInterface.H
index 3eca876d0..b2caa25f2 100644
--- a/src/foam/matrices/lduMatrix/lduAddressing/lduInterfaces/regionCoupleLduInterface/regionCoupleLduInterface.H
+++ b/src/foam/matrices/lduMatrix/lduAddressing/lduInterfaces/regionCoupleLduInterface/regionCoupleLduInterface.H
@@ -61,7 +61,6 @@ public:
 
     //- Destructor
     virtual ~regionCoupleLduInterface();
-
 };
 
 
diff --git a/src/foam/matrices/lduMatrix/solvers/AMG/AMGAgglomerations/GAMGAgglomeration/GAMGAgglomerateLduAddressing.C b/src/foam/matrices/lduMatrix/solvers/AMG/AMGAgglomerations/GAMGAgglomeration/GAMGAgglomerateLduAddressing.C
index 9ba20ce1f..d9f186b58 100644
--- a/src/foam/matrices/lduMatrix/solvers/AMG/AMGAgglomerations/GAMGAgglomeration/GAMGAgglomerateLduAddressing.C
+++ b/src/foam/matrices/lduMatrix/solvers/AMG/AMGAgglomerations/GAMGAgglomeration/GAMGAgglomerateLduAddressing.C
@@ -264,6 +264,7 @@ void Foam::GAMGAgglomeration::agglomerateLduAddressing
             nCoarseCells,
             coarseOwner,
             coarseNeighbour,
+            fineMesh.comm(),
             true
         )
     );
@@ -298,7 +299,8 @@ void Foam::GAMGAgglomeration::agglomerateLduAddressing
                         (
                             restrictMap
                         ),
-                        fineInterfaceAddr[inti]
+                        fineInterfaceAddr[inti],
+                        fineMesh.comm() // Set up comm per level?
                     ).ptr()
                 );
 
diff --git a/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/AMGInterface/AMGInterface.H b/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/AMGInterface/AMGInterface.H
index 639a04314..d6b311d51 100644
--- a/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/AMGInterface/AMGInterface.H
+++ b/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/AMGInterface/AMGInterface.H
@@ -132,14 +132,16 @@ public:
                 const lduInterfacePtrsList& coarseInterfaces,
                 const lduInterface& fineInterface,
                 const labelField& localRestrictAddressing,
-                const labelField& neighbourRestrictAddressing
+                const labelField& neighbourRestrictAddressing,
+                const label coarseComm
             ),
             (
                 lduMesh,
                 coarseInterfaces,
                 fineInterface,
                 localRestrictAddressing,
-                neighbourRestrictAddressing
+                neighbourRestrictAddressing,
+                coarseComm
             )
         );
 
@@ -154,7 +156,8 @@ public:
             const lduInterfacePtrsList& coarseInterfaces,
             const lduInterface& fineInterface,
             const labelField& localRestrictAddressing,
-            const labelField& neighbourRestrictAddressing
+            const labelField& neighbourRestrictAddressing,
+            const label coarseComm
         );
 
 
@@ -162,7 +165,10 @@ public:
 
         //- Construct from fine-level interface,
         //  local and neighbour restrict addressing
-        AMGInterface(const lduPrimitiveMesh& lduMesh)
+        AMGInterface
+        (
+            const lduPrimitiveMesh& lduMesh
+        )
         :
             lduMesh_(lduMesh)
         {}
diff --git a/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/AMGInterface/newAMGInterface.C b/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/AMGInterface/newAMGInterface.C
index 2f6dc2548..7bd9cffaf 100644
--- a/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/AMGInterface/newAMGInterface.C
+++ b/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/AMGInterface/newAMGInterface.C
@@ -35,7 +35,8 @@ Foam::autoPtr<Foam::AMGInterface> Foam::AMGInterface::New
     const lduInterfacePtrsList& coarseInterfaces,
     const lduInterface& fineInterface,
     const labelField& localRestrictAddressing,
-    const labelField& neighbourRestrictAddressing
+    const labelField& neighbourRestrictAddressing,
+    const label coarseComm
 )
 {
     word coupleType(fineInterface.type());
@@ -53,7 +54,8 @@ Foam::autoPtr<Foam::AMGInterface> Foam::AMGInterface::New
             "    const lduInterfacePtrsList& coarseInterfaces,\n"
             "    const lduInterface& fineInterface,\n"
             "    const labelField& localRestrictAddressing,\n"
-            "    const labelField& neighbourRestrictAddressing\n"
+            "    const labelField& neighbourRestrictAddressing,\n"
+            "    const label coarseComm\n"
             ")"
         )   << "Unknown AMGInterface type " << coupleType << ".\n"
             << "Valid AMGInterface types are :"
@@ -69,7 +71,8 @@ Foam::autoPtr<Foam::AMGInterface> Foam::AMGInterface::New
             coarseInterfaces,
             fineInterface,
             localRestrictAddressing,
-            neighbourRestrictAddressing
+            neighbourRestrictAddressing,
+            coarseComm
         )
     );
 }
diff --git a/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/cyclicAMGInterface/cyclicAMGInterface.C b/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/cyclicAMGInterface/cyclicAMGInterface.C
index 72fe4cf0b..5f8e0d1dc 100644
--- a/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/cyclicAMGInterface/cyclicAMGInterface.C
+++ b/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/cyclicAMGInterface/cyclicAMGInterface.C
@@ -48,7 +48,8 @@ Foam::cyclicAMGInterface::cyclicAMGInterface
     const lduInterfacePtrsList& coarseInterfaces,
     const lduInterface& fineInterface,
     const labelField& localRestrictAddressing,
-    const labelField& neighbourRestrictAddressing
+    const labelField& neighbourRestrictAddressing,
+    const label coarseComm // Not needed
 )
 :
     AMGInterface(lduMesh),
diff --git a/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/cyclicAMGInterface/cyclicAMGInterface.H b/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/cyclicAMGInterface/cyclicAMGInterface.H
index 0a3978e47..a3e75662a 100644
--- a/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/cyclicAMGInterface/cyclicAMGInterface.H
+++ b/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/cyclicAMGInterface/cyclicAMGInterface.H
@@ -84,13 +84,13 @@ public:
             const lduInterfacePtrsList& coarseInterfaces,
             const lduInterface& fineInterface,
             const labelField& localRestrictAddressing,
-            const labelField& neighbourRestrictAddressing
+            const labelField& neighbourRestrictAddressing,
+            const label coarseComm
         );
 
 
-    // Destructor
-
-        virtual ~cyclicAMGInterface();
+    //- Destructor
+    virtual ~cyclicAMGInterface();
 
 
     // Member Functions
diff --git a/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/cyclicGGIAMGInterface/cyclicGGIAMGInterface.C b/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/cyclicGGIAMGInterface/cyclicGGIAMGInterface.C
index c64227099..fa1c2e546 100644
--- a/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/cyclicGGIAMGInterface/cyclicGGIAMGInterface.C
+++ b/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/cyclicGGIAMGInterface/cyclicGGIAMGInterface.C
@@ -48,7 +48,8 @@ Foam::cyclicGGIAMGInterface::cyclicGGIAMGInterface
     const lduInterfacePtrsList& coarseInterfaces,
     const lduInterface& fineInterface,
     const labelField& localRestrictAddressing,
-    const labelField& neighbourRestrictAddressing
+    const labelField& neighbourRestrictAddressing,
+    const label coarseComm
 )
 :
     ggiAMGInterface
@@ -57,7 +58,8 @@ Foam::cyclicGGIAMGInterface::cyclicGGIAMGInterface
         coarseInterfaces,
         fineInterface,
         localRestrictAddressing,
-        neighbourRestrictAddressing
+        neighbourRestrictAddressing,
+        coarseComm
     )
 {}
 
diff --git a/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/cyclicGGIAMGInterface/cyclicGGIAMGInterface.H b/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/cyclicGGIAMGInterface/cyclicGGIAMGInterface.H
index 2498c2282..be62b18f3 100644
--- a/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/cyclicGGIAMGInterface/cyclicGGIAMGInterface.H
+++ b/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/cyclicGGIAMGInterface/cyclicGGIAMGInterface.H
@@ -70,7 +70,8 @@ public:
             const lduInterfacePtrsList& coarseInterfaces,
             const lduInterface& fineInterface,
             const labelField& localRestrictAddressing,
-            const labelField& neighbourRestrictAddressing
+            const labelField& neighbourRestrictAddressing,
+            const label coarseComm
         );
 
 
diff --git a/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/ggiAMGInterface/ggiAMGInterface.C b/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/ggiAMGInterface/ggiAMGInterface.C
index c529bab19..5c8eca04a 100644
--- a/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/ggiAMGInterface/ggiAMGInterface.C
+++ b/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/ggiAMGInterface/ggiAMGInterface.C
@@ -195,7 +195,8 @@ Foam::ggiAMGInterface::ggiAMGInterface
     const lduInterfacePtrsList& coarseInterfaces,
     const lduInterface& fineInterface,
     const labelField& localRestrictAddressing,
-    const labelField& neighbourRestrictAddressing
+    const labelField& neighbourRestrictAddressing,
+    const label coarseComm
 )
 :
     AMGInterface(lduMesh),
@@ -203,6 +204,8 @@ Foam::ggiAMGInterface::ggiAMGInterface
     zoneSize_(0),
     zoneAddressing_(),
     procMasterFaces_(),
+    comm_(coarseComm),
+    tag_(refCast<const ggiLduInterface>(fineInterface).tag()),
     mapPtr_(NULL)
 {
     // New algorithm will assemble local clusters on the master side and
diff --git a/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/ggiAMGInterface/ggiAMGInterface.H b/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/ggiAMGInterface/ggiAMGInterface.H
index a4ba6d917..2803de3e7 100644
--- a/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/ggiAMGInterface/ggiAMGInterface.H
+++ b/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/ggiAMGInterface/ggiAMGInterface.H
@@ -74,8 +74,15 @@ class ggiAMGInterface
         //  to allow the slave to insert faces in the same order
         labelListList procMasterFaces_;
 
+
         // Parallel communication
 
+            //- Communicator to use for parallel communication
+            const label comm_;
+
+            //- Message tag used for sending
+            const int tag_;
+
             //- Map-distribute comms tool
             mutable mapDistribute* mapPtr_;
 
@@ -118,7 +125,8 @@ public:
             const lduInterfacePtrsList& coarseInterfaces,
             const lduInterface& fineInterface,
             const labelField& localRestrictAddressing,
-            const labelField& neighbourRestrictAddressing
+            const labelField& neighbourRestrictAddressing,
+            const label coarseComm
         );
 
 
@@ -137,6 +145,21 @@ public:
             }
 
 
+        // Communications support
+
+            //- Return communicator used for parallel communication
+            virtual int comm() const
+            {
+                return comm_;
+            }
+
+            //- Return message tag used for sending
+            virtual int tag() const
+            {
+                return tag_;
+            }
+
+
         // Agglomeration
 
             //- Agglomerating the given fine-level coefficients and return
diff --git a/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/mixingPlaneAMGInterface/mixingPlaneAMGInterface.C b/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/mixingPlaneAMGInterface/mixingPlaneAMGInterface.C
index e8f95a673..2359ae6bd 100644
--- a/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/mixingPlaneAMGInterface/mixingPlaneAMGInterface.C
+++ b/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/mixingPlaneAMGInterface/mixingPlaneAMGInterface.C
@@ -54,14 +54,17 @@ Foam::mixingPlaneAMGInterface::mixingPlaneAMGInterface
     const lduInterfacePtrsList& coarseInterfaces,
     const lduInterface& fineInterface,
     const labelField& localRestrictAddressing,
-    const labelField& neighbourRestrictAddressing
+    const labelField& neighbourRestrictAddressing,
+    const label coarseComm
 )
 :
     AMGInterface(lduMesh),
     fineMixingPlaneInterface_
     (
         refCast<const mixingPlaneLduInterface>(fineInterface)
-    )
+    ),
+    comm_(coarseComm),
+    tag_(refCast<const mixingPlaneLduInterface>(fineInterface).tag())
 {}
 
 
diff --git a/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/mixingPlaneAMGInterface/mixingPlaneAMGInterface.H b/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/mixingPlaneAMGInterface/mixingPlaneAMGInterface.H
index 16b1ade06..518f5bfee 100644
--- a/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/mixingPlaneAMGInterface/mixingPlaneAMGInterface.H
+++ b/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/mixingPlaneAMGInterface/mixingPlaneAMGInterface.H
@@ -65,6 +65,15 @@ class mixingPlaneAMGInterface
         const mixingPlaneLduInterface& fineMixingPlaneInterface_;
 
 
+        // Parallel communication
+
+            //- Communicator to use for parallel communication
+            const label comm_;
+
+            //- Message tag used for sending
+            const int tag_;
+
+
     // Private Member Functions
 
         //- Disallow default bitwise copy construct
@@ -96,13 +105,13 @@ public:
             const lduInterfacePtrsList& coarseInterfaces,
             const lduInterface& fineInterface,
             const labelField& localRestrictAddressing,
-            const labelField& neighbourRestrictAddressing
+            const labelField& neighbourRestrictAddressing,
+            const label coarseComm
         );
 
 
-    // Destructor
-
-        virtual ~mixingPlaneAMGInterface();
+    // -Destructor
+    virtual ~mixingPlaneAMGInterface();
 
 
     // Member Functions
@@ -116,6 +125,21 @@ public:
             }
 
 
+        // Communications support
+
+            //- Return communicator used for parallel communication
+            virtual int comm() const
+            {
+                return comm_;
+            }
+
+            //- Return message tag used for sending
+            virtual int tag() const
+            {
+                return tag_;
+            }
+
+
         // Agglomeration
 
             //- Agglomerating the given fine-level coefficients and return
diff --git a/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/processorAMGInterface/processorAMGInterface.C b/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/processorAMGInterface/processorAMGInterface.C
index 8f60ee4b9..25bbb8552 100644
--- a/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/processorAMGInterface/processorAMGInterface.C
+++ b/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/processorAMGInterface/processorAMGInterface.C
@@ -48,11 +48,14 @@ Foam::processorAMGInterface::processorAMGInterface
     const lduInterfacePtrsList& coarseInterfaces,
     const lduInterface& fineInterface,
     const labelField& localRestrictAddressing,
-    const labelField& neighbourRestrictAddressing
+    const labelField& neighbourRestrictAddressing,
+    const label coarseComm
 )
 :
     AMGInterface(lduMesh),
-    fineProcInterface_(refCast<const processorLduInterface>(fineInterface))
+    fineProcInterface_(refCast<const processorLduInterface>(fineInterface)),
+    comm_(coarseComm),
+    tag_(refCast<const processorLduInterface>(fineInterface).tag())
 {
     // Make a lookup table of entries for owner/neighbour
     HashTable<SLList<label>, label, Hash<label> > neighboursTable
diff --git a/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/processorAMGInterface/processorAMGInterface.H b/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/processorAMGInterface/processorAMGInterface.H
index 50db8b0f7..4fbda6c03 100644
--- a/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/processorAMGInterface/processorAMGInterface.H
+++ b/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/processorAMGInterface/processorAMGInterface.H
@@ -59,6 +59,12 @@ class processorAMGInterface
         //  agglomerated
         const processorLduInterface& fineProcInterface_;
 
+        //- Communicator to use for parallel communication
+        const label comm_;
+
+        //- Message tag used for sending
+        const int tag_;
+
 
     // Private Member Functions
 
@@ -84,13 +90,13 @@ public:
             const lduInterfacePtrsList& coarseInterfaces,
             const lduInterface& fineInterface,
             const labelField& localRestrictAddressing,
-            const labelField& neighbourRestrictAddressing
+            const labelField& neighbourRestrictAddressing,
+            const label coarseComm
         );
 
 
-    // Destructor
-
-        virtual ~processorAMGInterface();
+    //- Destructor
+    virtual ~processorAMGInterface();
 
 
     // Member Functions
@@ -104,6 +110,21 @@ public:
             }
 
 
+        // Communications support
+
+            //- Return communicator used for parallel communication
+            virtual int comm() const
+            {
+                return comm_;
+            }
+
+            //- Return message tag used for sending
+            virtual int tag() const
+            {
+                return tag_;
+            }
+
+
         // Interface transfer functions
 
             //- Initialise interface data transfer
diff --git a/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/regionCoupleAMGInterface/regionCoupleAMGInterface.C b/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/regionCoupleAMGInterface/regionCoupleAMGInterface.C
index 4f1c1f3b8..79a749339 100644
--- a/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/regionCoupleAMGInterface/regionCoupleAMGInterface.C
+++ b/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/regionCoupleAMGInterface/regionCoupleAMGInterface.C
@@ -48,7 +48,8 @@ Foam::regionCoupleAMGInterface::regionCoupleAMGInterface
     const lduInterfacePtrsList& coarseInterfaces,
     const lduInterface& fineInterface,
     const labelField& localRestrictAddressing,
-    const labelField& neighbourRestrictAddressing
+    const labelField& neighbourRestrictAddressing,
+    const label coarseComm
 )
 :
     ggiAMGInterface
@@ -57,7 +58,8 @@ Foam::regionCoupleAMGInterface::regionCoupleAMGInterface
         coarseInterfaces,
         fineInterface,
         localRestrictAddressing,
-        neighbourRestrictAddressing
+        neighbourRestrictAddressing,
+        coarseComm
     ),
     coupled_(fineInterface.coupled())
 {}
diff --git a/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/regionCoupleAMGInterface/regionCoupleAMGInterface.H b/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/regionCoupleAMGInterface/regionCoupleAMGInterface.H
index d132df7ae..136b982f5 100644
--- a/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/regionCoupleAMGInterface/regionCoupleAMGInterface.H
+++ b/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/regionCoupleAMGInterface/regionCoupleAMGInterface.H
@@ -46,7 +46,7 @@ namespace Foam
 {
 
 /*---------------------------------------------------------------------------*\
-                     Class regionCoupleAMGInterface Declaration
+                   Class regionCoupleAMGInterface Declaration
 \*---------------------------------------------------------------------------*/
 
 class regionCoupleAMGInterface
@@ -76,13 +76,13 @@ public:
             const lduInterfacePtrsList& coarseInterfaces,
             const lduInterface& fineInterface,
             const labelField& localRestrictAddressing,
-            const labelField& neighbourRestrictAddressing
+            const labelField& neighbourRestrictAddressing,
+            const label coarseComm
         );
 
 
-    // Destructor
-
-        virtual ~regionCoupleAMGInterface();
+    //- Destructor
+    virtual ~regionCoupleAMGInterface();
 
 
     // Member Functions
diff --git a/src/foam/meshes/lduMesh/lduMesh.H b/src/foam/meshes/lduMesh/lduMesh.H
index afc2853b6..63de02741 100644
--- a/src/foam/meshes/lduMesh/lduMesh.H
+++ b/src/foam/meshes/lduMesh/lduMesh.H
@@ -59,10 +59,9 @@ public:
 
     // Constructors
 
-    // Destructor
-
-        virtual ~lduMesh()
-        {}
+    //- Destructor
+    virtual ~lduMesh()
+    {}
 
 
     // Member Functions
@@ -78,6 +77,10 @@ public:
             //- Return a list of pointers for each patch
             //  with only those pointing to interfaces being set
             virtual lduInterfacePtrsList interfaces() const = 0;
+
+            //- Return communicator used for parallel communication
+            virtual int comm() const = 0;
+
 };
 
 
diff --git a/src/foam/meshes/lduMesh/lduPrimitiveMesh.H b/src/foam/meshes/lduMesh/lduPrimitiveMesh.H
index 9f12228d9..f966fea70 100644
--- a/src/foam/meshes/lduMesh/lduPrimitiveMesh.H
+++ b/src/foam/meshes/lduMesh/lduPrimitiveMesh.H
@@ -69,6 +69,9 @@ class lduPrimitiveMesh
         //  Note this does must be held as a copy.  HJ, 20/Feb/2009
         lduSchedule patchSchedule_;
 
+        //- Communicator to use for parallel communication
+        const label comm_;
+
 
     // Private Member Functions
 
@@ -89,12 +92,14 @@ public:
         (
             const label nCells,
             const unallocLabelList& l,
-            const unallocLabelList& u
+            const unallocLabelList& u,
+            const label comm
         )
         :
             lduAddressing(nCells),
             lowerAddr_(l),
-            upperAddr_(u)
+            upperAddr_(u),
+            comm_(comm)
         {}
 
 
@@ -104,12 +109,14 @@ public:
             const label nCells,
             labelList& l,
             labelList& u,
+            const label comm,
             bool reUse
         )
         :
             lduAddressing(nCells),
             lowerAddr_(l, reUse),
-            upperAddr_(u, reUse)
+            upperAddr_(u, reUse),
+            comm_(comm)
         {}
 
 
@@ -194,6 +201,12 @@ public:
             {
                 return patchSchedule_;
             }
+
+            //- Return communicator used for parallel communication
+            virtual label comm() const
+            {
+                return comm_;
+            }
 };
 
 
diff --git a/src/foam/meshes/pointMesh/pointPatches/constraint/ggi/ggiPointPatch.H b/src/foam/meshes/pointMesh/pointPatches/constraint/ggi/ggiPointPatch.H
index 34bd9a909..faccaad0d 100644
--- a/src/foam/meshes/pointMesh/pointPatches/constraint/ggi/ggiPointPatch.H
+++ b/src/foam/meshes/pointMesh/pointPatches/constraint/ggi/ggiPointPatch.H
@@ -111,9 +111,8 @@ public:
         );
 
 
-    // Destructor
-
-        virtual ~ggiPointPatch();
+    //- Destructor
+    virtual ~ggiPointPatch();
 
 
     // Member Functions
@@ -122,6 +121,21 @@ public:
 
             //- Return true because this patch is coupled
             virtual bool coupled() const;
+
+
+        // Communications support
+
+            //- Return communicator used for communication
+            virtual label comm() const
+            {
+                return ggiPolyPatch_.comm();
+            }
+
+            //- Return message tag to use for communication
+            virtual int tag() const
+            {
+                return ggiPolyPatch_.tag();
+            }
 };
 
 
diff --git a/src/foam/meshes/pointMesh/pointPatches/constraint/processor/processorPointPatch.H b/src/foam/meshes/pointMesh/pointPatches/constraint/processor/processorPointPatch.H
index ab0957ee8..61d89faa5 100644
--- a/src/foam/meshes/pointMesh/pointPatches/constraint/processor/processorPointPatch.H
+++ b/src/foam/meshes/pointMesh/pointPatches/constraint/processor/processorPointPatch.H
@@ -124,9 +124,8 @@ public:
         );
 
 
-    // Destructor
-
-        virtual ~processorPointPatch();
+    //- Destructor
+    virtual ~processorPointPatch();
 
 
     // Member functions
@@ -175,6 +174,21 @@ public:
         }
 
 
+        // Communications support
+
+            //- Return communicator used for communication
+            virtual label comm() const
+            {
+                return procPolyPatch_.comm();
+            }
+
+            //- Return message tag to use for communication
+            virtual int tag() const
+            {
+                return procPolyPatch_.tag();
+            }
+
+
         // Access functions for demand driven data
 
             //- Return mesh points
diff --git a/src/foam/meshes/polyMesh/mapPolyMesh/mapDistribute/mapDistribute.C b/src/foam/meshes/polyMesh/mapPolyMesh/mapDistribute/mapDistribute.C
index 3d742cc84..e8142cbd7 100644
--- a/src/foam/meshes/polyMesh/mapPolyMesh/mapDistribute/mapDistribute.C
+++ b/src/foam/meshes/polyMesh/mapPolyMesh/mapDistribute/mapDistribute.C
@@ -33,7 +33,8 @@ License
 Foam::List<Foam::labelPair> Foam::mapDistribute::schedule
 (
     const labelListList& subMap,
-    const labelListList& constructMap
+    const labelListList& constructMap,
+    const int tag
 )
 {
     // Communications: send and receive processor
@@ -74,7 +75,7 @@ Foam::List<Foam::labelPair> Foam::mapDistribute::schedule
             slave++
         )
         {
-            IPstream fromSlave(Pstream::scheduled, slave);
+            IPstream fromSlave(Pstream::scheduled, slave, 0, tag);
             List<labelPair> nbrData(fromSlave);
 
             forAll (nbrData, i)
@@ -95,18 +96,24 @@ Foam::List<Foam::labelPair> Foam::mapDistribute::schedule
             slave++
         )
         {
-            OPstream toSlave(Pstream::scheduled, slave);
+            OPstream toSlave(Pstream::scheduled, slave, 0, tag);
             toSlave << allComms;
         }
     }
     else
     {
         {
-            OPstream toMaster(Pstream::scheduled, Pstream::masterNo());
+            OPstream toMaster(Pstream::scheduled, Pstream::masterNo(), 0, tag);
             toMaster << allComms;
         }
         {
-            IPstream fromMaster(Pstream::scheduled, Pstream::masterNo());
+            IPstream fromMaster
+            (
+                Pstream::scheduled,
+                Pstream::masterNo(),
+                0,
+                tag
+            );
             fromMaster >> allComms;
         }
     }
@@ -156,7 +163,7 @@ const Foam::List<Foam::labelPair>& Foam::mapDistribute::schedule() const
         (
             new List<labelPair>
             (
-                schedule(subMap_, constructMap_)
+                schedule(subMap_, constructMap_, Pstream::msgType())
             )
         );
     }
@@ -311,7 +318,7 @@ Foam::mapDistribute::mapDistribute(const mapDistribute& map)
 
 // * * * * * * * * * * * * * Private Member Functions  * * * * * * * * * * * //
 
-void Foam::mapDistribute::compact(const boolList& elemIsUsed)
+void Foam::mapDistribute::compact(const boolList& elemIsUsed, const int tag)
 {
     // 1. send back to sender. Have him delete the corresponding element
     //    from the submap and do the same to the constructMap locally
@@ -319,7 +326,12 @@ void Foam::mapDistribute::compact(const boolList& elemIsUsed)
 
     // Send elemIsUsed field to neighbour. Use nonblocking code from
     // mapDistribute but in reverse order.
+    if (Pstream::parRun())
     {
+        label startOfRequests = Pstream::nRequests();
+
+        // Set up receives from neighbours
+
         List<boolList> sendFields(Pstream::nProcs());
 
         for (label domain = 0; domain < Pstream::nProcs(); domain++)
@@ -340,7 +352,8 @@ void Foam::mapDistribute::compact(const boolList& elemIsUsed)
                     Pstream::nonBlocking,
                     domain,
                     reinterpret_cast<const char*>(subField.begin()),
-                    subField.size()*sizeof(bool)
+                    subField.size()*sizeof(bool),
+                    tag
                 );
             }
         }
@@ -361,7 +374,8 @@ void Foam::mapDistribute::compact(const boolList& elemIsUsed)
                     Pstream::nonBlocking,
                     domain,
                     reinterpret_cast<char*>(recvFields[domain].begin()),
-                    recvFields[domain].size()*sizeof(bool)
+                    recvFields[domain].size()*sizeof(bool),
+                    tag
                 );
             }
         }
@@ -382,8 +396,7 @@ void Foam::mapDistribute::compact(const boolList& elemIsUsed)
 
         // Wait for all to finish
 
-        OPstream::waitRequests();
-        IPstream::waitRequests();
+        Pstream::waitRequests(startOfRequests);
 
 
         // Compact out all submap entries that are referring to unused elements
diff --git a/src/foam/meshes/polyMesh/mapPolyMesh/mapDistribute/mapDistribute.H b/src/foam/meshes/polyMesh/mapPolyMesh/mapDistribute/mapDistribute.H
index 8919ed5aa..fe395d7a0 100644
--- a/src/foam/meshes/polyMesh/mapPolyMesh/mapDistribute/mapDistribute.H
+++ b/src/foam/meshes/polyMesh/mapPolyMesh/mapDistribute/mapDistribute.H
@@ -166,7 +166,8 @@ public:
             static List<labelPair> schedule
             (
                 const labelListList& subMap,
-                const labelListList& constructMap
+                const labelListList& constructMap,
+                const int tag
             );
 
             //- Return a schedule. Demand driven. See above.
@@ -178,8 +179,11 @@ public:
             //- Compact maps. Gets per field a bool whether it is used locally
             //  and works out itself what this side and sender side can remove
             //  from maps.
-            void compact(const boolList& elemIsUsed);
-
+            void compact
+            (
+                const boolList& elemIsUsed,
+                const int tag = Pstream::msgType()
+            );
 
             //- Distribute data. Note:schedule only used for Pstream::scheduled
             //  for now, all others just use send-to-all, receive-from-all.
@@ -191,7 +195,8 @@ public:
                 const label constructSize,
                 const labelListList& subMap,
                 const labelListList& constructMap,
-                List<T>&
+                List<T>&,
+                const int tag = Pstream::msgType()
             );
 
             //- Distribute data. If multiple processors writing to same
@@ -206,12 +211,17 @@ public:
                 const labelListList& constructMap,
                 List<T>&,
                 const CombineOp& cop,
-                const T& nullValue
+                const T& nullValue,
+                const int tag = Pstream::msgType()
             );
 
             //- Distribute data using default commsType.
             template<class T>
-            void distribute(List<T>& fld) const
+            void distribute
+            (
+                 List<T>& fld,
+                const int tag = Pstream::msgType()
+            ) const
             {
                 if
                 (
diff --git a/src/foam/meshes/polyMesh/mapPolyMesh/mapDistribute/mapDistributeTemplates.C b/src/foam/meshes/polyMesh/mapPolyMesh/mapDistribute/mapDistributeTemplates.C
index cccdfbe5d..75a4efe92 100644
--- a/src/foam/meshes/polyMesh/mapPolyMesh/mapDistribute/mapDistributeTemplates.C
+++ b/src/foam/meshes/polyMesh/mapPolyMesh/mapDistribute/mapDistributeTemplates.C
@@ -36,9 +36,35 @@ void Foam::mapDistribute::distribute
     const label constructSize,
     const labelListList& subMap,
     const labelListList& constructMap,
-    List<T>& field
+    List<T>& field,
+    const int tag
 )
 {
+    if (!Pstream::parRun())
+    {
+        // Do only me to me.
+
+        const labelList& mySubMap = subMap[Pstream::myProcNo()];
+
+        List<T> subField(mySubMap.size());
+        forAll(mySubMap, i)
+        {
+            subField[i] = field[mySubMap[i]];
+        }
+
+        // Receive sub field from myself (subField)
+        const labelList& map = constructMap[Pstream::myProcNo()];
+
+        field.setSize(constructSize);
+
+        forAll(map, i)
+        {
+            field[map[i]] = subField[i];
+        }
+
+        return;
+    }
+
     if (commsType == Pstream::blocking)
     {
         // Since buffered sending can reuse the field to collect the
@@ -51,7 +77,7 @@ void Foam::mapDistribute::distribute
 
             if (domain != Pstream::myProcNo() && map.size())
             {
-                OPstream toNbr(Pstream::blocking, domain);
+                OPstream toNbr(Pstream::blocking, domain, 0, tag);
                 toNbr << UIndirectList<T>(field, map);
             }
         }
@@ -82,7 +108,7 @@ void Foam::mapDistribute::distribute
 
             if (domain != Pstream::myProcNo() && map.size())
             {
-                IPstream fromNbr(Pstream::blocking, domain);
+                IPstream fromNbr(Pstream::blocking, domain, 0, tag);
                 List<T> subField(fromNbr);
 
                 checkReceivedSize(domain, map.size(), subField.size());
@@ -116,28 +142,52 @@ void Foam::mapDistribute::distribute
         forAll(schedule, i)
         {
             const labelPair& twoProcs = schedule[i];
+            // twoProcs is a swap pair of processors. The first one is the
+            // one that needs to send first and then receive.
+
             label sendProc = twoProcs[0];
             label recvProc = twoProcs[1];
 
             if (Pstream::myProcNo() == sendProc)
             {
-                // I am sender. Send to recvProc.
-                OPstream toNbr(Pstream::scheduled, recvProc);
-                toNbr << UIndirectList<T>(field, subMap[recvProc]);
+                // I am send first, receive next
+                {
+                    OPstream toNbr(Pstream::scheduled, recvProc, 0, tag);
+                    toNbr << UIndirectList<T>(field, subMap[recvProc]);
+                }
+                {
+                    IPstream fromNbr(Pstream::scheduled, recvProc, 0, tag);
+                    List<T> subField(fromNbr);
+
+                    const labelList& map = constructMap[recvProc];
+
+                    checkReceivedSize(recvProc, map.size(), subField.size());
+
+                    forAll(map, i)
+                    {
+                        newField[map[i]] = subField[i];
+                    }
+                }
             }
             else
             {
-                // I am receiver. Receive from sendProc.
-                IPstream fromNbr(Pstream::scheduled, sendProc);
-                List<T> subField(fromNbr);
-
-                const labelList& map = constructMap[sendProc];
-
-                checkReceivedSize(recvProc, map.size(), subField.size());
-
-                forAll(map, i)
+                // I am receive first, send next
                 {
-                    newField[map[i]] = subField[i];
+                    IPstream fromNbr(Pstream::scheduled, sendProc, 0, tag);
+                    List<T> subField(fromNbr);
+
+                    const labelList& map = constructMap[sendProc];
+
+                    checkReceivedSize(sendProc, map.size(), subField.size());
+
+                    forAll(map, i)
+                    {
+                        newField[map[i]] = subField[i];
+                    }
+                }
+                {
+                    OPstream toNbr(Pstream::scheduled, sendProc, 0, tag);
+                    toNbr << UIndirectList<T>(field, subMap[sendProc]);
                 }
             }
         }
@@ -146,129 +196,173 @@ void Foam::mapDistribute::distribute
     }
     else if (commsType == Pstream::nonBlocking)
     {
+        label nOutstanding = Pstream::nRequests();
+
         if (!contiguous<T>())
         {
-            FatalErrorIn
-            (
-                "template<class T>\n"
-                "void mapDistribute::distribute\n"
-                "(\n"
-                "    const Pstream::commsTypes commsType,\n"
-                "    const List<labelPair>& schedule,\n"
-                "    const label constructSize,\n"
-                "    const labelListList& subMap,\n"
-                "    const labelListList& constructMap,\n"
-                "    List<T>& field\n"
-                ")\n"
-            )   << "Non-blocking only supported for contiguous data."
-                << exit(FatalError);
-        }
-
-        // Set up sends to neighbours
-
-        List<List<T> > sendFields(Pstream::nProcs());
-
-        for (label domain = 0; domain < Pstream::nProcs(); domain++)
-        {
-            const labelList& map = subMap[domain];
-
-            if (domain != Pstream::myProcNo() && map.size())
+            // Stream data into buffer
+            for (label domain = 0; domain < Pstream::nProcs(); domain++)
             {
-                List<T>& subField = sendFields[domain];
+                const labelList& map = subMap[domain];
 
+                if (domain != Pstream::myProcNo() && map.size())
+                {
+                    // Put data into send buffer
+                    OPstream toDomain(Pstream::nonBlocking, domain, 0, tag);
+                    toDomain << UIndirectList<T>(field, map);
+                }
+            }
+
+            // Start receiving. Do not block.
+
+            {
+                // Set up 'send' to myself
+                const labelList& mySubMap = subMap[Pstream::myProcNo()];
+                List<T> mySubField(mySubMap.size());
+                forAll(mySubMap, i)
+                {
+                    mySubField[i] = field[mySubMap[i]];
+                }
+                // Combine bits. Note that can reuse field storage
+                field.setSize(constructSize);
+                // Receive sub field from myself
+                {
+                    const labelList& map = constructMap[Pstream::myProcNo()];
+
+                    forAll(map, i)
+                    {
+                        field[map[i]] = mySubField[i];
+                    }
+                }
+            }
+
+            // Block ourselves, waiting only for the current comms
+            Pstream::waitRequests(nOutstanding);
+
+            // Consume
+            for (label domain = 0; domain < Pstream::nProcs(); domain++)
+            {
+                const labelList& map = constructMap[domain];
+
+                if (domain != Pstream::myProcNo() && map.size())
+                {
+                    IPstream str(Pstream::nonBlocking, domain, 0, tag);
+                    List<T> recvField(str);
+
+                    checkReceivedSize(domain, map.size(), recvField.size());
+
+                    forAll(map, i)
+                    {
+                        field[map[i]] = recvField[i];
+                    }
+                }
+            }
+        }
+        else
+        {
+            // Set up sends to neighbours
+
+            List<List<T > > sendFields(Pstream::nProcs());
+
+            for (label domain = 0; domain < Pstream::nProcs(); domain++)
+            {
+                const labelList& map = subMap[domain];
+
+                if (domain != Pstream::myProcNo() && map.size())
+                {
+                    List<T>& subField = sendFields[domain];
+                    subField.setSize(map.size());
+                    forAll(map, i)
+                    {
+                        subField[i] = field[map[i]];
+                    }
+
+                    OPstream::write
+                    (
+                        Pstream::nonBlocking,
+                        domain,
+                        reinterpret_cast<const char*>(subField.begin()),
+                        subField.byteSize(),
+                        tag
+                    );
+                }
+            }
+
+            // Set up receives from neighbours
+
+            List<List<T > > recvFields(Pstream::nProcs());
+
+            for (label domain = 0; domain < Pstream::nProcs(); domain++)
+            {
+                const labelList& map = constructMap[domain];
+
+                if (domain != Pstream::myProcNo() && map.size())
+                {
+                    recvFields[domain].setSize(map.size());
+                    IPstream::read
+                    (
+                        Pstream::nonBlocking,
+                        domain,
+                        reinterpret_cast<char*>(recvFields[domain].begin()),
+                        recvFields[domain].byteSize(),
+                        tag
+                    );
+                }
+            }
+
+
+            // Set up 'send' to myself
+
+            {
+                const labelList& map = subMap[Pstream::myProcNo()];
+
+                List<T>& subField = sendFields[Pstream::myProcNo()];
                 subField.setSize(map.size());
 
                 forAll(map, i)
                 {
                     subField[i] = field[map[i]];
                 }
-
-                OPstream::write
-                (
-                    Pstream::nonBlocking,
-                    domain,
-                    reinterpret_cast<const char*>(subField.begin()),
-                    subField.size()*sizeof(T)
-                );
             }
-        }
 
-        // Set up receives from neighbours
 
-        List<List<T> > recvFields(Pstream::nProcs());
+            // Combine bits. Note that can reuse field storage
 
-        for (label domain = 0; domain < Pstream::nProcs(); domain++)
-        {
-            const labelList& map = constructMap[domain];
+            field.setSize(constructSize);
 
-            if (domain != Pstream::myProcNo() && map.size())
+            // Receive sub field from myself (sendFields[Pstream::myProcNo()])
             {
-                recvFields[domain].setSize(map.size());
-                IPstream::read
-                (
-                    Pstream::nonBlocking,
-                    domain,
-                    reinterpret_cast<char*>(recvFields[domain].begin()),
-                    recvFields[domain].size()*sizeof(T)
-                );
-            }
-        }
-
-
-        // Set up 'send' to myself
-
-        {
-            const labelList& map = subMap[Pstream::myProcNo()];
-
-            List<T>& subField = sendFields[Pstream::myProcNo()];
-            subField.setSize(map.size());
-
-            forAll(map, i)
-            {
-                subField[i] = field[map[i]];
-            }
-        }
-
-
-        // Combine bits. Note that can reuse field storage
-
-        field.setSize(constructSize);
-
-        // Receive sub field from myself (sendFields[Pstream::myProcNo()])
-        {
-            const labelList& map = constructMap[Pstream::myProcNo()];
-            const List<T>& subField = sendFields[Pstream::myProcNo()];
-
-            forAll(map, i)
-            {
-                field[map[i]] = subField[i];
-            }
-        }
-
-
-        // Wait for all to finish
-
-        OPstream::waitRequests();
-        IPstream::waitRequests();
-
-        // Collect neighbour fields
-
-        for (label domain = 0; domain < Pstream::nProcs(); domain++)
-        {
-            const labelList& map = constructMap[domain];
-
-            if (domain != Pstream::myProcNo() && map.size())
-            {
-                checkReceivedSize
-                (
-                    domain,
-                    map.size(),
-                    recvFields[domain].size()
-                );
+                const labelList& map = constructMap[Pstream::myProcNo()];
+                const List<T>& subField = sendFields[Pstream::myProcNo()];
 
                 forAll(map, i)
                 {
-                    field[map[i]] = recvFields[domain][i];
+                    field[map[i]] = subField[i];
+                }
+            }
+
+
+            // Wait for all to finish
+
+            Pstream::waitRequests(nOutstanding);
+
+
+            // Collect neighbour fields
+
+            for (label domain = 0; domain < Pstream::nProcs(); domain++)
+            {
+                const labelList& map = constructMap[domain];
+
+                if (domain != Pstream::myProcNo() && map.size())
+                {
+                    const List<T>& subField = recvFields[domain];
+
+                    checkReceivedSize(domain, map.size(), subField.size());
+
+                    forAll(map, i)
+                    {
+                        field[map[i]] = subField[i];
+                    }
                 }
             }
         }
@@ -294,9 +388,35 @@ void Foam::mapDistribute::distribute
     const labelListList& constructMap,
     List<T>& field,
     const CombineOp& cop,
-    const T& nullValue
+    const T& nullValue,
+    const int tag
 )
 {
+    if (!Pstream::parRun())
+    {
+        // Do only me to me.
+
+        const labelList& mySubMap = subMap[Pstream::myProcNo()];
+
+        List<T> subField(mySubMap.size());
+        forAll(mySubMap, i)
+        {
+            subField[i] = field[mySubMap[i]];
+        }
+
+        // Receive sub field from myself (subField)
+        const labelList& map = constructMap[Pstream::myProcNo()];
+
+        field.setSize(constructSize);
+        field = nullValue;
+
+        forAll(map, i)
+        {
+            cop(field[map[i]], subField[i]);
+        }
+        return;
+    }
+
     if (commsType == Pstream::blocking)
     {
         // Since buffered sending can reuse the field to collect the
@@ -309,7 +429,7 @@ void Foam::mapDistribute::distribute
 
             if (domain != Pstream::myProcNo() && map.size())
             {
-                OPstream toNbr(Pstream::blocking, domain);
+                OPstream toNbr(Pstream::blocking, domain, 0, tag);
                 toNbr << UIndirectList<T>(field, map);
             }
         }
@@ -341,28 +461,10 @@ void Foam::mapDistribute::distribute
 
             if (domain != Pstream::myProcNo() && map.size())
             {
-                IPstream fromNbr(Pstream::blocking, domain);
+                IPstream fromNbr(Pstream::blocking, domain, 0, tag);
                 List<T> subField(fromNbr);
 
-                if (subField.size() != map.size())
-                {
-                    FatalErrorIn
-                    (
-                        "template<class T>\n"
-                        "void mapDistribute::distribute\n"
-                        "(\n"
-                        "    const Pstream::commsTypes commsType,\n"
-                        "    const List<labelPair>& schedule,\n"
-                        "    const label constructSize,\n"
-                        "    const labelListList& subMap,\n"
-                        "    const labelListList& constructMap,\n"
-                        "    List<T>& field\n"
-                        ")\n"
-                    )   << "Expected from processor " << domain
-                        << " " << map.size() << " but received "
-                        << subField.size() << " elements."
-                        << abort(FatalError);
-                }
+                checkReceivedSize(domain, map.size(), subField.size());
 
                 forAll(map, i)
                 {
@@ -393,195 +495,227 @@ void Foam::mapDistribute::distribute
         forAll(schedule, i)
         {
             const labelPair& twoProcs = schedule[i];
+            // twoProcs is a swap pair of processors. The first one is the
+            // one that needs to send first and then receive.
+
             label sendProc = twoProcs[0];
             label recvProc = twoProcs[1];
 
             if (Pstream::myProcNo() == sendProc)
             {
-                // I am sender. Send to recvProc.
-                OPstream toNbr(Pstream::scheduled, recvProc);
-                toNbr << UIndirectList<T>(field, subMap[recvProc]);
+                // I am send first, receive next
+                {
+                    OPstream toNbr(Pstream::scheduled, recvProc, 0, tag);
+                    toNbr << UIndirectList<T>(field, subMap[recvProc]);
+                }
+                {
+                    IPstream fromNbr(Pstream::scheduled, recvProc, 0, tag);
+                    List<T> subField(fromNbr);
+                    const labelList& map = constructMap[recvProc];
+
+                    checkReceivedSize(recvProc, map.size(), subField.size());
+
+                    forAll(map, i)
+                    {
+                        cop(newField[map[i]], subField[i]);
+                    }
+                }
             }
             else
             {
-                // I am receiver. Receive from sendProc.
-                IPstream fromNbr(Pstream::scheduled, sendProc);
-                List<T> subField(fromNbr);
-
-                const labelList& map = constructMap[sendProc];
-
-                if (subField.size() != map.size())
+                // I am receive first, send next
                 {
-                    FatalErrorIn
-                    (
-                        "template<class T>\n"
-                        "void mapDistribute::distribute\n"
-                        "(\n"
-                        "    const Pstream::commsTypes commsType,\n"
-                        "    const List<labelPair>& schedule,\n"
-                        "    const label constructSize,\n"
-                        "    const labelListList& subMap,\n"
-                        "    const labelListList& constructMap,\n"
-                        "    List<T>& field\n"
-                        ")\n"
-                    )   << "Expected from processor " << sendProc
-                        << " " << map.size() << " but received "
-                        << subField.size() << " elements."
-                        << abort(FatalError);
+                    IPstream fromNbr(Pstream::scheduled, sendProc, 0, tag);
+                    List<T> subField(fromNbr);
+                    const labelList& map = constructMap[sendProc];
+
+                    checkReceivedSize(sendProc, map.size(), subField.size());
+
+                    forAll(map, i)
+                    {
+                        cop(newField[map[i]], subField[i]);
+                    }
                 }
-
-                forAll(map, i)
                 {
-                    cop(newField[map[i]], subField[i]);
+                    OPstream toNbr(Pstream::scheduled, sendProc, 0, tag);
+                    toNbr << UIndirectList<T>(field, subMap[sendProc]);
                 }
             }
         }
+
         field.transfer(newField);
     }
     else if (commsType == Pstream::nonBlocking)
     {
+        label nOutstanding = Pstream::nRequests();
+
         if (!contiguous<T>())
         {
-            FatalErrorIn
-            (
-                "template<class T>\n"
-                "void mapDistribute::distribute\n"
-                "(\n"
-                "    const Pstream::commsTypes commsType,\n"
-                "    const List<labelPair>& schedule,\n"
-                "    const label constructSize,\n"
-                "    const labelListList& subMap,\n"
-                "    const labelListList& constructMap,\n"
-                "    List<T>& field\n"
-                ")\n"
-            )   << "Non-blocking only supported for contiguous data."
-                << exit(FatalError);
-        }
-
-        // Set up sends to neighbours
-
-        List<List<T> > sendFields(Pstream::nProcs());
-
-        for (label domain = 0; domain < Pstream::nProcs(); domain++)
-        {
-            const labelList& map = subMap[domain];
-
-            if (domain != Pstream::myProcNo() && map.size())
+            // Stream data into buffer
+            for (label domain = 0; domain < Pstream::nProcs(); domain++)
             {
-                List<T>& subField = sendFields[domain];
+                const labelList& map = subMap[domain];
+
+                if (domain != Pstream::myProcNo() && map.size())
+                {
+                    // Put data into send buffer
+                    OPstream toDomain(Pstream::nonBlocking, domain, 0, tag);
+                    toDomain << UIndirectList<T>(field, map);
+                }
+            }
+
+            // Start receiving. Do not block.
+
+            {
+                // Set up 'send' to myself
+                List<T> mySubField(field, subMap[Pstream::myProcNo()]);
+                // Combine bits. Note that can reuse field storage
+                field.setSize(constructSize);
+                field = nullValue;
+                // Receive sub field from myself
+                {
+                    const labelList& map = constructMap[Pstream::myProcNo()];
+
+                    forAll(map, i)
+                    {
+                        cop(field[map[i]], mySubField[i]);
+                    }
+                }
+            }
+
+            // Block ourselves, waiting only for the current comms
+            Pstream::waitRequests(nOutstanding);
+
+            // Consume
+            for (label domain = 0; domain < Pstream::nProcs(); domain++)
+            {
+                const labelList& map = constructMap[domain];
+
+                if (domain != Pstream::myProcNo() && map.size())
+                {
+                    IPstream str(Pstream::nonBlocking, domain, 0, tag);
+                    List<T> recvField(str);
+
+                    checkReceivedSize(domain, map.size(), recvField.size());
+
+                    forAll(map, i)
+                    {
+                        cop(field[map[i]], recvField[i]);
+                    }
+                }
+            }
+        }
+        else
+        {
+            // Set up sends to neighbours
+
+            List<List<T> > sendFields(Pstream::nProcs());
+
+            for (label domain = 0; domain < Pstream::nProcs(); domain++)
+            {
+                const labelList& map = subMap[domain];
+
+                if (domain != Pstream::myProcNo() && map.size())
+                {
+                    List<T>& subField = sendFields[domain];
+                    subField.setSize(map.size());
+                    forAll(map, i)
+                    {
+                        subField[i] = field[map[i]];
+                    }
+
+                    OPstream::write
+                    (
+                        Pstream::nonBlocking,
+                        domain,
+                        reinterpret_cast<const char*>(subField.begin()),
+                        subField.size()*sizeof(T),
+                        tag
+                    );
+                }
+            }
+
+            // Set up receives from neighbours
+
+            List<List<T > > recvFields(Pstream::nProcs());
+
+            for (label domain = 0; domain < Pstream::nProcs(); domain++)
+            {
+                const labelList& map = constructMap[domain];
+
+                if (domain != Pstream::myProcNo() && map.size())
+                {
+                    recvFields[domain].setSize(map.size());
+                    IPstream::read
+                    (
+                        Pstream::nonBlocking,
+                        domain,
+                        reinterpret_cast<char*>(recvFields[domain].begin()),
+                        recvFields[domain].size()*sizeof(T),
+                        tag
+                    );
+                }
+            }
+
+            // Set up 'send' to myself
+
+            {
+                const labelList& map = subMap[Pstream::myProcNo()];
+
+                List<T>& subField = sendFields[Pstream::myProcNo()];
                 subField.setSize(map.size());
                 forAll(map, i)
                 {
                     subField[i] = field[map[i]];
                 }
-
-                OPstream::write
-                (
-                    Pstream::nonBlocking,
-                    domain,
-                    reinterpret_cast<const char*>(subField.begin()),
-                    subField.size()*sizeof(T)
-                );
             }
-        }
 
-        // Set up receives from neighbours
 
-        List<List<T> > recvFields(Pstream::nProcs());
+            // Combine bits. Note that can reuse field storage
 
-        for (label domain = 0; domain < Pstream::nProcs(); domain++)
-        {
-            const labelList& map = constructMap[domain];
+            field.setSize(constructSize);
+            field = nullValue;
 
-            if (domain != Pstream::myProcNo() && map.size())
+            // Receive sub field from myself (subField)
             {
-                recvFields[domain].setSize(map.size());
-                IPstream::read
-                (
-                    Pstream::nonBlocking,
-                    domain,
-                    reinterpret_cast<char*>(recvFields[domain].begin()),
-                    recvFields[domain].size()*sizeof(T)
-                );
-            }
-        }
-
-        // Set up 'send' to myself
-
-        {
-            const labelList& map = subMap[Pstream::myProcNo()];
-
-            List<T>& subField = sendFields[Pstream::myProcNo()];
-            subField.setSize(map.size());
-            forAll(map, i)
-            {
-                subField[i] = field[map[i]];
-            }
-        }
-
-
-        // Combine bits. Note that can reuse field storage
-
-        field.setSize(constructSize);
-        field = nullValue;
-
-        // Receive sub field from myself (subField)
-        {
-            const labelList& map = constructMap[Pstream::myProcNo()];
-            const List<T>& subField = sendFields[Pstream::myProcNo()];
-
-            forAll(map, i)
-            {
-                cop(field[map[i]], subField[i]);
-            }
-        }
-
-
-        // Wait for all to finish
-
-        OPstream::waitRequests();
-        IPstream::waitRequests();
-
-        // Collect neighbour fields
-
-        for (label domain = 0; domain < Pstream::nProcs(); domain++)
-        {
-            const labelList& map = constructMap[domain];
-
-            if (domain != Pstream::myProcNo() && map.size())
-            {
-                if (recvFields[domain].size() != map.size())
-                {
-                    FatalErrorIn
-                    (
-                        "template<class T>\n"
-                        "void mapDistribute::distribute\n"
-                        "(\n"
-                        "    const Pstream::commsTypes commsType,\n"
-                        "    const List<labelPair>& schedule,\n"
-                        "    const label constructSize,\n"
-                        "    const labelListList& subMap,\n"
-                        "    const labelListList& constructMap,\n"
-                        "    List<T>& field\n"
-                        ")\n"
-                    )   << "Expected from processor " << domain
-                        << " " << map.size() << " but received "
-                        << recvFields[domain].size() << " elements."
-                        << abort(FatalError);
-                }
+                const labelList& map = constructMap[Pstream::myProcNo()];
+                const List<T>& subField = sendFields[Pstream::myProcNo()];
 
                 forAll(map, i)
                 {
-                    cop(field[map[i]], recvFields[domain][i]);
+                    cop(field[map[i]], subField[i]);
+                }
+            }
+
+
+            // Wait for all to finish
+            Pstream::waitRequests(nOutstanding);
+
+
+            // Collect neighbour fields
+
+            for (label domain = 0; domain < Pstream::nProcs(); domain++)
+            {
+                const labelList& map = constructMap[domain];
+
+                if (domain != Pstream::myProcNo() && map.size())
+                {
+                    const List<T>& subField = recvFields[domain];
+
+                    checkReceivedSize(domain, map.size(), subField.size());
+
+                    forAll(map, i)
+                    {
+                        cop(field[map[i]], subField[i]);
+                    }
                 }
             }
         }
     }
     else
     {
-        // This needs to be cleaned up: temporary solution.  HJ, 15/Jun/2014
         FatalErrorIn("mapDistribute::distribute(..)")
-            << "Unknown communication schedule " << label(commsType)
+            << "Unknown communication schedule " << commsType
             << abort(FatalError);
     }
 }
diff --git a/src/foam/meshes/polyMesh/polyMesh.C b/src/foam/meshes/polyMesh/polyMesh.C
index a1c015b87..054113c87 100644
--- a/src/foam/meshes/polyMesh/polyMesh.C
+++ b/src/foam/meshes/polyMesh/polyMesh.C
@@ -244,6 +244,7 @@ Foam::polyMesh::polyMesh(const IOobject& io)
     bounds_(allPoints_),
     geometricD_(Vector<label>::zero),
     solutionD_(Vector<label>::zero),
+    comm_(Pstream::worldComm),
     pointZones_
     (
         IOobject
@@ -441,6 +442,7 @@ Foam::polyMesh::polyMesh
     bounds_(allPoints_, syncPar),
     geometricD_(Vector<label>::zero),
     solutionD_(Vector<label>::zero),
+    comm_(Pstream::worldComm),
     pointZones_
     (
         IOobject
@@ -601,6 +603,7 @@ Foam::polyMesh::polyMesh
     bounds_(allPoints_, syncPar),
     geometricD_(Vector<label>::zero),
     solutionD_(Vector<label>::zero),
+    comm_(Pstream::worldComm),
     pointZones_
     (
         IOobject
@@ -925,6 +928,18 @@ Foam::label Foam::polyMesh::nSolutionD() const
 }
 
 
+Foam::label Foam::polyMesh::comm() const
+{
+    return comm_;
+}
+
+
+Foam::label& Foam::polyMesh::comm()
+{
+    return comm_;
+}
+
+
 // Add boundary patches. Constructor helper
 void Foam::polyMesh::addPatches
 (
diff --git a/src/foam/meshes/polyMesh/polyMesh.H b/src/foam/meshes/polyMesh/polyMesh.H
index 01fe6e65e..88128cbb8 100644
--- a/src/foam/meshes/polyMesh/polyMesh.H
+++ b/src/foam/meshes/polyMesh/polyMesh.H
@@ -134,6 +134,9 @@ private:
             //  Defined according to the presence of empty patches
             mutable Vector<label> solutionD_;
 
+            //- Communicator used for parallel communication
+            label comm_;
+
 
         // Zoning information
 
@@ -293,9 +296,8 @@ public:
         );
 
 
-    // Destructor
-
-        virtual ~polyMesh();
+    //- Destructor
+    virtual ~polyMesh();
 
 
     // Member Functions
@@ -386,23 +388,36 @@ public:
             //- Return the number of valid solved-for dimensions in the mesh
             label nSolutionD() const;
 
-            //- Return point zone mesh
-            const pointZoneMesh& pointZones() const
-            {
-                return pointZones_;
-            }
 
-            //- Return face zone mesh
-            const faceZoneMesh& faceZones() const
-            {
-                return faceZones_;
-            }
+            // Communication support
+
+                //- Return communicator used for parallel communication
+                label comm() const;
+
+                //- Return communicator used for parallel communication
+                label& comm();
+
+
+            // Point, face and cell zones
+
+                //- Return point zone mesh
+                const pointZoneMesh& pointZones() const
+                {
+                    return pointZones_;
+                }
+
+                //- Return face zone mesh
+                const faceZoneMesh& faceZones() const
+                {
+                    return faceZones_;
+                }
+
+                //- Return cell zone mesh
+                const cellZoneMesh& cellZones() const
+                {
+                    return cellZones_;
+                }
 
-            //- Return cell zone mesh
-            const cellZoneMesh& cellZones() const
-            {
-                return cellZones_;
-            }
 
             //- Return parallel info
             const globalMeshData& globalData() const;
diff --git a/src/foam/meshes/polyMesh/polyPatches/constraint/ggi/ggiPolyPatch.C b/src/foam/meshes/polyMesh/polyPatches/constraint/ggi/ggiPolyPatch.C
index 71181bcb5..22751734f 100644
--- a/src/foam/meshes/polyMesh/polyPatches/constraint/ggi/ggiPolyPatch.C
+++ b/src/foam/meshes/polyMesh/polyPatches/constraint/ggi/ggiPolyPatch.C
@@ -31,6 +31,7 @@ Contributor
 
 #include "ggiPolyPatch.H"
 #include "polyBoundaryMesh.H"
+#include "polyMesh.H"
 #include "addToRunTimeSelectionTable.H"
 #include "demandDrivenData.H"
 #include "polyPatchID.H"
@@ -65,7 +66,7 @@ bool Foam::ggiPolyPatch::active() const
     // For decomposition and reconstruction
     // If not runing in parallel and the patch is not local, this is a serial
     // operation on a piece of a parallel decomposition and is therefore
-    // inactive.  HJ, 5/Spe/2016
+    // inactive.  HJ, 5/Sep/2016
     if (!Pstream::parRun() && !localParallel())
     {
         return false;
@@ -770,6 +771,18 @@ const Foam::faceZone& Foam::ggiPolyPatch::zone() const
 }
 
 
+Foam::label Foam::ggiPolyPatch::comm() const
+{
+    return boundaryMesh().mesh().comm();
+}
+
+
+int Foam::ggiPolyPatch::tag() const
+{
+    return Pstream::msgType();
+}
+
+
 const Foam::labelList& Foam::ggiPolyPatch::zoneAddressing() const
 {
     if (!zoneAddressingPtr_)
diff --git a/src/foam/meshes/polyMesh/polyPatches/constraint/ggi/ggiPolyPatch.H b/src/foam/meshes/polyMesh/polyPatches/constraint/ggi/ggiPolyPatch.H
index ab6ca49b0..f17e3636b 100644
--- a/src/foam/meshes/polyMesh/polyPatches/constraint/ggi/ggiPolyPatch.H
+++ b/src/foam/meshes/polyMesh/polyPatches/constraint/ggi/ggiPolyPatch.H
@@ -303,6 +303,15 @@ public:
             const faceZone& zone() const;
 
 
+        // Communications support
+
+            //- Return communicator used for communication
+            virtual label comm() const;
+
+            //- Return message tag to use for communication
+            virtual int tag() const;
+
+
         // Interpolation data
 
             //- Is this the master side?
diff --git a/src/foam/meshes/polyMesh/polyPatches/constraint/mixingPlane/mixingPlanePolyPatch.C b/src/foam/meshes/polyMesh/polyPatches/constraint/mixingPlane/mixingPlanePolyPatch.C
index 298853762..ca4bd8a93 100644
--- a/src/foam/meshes/polyMesh/polyPatches/constraint/mixingPlane/mixingPlanePolyPatch.C
+++ b/src/foam/meshes/polyMesh/polyPatches/constraint/mixingPlane/mixingPlanePolyPatch.C
@@ -805,6 +805,18 @@ Foam::mixingPlanePolyPatch::patchToPatch() const
 }
 
 
+Foam::label Foam::mixingPlanePolyPatch::comm() const
+{
+    return boundaryMesh().mesh().comm();
+}
+
+
+int Foam::mixingPlanePolyPatch::tag() const
+{
+    return Pstream::msgType();
+}
+
+
 const Foam::vectorField&
 Foam::mixingPlanePolyPatch::reconFaceCellCentres() const
 {
diff --git a/src/foam/meshes/polyMesh/polyPatches/constraint/mixingPlane/mixingPlanePolyPatch.H b/src/foam/meshes/polyMesh/polyPatches/constraint/mixingPlane/mixingPlanePolyPatch.H
index 2f4b75b54..58173e15d 100644
--- a/src/foam/meshes/polyMesh/polyPatches/constraint/mixingPlane/mixingPlanePolyPatch.H
+++ b/src/foam/meshes/polyMesh/polyPatches/constraint/mixingPlane/mixingPlanePolyPatch.H
@@ -278,9 +278,8 @@ public:
         }
 
 
-    // Destructor
-
-        virtual ~mixingPlanePolyPatch();
+    //- Destructor
+    virtual ~mixingPlanePolyPatch();
 
 
     // Member functions
@@ -346,6 +345,15 @@ public:
             const mixingPlaneZoneInterpolation& patchToPatch() const;
 
 
+        // Communications support
+
+            //- Return communicator used for communication
+            virtual label comm() const;
+
+            //- Return message tag to use for communication
+            virtual int tag() const;
+
+
         // Interpolation functions
 
             //- Interpolate face field to profile: given field on a
diff --git a/src/foam/meshes/polyMesh/polyPatches/constraint/processor/processorPolyPatch.C b/src/foam/meshes/polyMesh/polyPatches/constraint/processor/processorPolyPatch.C
index 8f8142d98..48b1e9397 100644
--- a/src/foam/meshes/polyMesh/polyPatches/constraint/processor/processorPolyPatch.C
+++ b/src/foam/meshes/polyMesh/polyPatches/constraint/processor/processorPolyPatch.C
@@ -27,12 +27,13 @@ License
 #include "addToRunTimeSelectionTable.H"
 #include "dictionary.H"
 #include "SubField.H"
-#include "demandDrivenData.H"
 #include "matchPoints.H"
 #include "OFstream.H"
+#include "polyBoundaryMesh.H"
 #include "polyMesh.H"
 #include "foamTime.H"
 #include "transformList.H"
+#include "demandDrivenData.H"
 
 // * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
 
@@ -134,6 +135,18 @@ Foam::processorPolyPatch::~processorPolyPatch()
 
 // * * * * * * * * * * * * * * * Member Functions  * * * * * * * * * * * * * //
 
+Foam::label Foam::processorPolyPatch::comm() const
+{
+    return boundaryMesh().mesh().comm();
+}
+
+
+int Foam::processorPolyPatch::tag() const
+{
+    return Pstream::msgType();
+}
+
+
 void Foam::processorPolyPatch::initAddressing()
 {
     polyPatch::initAddressing();
diff --git a/src/foam/meshes/polyMesh/polyPatches/constraint/processor/processorPolyPatch.H b/src/foam/meshes/polyMesh/polyPatches/constraint/processor/processorPolyPatch.H
index 6dde074f8..f90a286d1 100644
--- a/src/foam/meshes/polyMesh/polyPatches/constraint/processor/processorPolyPatch.H
+++ b/src/foam/meshes/polyMesh/polyPatches/constraint/processor/processorPolyPatch.H
@@ -43,6 +43,7 @@ SourceFiles
 #define processorPolyPatch_H
 
 #include "coupledPolyPatch.H"
+#include "polyBoundaryMesh.H"
 
 // * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
 
@@ -189,9 +190,8 @@ public:
         }
 
 
-    // Destructor
-
-        virtual ~processorPolyPatch();
+    //- Destructor
+    virtual ~processorPolyPatch();
 
 
     // Member functions
@@ -208,6 +208,15 @@ public:
             return neighbProcNo_;
         }
 
+        // Communications support
+
+            //- Return communicator used for communication
+            virtual label comm() const;
+
+            //- Return message tag to use for communication
+            virtual int tag() const;
+
+
         //- Does the processor own the patch ?
         bool owner() const
         {
diff --git a/src/foam/meshes/polyMesh/polyPatches/constraint/regionCouple/regionCouplePolyPatch.C b/src/foam/meshes/polyMesh/polyPatches/constraint/regionCouple/regionCouplePolyPatch.C
index 7471e348e..233c9b697 100644
--- a/src/foam/meshes/polyMesh/polyPatches/constraint/regionCouple/regionCouplePolyPatch.C
+++ b/src/foam/meshes/polyMesh/polyPatches/constraint/regionCouple/regionCouplePolyPatch.C
@@ -921,6 +921,18 @@ void Foam::regionCouplePolyPatch::detach() const
 }
 
 
+Foam::label Foam::regionCouplePolyPatch::comm() const
+{
+    return boundaryMesh().mesh().comm();
+}
+
+
+int Foam::regionCouplePolyPatch::tag() const
+{
+    return Pstream::msgType();
+}
+
+
 const Foam::labelList& Foam::regionCouplePolyPatch::zoneAddressing() const
 {
     if (!zoneAddressingPtr_)
diff --git a/src/foam/meshes/polyMesh/polyPatches/constraint/regionCouple/regionCouplePolyPatch.H b/src/foam/meshes/polyMesh/polyPatches/constraint/regionCouple/regionCouplePolyPatch.H
index 5a9dacee6..b6cb05589 100644
--- a/src/foam/meshes/polyMesh/polyPatches/constraint/regionCouple/regionCouplePolyPatch.H
+++ b/src/foam/meshes/polyMesh/polyPatches/constraint/regionCouple/regionCouplePolyPatch.H
@@ -291,7 +291,7 @@ public:
         }
 
 
-    //-Destructor
+    //- Destructor
     virtual ~regionCouplePolyPatch();
 
 
@@ -351,6 +351,15 @@ public:
             void detach() const;
 
 
+        // Communications support
+
+            //- Return communicator used for communication
+            virtual label comm() const;
+
+            //- Return message tag to use for communication
+            virtual int tag() const;
+
+
         // Interpolation data
 
             //- Is this the master side?
diff --git a/src/lduSolvers/amg/amgPolicy/pamgPolicy.C b/src/lduSolvers/amg/amgPolicy/pamgPolicy.C
index a4d302e32..bf154925f 100644
--- a/src/lduSolvers/amg/amgPolicy/pamgPolicy.C
+++ b/src/lduSolvers/amg/amgPolicy/pamgPolicy.C
@@ -651,7 +651,8 @@ Foam::autoPtr<Foam::amgMatrix> Foam::pamgPolicy::restrictMatrix
                     coarseInterfaces,
                     fineInterface,
                     fineInterface.interfaceInternalField(child_),
-                    fineInterfaceAddr[intI]
+                    fineInterfaceAddr[intI],
+                    Pstream::worldComm //HJ, AMG Comm fineMesh.comm()
                 ).ptr()
             );
         }
diff --git a/src/tetFiniteElement/tetPolyMesh/tetPolyMesh.C b/src/tetFiniteElement/tetPolyMesh/tetPolyMesh.C
index 791f4c9f0..a7bbbef8c 100644
--- a/src/tetFiniteElement/tetPolyMesh/tetPolyMesh.C
+++ b/src/tetFiniteElement/tetPolyMesh/tetPolyMesh.C
@@ -61,6 +61,7 @@ void tetPolyMesh::clearOut() const
     clearOutParPointData();
 }
 
+
 // * * * * * * * * * * * * * * * * Constructors  * * * * * * * * * * * * * * //
 
 // Construct from components
@@ -71,6 +72,7 @@ tetPolyMesh::tetPolyMesh(const polyMesh& pMesh)
     boundary_(*this, pMesh.boundaryMesh()),
     faceOffset_(mesh_.nPoints()),
     cellOffset_(faceOffset_ + mesh_.nFaces()),
+    comm_(Pstream::worldComm),
     nPoints_(-1),
     nEdges_(-1),
     nTets_(-1),
@@ -105,6 +107,18 @@ tetPolyMesh::~tetPolyMesh()
 
 // * * * * * * * * * * * * * * * Member Functions  * * * * * * * * * * * * * //
 
+Foam::label Foam::tetPolyMesh::comm() const
+{
+    return comm_;
+}
+
+
+Foam::label& Foam::tetPolyMesh::comm()
+{
+    return comm_;
+}
+
+
 // Return number of edges in decomposition for a face
 label tetPolyMesh::nEdgesForFace(const label faceID) const
 {
diff --git a/src/tetFiniteElement/tetPolyMesh/tetPolyMesh.H b/src/tetFiniteElement/tetPolyMesh/tetPolyMesh.H
index ec93bfa24..7727db5ed 100644
--- a/src/tetFiniteElement/tetPolyMesh/tetPolyMesh.H
+++ b/src/tetFiniteElement/tetPolyMesh/tetPolyMesh.H
@@ -116,7 +116,7 @@ class tetPolyMeshMapper;
 class tetPolyMeshLduAddressing;
 
 /*---------------------------------------------------------------------------*\
-                     Class tetPolyMesh Declaration
+                           Class tetPolyMesh Declaration
 \*---------------------------------------------------------------------------*/
 
 class tetPolyMesh
@@ -138,6 +138,9 @@ class tetPolyMesh
         //- Offset in numbering to first cell centre
         label cellOffset_;
 
+        //- Communicator used for parallel communication
+        label comm_;
+
 
     // Demand-driven data
 
@@ -205,9 +208,9 @@ public:
         //- Construct from components
         explicit tetPolyMesh(const polyMesh& pMesh);
 
-    // Destructor
 
-        ~tetPolyMesh();
+    //- Destructor
+    virtual ~tetPolyMesh();
 
 
     // Member Functions
@@ -370,6 +373,15 @@ public:
                 const edgeList& parallelEdges() const;
 
 
+            // Communication support
+
+                //- Return communicator used for parallel communication
+                label comm() const;
+
+                //- Return communicator used for parallel communication
+                label& comm();
+
+
         // Edit
 
             //- Update mesh topology using the morph engine

From e7e1de3460e6a700f19cd6ef7f9783515f871efb Mon Sep 17 00:00:00 2001
From: Hrvoje Jasak <h.jasak@wikki.co.uk>
Date: Wed, 14 Sep 2016 16:18:45 +0100
Subject: [PATCH 038/130] Update decomposePar

---
 .../system/decomposeParDict                   | 40 ++++---------------
 1 file changed, 8 insertions(+), 32 deletions(-)

diff --git a/tutorials/basic/laplacianFoam/Case1.1_mixingPlane_sector_AB_60deg/system/decomposeParDict b/tutorials/basic/laplacianFoam/Case1.1_mixingPlane_sector_AB_60deg/system/decomposeParDict
index ec1ba024d..536b423e1 100644
--- a/tutorials/basic/laplacianFoam/Case1.1_mixingPlane_sector_AB_60deg/system/decomposeParDict
+++ b/tutorials/basic/laplacianFoam/Case1.1_mixingPlane_sector_AB_60deg/system/decomposeParDict
@@ -16,40 +16,16 @@ FoamFile
 
 numberOfSubdomains 12;
 
+globalFaceZones
+(
+    sideWallA_BB_cyclic1Zone
+    sideWallA_BB_cyclic2Zone
+    B_UPSTREAMZone
+    B_DOWNSTREAMZone
+);
+
 method          metis;
 
-simpleCoeffs
-{
-    n               (2 2 1);
-    delta           0.001;
-}
-
-hierarchicalCoeffs
-{
-    n               (1 1 1);
-    delta           0.001;
-    order           xyz;
-}
-
-metisCoeffs
-{
-    processorWeights
-    (
-        1
-        1
-        1
-        1
-        1
-        1
-        1
-        1
-        1
-        1
-        1
-        1
-    );
-}
-
 manualCoeffs
 {
     dataFile        "";

From e3ffddaf2a29d4cd61fca0dbe4d2e32c73bcb4b8 Mon Sep 17 00:00:00 2001
From: Hrvoje Jasak <h.jasak@wikki.co.uk>
Date: Wed, 14 Sep 2016 16:19:38 +0100
Subject: [PATCH 039/130] Remove debug message

---
 src/foam/db/IOstreams/Pstreams/Pstream.C | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/src/foam/db/IOstreams/Pstreams/Pstream.C b/src/foam/db/IOstreams/Pstreams/Pstream.C
index 14fc149ac..1d8bab960 100644
--- a/src/foam/db/IOstreams/Pstreams/Pstream.C
+++ b/src/foam/db/IOstreams/Pstreams/Pstream.C
@@ -108,7 +108,7 @@ Foam::List<Foam::Pstream::commsStruct> Foam::Pstream::calcLinearComm
         belowIDs,
         labelList(0)
     );
-    Pout<< "SIZE: " << linearCommunication.size() << " nProcs: " << nProcs << endl;
+
     // Slaves. Have no below processors, only communicate up to master
     for (label procID = 1; procID < nProcs; procID++)
     {
@@ -166,8 +166,6 @@ Foam::List<Foam::Pstream::commsStruct> Foam::Pstream::calcTreeComm
     List<dynamicLabelList> receives(nProcs);
     labelList sends(nProcs, -1);
 
-    // Info<< "Using " << nLevels << " communication levels" << endl;
-
     label offset = 2;
     label childOffset = offset/2;
 

From 0ffa52a02910c19752f2402cebf7fe8557cca82d Mon Sep 17 00:00:00 2001
From: Hrvoje Jasak <h.jasak@wikki.co.uk>
Date: Wed, 14 Sep 2016 16:19:46 +0100
Subject: [PATCH 040/130] Formatting

---
 .../polyPatches/constraint/cyclicGgi/cyclicGgiPolyPatch.H    | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/src/foam/meshes/polyMesh/polyPatches/constraint/cyclicGgi/cyclicGgiPolyPatch.H b/src/foam/meshes/polyMesh/polyPatches/constraint/cyclicGgi/cyclicGgiPolyPatch.H
index e869fdcaa..1a29e239d 100644
--- a/src/foam/meshes/polyMesh/polyPatches/constraint/cyclicGgi/cyclicGgiPolyPatch.H
+++ b/src/foam/meshes/polyMesh/polyPatches/constraint/cyclicGgi/cyclicGgiPolyPatch.H
@@ -171,9 +171,8 @@ public:
         }
 
 
-    // Destructor
-
-        virtual ~cyclicGgiPolyPatch();
+    //- Destructor
+    virtual ~cyclicGgiPolyPatch();
 
 
     // Member functions

From f28283c743eda48f80644e1d5b5dd3ad72895d71 Mon Sep 17 00:00:00 2001
From: Hrvoje Jasak <h.jasak@wikki.co.uk>
Date: Wed, 14 Sep 2016 16:20:07 +0100
Subject: [PATCH 041/130] Update decompose and reconstruct: patch cannot be
 active

---
 .../polyPatches/constraint/ggi/ggiPolyPatch.C | 34 +++++++++++++------
 1 file changed, 23 insertions(+), 11 deletions(-)

diff --git a/src/foam/meshes/polyMesh/polyPatches/constraint/ggi/ggiPolyPatch.C b/src/foam/meshes/polyMesh/polyPatches/constraint/ggi/ggiPolyPatch.C
index 22751734f..b94a52083 100644
--- a/src/foam/meshes/polyMesh/polyPatches/constraint/ggi/ggiPolyPatch.C
+++ b/src/foam/meshes/polyMesh/polyPatches/constraint/ggi/ggiPolyPatch.C
@@ -63,16 +63,28 @@ bool Foam::ggiPolyPatch::active() const
     polyPatchID shadow(shadowName_, boundaryMesh());
     faceZoneID zone(zoneName_, boundaryMesh().mesh().faceZones());
 
-    // For decomposition and reconstruction
-    // If not runing in parallel and the patch is not local, this is a serial
-    // operation on a piece of a parallel decomposition and is therefore
-    // inactive.  HJ, 5/Sep/2016
-    if (!Pstream::parRun() && !localParallel())
+    if (shadow.active() && zone.active())
     {
+        if
+        (
+            !Pstream::parRun()
+         && size() != boundaryMesh().mesh().faceZones()[zone.index()].size()
+        )
+        {
+            // Patch is present in serial run, but zone is not the same size
+            // Probably doing decomposition and reconstruction
+            // HJ, 14/Sep/2016
+
+            return false;
+        }
+
+        return true;
+    }
+    else
+    {
+        // Zones not active.  Nothing to check
         return false;
     }
-
-    return shadow.active() && zone.active();
 }
 
 
@@ -960,11 +972,11 @@ void Foam::ggiPolyPatch::initMovePoints(const pointField& p)
         {
             map();
         }
-    }
 
-    if (active() && master())
-    {
-        reconFaceCellCentres();
+        if (master())
+        {
+            reconFaceCellCentres();
+        }
     }
 
     polyPatch::initMovePoints(p);

From dbe26da7bd53feb3b4d48235ccc85a126ba87919 Mon Sep 17 00:00:00 2001
From: Hrvoje Jasak <h.jasak@wikki.co.uk>
Date: Thu, 15 Sep 2016 15:28:43 +0100
Subject: [PATCH 042/130] Removed unnecessary files

---
 src/foam/db/IOstreams/Pstreams/Pstream.H      |  1 -
 .../db/IOstreams/Pstreams/PstreamReduceOps.C  | 31 -------------------
 2 files changed, 32 deletions(-)
 delete mode 100644 src/foam/db/IOstreams/Pstreams/PstreamReduceOps.C

diff --git a/src/foam/db/IOstreams/Pstreams/Pstream.H b/src/foam/db/IOstreams/Pstreams/Pstream.H
index 5d0110e24..2f44eca57 100644
--- a/src/foam/db/IOstreams/Pstreams/Pstream.H
+++ b/src/foam/db/IOstreams/Pstreams/Pstream.H
@@ -29,7 +29,6 @@ Description
 
 SourceFiles
     Pstream.C
-    PstreamReduceOps.C
     PstreamCommsStruct.C
     gatherScatter.C
     combineGatherScatter.C
diff --git a/src/foam/db/IOstreams/Pstreams/PstreamReduceOps.C b/src/foam/db/IOstreams/Pstreams/PstreamReduceOps.C
deleted file mode 100644
index be693beb8..000000000
--- a/src/foam/db/IOstreams/Pstreams/PstreamReduceOps.C
+++ /dev/null
@@ -1,31 +0,0 @@
-/*---------------------------------------------------------------------------*\
-  =========                 |
-  \\      /  F ield         | foam-extend: Open Source CFD
-   \\    /   O peration     | Version:     4.0
-    \\  /    A nd           | Web:         http://www.foam-extend.org
-     \\/     M anipulation  | For copyright notice see file Copyright
--------------------------------------------------------------------------------
-License
-    This file is part of foam-extend.
-
-    foam-extend is free software: you can redistribute it and/or modify it
-    under the terms of the GNU General Public License as published by the
-    Free Software Foundation, either version 3 of the License, or (at your
-    option) any later version.
-
-    foam-extend is distributed in the hope that it will be useful, but
-    WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-    General Public License for more details.
-
-    You should have received a copy of the GNU General Public License
-    along with foam-extend.  If not, see <http://www.gnu.org/licenses/>.
-
-\*---------------------------------------------------------------------------*/
-
-#include "PstreamReduceOps.H"
-#include "allReduce.H"
-
-// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
-
-// ************************************************************************* //

From b514e95f1b7c206ce89b5857b52c216d318f696c Mon Sep 17 00:00:00 2001
From: Hrvoje Jasak <h.jasak@wikki.co.uk>
Date: Thu, 15 Sep 2016 15:35:48 +0100
Subject: [PATCH 043/130] Activate local comms in GGI

---
 .../BlockMatrixAgglomeration.C                |  3 +-
 .../GAMGAgglomerateLduAddressing.C            |  3 +-
 .../interfaces/AMGInterface/AMGInterface.H    |  9 ++--
 .../interfaces/AMGInterface/newAMGInterface.C |  9 ++--
 .../cyclicAMGInterface/cyclicAMGInterface.C   |  3 +-
 .../cyclicAMGInterface/cyclicAMGInterface.H   |  3 +-
 .../cyclicGGIAMGInterface.C                   |  6 +--
 .../cyclicGGIAMGInterface.H                   |  3 +-
 .../ggiAMGInterface/ggiAMGInterface.C         | 11 ++--
 .../ggiAMGInterface/ggiAMGInterface.H         |  3 +-
 .../mixingPlaneAMGInterface.C                 |  7 ++-
 .../mixingPlaneAMGInterface.H                 |  3 +-
 .../processorAMGInterface.C                   |  7 ++-
 .../processorAMGInterface.H                   |  3 +-
 .../regionCoupleAMGInterface.C                |  6 +--
 .../regionCoupleAMGInterface.H                |  3 +-
 .../polyPatches/constraint/ggi/ggiPolyPatch.C | 52 ++++++++++++++++++-
 .../polyPatches/constraint/ggi/ggiPolyPatch.H |  6 +++
 src/lduSolvers/amg/amgPolicy/pamgPolicy.C     |  3 +-
 19 files changed, 88 insertions(+), 55 deletions(-)

diff --git a/src/foam/matrices/blockLduMatrix/BlockAMG/BlockMatrixCoarsening/BlockMatrixAgglomeration/BlockMatrixAgglomeration.C b/src/foam/matrices/blockLduMatrix/BlockAMG/BlockMatrixCoarsening/BlockMatrixAgglomeration/BlockMatrixAgglomeration.C
index bfcef67c0..b42f757af 100644
--- a/src/foam/matrices/blockLduMatrix/BlockAMG/BlockMatrixCoarsening/BlockMatrixAgglomeration/BlockMatrixAgglomeration.C
+++ b/src/foam/matrices/blockLduMatrix/BlockAMG/BlockMatrixCoarsening/BlockMatrixAgglomeration/BlockMatrixAgglomeration.C
@@ -954,8 +954,7 @@ Foam::BlockMatrixAgglomeration<Type>::restrictMatrix() const
                     coarseInterfaces,
                     fineInterface,
                     fineInterface.interfaceInternalField(agglomIndex_),
-                    fineInterfaceAddr[intI],
-                    Pstream::worldComm //HJ, AMG Comm fineMesh.comm(),
+                    fineInterfaceAddr[intI]
                 ).ptr()
             );
         }
diff --git a/src/foam/matrices/lduMatrix/solvers/AMG/AMGAgglomerations/GAMGAgglomeration/GAMGAgglomerateLduAddressing.C b/src/foam/matrices/lduMatrix/solvers/AMG/AMGAgglomerations/GAMGAgglomeration/GAMGAgglomerateLduAddressing.C
index d9f186b58..6a6b7ac66 100644
--- a/src/foam/matrices/lduMatrix/solvers/AMG/AMGAgglomerations/GAMGAgglomeration/GAMGAgglomerateLduAddressing.C
+++ b/src/foam/matrices/lduMatrix/solvers/AMG/AMGAgglomerations/GAMGAgglomeration/GAMGAgglomerateLduAddressing.C
@@ -299,8 +299,7 @@ void Foam::GAMGAgglomeration::agglomerateLduAddressing
                         (
                             restrictMap
                         ),
-                        fineInterfaceAddr[inti],
-                        fineMesh.comm() // Set up comm per level?
+                        fineInterfaceAddr[inti]
                     ).ptr()
                 );
 
diff --git a/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/AMGInterface/AMGInterface.H b/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/AMGInterface/AMGInterface.H
index d6b311d51..090cca366 100644
--- a/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/AMGInterface/AMGInterface.H
+++ b/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/AMGInterface/AMGInterface.H
@@ -132,16 +132,14 @@ public:
                 const lduInterfacePtrsList& coarseInterfaces,
                 const lduInterface& fineInterface,
                 const labelField& localRestrictAddressing,
-                const labelField& neighbourRestrictAddressing,
-                const label coarseComm
+                const labelField& neighbourRestrictAddressing
             ),
             (
                 lduMesh,
                 coarseInterfaces,
                 fineInterface,
                 localRestrictAddressing,
-                neighbourRestrictAddressing,
-                coarseComm
+                neighbourRestrictAddressing
             )
         );
 
@@ -156,8 +154,7 @@ public:
             const lduInterfacePtrsList& coarseInterfaces,
             const lduInterface& fineInterface,
             const labelField& localRestrictAddressing,
-            const labelField& neighbourRestrictAddressing,
-            const label coarseComm
+            const labelField& neighbourRestrictAddressing
         );
 
 
diff --git a/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/AMGInterface/newAMGInterface.C b/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/AMGInterface/newAMGInterface.C
index 7bd9cffaf..2f6dc2548 100644
--- a/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/AMGInterface/newAMGInterface.C
+++ b/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/AMGInterface/newAMGInterface.C
@@ -35,8 +35,7 @@ Foam::autoPtr<Foam::AMGInterface> Foam::AMGInterface::New
     const lduInterfacePtrsList& coarseInterfaces,
     const lduInterface& fineInterface,
     const labelField& localRestrictAddressing,
-    const labelField& neighbourRestrictAddressing,
-    const label coarseComm
+    const labelField& neighbourRestrictAddressing
 )
 {
     word coupleType(fineInterface.type());
@@ -54,8 +53,7 @@ Foam::autoPtr<Foam::AMGInterface> Foam::AMGInterface::New
             "    const lduInterfacePtrsList& coarseInterfaces,\n"
             "    const lduInterface& fineInterface,\n"
             "    const labelField& localRestrictAddressing,\n"
-            "    const labelField& neighbourRestrictAddressing,\n"
-            "    const label coarseComm\n"
+            "    const labelField& neighbourRestrictAddressing\n"
             ")"
         )   << "Unknown AMGInterface type " << coupleType << ".\n"
             << "Valid AMGInterface types are :"
@@ -71,8 +69,7 @@ Foam::autoPtr<Foam::AMGInterface> Foam::AMGInterface::New
             coarseInterfaces,
             fineInterface,
             localRestrictAddressing,
-            neighbourRestrictAddressing,
-            coarseComm
+            neighbourRestrictAddressing
         )
     );
 }
diff --git a/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/cyclicAMGInterface/cyclicAMGInterface.C b/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/cyclicAMGInterface/cyclicAMGInterface.C
index 5f8e0d1dc..72fe4cf0b 100644
--- a/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/cyclicAMGInterface/cyclicAMGInterface.C
+++ b/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/cyclicAMGInterface/cyclicAMGInterface.C
@@ -48,8 +48,7 @@ Foam::cyclicAMGInterface::cyclicAMGInterface
     const lduInterfacePtrsList& coarseInterfaces,
     const lduInterface& fineInterface,
     const labelField& localRestrictAddressing,
-    const labelField& neighbourRestrictAddressing,
-    const label coarseComm // Not needed
+    const labelField& neighbourRestrictAddressing
 )
 :
     AMGInterface(lduMesh),
diff --git a/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/cyclicAMGInterface/cyclicAMGInterface.H b/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/cyclicAMGInterface/cyclicAMGInterface.H
index a3e75662a..90d260e35 100644
--- a/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/cyclicAMGInterface/cyclicAMGInterface.H
+++ b/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/cyclicAMGInterface/cyclicAMGInterface.H
@@ -84,8 +84,7 @@ public:
             const lduInterfacePtrsList& coarseInterfaces,
             const lduInterface& fineInterface,
             const labelField& localRestrictAddressing,
-            const labelField& neighbourRestrictAddressing,
-            const label coarseComm
+            const labelField& neighbourRestrictAddressing
         );
 
 
diff --git a/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/cyclicGGIAMGInterface/cyclicGGIAMGInterface.C b/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/cyclicGGIAMGInterface/cyclicGGIAMGInterface.C
index fa1c2e546..c64227099 100644
--- a/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/cyclicGGIAMGInterface/cyclicGGIAMGInterface.C
+++ b/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/cyclicGGIAMGInterface/cyclicGGIAMGInterface.C
@@ -48,8 +48,7 @@ Foam::cyclicGGIAMGInterface::cyclicGGIAMGInterface
     const lduInterfacePtrsList& coarseInterfaces,
     const lduInterface& fineInterface,
     const labelField& localRestrictAddressing,
-    const labelField& neighbourRestrictAddressing,
-    const label coarseComm
+    const labelField& neighbourRestrictAddressing
 )
 :
     ggiAMGInterface
@@ -58,8 +57,7 @@ Foam::cyclicGGIAMGInterface::cyclicGGIAMGInterface
         coarseInterfaces,
         fineInterface,
         localRestrictAddressing,
-        neighbourRestrictAddressing,
-        coarseComm
+        neighbourRestrictAddressing
     )
 {}
 
diff --git a/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/cyclicGGIAMGInterface/cyclicGGIAMGInterface.H b/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/cyclicGGIAMGInterface/cyclicGGIAMGInterface.H
index be62b18f3..2498c2282 100644
--- a/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/cyclicGGIAMGInterface/cyclicGGIAMGInterface.H
+++ b/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/cyclicGGIAMGInterface/cyclicGGIAMGInterface.H
@@ -70,8 +70,7 @@ public:
             const lduInterfacePtrsList& coarseInterfaces,
             const lduInterface& fineInterface,
             const labelField& localRestrictAddressing,
-            const labelField& neighbourRestrictAddressing,
-            const label coarseComm
+            const labelField& neighbourRestrictAddressing
         );
 
 
diff --git a/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/ggiAMGInterface/ggiAMGInterface.C b/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/ggiAMGInterface/ggiAMGInterface.C
index 5c8eca04a..9222720fe 100644
--- a/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/ggiAMGInterface/ggiAMGInterface.C
+++ b/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/ggiAMGInterface/ggiAMGInterface.C
@@ -78,7 +78,7 @@ void Foam::ggiAMGInterface::initFastReduce() const
         zoneProcID[za[zaI]] = Pstream::myProcNo();
     }
 
-    reduce(zoneProcID, maxOp<labelField>());
+    reduce(zoneProcID, maxOp<labelField>(), tag(), comm());
 
     // Find out where my zone data is coming from
     labelList nRecv(Pstream::nProcs(), 0);
@@ -195,8 +195,7 @@ Foam::ggiAMGInterface::ggiAMGInterface
     const lduInterfacePtrsList& coarseInterfaces,
     const lduInterface& fineInterface,
     const labelField& localRestrictAddressing,
-    const labelField& neighbourRestrictAddressing,
-    const label coarseComm
+    const labelField& neighbourRestrictAddressing
 )
 :
     AMGInterface(lduMesh),
@@ -204,8 +203,8 @@ Foam::ggiAMGInterface::ggiAMGInterface
     zoneSize_(0),
     zoneAddressing_(),
     procMasterFaces_(),
-    comm_(coarseComm),
-    tag_(refCast<const ggiLduInterface>(fineInterface).tag()),
+    comm_(fineGgiInterface_.comm()),
+    tag_(fineGgiInterface_.tag()),
     mapPtr_(NULL)
 {
     // New algorithm will assemble local clusters on the master side and
@@ -688,7 +687,7 @@ Foam::ggiAMGInterface::ggiAMGInterface
 
     nCoarseFacesPerProc[Pstream::myProcNo()] = nCoarseFaces;
 
-    reduce(nCoarseFacesPerProc, sumOp<labelList>());
+    reduce(nCoarseFacesPerProc, sumOp<labelList>(), tag(), comm());
 
     // Coarse global face zone is assembled by adding all faces from proc0,
     // followed by all faces from proc1 etc.
diff --git a/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/ggiAMGInterface/ggiAMGInterface.H b/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/ggiAMGInterface/ggiAMGInterface.H
index 2803de3e7..e18fe220f 100644
--- a/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/ggiAMGInterface/ggiAMGInterface.H
+++ b/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/ggiAMGInterface/ggiAMGInterface.H
@@ -125,8 +125,7 @@ public:
             const lduInterfacePtrsList& coarseInterfaces,
             const lduInterface& fineInterface,
             const labelField& localRestrictAddressing,
-            const labelField& neighbourRestrictAddressing,
-            const label coarseComm
+            const labelField& neighbourRestrictAddressing
         );
 
 
diff --git a/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/mixingPlaneAMGInterface/mixingPlaneAMGInterface.C b/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/mixingPlaneAMGInterface/mixingPlaneAMGInterface.C
index 2359ae6bd..8ad4cc723 100644
--- a/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/mixingPlaneAMGInterface/mixingPlaneAMGInterface.C
+++ b/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/mixingPlaneAMGInterface/mixingPlaneAMGInterface.C
@@ -54,8 +54,7 @@ Foam::mixingPlaneAMGInterface::mixingPlaneAMGInterface
     const lduInterfacePtrsList& coarseInterfaces,
     const lduInterface& fineInterface,
     const labelField& localRestrictAddressing,
-    const labelField& neighbourRestrictAddressing,
-    const label coarseComm
+    const labelField& neighbourRestrictAddressing
 )
 :
     AMGInterface(lduMesh),
@@ -63,8 +62,8 @@ Foam::mixingPlaneAMGInterface::mixingPlaneAMGInterface
     (
         refCast<const mixingPlaneLduInterface>(fineInterface)
     ),
-    comm_(coarseComm),
-    tag_(refCast<const mixingPlaneLduInterface>(fineInterface).tag())
+    comm_(fineMixingPlaneInterface_.comm()),
+    tag_(fineMixingPlaneInterface_.tag())
 {}
 
 
diff --git a/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/mixingPlaneAMGInterface/mixingPlaneAMGInterface.H b/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/mixingPlaneAMGInterface/mixingPlaneAMGInterface.H
index 518f5bfee..47a8bc47b 100644
--- a/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/mixingPlaneAMGInterface/mixingPlaneAMGInterface.H
+++ b/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/mixingPlaneAMGInterface/mixingPlaneAMGInterface.H
@@ -105,8 +105,7 @@ public:
             const lduInterfacePtrsList& coarseInterfaces,
             const lduInterface& fineInterface,
             const labelField& localRestrictAddressing,
-            const labelField& neighbourRestrictAddressing,
-            const label coarseComm
+            const labelField& neighbourRestrictAddressing
         );
 
 
diff --git a/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/processorAMGInterface/processorAMGInterface.C b/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/processorAMGInterface/processorAMGInterface.C
index 25bbb8552..5d99c36eb 100644
--- a/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/processorAMGInterface/processorAMGInterface.C
+++ b/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/processorAMGInterface/processorAMGInterface.C
@@ -48,14 +48,13 @@ Foam::processorAMGInterface::processorAMGInterface
     const lduInterfacePtrsList& coarseInterfaces,
     const lduInterface& fineInterface,
     const labelField& localRestrictAddressing,
-    const labelField& neighbourRestrictAddressing,
-    const label coarseComm
+    const labelField& neighbourRestrictAddressing
 )
 :
     AMGInterface(lduMesh),
     fineProcInterface_(refCast<const processorLduInterface>(fineInterface)),
-    comm_(coarseComm),
-    tag_(refCast<const processorLduInterface>(fineInterface).tag())
+    comm_(fineProcInterface_.comm()),
+    tag_(fineProcInterface_.tag())
 {
     // Make a lookup table of entries for owner/neighbour
     HashTable<SLList<label>, label, Hash<label> > neighboursTable
diff --git a/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/processorAMGInterface/processorAMGInterface.H b/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/processorAMGInterface/processorAMGInterface.H
index 4fbda6c03..60aff5b78 100644
--- a/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/processorAMGInterface/processorAMGInterface.H
+++ b/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/processorAMGInterface/processorAMGInterface.H
@@ -90,8 +90,7 @@ public:
             const lduInterfacePtrsList& coarseInterfaces,
             const lduInterface& fineInterface,
             const labelField& localRestrictAddressing,
-            const labelField& neighbourRestrictAddressing,
-            const label coarseComm
+            const labelField& neighbourRestrictAddressing
         );
 
 
diff --git a/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/regionCoupleAMGInterface/regionCoupleAMGInterface.C b/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/regionCoupleAMGInterface/regionCoupleAMGInterface.C
index 79a749339..4f1c1f3b8 100644
--- a/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/regionCoupleAMGInterface/regionCoupleAMGInterface.C
+++ b/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/regionCoupleAMGInterface/regionCoupleAMGInterface.C
@@ -48,8 +48,7 @@ Foam::regionCoupleAMGInterface::regionCoupleAMGInterface
     const lduInterfacePtrsList& coarseInterfaces,
     const lduInterface& fineInterface,
     const labelField& localRestrictAddressing,
-    const labelField& neighbourRestrictAddressing,
-    const label coarseComm
+    const labelField& neighbourRestrictAddressing
 )
 :
     ggiAMGInterface
@@ -58,8 +57,7 @@ Foam::regionCoupleAMGInterface::regionCoupleAMGInterface
         coarseInterfaces,
         fineInterface,
         localRestrictAddressing,
-        neighbourRestrictAddressing,
-        coarseComm
+        neighbourRestrictAddressing
     ),
     coupled_(fineInterface.coupled())
 {}
diff --git a/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/regionCoupleAMGInterface/regionCoupleAMGInterface.H b/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/regionCoupleAMGInterface/regionCoupleAMGInterface.H
index 136b982f5..4701acdf5 100644
--- a/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/regionCoupleAMGInterface/regionCoupleAMGInterface.H
+++ b/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/regionCoupleAMGInterface/regionCoupleAMGInterface.H
@@ -76,8 +76,7 @@ public:
             const lduInterfacePtrsList& coarseInterfaces,
             const lduInterface& fineInterface,
             const labelField& localRestrictAddressing,
-            const labelField& neighbourRestrictAddressing,
-            const label coarseComm
+            const labelField& neighbourRestrictAddressing
         );
 
 
diff --git a/src/foam/meshes/polyMesh/polyPatches/constraint/ggi/ggiPolyPatch.C b/src/foam/meshes/polyMesh/polyPatches/constraint/ggi/ggiPolyPatch.C
index b94a52083..c4f748d07 100644
--- a/src/foam/meshes/polyMesh/polyPatches/constraint/ggi/ggiPolyPatch.C
+++ b/src/foam/meshes/polyMesh/polyPatches/constraint/ggi/ggiPolyPatch.C
@@ -359,6 +359,45 @@ void Foam::ggiPolyPatch::calcLocalParallel() const
         reduce(emptyOrComplete, andOp<bool>());
     }
 
+    if (emptyOrComplete)
+    {
+        comm_ = boundaryMesh().mesh().comm();
+    }
+    else
+    {
+        // Count how many patch faces exist on each processor
+        labelList nFacesPerProc(Pstream::nProcs(), 0);
+        nFacesPerProc[Pstream::myProcNo()] = size();
+//         reduce(nFacesPerProc, maxOp<label>());
+        Pstream::gatherList(nFacesPerProc);
+        Pstream::scatterList(nFacesPerProc);
+
+        Pout<< "nFacesPerProc: " << nFacesPerProc << endl;
+        // Make a comm, from all processors that contain the ggi faces
+        labelList ggiCommProcs(Pstream::nProcs());
+        label nGgiCommProcs = 0;
+
+        forAll (nFacesPerProc, procI)
+        {
+            if (nFacesPerProc[procI] > 0)
+            {
+                ggiCommProcs[nGgiCommProcs] = procI;
+                nGgiCommProcs++;
+            }
+        }
+        ggiCommProcs.setSize(nGgiCommProcs);
+
+        // Allocate communicator
+        Info<< "Allocating communicator for GGI patch " << name()
+            << " with " << ggiCommProcs
+            << endl;
+        comm_ = Pstream::allocateCommunicator
+        (
+            Pstream::worldComm,
+            ggiCommProcs
+        );
+    }
+
     if (debug && Pstream::parRun())
     {
         Info<< "GGI patch Master: " << name()
@@ -785,7 +824,18 @@ const Foam::faceZone& Foam::ggiPolyPatch::zone() const
 
 Foam::label Foam::ggiPolyPatch::comm() const
 {
-    return boundaryMesh().mesh().comm();
+    //HJ, Testing.  Use optimised comm or a local one
+
+    // Note: comm is calculated with localParallel and will use the
+    // localParallelPtr_ for signalling.  HJ, 10/Sep2016
+    if (!localParallelPtr_)
+    {
+        calcLocalParallel();
+    }
+
+    return comm_;
+
+//     return boundaryMesh().mesh().comm();
 }
 
 
diff --git a/src/foam/meshes/polyMesh/polyPatches/constraint/ggi/ggiPolyPatch.H b/src/foam/meshes/polyMesh/polyPatches/constraint/ggi/ggiPolyPatch.H
index f17e3636b..6c5e0415e 100644
--- a/src/foam/meshes/polyMesh/polyPatches/constraint/ggi/ggiPolyPatch.H
+++ b/src/foam/meshes/polyMesh/polyPatches/constraint/ggi/ggiPolyPatch.H
@@ -105,6 +105,12 @@ class ggiPolyPatch
             //  Used for parallel optimisation
             mutable bool* localParallelPtr_;
 
+            //- Communicator to use for parallel communication
+            mutable label comm_;
+
+            //- Message tag used for sending
+            mutable int tag_;
+
             //- Map-distribute comms tool
             mutable mapDistribute* mapPtr_;
 
diff --git a/src/lduSolvers/amg/amgPolicy/pamgPolicy.C b/src/lduSolvers/amg/amgPolicy/pamgPolicy.C
index bf154925f..a4d302e32 100644
--- a/src/lduSolvers/amg/amgPolicy/pamgPolicy.C
+++ b/src/lduSolvers/amg/amgPolicy/pamgPolicy.C
@@ -651,8 +651,7 @@ Foam::autoPtr<Foam::amgMatrix> Foam::pamgPolicy::restrictMatrix
                     coarseInterfaces,
                     fineInterface,
                     fineInterface.interfaceInternalField(child_),
-                    fineInterfaceAddr[intI],
-                    Pstream::worldComm //HJ, AMG Comm fineMesh.comm()
+                    fineInterfaceAddr[intI]
                 ).ptr()
             );
         }

From e73480637ba27bff6e7dad12d9fda1d017634284 Mon Sep 17 00:00:00 2001
From: Hrvoje Jasak <h.jasak@wikki.co.uk>
Date: Mon, 19 Sep 2016 12:11:06 +0100
Subject: [PATCH 044/130] Fast comms, update

---
 src/foam/db/IOstreams/Pstreams/OPwrite.C      |  5 +++-
 src/foam/db/IOstreams/Pstreams/Pstream.C      | 21 ++++++++-------
 .../db/IOstreams/Pstreams/PstreamReduceOps.H  |  2 +-
 src/foam/db/IOstreams/Pstreams/allReduce.H    |  1 +
 .../IOstreams/Pstreams/allReduceTemplates.C   | 10 ++++---
 .../db/IOstreams/Pstreams/gatherScatter.C     | 20 +++++++-------
 .../ggiAMGInterface/ggiAMGInterface.C         |  5 +++-
 .../polyPatches/constraint/ggi/ggiPolyPatch.C | 27 +++++++++++++++----
 8 files changed, 59 insertions(+), 32 deletions(-)

diff --git a/src/foam/db/IOstreams/Pstreams/OPwrite.C b/src/foam/db/IOstreams/Pstreams/OPwrite.C
index 8fee41230..5f3302bc9 100644
--- a/src/foam/db/IOstreams/Pstreams/OPwrite.C
+++ b/src/foam/db/IOstreams/Pstreams/OPwrite.C
@@ -42,7 +42,9 @@ Foam::OPstream::~OPstream()
             commsType_,
             toProcNo_,
             buf_.begin(),
-            bufPosition_
+            bufPosition_,
+            tag_,
+            comm_
         )
     )
     {
@@ -81,6 +83,7 @@ bool Foam::OPstream::write
             << " commsType:" << Pstream::commsTypeNames[commsType]
             << " warnComm:" << Pstream::warnComm
             << Foam::endl;
+
         error::printStack(Pout);
     }
 
diff --git a/src/foam/db/IOstreams/Pstreams/Pstream.C b/src/foam/db/IOstreams/Pstreams/Pstream.C
index 1d8bab960..4691c802e 100644
--- a/src/foam/db/IOstreams/Pstreams/Pstream.C
+++ b/src/foam/db/IOstreams/Pstreams/Pstream.C
@@ -106,7 +106,7 @@ Foam::List<Foam::Pstream::commsStruct> Foam::Pstream::calcLinearComm
         0,
         -1,
         belowIDs,
-        labelList(0)
+        labelList()
     );
 
     // Slaves. Have no below processors, only communicate up to master
@@ -117,8 +117,8 @@ Foam::List<Foam::Pstream::commsStruct> Foam::Pstream::calcLinearComm
             nProcs,
             procID,
             0,
-            labelList(0),
-            labelList(0)
+            labelList(),
+            labelList()
         );
     }
 
@@ -280,7 +280,7 @@ void Foam::Pstream::allocatePstreamCommunicator
         {
             FatalErrorIn
             (
-                "Pstream::allocateCommunicator\n"
+                "Pstream::allocatePstreamCommunicator\n"
                 "(\n"
                 "    const label parentIndex,\n"
                 "    const labelList& subRanks\n"
@@ -391,10 +391,10 @@ Foam::label Foam::Pstream::allocateCommunicator
         index = parentCommunicator_.size();
 
         myProcNo_.append(-1);
-        procIDs_.append(List<int>(0));
+        procIDs_.append(List<int>());
         parentCommunicator_.append(-1);
-        linearCommunication_.append(List<commsStruct>(0));
-        treeCommunication_.append(List<commsStruct>(0));
+        linearCommunication_.append(List<commsStruct>());
+        treeCommunication_.append(List<commsStruct>());
     }
 
     if (debug)
@@ -415,7 +415,7 @@ Foam::label Foam::Pstream::allocateCommunicator
         procIDs_[index][i] = subRanks[i];
 
         // Enforce incremental order (so index is rank in next communicator)
-        if (i >= 1 && subRanks[i] <= subRanks[i-1])
+        if (i >= 1 && subRanks[i] <= subRanks[i - 1])
         {
             FatalErrorIn
             (
@@ -674,6 +674,7 @@ void Foam::reduce
             << endl;
         error::printStack(Pout);
     }
+
     allReduce(Value, 1, MPI_SCALAR, MPI_SUM, bop, tag, comm);
 }
 
@@ -1036,9 +1037,9 @@ Foam::debug::optimisationSwitch
 Foam::Pstream::defaultCommsType
 (
     "commsType",
-//     "nonBlocking",
+    "nonBlocking",
 //     "scheduled",
-    "blocking",
+//     "blocking",
     "blocking, nonBlocking, scheduled"
 );
 
diff --git a/src/foam/db/IOstreams/Pstreams/PstreamReduceOps.H b/src/foam/db/IOstreams/Pstreams/PstreamReduceOps.H
index 2b7f45263..f92ed0bca 100644
--- a/src/foam/db/IOstreams/Pstreams/PstreamReduceOps.H
+++ b/src/foam/db/IOstreams/Pstreams/PstreamReduceOps.H
@@ -134,7 +134,7 @@ void reduce
 {
     notImplemented
     (
-        "reduce(T&, const BinaryOp&, const int, const label, label&"
+        "reduce(T&, const BinaryOp&, const int, const label, label&)"
     );
 }
 
diff --git a/src/foam/db/IOstreams/Pstreams/allReduce.H b/src/foam/db/IOstreams/Pstreams/allReduce.H
index 792dcea54..41090a48f 100644
--- a/src/foam/db/IOstreams/Pstreams/allReduce.H
+++ b/src/foam/db/IOstreams/Pstreams/allReduce.H
@@ -54,6 +54,7 @@ void allReduce
     const int communicator
 );
 
+
 // * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
 
 } // End namespace Foam
diff --git a/src/foam/db/IOstreams/Pstreams/allReduceTemplates.C b/src/foam/db/IOstreams/Pstreams/allReduceTemplates.C
index ab16055a6..13b6eddfc 100644
--- a/src/foam/db/IOstreams/Pstreams/allReduceTemplates.C
+++ b/src/foam/db/IOstreams/Pstreams/allReduceTemplates.C
@@ -64,7 +64,7 @@ void Foam::allReduce
                         &value,
                         MPICount,
                         MPIType,
-                        slave,  //Pstream::procID(slave),
+                        slave,
                         tag,
                         PstreamGlobals::MPICommunicators_[comm],
                         MPI_STATUS_IGNORE
@@ -98,7 +98,7 @@ void Foam::allReduce
                     &Value,
                     MPICount,
                     MPIType,
-                    Pstream::masterNo(),//Pstream::procID(masterNo()),
+                    Pstream::masterNo(),
                     tag,
                     PstreamGlobals::MPICommunicators_[comm]
                 )
@@ -137,7 +137,7 @@ void Foam::allReduce
                         &Value,
                         MPICount,
                         MPIType,
-                        slave,      //Pstream::procID(slave),
+                        slave,
                         tag,
                         PstreamGlobals::MPICommunicators_[comm]
                     )
@@ -168,7 +168,7 @@ void Foam::allReduce
                     &Value,
                     MPICount,
                     MPIType,
-                    Pstream::masterNo(),//Pstream::procID(masterNo()),
+                    Pstream::masterNo(),
                     tag,
                     PstreamGlobals::MPICommunicators_[comm],
                     MPI_STATUS_IGNORE
@@ -194,6 +194,7 @@ void Foam::allReduce
     else
     {
         Type sum;
+
         MPI_Allreduce
         (
             &Value,
@@ -203,6 +204,7 @@ void Foam::allReduce
             MPIOp,
             PstreamGlobals::MPICommunicators_[comm]
         );
+
         Value = sum;
     }
 }
diff --git a/src/foam/db/IOstreams/Pstreams/gatherScatter.C b/src/foam/db/IOstreams/Pstreams/gatherScatter.C
index d182b830f..a98cf9792 100644
--- a/src/foam/db/IOstreams/Pstreams/gatherScatter.C
+++ b/src/foam/db/IOstreams/Pstreams/gatherScatter.C
@@ -50,10 +50,10 @@ void Pstream::gather
     const label comm
 )
 {
-    if (Pstream::nProcs(comm) > 1)
+    if (Pstream::nProcs() > 1)
     {
         // Get my communication order
-        const commsStruct& myComm = comms[Pstream::myProcNo(comm)];
+        const commsStruct& myComm = comms[Pstream::myProcNo()];
 
         // Receive from my downstairs neighbours
         forAll (myComm.below(), belowI)
@@ -129,13 +129,13 @@ void Pstream::gather
     const label comm
 )
 {
-    if (Pstream::nProcs(comm) < Pstream::nProcsSimpleSum())
+    if (Pstream::nProcs() < Pstream::nProcsSimpleSum())
     {
-        gather(Pstream::linearCommunication(comm), Value, bop, tag, comm);
+        gather(Pstream::linearCommunication(), Value, bop, tag, comm);
     }
     else
     {
-        gather(Pstream::treeCommunication(comm), Value, bop, tag, comm);
+        gather(Pstream::treeCommunication(), Value, bop, tag, comm);
     }
 }
 
@@ -149,10 +149,10 @@ void Pstream::scatter
     const label comm
 )
 {
-    if (Pstream::nProcs(comm) > 1)
+    if (Pstream::nProcs() > 1)
     {
         // Get my communication order
-        const commsStruct& myComm = comms[Pstream::myProcNo(comm)];
+        const commsStruct& myComm = comms[Pstream::myProcNo()];
 
         // Receive from up
         if (myComm.above() != -1)
@@ -218,13 +218,13 @@ void Pstream::scatter
 template <class T>
 void Pstream::scatter(T& Value, const int tag, const label comm)
 {
-    if (Pstream::nProcs(comm) < Pstream::nProcsSimpleSum())
+    if (Pstream::nProcs() < Pstream::nProcsSimpleSum())
     {
-        scatter(Pstream::linearCommunication(comm), Value, tag, comm);
+        scatter(Pstream::linearCommunication(), Value, tag, comm);
     }
     else
     {
-        scatter(Pstream::treeCommunication(comm), Value, tag, comm);
+        scatter(Pstream::treeCommunication(), Value, tag, comm);
     }
 }
 
diff --git a/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/ggiAMGInterface/ggiAMGInterface.C b/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/ggiAMGInterface/ggiAMGInterface.C
index 9222720fe..26402a570 100644
--- a/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/ggiAMGInterface/ggiAMGInterface.C
+++ b/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/ggiAMGInterface/ggiAMGInterface.C
@@ -78,6 +78,8 @@ void Foam::ggiAMGInterface::initFastReduce() const
         zoneProcID[za[zaI]] = Pstream::myProcNo();
     }
 
+//     reduce(zoneProcID, maxOp<labelField>());
+
     reduce(zoneProcID, maxOp<labelField>(), tag(), comm());
 
     // Find out where my zone data is coming from
@@ -687,7 +689,8 @@ Foam::ggiAMGInterface::ggiAMGInterface
 
     nCoarseFacesPerProc[Pstream::myProcNo()] = nCoarseFaces;
 
-    reduce(nCoarseFacesPerProc, sumOp<labelList>(), tag(), comm());
+//     reduce(nCoarseFacesPerProc, sumOp<labelList>(), tag(), comm());
+    reduce(nCoarseFacesPerProc, sumOp<labelList>());
 
     // Coarse global face zone is assembled by adding all faces from proc0,
     // followed by all faces from proc1 etc.
diff --git a/src/foam/meshes/polyMesh/polyPatches/constraint/ggi/ggiPolyPatch.C b/src/foam/meshes/polyMesh/polyPatches/constraint/ggi/ggiPolyPatch.C
index c4f748d07..d7c63c6a6 100644
--- a/src/foam/meshes/polyMesh/polyPatches/constraint/ggi/ggiPolyPatch.C
+++ b/src/foam/meshes/polyMesh/polyPatches/constraint/ggi/ggiPolyPatch.C
@@ -368,11 +368,10 @@ void Foam::ggiPolyPatch::calcLocalParallel() const
         // Count how many patch faces exist on each processor
         labelList nFacesPerProc(Pstream::nProcs(), 0);
         nFacesPerProc[Pstream::myProcNo()] = size();
-//         reduce(nFacesPerProc, maxOp<label>());
+
         Pstream::gatherList(nFacesPerProc);
         Pstream::scatterList(nFacesPerProc);
 
-        Pout<< "nFacesPerProc: " << nFacesPerProc << endl;
         // Make a comm, from all processors that contain the ggi faces
         labelList ggiCommProcs(Pstream::nProcs());
         label nGgiCommProcs = 0;
@@ -388,14 +387,15 @@ void Foam::ggiPolyPatch::calcLocalParallel() const
         ggiCommProcs.setSize(nGgiCommProcs);
 
         // Allocate communicator
-        Info<< "Allocating communicator for GGI patch " << name()
-            << " with " << ggiCommProcs
-            << endl;
         comm_ = Pstream::allocateCommunicator
         (
             Pstream::worldComm,
             ggiCommProcs
         );
+
+        Info<< "Allocating communicator for GGI patch " << name()
+            << " with " << ggiCommProcs << ": " << comm_
+            << endl;
     }
 
     if (debug && Pstream::parRun())
@@ -584,6 +584,13 @@ void Foam::ggiPolyPatch::clearGeom() const
     // HR, 11/Jul/2013
     deleteDemandDrivenData(localParallelPtr_);
 
+    // Clear communicator
+    if (comm_ != Pstream::worldComm)
+    {
+        Pstream::freeCommunicator(comm_);
+        comm_ = Pstream::worldComm;
+    }
+
     deleteDemandDrivenData(mapPtr_);
 }
 
@@ -623,6 +630,8 @@ Foam::ggiPolyPatch::ggiPolyPatch
     remoteZoneAddressingPtr_(NULL),
     reconFaceCellCentresPtr_(NULL),
     localParallelPtr_(NULL),
+    comm_(Pstream::worldComm),
+    tag_(Pstream::msgType()),
     mapPtr_(NULL)
 {}
 
@@ -652,6 +661,8 @@ Foam::ggiPolyPatch::ggiPolyPatch
     remoteZoneAddressingPtr_(NULL),
     reconFaceCellCentresPtr_(NULL),
     localParallelPtr_(NULL),
+    comm_(Pstream::worldComm),
+    tag_(-1),
     mapPtr_(NULL)
 {}
 
@@ -676,6 +687,8 @@ Foam::ggiPolyPatch::ggiPolyPatch
     remoteZoneAddressingPtr_(NULL),
     reconFaceCellCentresPtr_(NULL),
     localParallelPtr_(NULL),
+    comm_(Pstream::worldComm),
+    tag_(-1),
     mapPtr_(NULL)
 {
     if (dict.found("quickReject"))
@@ -706,6 +719,8 @@ Foam::ggiPolyPatch::ggiPolyPatch
     remoteZoneAddressingPtr_(NULL),
     reconFaceCellCentresPtr_(NULL),
     localParallelPtr_(NULL),
+    comm_(pp.comm_),
+    tag_(pp.tag_),
     mapPtr_(NULL)
 {}
 
@@ -732,6 +747,8 @@ Foam::ggiPolyPatch::ggiPolyPatch
     remoteZoneAddressingPtr_(NULL),
     reconFaceCellCentresPtr_(NULL),
     localParallelPtr_(NULL),
+    comm_(pp.comm_),
+    tag_(pp.tag_),
     mapPtr_(NULL)
 {}
 

From 358c854945134fdec4798261fa4ebaa87867fda2 Mon Sep 17 00:00:00 2001
From: Hrvoje Jasak <h.jasak@wikki.co.uk>
Date: Mon, 19 Sep 2016 15:42:09 +0100
Subject: [PATCH 045/130] ESI Comms optimisation: send to neighbours in reverse
 order

---
 src/foam/db/IOstreams/Pstreams/Pstream.C      | 49 +++++++++++++------
 .../IOstreams/Pstreams/combineGatherScatter.C | 21 +++++---
 .../db/IOstreams/Pstreams/gatherScatter.C     |  7 ++-
 .../db/IOstreams/Pstreams/gatherScatterList.C |  7 ++-
 4 files changed, 60 insertions(+), 24 deletions(-)

diff --git a/src/foam/db/IOstreams/Pstreams/Pstream.C b/src/foam/db/IOstreams/Pstreams/Pstream.C
index 4691c802e..e74c72f6e 100644
--- a/src/foam/db/IOstreams/Pstreams/Pstream.C
+++ b/src/foam/db/IOstreams/Pstreams/Pstream.C
@@ -67,22 +67,43 @@ const Foam::NamedEnum<Foam::Pstream::commsTypes, 3>
 
 void Foam::Pstream::setParRun(const label nProcs)
 {
-    parRun_ = true;
-
-    // Redo worldComm communicator (created at static initialisation)
-    freeCommunicator(Pstream::worldComm);
-    label comm = allocateCommunicator(-1, identity(nProcs), true);
-
-    if (comm != Pstream::worldComm)
+    if (nProcs == 0)
     {
-        FatalErrorIn("Pstream::setParRun(const label)")
-            << "problem : comm:" << comm
-            << "  Pstream::worldComm:" << Pstream::worldComm
-            << Foam::exit(FatalError);
-    }
+        parRun_ = false;
+        freeCommunicator(Pstream::worldComm);
 
-    Pout.prefix() = '[' +  name(myProcNo()) + "] ";
-    Perr.prefix() = '[' +  name(myProcNo()) + "] ";
+        label comm = allocateCommunicator(-1, labelList(1, label(0)), false);
+
+        if (comm != Pstream::worldComm)
+        {
+            FatalErrorIn("Pstream::setParRun(const label)")
+                << "problem : comm:" << comm
+                << "  Pstream::worldComm:" << Pstream::worldComm
+                << Foam::exit(FatalError);
+        }
+
+        Pout.prefix() = "";
+        Perr.prefix() = "";
+    }
+    else
+    {
+        parRun_ = true;
+
+        // Redo worldComm communicator (created at static initialisation)
+        freeCommunicator(Pstream::worldComm);
+        label comm = allocateCommunicator(-1, identity(nProcs), true);
+
+        if (comm != Pstream::worldComm)
+        {
+            FatalErrorIn("Pstream::setParRun(const label)")
+                << "problem : comm:" << comm
+                << "  Pstream::worldComm:" << Pstream::worldComm
+                << Foam::exit(FatalError);
+        }
+
+        Pout.prefix() = '[' +  name(myProcNo()) + "] ";
+        Perr.prefix() = '[' +  name(myProcNo()) + "] ";
+    }
 }
 
 
diff --git a/src/foam/db/IOstreams/Pstreams/combineGatherScatter.C b/src/foam/db/IOstreams/Pstreams/combineGatherScatter.C
index 0c9c59b41..7c847264c 100644
--- a/src/foam/db/IOstreams/Pstreams/combineGatherScatter.C
+++ b/src/foam/db/IOstreams/Pstreams/combineGatherScatter.C
@@ -222,8 +222,11 @@ void Pstream::combineScatter
             }
         }
 
-        // Send to my downstairs neighbours
-        forAll (myComm.below(), belowI)
+        // Send to my downstairs neighbours.  Note reverse order (compared to
+        // receiving). This is to make sure to send to the critical path
+        // (only when using a tree schedule!) first.
+        // This is ESI Comms optimisation, v16.06.  HJ, 19/Sep/2016
+        forAllReverse (myComm.below(), belowI)
         {
             label belowID = myComm.below()[belowI];
 
@@ -461,8 +464,11 @@ void Pstream::listCombineScatter
             }
         }
 
-        // Send to my downstairs neighbours
-        forAll (myComm.below(), belowI)
+        // Send to my downstairs neighbours.  Note reverse order (compared to
+        // receiving). This is to make sure to send to the critical path
+        // (only when using a tree schedule!) first.
+        // This is ESI Comms optimisation, v16.06.  HJ, 19/Sep/2016
+        forAllReverse (myComm.below(), belowI)
         {
             label belowID = myComm.below()[belowI];
 
@@ -662,8 +668,11 @@ void Pstream::mapCombineScatter
             }
         }
 
-        // Send to my downstairs neighbours
-        forAll (myComm.below(), belowI)
+        // Send to my downstairs neighbours.  Note reverse order (compared to
+        // receiving). This is to make sure to send to the critical path
+        // (only when using a tree schedule!) first.
+        // This is ESI Comms optimisation, v16.06.  HJ, 19/Sep/2016
+        forAllReverse (myComm.below(), belowI)
         {
             label belowID = myComm.below()[belowI];
 
diff --git a/src/foam/db/IOstreams/Pstreams/gatherScatter.C b/src/foam/db/IOstreams/Pstreams/gatherScatter.C
index a98cf9792..33e469f4d 100644
--- a/src/foam/db/IOstreams/Pstreams/gatherScatter.C
+++ b/src/foam/db/IOstreams/Pstreams/gatherScatter.C
@@ -183,8 +183,11 @@ void Pstream::scatter
             }
         }
 
-        // Send to my downstairs neighbours
-        forAll (myComm.below(), belowI)
+        // Send to my downstairs neighbours.  Note reverse order (compared to
+        // receiving). This is to make sure to send to the critical path
+        // (only when using a tree schedule!) first.
+        // This is ESI Comms optimisation, v16.06.  HJ, 19/Sep/2016
+        forAllReverse (myComm.below(), belowI)
         {
             if (contiguous<T>())
             {
diff --git a/src/foam/db/IOstreams/Pstreams/gatherScatterList.C b/src/foam/db/IOstreams/Pstreams/gatherScatterList.C
index b056256b1..d6e7a83e9 100644
--- a/src/foam/db/IOstreams/Pstreams/gatherScatterList.C
+++ b/src/foam/db/IOstreams/Pstreams/gatherScatterList.C
@@ -290,8 +290,11 @@ void Pstream::scatterList
             }
         }
 
-        // Send to my downstairs neighbours
-        forAll (myComm.below(), belowI)
+        // Send to my downstairs neighbours.  Note reverse order (compared to
+        // receiving). This is to make sure to send to the critical path
+        // (only when using a tree schedule!) first.
+        // This is ESI Comms optimisation, v16.06.  HJ, 19/Sep/2016
+        forAllReverse (myComm.below(), belowI)
         {
             label belowID = myComm.below()[belowI];
             const labelList& notBelowLeaves = comms[belowID].allNotBelow();

From 1848aaeeb214e41a267d03c4f9f983146a5b4204 Mon Sep 17 00:00:00 2001
From: Hrvoje Jasak <h.jasak@wikki.co.uk>
Date: Mon, 19 Sep 2016 15:42:34 +0100
Subject: [PATCH 046/130] Optimisation: inline functions in dimensionedType

---
 .../dimensionedType/dimensionedType.C         | 39 -------------------
 .../dimensionedType/dimensionedType.H         | 31 ++++++++++++---
 2 files changed, 25 insertions(+), 45 deletions(-)

diff --git a/src/foam/dimensionedTypes/dimensionedType/dimensionedType.C b/src/foam/dimensionedTypes/dimensionedType/dimensionedType.C
index 24439b85d..028e32f23 100644
--- a/src/foam/dimensionedTypes/dimensionedType/dimensionedType.C
+++ b/src/foam/dimensionedTypes/dimensionedType/dimensionedType.C
@@ -132,45 +132,6 @@ dimensioned<Type>::dimensioned
 
 // * * * * * * * * * * * * * * * Member Functions  * * * * * * * * * * * * * //
 
-template <class Type>
-const word& dimensioned<Type>::name() const
-{
-    return name_;
-}
-
-template <class Type>
-word& dimensioned<Type>::name()
-{
-    return name_;
-}
-
-
-template <class Type>
-const dimensionSet& dimensioned<Type>::dimensions() const
-{
-    return dimensions_;
-}
-
-template <class Type>
-dimensionSet& dimensioned<Type>::dimensions()
-{
-    return dimensions_;
-}
-
-
-template <class Type>
-const Type& dimensioned<Type>::value() const
-{
-    return value_;
-}
-
-template <class Type>
-Type& dimensioned<Type>::value()
-{
-    return value_;
-}
-
-
 template <class Type>
 dimensioned<typename dimensioned<Type>::cmptType> dimensioned<Type>::component
 (
diff --git a/src/foam/dimensionedTypes/dimensionedType/dimensionedType.H b/src/foam/dimensionedTypes/dimensionedType/dimensionedType.H
index e37db682c..6b7c41c92 100644
--- a/src/foam/dimensionedTypes/dimensionedType/dimensionedType.H
+++ b/src/foam/dimensionedTypes/dimensionedType/dimensionedType.H
@@ -130,22 +130,41 @@ public:
     // Member functions
 
         //- Return const reference to name.
-        const word& name() const;
+        inline const word& name() const
+        {
+            return name_;
+        }
 
         //- Return non-const reference to name.
-        word& name();
+        inline word& name()
+        {
+            return name_;
+        }
+
 
         //- Return const reference to dimensions.
-        const dimensionSet& dimensions() const;
+        const dimensionSet& dimensions() const
+        {
+            return dimensions_;
+        }
 
         //- Return non-const reference to dimensions.
-        dimensionSet& dimensions();
+        inline dimensionSet& dimensions()
+        {
+            return dimensions_;
+        }
 
         //- Return const reference to value.
-        const Type& value() const;
+        inline const Type& value() const
+        {
+            return value_;
+        }
 
         //- Return non-const reference to value.
-        Type& value();
+        inline Type& value()
+        {
+            return value_;
+        }
 
         //- Return a component as a dimensioned<cmptType>
         dimensioned<cmptType> component(const direction) const;

From ceebeca087b62d4a634d2e95ddf72e0d035df991 Mon Sep 17 00:00:00 2001
From: Hrvoje Jasak <h.jasak@wikki.co.uk>
Date: Mon, 19 Sep 2016 16:20:45 +0100
Subject: [PATCH 047/130] GGI check changed to real value

---
 .../GGIInterpolation/GGIInterpolationWeights.C   | 16 +++++++++++++++-
 1 file changed, 15 insertions(+), 1 deletion(-)

diff --git a/src/foam/interpolations/GGIInterpolation/GGIInterpolationWeights.C b/src/foam/interpolations/GGIInterpolation/GGIInterpolationWeights.C
index 236276e3c..c22340401 100644
--- a/src/foam/interpolations/GGIInterpolation/GGIInterpolationWeights.C
+++ b/src/foam/interpolations/GGIInterpolation/GGIInterpolationWeights.C
@@ -418,7 +418,21 @@ void GGIInterpolation<MasterPatch, SlavePatch>::calcAddressing() const
                         neighbPointsInUV
                     );
 
-                if (intersectionArea > VSMALL) // Or > areaErrorTol_ ???
+                scalar intersectionTestArea =
+                    Foam::max
+                    (
+                        VSMALL,
+                        areaErrorTol_()*
+                        Foam::max
+                        (
+                            surfaceAreaMasterPointsInUV,
+                            surfaceAreaNeighbPointsInUV
+                        )
+                    );
+
+                // Fix: previously checked for VSMALL.
+                // HJ, 19/Sep2/106
+                if (intersectionArea > intersectionTestArea)
                 {
                     // We compute the GGI weights based on this
                     // intersection area, and on the individual face

From 913aa4a4431c3115e77e5547c49f5ba6b4806eba Mon Sep 17 00:00:00 2001
From: Hrvoje Jasak <h.jasak@wikki.co.uk>
Date: Mon, 19 Sep 2016 16:21:03 +0100
Subject: [PATCH 048/130] Bugfix: parallel comms usage in gather-scatter

---
 .../db/IOstreams/Pstreams/gatherScatter.C     | 20 +++++++++----------
 .../db/IOstreams/Pstreams/gatherScatterList.C |  2 +-
 2 files changed, 11 insertions(+), 11 deletions(-)

diff --git a/src/foam/db/IOstreams/Pstreams/gatherScatter.C b/src/foam/db/IOstreams/Pstreams/gatherScatter.C
index 33e469f4d..24bab91a5 100644
--- a/src/foam/db/IOstreams/Pstreams/gatherScatter.C
+++ b/src/foam/db/IOstreams/Pstreams/gatherScatter.C
@@ -50,10 +50,10 @@ void Pstream::gather
     const label comm
 )
 {
-    if (Pstream::nProcs() > 1)
+    if (Pstream::nProcs(comm) > 1)
     {
         // Get my communication order
-        const commsStruct& myComm = comms[Pstream::myProcNo()];
+        const commsStruct& myComm = comms[Pstream::myProcNo(comm)];
 
         // Receive from my downstairs neighbours
         forAll (myComm.below(), belowI)
@@ -129,13 +129,13 @@ void Pstream::gather
     const label comm
 )
 {
-    if (Pstream::nProcs() < Pstream::nProcsSimpleSum())
+    if (Pstream::nProcs(comm) < Pstream::nProcsSimpleSum())
     {
-        gather(Pstream::linearCommunication(), Value, bop, tag, comm);
+        gather(Pstream::linearCommunication(comm), Value, bop, tag, comm);
     }
     else
     {
-        gather(Pstream::treeCommunication(), Value, bop, tag, comm);
+        gather(Pstream::treeCommunication(comm), Value, bop, tag, comm);
     }
 }
 
@@ -149,10 +149,10 @@ void Pstream::scatter
     const label comm
 )
 {
-    if (Pstream::nProcs() > 1)
+    if (Pstream::nProcs(comm) > 1)
     {
         // Get my communication order
-        const commsStruct& myComm = comms[Pstream::myProcNo()];
+        const commsStruct& myComm = comms[Pstream::myProcNo(comm)];
 
         // Receive from up
         if (myComm.above() != -1)
@@ -221,13 +221,13 @@ void Pstream::scatter
 template <class T>
 void Pstream::scatter(T& Value, const int tag, const label comm)
 {
-    if (Pstream::nProcs() < Pstream::nProcsSimpleSum())
+    if (Pstream::nProcs(comm) < Pstream::nProcsSimpleSum())
     {
-        scatter(Pstream::linearCommunication(), Value, tag, comm);
+        scatter(Pstream::linearCommunication(comm), Value, tag, comm);
     }
     else
     {
-        scatter(Pstream::treeCommunication(), Value, tag, comm);
+        scatter(Pstream::treeCommunication(comm), Value, tag, comm);
     }
 }
 
diff --git a/src/foam/db/IOstreams/Pstreams/gatherScatterList.C b/src/foam/db/IOstreams/Pstreams/gatherScatterList.C
index d6e7a83e9..a017ecb55 100644
--- a/src/foam/db/IOstreams/Pstreams/gatherScatterList.C
+++ b/src/foam/db/IOstreams/Pstreams/gatherScatterList.C
@@ -233,7 +233,7 @@ void Pstream::scatterList
                 ")"
             )   << "Size of list:" << Values.size()
                 << " does not equal the number of processors:"
-                << Pstream::nProcs()
+                << Pstream::nProcs(comm)
                 << Foam::abort(FatalError);
         }
 

From 7261ad517723b65bbd6155377ffe9719d4830698 Mon Sep 17 00:00:00 2001
From: Hrvoje Jasak <h.jasak@wikki.co.uk>
Date: Tue, 20 Sep 2016 22:11:52 +0100
Subject: [PATCH 049/130] Formatting

---
 src/foam/db/IOstreams/Pstreams/Pstream.C | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/foam/db/IOstreams/Pstreams/Pstream.C b/src/foam/db/IOstreams/Pstreams/Pstream.C
index e74c72f6e..0177a39ee 100644
--- a/src/foam/db/IOstreams/Pstreams/Pstream.C
+++ b/src/foam/db/IOstreams/Pstreams/Pstream.C
@@ -431,6 +431,7 @@ Foam::label Foam::Pstream::allocateCommunicator
 
     // Convert from label to int
     procIDs_[index].setSize(subRanks.size());
+
     forAll (procIDs_[index], i)
     {
         procIDs_[index][i] = subRanks[i];

From 3ecf7074fe152a8465e97c5348e105ff37edf2f1 Mon Sep 17 00:00:00 2001
From: Hrvoje Jasak <h.jasak@wikki.co.uk>
Date: Tue, 20 Sep 2016 22:12:22 +0100
Subject: [PATCH 050/130] Exclude processors outside of comm from gatherScatter
 without failure

---
 src/foam/db/IOstreams/Pstreams/gatherScatter.C     | 14 ++++++++++++++
 src/foam/db/IOstreams/Pstreams/gatherScatterList.C | 14 ++++++++++++++
 2 files changed, 28 insertions(+)

diff --git a/src/foam/db/IOstreams/Pstreams/gatherScatter.C b/src/foam/db/IOstreams/Pstreams/gatherScatter.C
index 24bab91a5..d232c0e02 100644
--- a/src/foam/db/IOstreams/Pstreams/gatherScatter.C
+++ b/src/foam/db/IOstreams/Pstreams/gatherScatter.C
@@ -50,6 +50,13 @@ void Pstream::gather
     const label comm
 )
 {
+    // Return if not active in this comm
+    // HJ, 12/Sep/2016
+    if (Pstream::myProcNo(comm) == -1)
+    {
+        return;
+    }
+
     if (Pstream::nProcs(comm) > 1)
     {
         // Get my communication order
@@ -149,6 +156,13 @@ void Pstream::scatter
     const label comm
 )
 {
+    // Return if not active in this comm
+    // HJ, 12/Sep/2016
+    if (Pstream::myProcNo(comm) == -1)
+    {
+        return;
+    }
+
     if (Pstream::nProcs(comm) > 1)
     {
         // Get my communication order
diff --git a/src/foam/db/IOstreams/Pstreams/gatherScatterList.C b/src/foam/db/IOstreams/Pstreams/gatherScatterList.C
index a017ecb55..8f872e384 100644
--- a/src/foam/db/IOstreams/Pstreams/gatherScatterList.C
+++ b/src/foam/db/IOstreams/Pstreams/gatherScatterList.C
@@ -54,6 +54,13 @@ void Pstream::gatherList
     const label comm
 )
 {
+    // Return if not active in this comm
+    // HJ, 12/Sep/2016
+    if (Pstream::myProcNo(comm) == -1)
+    {
+        return;
+    }
+
     if (Pstream::nProcs(comm) > 1)
     {
         if (Values.size() != Pstream::nProcs(comm))
@@ -218,6 +225,13 @@ void Pstream::scatterList
     const label comm
 )
 {
+    // Return if not active in this comm
+    // HJ, 12/Sep/2016
+    if (Pstream::myProcNo(comm) == -1)
+    {
+        return;
+    }
+
     if (Pstream::nProcs(comm) > 1)
     {
         if (Values.size() != Pstream::nProcs(comm))

From 38a8732e4e2e0eea4d0370602e9b16379c58f831 Mon Sep 17 00:00:00 2001
From: Hrvoje Jasak <h.jasak@wikki.co.uk>
Date: Tue, 20 Sep 2016 22:13:37 +0100
Subject: [PATCH 051/130] Bugfix: localParallel and comms allocation per GGI
 pair

---
 .../polyPatches/constraint/ggi/ggiPolyPatch.C | 95 ++++++++++---------
 .../polyPatches/constraint/ggi/ggiPolyPatch.H |  1 +
 2 files changed, 50 insertions(+), 46 deletions(-)

diff --git a/src/foam/meshes/polyMesh/polyPatches/constraint/ggi/ggiPolyPatch.C b/src/foam/meshes/polyMesh/polyPatches/constraint/ggi/ggiPolyPatch.C
index d7c63c6a6..621039612 100644
--- a/src/foam/meshes/polyMesh/polyPatches/constraint/ggi/ggiPolyPatch.C
+++ b/src/foam/meshes/polyMesh/polyPatches/constraint/ggi/ggiPolyPatch.C
@@ -65,16 +65,11 @@ bool Foam::ggiPolyPatch::active() const
 
     if (shadow.active() && zone.active())
     {
-        if
-        (
-            !Pstream::parRun()
-         && size() != boundaryMesh().mesh().faceZones()[zone.index()].size()
-        )
+        if (!Pstream::parRun() && !localParallel())
         {
             // Patch is present in serial run, but zone is not the same size
             // Probably doing decomposition and reconstruction
             // HJ, 14/Sep/2016
-
             return false;
         }
 
@@ -325,49 +320,46 @@ void Foam::ggiPolyPatch::calcLocalParallel() const
     localParallelPtr_ = new bool(false);
     bool& emptyOrComplete = *localParallelPtr_;
 
-    // If running in serial, all GGIs are expanded to zone size.
-    // This happens on decomposition and reconstruction where
-    // size and shadow size may be zero, but zone size may not
-    // HJ, 1/Jun/2011
-    if (!Pstream::parRun())
+    if (size() > zone().size())
     {
+        FatalErrorIn("void ggiPolyPatch::calcLocalParallel() const")
+            << "Patch size is greater than zone size for GGI patch "
+            << name() << ".  This is not allowerd: "
+            << "the face zone must contain all patch faces and be "
+            << "global in parallel runs"
+            << abort(FatalError);
+    }
+
+    // Calculate localisation on master and shadow
+    if ((size() == 0 && shadow().size() == 0))
+    {
+        // No ggi on this processor
+        emptyOrComplete = true;
+    }
+    else if (!zone().empty() || !shadow().zone().empty())
+    {
+        // GGI present on the processor and complete for both
+        emptyOrComplete =
+        (
+            zone().size() == size()
+         && shadow().zone().size() == shadow().size()
+        );
+    }
+    else
+    {
+        // Master and shadow on different processors
         emptyOrComplete = false;
     }
-    else
-    {
-        // Check that patch size is greater than the zone size.
-        // This is an indication of the error where the face zone is not global
-        // in a parallel run.  HJ, 9/Nov/2014
-        if (size() > zone().size())
-        {
-            FatalErrorIn("void ggiPolyPatch::calcLocalParallel() const")
-                << "Patch size is greater than zone size for GGI patch "
-                << name() << ".  This is not allowerd: "
-                << "the face zone must contain all patch faces and be "
-                << "global in parallel runs"
-                << abort(FatalError);
-        }
 
-        // Calculate localisation on master and shadow
-        emptyOrComplete =
-            (
-                zone().size() == size()
-             && shadow().zone().size() == shadow().size()
-            )
-         || (size() == 0 && shadow().size() == 0);
+    reduce(emptyOrComplete, andOp<bool>());
 
-        reduce(emptyOrComplete, andOp<bool>());
-    }
-
-    if (emptyOrComplete)
-    {
-        comm_ = boundaryMesh().mesh().comm();
-    }
-    else
+    // Note: only master allocates the comm_
+    // HJ, 20/Sep/2016
+    if (!emptyOrComplete && Pstream::parRun() && master())
     {
         // Count how many patch faces exist on each processor
         labelList nFacesPerProc(Pstream::nProcs(), 0);
-        nFacesPerProc[Pstream::myProcNo()] = size();
+        nFacesPerProc[Pstream::myProcNo()] = size() + shadow().size();
 
         Pstream::gatherList(nFacesPerProc);
         Pstream::scatterList(nFacesPerProc);
@@ -397,6 +389,10 @@ void Foam::ggiPolyPatch::calcLocalParallel() const
             << " with " << ggiCommProcs << ": " << comm_
             << endl;
     }
+    else
+    {
+        comm_ = boundaryMesh().mesh().comm();
+    }
 
     if (debug && Pstream::parRun())
     {
@@ -844,13 +840,20 @@ Foam::label Foam::ggiPolyPatch::comm() const
     //HJ, Testing.  Use optimised comm or a local one
 
     // Note: comm is calculated with localParallel and will use the
-    // localParallelPtr_ for signalling.  HJ, 10/Sep2016
-    if (!localParallelPtr_)
+    // localParallelPtr_ for signalling.  HJ, 10/Sep/2016
+    if (master())
     {
-        calcLocalParallel();
-    }
+        if (!localParallelPtr_)
+        {
+            calcLocalParallel();
+        }
 
-    return comm_;
+        return comm_;
+    }
+    else
+    {
+        return shadow().comm();
+    }
 
 //     return boundaryMesh().mesh().comm();
 }
diff --git a/src/foam/meshes/polyMesh/polyPatches/constraint/ggi/ggiPolyPatch.H b/src/foam/meshes/polyMesh/polyPatches/constraint/ggi/ggiPolyPatch.H
index 6c5e0415e..8e585b6dc 100644
--- a/src/foam/meshes/polyMesh/polyPatches/constraint/ggi/ggiPolyPatch.H
+++ b/src/foam/meshes/polyMesh/polyPatches/constraint/ggi/ggiPolyPatch.H
@@ -106,6 +106,7 @@ class ggiPolyPatch
             mutable bool* localParallelPtr_;
 
             //- Communicator to use for parallel communication
+            //  Note: only master side allocates the comm
             mutable label comm_;
 
             //- Message tag used for sending

From 96b1b666407b5405f9db64d7a5f97d4fbc5c89ec Mon Sep 17 00:00:00 2001
From: Hrvoje Jasak <h.jasak@wikki.co.uk>
Date: Tue, 20 Sep 2016 22:14:58 +0100
Subject: [PATCH 052/130] Formatting

---
 .../meshes/polyMesh/mapPolyMesh/mapDistribute/mapDistribute.H  | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/foam/meshes/polyMesh/mapPolyMesh/mapDistribute/mapDistribute.H b/src/foam/meshes/polyMesh/mapPolyMesh/mapDistribute/mapDistribute.H
index fe395d7a0..88049e56c 100644
--- a/src/foam/meshes/polyMesh/mapPolyMesh/mapDistribute/mapDistribute.H
+++ b/src/foam/meshes/polyMesh/mapPolyMesh/mapDistribute/mapDistribute.H
@@ -265,7 +265,7 @@ public:
                 }
             }
 
-            //- Correct for topo change.
+            //- Correct for topo change
             void updateMesh(const mapPolyMesh&)
             {
                 notImplemented
@@ -274,6 +274,7 @@ public:
                 );
             }
 
+
     // Member Operators
 
         void operator=(const mapDistribute&);

From dee7645a8f90bd6c7be4a863abff974992051a6a Mon Sep 17 00:00:00 2001
From: Hrvoje Jasak <h.jasak@wikki.co.uk>
Date: Tue, 20 Sep 2016 22:15:16 +0100
Subject: [PATCH 053/130] Bugfix: Better messaging on invalid choice for stream

---
 .../mapPolyMesh/mapDistribute/mapDistributeTemplates.C        | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/foam/meshes/polyMesh/mapPolyMesh/mapDistribute/mapDistributeTemplates.C b/src/foam/meshes/polyMesh/mapPolyMesh/mapDistribute/mapDistributeTemplates.C
index 75a4efe92..43aa235a2 100644
--- a/src/foam/meshes/polyMesh/mapPolyMesh/mapDistribute/mapDistributeTemplates.C
+++ b/src/foam/meshes/polyMesh/mapPolyMesh/mapDistribute/mapDistributeTemplates.C
@@ -369,9 +369,9 @@ void Foam::mapDistribute::distribute
     }
     else
     {
-        // This needs to be cleaned up: temporary solution.  HJ, 15/Jun/2014
         FatalErrorIn("mapDistribute::distribute(..)")
-            << "Unknown communication schedule " << label(commsType)
+            << "Unknown communication schedule "
+            << Pstream::commsTypeNames[commsType]
             << abort(FatalError);
     }
 }

From 0b1dc2b2cd9d027bfeb7e29b52154d85bc176541 Mon Sep 17 00:00:00 2001
From: Hrvoje Jasak <h.jasak@wikki.co.uk>
Date: Tue, 20 Sep 2016 22:17:16 +0100
Subject: [PATCH 054/130] Removed compressedSend/compressedReceive

---
 .../processor/processorFaPatchField.C         |   8 +-
 .../processor/processorFaPatchScalarField.C   |   4 +-
 .../processor/processorFvPatchField.C         |  12 +-
 .../processor/processorFvPatchScalarField.C   |   8 +-
 .../ProcessorBlockAMGInterfaceField.C         |   4 +-
 .../processorLduInterface.H                   |  26 ----
 .../processorLduInterfaceTemplates.C          | 142 ------------------
 .../processorAMGInterfaceField.C              |   4 +-
 8 files changed, 20 insertions(+), 188 deletions(-)

diff --git a/src/finiteArea/fields/faPatchFields/constraint/processor/processorFaPatchField.C b/src/finiteArea/fields/faPatchFields/constraint/processor/processorFaPatchField.C
index 4e155a0a6..3537b4c27 100644
--- a/src/finiteArea/fields/faPatchFields/constraint/processor/processorFaPatchField.C
+++ b/src/finiteArea/fields/faPatchFields/constraint/processor/processorFaPatchField.C
@@ -176,7 +176,7 @@ void processorFaPatchField<Type>::initEvaluate
 {
     if (Pstream::parRun())
     {
-        procPatch_.compressedSend(commsType, this->patchInternalField()());
+        procPatch_.send(commsType, this->patchInternalField()());
     }
 }
 
@@ -189,7 +189,7 @@ void processorFaPatchField<Type>::evaluate
 {
     if (Pstream::parRun())
     {
-        procPatch_.compressedReceive<Type>(commsType, *this);
+        procPatch_.receive<Type>(commsType, *this);
 
         if (doTransform())
         {
@@ -218,7 +218,7 @@ void processorFaPatchField<Type>::initInterfaceMatrixUpdate
     const bool switchToLhs
 ) const
 {
-    procPatch_.compressedSend
+    procPatch_.send
     (
         commsType,
         this->patch().patchInternalField(psiInternal)()
@@ -240,7 +240,7 @@ void processorFaPatchField<Type>::updateInterfaceMatrix
 {
     scalarField pnf
     (
-        procPatch_.compressedReceive<scalar>(commsType, this->size())()
+        procPatch_.receive<scalar>(commsType, this->size())()
     );
 
     // Transform according to the transformation tensor
diff --git a/src/finiteArea/fields/faPatchFields/constraint/processor/processorFaPatchScalarField.C b/src/finiteArea/fields/faPatchFields/constraint/processor/processorFaPatchScalarField.C
index 11be0915b..fe5b4dc27 100644
--- a/src/finiteArea/fields/faPatchFields/constraint/processor/processorFaPatchScalarField.C
+++ b/src/finiteArea/fields/faPatchFields/constraint/processor/processorFaPatchScalarField.C
@@ -52,7 +52,7 @@ void processorFaPatchField<scalar>::initInterfaceMatrixUpdate
     const bool switchToLhs
 ) const
 {
-    procPatch_.compressedSend
+    procPatch_.send
     (
         commsType,
         patch().patchInternalField(psiInternal)()
@@ -74,7 +74,7 @@ void processorFaPatchField<scalar>::updateInterfaceMatrix
 {
     scalarField pnf
     (
-        procPatch_.compressedReceive<scalar>(commsType, this->size())()
+        procPatch_.receive<scalar>(commsType, this->size())()
     );
 
     const unallocLabelList& edgeFaces = patch().edgeFaces();
diff --git a/src/finiteVolume/fields/fvPatchFields/constraint/processor/processorFvPatchField.C b/src/finiteVolume/fields/fvPatchFields/constraint/processor/processorFvPatchField.C
index 0a92f62b1..e6ce38d34 100644
--- a/src/finiteVolume/fields/fvPatchFields/constraint/processor/processorFvPatchField.C
+++ b/src/finiteVolume/fields/fvPatchFields/constraint/processor/processorFvPatchField.C
@@ -180,7 +180,7 @@ void processorFvPatchField<Type>::initEvaluate
 {
     if (Pstream::parRun())
     {
-        procPatch_.compressedSend(commsType, this->patchInternalField()());
+        procPatch_.send(commsType, this->patchInternalField()());
     }
 }
 
@@ -193,7 +193,7 @@ void processorFvPatchField<Type>::evaluate
 {
     if (Pstream::parRun())
     {
-        procPatch_.compressedReceive<Type>(commsType, *this);
+        procPatch_.receive<Type>(commsType, *this);
 
         if (doTransform())
         {
@@ -222,7 +222,7 @@ void processorFvPatchField<Type>::initInterfaceMatrixUpdate
     const bool switchToLhs
 ) const
 {
-    procPatch_.compressedSend
+    procPatch_.send
     (
         commsType,
         this->patch().patchInternalField(psiInternal)()
@@ -244,7 +244,7 @@ void processorFvPatchField<Type>::updateInterfaceMatrix
 {
     scalarField pnf
     (
-        procPatch_.compressedReceive<scalar>(commsType, this->size())()
+        procPatch_.receive<scalar>(commsType, this->size())()
     );
 
     // Transform according to the transformation tensor
@@ -281,7 +281,7 @@ void processorFvPatchField<Type>::initInterfaceMatrixUpdate
     const bool switchToLhs
 ) const
 {
-    procPatch_.compressedSend
+    procPatch_.send
     (
         commsType,
         this->patch().patchInternalField(psiInternal)()
@@ -302,7 +302,7 @@ void processorFvPatchField<Type>::updateInterfaceMatrix
 {
     Field<Type> pnf
     (
-        procPatch_.compressedReceive<Type>(commsType, this->size())()
+        procPatch_.receive<Type>(commsType, this->size())()
     );
 
     // Multiply neighbour field with coeffs and re-use pnf for result
diff --git a/src/finiteVolume/fields/fvPatchFields/constraint/processor/processorFvPatchScalarField.C b/src/finiteVolume/fields/fvPatchFields/constraint/processor/processorFvPatchScalarField.C
index 105c6b466..36e3392d9 100644
--- a/src/finiteVolume/fields/fvPatchFields/constraint/processor/processorFvPatchScalarField.C
+++ b/src/finiteVolume/fields/fvPatchFields/constraint/processor/processorFvPatchScalarField.C
@@ -44,7 +44,7 @@ void processorFvPatchField<scalar>::initInterfaceMatrixUpdate
     const bool switchToLhs
 ) const
 {
-    procPatch_.compressedSend
+    procPatch_.send
     (
         commsType,
         patch().patchInternalField(psiInternal)()
@@ -66,7 +66,7 @@ void processorFvPatchField<scalar>::updateInterfaceMatrix
 {
     scalarField pnf
     (
-        procPatch_.compressedReceive<scalar>(commsType, this->size())()
+        procPatch_.receive<scalar>(commsType, this->size())()
     );
 
     const unallocLabelList& faceCells = patch().faceCells();
@@ -99,7 +99,7 @@ void processorFvPatchField<scalar>::initInterfaceMatrixUpdate
     const bool switchToLhs
 ) const
 {
-    procPatch_.compressedSend
+    procPatch_.send
     (
         commsType,
         patch().patchInternalField(psiInternal)()
@@ -120,7 +120,7 @@ void processorFvPatchField<scalar>::updateInterfaceMatrix
 {
     scalarField pnf
     (
-        procPatch_.compressedReceive<scalar>(commsType, this->size())()
+        procPatch_.receive<scalar>(commsType, this->size())()
     );
 
     const unallocLabelList& faceCells = patch().faceCells();
diff --git a/src/foam/matrices/blockLduMatrix/BlockAMG/BlockAMGInterfaceFields/ProcessorBlockAMGInterfaceField/ProcessorBlockAMGInterfaceField.C b/src/foam/matrices/blockLduMatrix/BlockAMG/BlockAMGInterfaceFields/ProcessorBlockAMGInterfaceField/ProcessorBlockAMGInterfaceField.C
index bc50d1a43..483cbe802 100644
--- a/src/foam/matrices/blockLduMatrix/BlockAMG/BlockAMGInterfaceFields/ProcessorBlockAMGInterfaceField/ProcessorBlockAMGInterfaceField.C
+++ b/src/foam/matrices/blockLduMatrix/BlockAMG/BlockAMGInterfaceFields/ProcessorBlockAMGInterfaceField/ProcessorBlockAMGInterfaceField.C
@@ -94,7 +94,7 @@ void Foam::ProcessorBlockAMGInterfaceField<Type>::initInterfaceMatrixUpdate
     const bool switchToLhs
 ) const
 {
-    procInterface_.compressedSend
+    procInterface_.send
     (
         commsType,
         procInterface_.interfaceInternalField(psiInternal)()
@@ -115,7 +115,7 @@ void Foam::ProcessorBlockAMGInterfaceField<Type>::updateInterfaceMatrix
 {
     Field<Type> pnf
     (
-        procInterface_.compressedReceive<Type>(commsType, this->size())
+        procInterface_.receive<Type>(commsType, this->size())
     );
 
     // Multiply neighbour field with coeffs and re-use pnf for result
diff --git a/src/foam/matrices/lduMatrix/lduAddressing/lduInterfaces/processorLduInterface/processorLduInterface.H b/src/foam/matrices/lduMatrix/lduAddressing/lduInterfaces/processorLduInterface/processorLduInterface.H
index a600d27a8..b2b390dcf 100644
--- a/src/foam/matrices/lduMatrix/lduAddressing/lduInterfaces/processorLduInterface/processorLduInterface.H
+++ b/src/foam/matrices/lduMatrix/lduAddressing/lduInterfaces/processorLduInterface/processorLduInterface.H
@@ -128,32 +128,6 @@ public:
                 const Pstream::commsTypes commsType,
                 const label size
             ) const;
-
-
-            //- Raw field send function with data compression
-            template<class Type>
-            void compressedSend
-            (
-                const Pstream::commsTypes commsType,
-                const UList<Type>&
-            ) const;
-
-            //- Raw field receive function with data compression
-            template<class Type>
-            void compressedReceive
-            (
-                const Pstream::commsTypes commsType,
-                UList<Type>&
-            ) const;
-
-            //- Raw field receive function with data compression,
-            //  returning a field
-            template<class Type>
-            tmp<Field<Type> > compressedReceive
-            (
-                const Pstream::commsTypes commsType,
-                const label size
-            ) const;
 };
 
 
diff --git a/src/foam/matrices/lduMatrix/lduAddressing/lduInterfaces/processorLduInterface/processorLduInterfaceTemplates.C b/src/foam/matrices/lduMatrix/lduAddressing/lduInterfaces/processorLduInterface/processorLduInterfaceTemplates.C
index 1645f505c..2a471f2af 100644
--- a/src/foam/matrices/lduMatrix/lduAddressing/lduInterfaces/processorLduInterface/processorLduInterfaceTemplates.C
+++ b/src/foam/matrices/lduMatrix/lduAddressing/lduInterfaces/processorLduInterface/processorLduInterfaceTemplates.C
@@ -131,146 +131,4 @@ Foam::tmp<Foam::Field<Type> > Foam::processorLduInterface::receive
 }
 
 
-template<class Type>
-void Foam::processorLduInterface::compressedSend
-(
-    const Pstream::commsTypes commsType,
-    const UList<Type>& f
-) const
-{
-    if (sizeof(scalar) != sizeof(float) && f.size())
-    {
-        static const label nCmpts = sizeof(Type)/sizeof(scalar);
-        label nm1 = (f.size() - 1)*nCmpts;
-        label nlast = sizeof(Type)/sizeof(float);
-        label nFloats = nm1 + nlast;
-        label nBytes = nFloats*sizeof(float);
-
-        const scalar *sArray = reinterpret_cast<const scalar*>(f.begin());
-        const scalar *slast = &sArray[nm1];
-        resizeBuf(sendBuf_, nBytes);
-        float *fArray = reinterpret_cast<float*>(sendBuf_.begin());
-
-        for (register label i = 0; i < nm1; i++)
-        {
-            fArray[i] = sArray[i] - slast[i%nCmpts];
-        }
-
-        reinterpret_cast<Type&>(fArray[nm1]) = f[f.size() - 1];
-
-        if (commsType == Pstream::blocking || commsType == Pstream::scheduled)
-        {
-            OPstream::write
-            (
-                commsType,
-                neighbProcNo(),
-                sendBuf_.begin(),
-                nBytes,
-                tag(),
-                comm()
-            );
-        }
-        else if (commsType == Pstream::nonBlocking)
-        {
-            resizeBuf(receiveBuf_, nBytes);
-
-            IPstream::read
-            (
-                commsType,
-                neighbProcNo(),
-                receiveBuf_.begin(),
-                receiveBuf_.size(),
-                tag(),
-                comm()
-            );
-
-            OPstream::write
-            (
-                commsType,
-                neighbProcNo(),
-                sendBuf_.begin(),
-                nBytes,
-                tag(),
-                comm()
-            );
-        }
-        else
-        {
-            FatalErrorIn("processorLduInterface::compressedSend")
-                << "Unsupported communications type " << commsType
-                << exit(FatalError);
-        }
-    }
-    else
-    {
-        this->send(commsType, f);
-    }
-}
-
-template<class Type>
-void Foam::processorLduInterface::compressedReceive
-(
-    const Pstream::commsTypes commsType,
-    UList<Type>& f
-) const
-{
-    if (sizeof(scalar) != sizeof(float) && f.size())
-    {
-        static const label nCmpts = sizeof(Type)/sizeof(scalar);
-        label nm1 = (f.size() - 1)*nCmpts;
-        label nlast = sizeof(Type)/sizeof(float);
-        label nFloats = nm1 + nlast;
-        label nBytes = nFloats*sizeof(float);
-
-        if (commsType == Pstream::blocking || commsType == Pstream::scheduled)
-        {
-            resizeBuf(receiveBuf_, nBytes);
-
-            IPstream::read
-            (
-                commsType,
-                neighbProcNo(),
-                receiveBuf_.begin(),
-                nBytes,
-                tag(),
-                comm()
-            );
-        }
-        else if (commsType != Pstream::nonBlocking)
-        {
-            FatalErrorIn("processorLduInterface::compressedReceive")
-                << "Unsupported communications type " << commsType
-                << exit(FatalError);
-        }
-
-        const float *fArray =
-            reinterpret_cast<const float*>(receiveBuf_.begin());
-        f[f.size() - 1] = reinterpret_cast<const Type&>(fArray[nm1]);
-        scalar *sArray = reinterpret_cast<scalar*>(f.begin());
-        const scalar *slast = &sArray[nm1];
-
-        for (register label i = 0; i < nm1; i++)
-        {
-            sArray[i] = fArray[i] + slast[i%nCmpts];
-        }
-    }
-    else
-    {
-        this->receive<Type>(commsType, f);
-    }
-}
-
-template<class Type>
-Foam::tmp<Foam::Field<Type> > Foam::processorLduInterface::compressedReceive
-(
-    const Pstream::commsTypes commsType,
-    const label size
-) const
-{
-    tmp<Field<Type> > tf(new Field<Type>(size));
-    compressedReceive(commsType, tf());
-    return tf;
-}
-
-
 // ************************************************************************* //
diff --git a/src/foam/matrices/lduMatrix/solvers/AMG/interfaceFields/processorAMGInterfaceField/processorAMGInterfaceField.C b/src/foam/matrices/lduMatrix/solvers/AMG/interfaceFields/processorAMGInterfaceField/processorAMGInterfaceField.C
index b258ebb80..e87076154 100644
--- a/src/foam/matrices/lduMatrix/solvers/AMG/interfaceFields/processorAMGInterfaceField/processorAMGInterfaceField.C
+++ b/src/foam/matrices/lduMatrix/solvers/AMG/interfaceFields/processorAMGInterfaceField/processorAMGInterfaceField.C
@@ -81,7 +81,7 @@ void Foam::processorAMGInterfaceField::initInterfaceMatrixUpdate
     const bool switchToLhs
 ) const
 {
-    procInterface_.compressedSend
+    procInterface_.send
     (
         commsType,
         procInterface_.interfaceInternalField(psiInternal)()
@@ -102,7 +102,7 @@ void Foam::processorAMGInterfaceField::updateInterfaceMatrix
 {
     scalarField pnf
     (
-        procInterface_.compressedReceive<scalar>(commsType, coeffs.size())
+        procInterface_.receive<scalar>(commsType, coeffs.size())
     );
     transformCoupleField(pnf, cmpt);
 

From 33f13b59c8b5002f10200d50522281eb772b91f3 Mon Sep 17 00:00:00 2001
From: Hrvoje Jasak <h.jasak@wikki.co.uk>
Date: Tue, 20 Sep 2016 22:41:39 +0100
Subject: [PATCH 055/130] Consistent comms in parallel ggi preparation

---
 .../ggiAMGInterface/ggiAMGInterface.C         | 118 ++++++++++--------
 1 file changed, 64 insertions(+), 54 deletions(-)

diff --git a/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/ggiAMGInterface/ggiAMGInterface.C b/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/ggiAMGInterface/ggiAMGInterface.C
index 26402a570..9ad831eb4 100644
--- a/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/ggiAMGInterface/ggiAMGInterface.C
+++ b/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/ggiAMGInterface/ggiAMGInterface.C
@@ -59,6 +59,22 @@ void Foam::ggiAMGInterface::initFastReduce() const
             << abort(FatalError);
     }
 
+    // If the processor is not in the GGI comm, create a dummy map
+    if (Pstream::myProcNo(comm()) == -1)
+    {
+        mapPtr_ = new mapDistribute
+        (
+            zoneSize(),  // This is zero if there are no local GGI faces
+            labelListList(Pstream::nProcs()),
+            labelListList(Pstream::nProcs())
+        );
+
+        return;
+    }
+
+    // From here on, work on processors within the communicator
+    // HJ, 20/Sep/2016
+
     // Note: this is different from ggiPolyPatch comms because zone
     // is the same on master the slave side.
     // HJ, 31/May/2016
@@ -78,8 +94,9 @@ void Foam::ggiAMGInterface::initFastReduce() const
         zoneProcID[za[zaI]] = Pstream::myProcNo();
     }
 
-//     reduce(zoneProcID, maxOp<labelField>());
-
+    // Note: reduce with a comm will only be present on processors containing
+    // master or slave faces.  Other processors created a dummy map above
+    // HJ, 20/Sep/2016
     reduce(zoneProcID, maxOp<labelField>(), tag(), comm());
 
     // Find out where my zone data is coming from
@@ -128,13 +145,19 @@ void Foam::ggiAMGInterface::initFastReduce() const
         localZoneIndices[za[zaI]] = zaI;
     }
 
-    labelListList shadowToReceiveAddr(Pstream::nProcs());
+    // Note: rewrite this to use comm().  However, all proc indicators will be
+    // done with local comms addressing instead of the global processor number
+    // HJ, 20/Sep/2016
+    labelListList shadowToReceiveAddr(Pstream::nProcs(comm()));
 
     // Get the list of what my shadow needs to receive from my zone
     // on all other processors
-    shadowToReceiveAddr[Pstream::myProcNo()] = shadowZa;
-    Pstream::gatherList(shadowToReceiveAddr);
-    Pstream::scatterList(shadowToReceiveAddr);
+    shadowToReceiveAddr[Pstream::myProcNo(comm())] = shadowZa;
+
+    // Note gather-scatter with a comm.  Processors without GGI faces
+    // have been eliminated beforehand
+    Pstream::gatherList(shadowToReceiveAddr, tag(), comm());
+    Pstream::scatterList(shadowToReceiveAddr, tag(), comm());
 
     // Now local zone indices contain the index of a local face that will
     // provide the data.  For faces that are not local, the index will be -1
@@ -146,40 +169,46 @@ void Foam::ggiAMGInterface::initFastReduce() const
     labelListList sendMap(Pstream::nProcs());
 
     // Collect local labels to be sent to each processor
-    forAll (shadowToReceiveAddr, procI)
+    for (label procI = 0; procI < Pstream::nProcs(); procI++)
     {
-        const labelList& curProcSend = shadowToReceiveAddr[procI];
-
-        // Find out how much of my data is going to this processor
-        label nProcSend = 0;
-
-        forAll (curProcSend, sendI)
+        // Look for processors that are in the current comm, ie. containing
+        // faces from the ggi patch
+        if (Pstream::procNo(comm(), procI) != -1)
         {
-            if (localZoneIndices[curProcSend[sendI]] > -1)
-            {
-                nProcSend++;
-            }
-        }
+            const labelList& curProcSend =
+                shadowToReceiveAddr[Pstream::procNo(comm(), procI)];
 
-        if (nProcSend > 0)
-        {
-            // Collect the indices
-            labelList& curSendMap = sendMap[procI];
-
-            curSendMap.setSize(nProcSend);
-
-            // Reset counter
-            nProcSend = 0;
+            // Find out how much of my data is going to this processor
+            label nProcSend = 0;
 
             forAll (curProcSend, sendI)
             {
                 if (localZoneIndices[curProcSend[sendI]] > -1)
                 {
-                    curSendMap[nProcSend] =
-                        localZoneIndices[curProcSend[sendI]];
                     nProcSend++;
                 }
             }
+
+            if (nProcSend > 0)
+            {
+                // Collect the indices
+                labelList& curSendMap = sendMap[procI];
+
+                curSendMap.setSize(nProcSend);
+
+                // Reset counter
+                nProcSend = 0;
+
+                forAll (curProcSend, sendI)
+                {
+                    if (localZoneIndices[curProcSend[sendI]] > -1)
+                    {
+                        curSendMap[nProcSend] =
+                            localZoneIndices[curProcSend[sendI]];
+                        nProcSend++;
+                    }
+                }
+            }
         }
     }
 
@@ -689,8 +718,12 @@ Foam::ggiAMGInterface::ggiAMGInterface
 
     nCoarseFacesPerProc[Pstream::myProcNo()] = nCoarseFaces;
 
-//     reduce(nCoarseFacesPerProc, sumOp<labelList>(), tag(), comm());
-    reduce(nCoarseFacesPerProc, sumOp<labelList>());
+    // Reduce number of coarse faces per proc
+    // Note: reducing with comms means that the processors with no
+    // contact with the GGI interface will have zero zone size.
+    // This needs to be handled separately in the initFastReduce
+    // HJ, 20/Sep/2016
+    reduce(nCoarseFacesPerProc, sumOp<labelList>(), tag(), comm());
 
     // Coarse global face zone is assembled by adding all faces from proc0,
     // followed by all faces from proc1 etc.
@@ -888,29 +921,6 @@ Foam::ggiAMGInterface::ggiAMGInterface
                 }
             }
         }
-
-//         // First index: master proc
-//         // Second index: slave proc
-//         // List contents: global faces in order
-//         labelListListList crissCrossList(Pstream::nProcs());
-
-//         labelListList& crissList = crissCrossList[Pstream::myProcNo()];
-
-//         crissList.setSize(Pstream::nProcs());
-
-//         forAll (crissList, procI)
-//         {
-//             crissList[procI] = procMasterFacesLL[procI];
-//         }
-
-//         Pstream::gatherList(crissCrossList);
-//         Pstream::scatterList(crissCrossList);
-
-//         forAll (procMasterFaces_, procI)
-//         {
-//             procMasterFaces_[procI] =
-//                 crissCrossList[procI][Pstream::myProcNo()];
-//         }
     }
     // Agglomerate slave
     else

From 65bb64442d73be5ecc173d2e5325617b40e9ae10 Mon Sep 17 00:00:00 2001
From: Hrvoje Jasak <h.jasak@wikki.co.uk>
Date: Fri, 23 Sep 2016 10:13:05 +0100
Subject: [PATCH 056/130] Git clean-up: Pstream changes

---
 src/foam/db/IOstreams/Pstreams/Pstream.C | 1092 +++++++++++++++-------
 src/foam/db/IOstreams/Pstreams/Pstream.H |    5 -
 2 files changed, 763 insertions(+), 334 deletions(-)

diff --git a/src/foam/db/IOstreams/Pstreams/Pstream.C b/src/foam/db/IOstreams/Pstreams/Pstream.C
index 09e7fb5e7..e74c72f6e 100644
--- a/src/foam/db/IOstreams/Pstreams/Pstream.C
+++ b/src/foam/db/IOstreams/Pstreams/Pstream.C
@@ -1,7 +1,7 @@
 /*---------------------------------------------------------------------------*\
   =========                 |
   \\      /  F ield         | foam-extend: Open Source CFD
-   \\    /   O peration     | Version:     3.2
+   \\    /   O peration     | Version:     4.0
     \\  /    A nd           | Web:         http://www.foam-extend.org
      \\/     M anipulation  | For copyright notice see file Copyright
 -------------------------------------------------------------------------------
@@ -23,11 +23,6 @@ License
 
 \*---------------------------------------------------------------------------*/
 
-
-#include <cstring>
-#include <cstdlib>
-#include <csignal>
-
 #include "mpi.h"
 
 #include "Pstream.H"
@@ -35,6 +30,13 @@ License
 #include "debug.H"
 #include "dictionary.H"
 #include "OSspecific.H"
+#include "PstreamGlobals.H"
+#include "SubList.H"
+#include "allReduce.H"
+
+#include <cstring>
+#include <cstdlib>
+#include <csignal>
 
 #if defined(WM_SP)
 #   define MPI_SCALAR MPI_FLOAT
@@ -63,65 +65,85 @@ const Foam::NamedEnum<Foam::Pstream::commsTypes, 3>
 
 // * * * * * * * * * * * * * Private Member Functions  * * * * * * * * * * * //
 
-void Foam::Pstream::setParRun()
+void Foam::Pstream::setParRun(const label nProcs)
 {
-    parRun_ = true;
+    if (nProcs == 0)
+    {
+        parRun_ = false;
+        freeCommunicator(Pstream::worldComm);
 
-    Pout.prefix() = '[' +  name(myProcNo()) + "] ";
-    Perr.prefix() = '[' +  name(myProcNo()) + "] ";
+        label comm = allocateCommunicator(-1, labelList(1, label(0)), false);
+
+        if (comm != Pstream::worldComm)
+        {
+            FatalErrorIn("Pstream::setParRun(const label)")
+                << "problem : comm:" << comm
+                << "  Pstream::worldComm:" << Pstream::worldComm
+                << Foam::exit(FatalError);
+        }
+
+        Pout.prefix() = "";
+        Perr.prefix() = "";
+    }
+    else
+    {
+        parRun_ = true;
+
+        // Redo worldComm communicator (created at static initialisation)
+        freeCommunicator(Pstream::worldComm);
+        label comm = allocateCommunicator(-1, identity(nProcs), true);
+
+        if (comm != Pstream::worldComm)
+        {
+            FatalErrorIn("Pstream::setParRun(const label)")
+                << "problem : comm:" << comm
+                << "  Pstream::worldComm:" << Pstream::worldComm
+                << Foam::exit(FatalError);
+        }
+
+        Pout.prefix() = '[' +  name(myProcNo()) + "] ";
+        Perr.prefix() = '[' +  name(myProcNo()) + "] ";
+    }
 }
 
 
-void Foam::Pstream::calcLinearComm(const label nProcs)
+Foam::List<Foam::Pstream::commsStruct> Foam::Pstream::calcLinearComm
+(
+    const label nProcs
+)
 {
-    linearCommunication_.setSize(nProcs);
+    List<commsStruct> linearCommunication(nProcs);
 
     // Master
     labelList belowIDs(nProcs - 1);
-    forAll(belowIDs, i)
+    forAll (belowIDs, i)
     {
         belowIDs[i] = i + 1;
     }
 
-    linearCommunication_[0] = commsStruct
+    linearCommunication[0] = commsStruct
     (
         nProcs,
         0,
         -1,
         belowIDs,
-        labelList(0)
+        labelList()
     );
 
     // Slaves. Have no below processors, only communicate up to master
     for (label procID = 1; procID < nProcs; procID++)
     {
-        linearCommunication_[procID] = commsStruct
+        linearCommunication[procID] = commsStruct
         (
             nProcs,
             procID,
             0,
-            labelList(0),
-            labelList(0)
+            labelList(),
+            labelList()
         );
     }
-}
 
-
-// Append my children (and my children children etc.) to allReceives.
-void Foam::Pstream::collectReceives
-(
-    const label procID,
-    const List<dynamicLabelList >& receives,
-    dynamicLabelList& allReceives
-)
-{
-    const dynamicLabelList& myChildren = receives[procID];
-
-    forAll(myChildren, childI)
-    {
-        allReceives.append(myChildren[childI]);
-        collectReceives(myChildren[childI], receives, allReceives);
-    }
+    return linearCommunication;
 }
 
 
@@ -151,7 +173,10 @@ void Foam::Pstream::collectReceives
 //  5       -               4
 //  6       7               4
 //  7       -               6
-void Foam::Pstream::calcTreeComm(label nProcs)
+Foam::List<Foam::Pstream::commsStruct> Foam::Pstream::calcTreeComm
+(
+    const label nProcs
+)
 {
     label nLevels = 1;
     while ((1 << nLevels) < nProcs)
@@ -159,11 +184,9 @@ void Foam::Pstream::calcTreeComm(label nProcs)
         nLevels++;
     }
 
-    List<dynamicLabelList > receives(nProcs);
+    List<dynamicLabelList> receives(nProcs);
     labelList sends(nProcs, -1);
 
-    // Info<< "Using " << nLevels << " communication levels" << endl;
-
     label offset = 2;
     label childOffset = offset/2;
 
@@ -190,18 +213,18 @@ void Foam::Pstream::calcTreeComm(label nProcs)
 
     // For all processors find the processors it receives data from
     // (and the processors they receive data from etc.)
-    List<dynamicLabelList > allReceives(nProcs);
+    List<dynamicLabelList> allReceives(nProcs);
     for (label procID = 0; procID < nProcs; procID++)
     {
         collectReceives(procID, receives, allReceives[procID]);
     }
 
 
-    treeCommunication_.setSize(nProcs);
+    List<commsStruct> treeCommunication(nProcs);
 
     for (label procID = 0; procID < nProcs; procID++)
     {
-        treeCommunication_[procID] = commsStruct
+        treeCommunication[procID] = commsStruct
         (
             nProcs,
             procID,
@@ -210,6 +233,26 @@ void Foam::Pstream::calcTreeComm(label nProcs)
             allReceives[procID].shrink()
         );
     }
+
+    return treeCommunication;
+}
+
+
+// Append my children (and my children;s children etc.) to allReceives.
+void Foam::Pstream::collectReceives
+(
+    const label procID,
+    const List<dynamicLabelList>& receives,
+    dynamicLabelList& allReceives
+)
+{
+    const dynamicLabelList& myChildren = receives[procID];
+
+    forAll (myChildren, childI)
+    {
+        allReceives.append(myChildren[childI]);
+        collectReceives(myChildren[childI], receives, allReceives);
+    }
 }
 
 
@@ -222,6 +265,294 @@ void Foam::Pstream::initCommunicationSchedule()
 }
 
 
+void Foam::Pstream::allocatePstreamCommunicator
+(
+    const label parentIndex,
+    const label index
+)
+{
+    if (index == PstreamGlobals::MPIGroups_.size())
+    {
+        // Extend storage with dummy values
+        MPI_Group newGroup;
+        PstreamGlobals::MPIGroups_.append(newGroup);
+        MPI_Comm newComm;
+        PstreamGlobals::MPICommunicators_.append(newComm);
+    }
+    else if (index > PstreamGlobals::MPIGroups_.size())
+    {
+        FatalErrorIn
+        (
+            "Pstream::allocatePstreamCommunicator\n"
+            "(\n"
+            "    const label parentIndex,\n"
+            "    const labelList& subRanks\n"
+            ")\n"
+        )   << "PstreamGlobals out of sync with Pstream data. Problem."
+            << Foam::exit(FatalError);
+    }
+
+
+    if (parentIndex == -1)
+    {
+        // Allocate world communicator
+
+        if (index != Pstream::worldComm)
+        {
+            FatalErrorIn
+            (
+                "Pstream::allocatePstreamCommunicator\n"
+                "(\n"
+                "    const label parentIndex,\n"
+                "    const labelList& subRanks\n"
+                ")\n"
+            )   << "world communicator should always be index "
+                << Pstream::worldComm << Foam::exit(FatalError);
+        }
+
+        PstreamGlobals::MPICommunicators_[index] = MPI_COMM_WORLD;
+        MPI_Comm_group(MPI_COMM_WORLD, &PstreamGlobals::MPIGroups_[index]);
+        MPI_Comm_rank
+        (
+            PstreamGlobals::MPICommunicators_[index],
+           &myProcNo_[index]
+        );
+
+        // Set the number of processes to the actual number
+        int numProcs;
+        MPI_Comm_size(PstreamGlobals::MPICommunicators_[index], &numProcs);
+        procIDs_[index] = identity(numProcs);
+    }
+    else
+    {
+        // Create new group
+        MPI_Group_incl
+        (
+            PstreamGlobals::MPIGroups_[parentIndex],
+            procIDs_[index].size(),
+            procIDs_[index].begin(),
+            &PstreamGlobals::MPIGroups_[index]
+        );
+
+        // Create new communicator
+        MPI_Comm_create
+        (
+            PstreamGlobals::MPICommunicators_[parentIndex],
+            PstreamGlobals::MPIGroups_[index],
+            &PstreamGlobals::MPICommunicators_[index]
+        );
+
+        if (PstreamGlobals::MPICommunicators_[index] == MPI_COMM_NULL)
+        {
+            myProcNo_[index] = -1;
+        }
+        else
+        {
+            if
+            (
+                MPI_Comm_rank
+                (
+                    PstreamGlobals::MPICommunicators_[index],
+                   &myProcNo_[index]
+                )
+            )
+            {
+                FatalErrorIn
+                (
+                    "Pstream::allocatePstreamCommunicator\n"
+                    "(\n"
+                    "    const label,\n"
+                    "    const labelList&\n"
+                    ")\n"
+                )   << "Problem :"
+                    << " when allocating communicator at " << index
+                    << " from ranks " << procIDs_[index]
+                    << " of parent " << parentIndex
+                    << " cannot find my own rank"
+                    << Foam::exit(FatalError);
+            }
+        }
+    }
+}
+
+
+void Foam::Pstream::freePstreamCommunicator(const label communicator)
+{
+    if (communicator != Pstream::worldComm)
+    {
+        if (PstreamGlobals::MPICommunicators_[communicator] != MPI_COMM_NULL)
+        {
+            // Free communicator. Sets communicator to MPI_COMM_NULL
+            MPI_Comm_free(&PstreamGlobals::MPICommunicators_[communicator]);
+        }
+        if (PstreamGlobals::MPIGroups_[communicator] != MPI_GROUP_NULL)
+        {
+            // Free greoup. Sets group to MPI_GROUP_NULL
+            MPI_Group_free(&PstreamGlobals::MPIGroups_[communicator]);
+        }
+    }
+}
+
+
+Foam::label Foam::Pstream::allocateCommunicator
+(
+    const label parentIndex,
+    const labelList& subRanks,
+    const bool doPstream
+)
+{
+    label index;
+    if (!freeComms_.empty())
+    {
+        index = freeComms_.pop();
+    }
+    else
+    {
+        // Extend storage
+        index = parentCommunicator_.size();
+
+        myProcNo_.append(-1);
+        procIDs_.append(List<int>());
+        parentCommunicator_.append(-1);
+        linearCommunication_.append(List<commsStruct>());
+        treeCommunication_.append(List<commsStruct>());
+    }
+
+    if (debug)
+    {
+        Pout<< "Communicators : Allocating communicator " << index << endl
+            << "    parent : " << parentIndex << endl
+            << "    procs  : " << subRanks << endl
+            << endl;
+    }
+
+    // Initialise; overwritten by allocatePstreamCommunicator
+    myProcNo_[index] = 0;
+
+    // Convert from label to int
+    procIDs_[index].setSize(subRanks.size());
+    forAll (procIDs_[index], i)
+    {
+        procIDs_[index][i] = subRanks[i];
+
+        // Enforce incremental order (so index is rank in next communicator)
+        if (i >= 1 && subRanks[i] <= subRanks[i - 1])
+        {
+            FatalErrorIn
+            (
+                "Pstream::allocateCommunicator"
+                "(const label, const labelList&, const bool)"
+            )   << "subranks not sorted : " << subRanks
+                << " when allocating subcommunicator from parent "
+                << parentIndex
+                << Foam::abort(FatalError);
+        }
+    }
+    parentCommunicator_[index] = parentIndex;
+
+    linearCommunication_[index] = calcLinearComm(procIDs_[index].size());
+    treeCommunication_[index] = calcTreeComm(procIDs_[index].size());
+
+
+    if (doPstream && parRun())
+    {
+        allocatePstreamCommunicator(parentIndex, index);
+    }
+
+    return index;
+}
+
+
+void Foam::Pstream::freeCommunicator
+(
+    const label communicator,
+    const bool doPstream
+)
+{
+    if (debug)
+    {
+        Pout<< "Communicators : Freeing communicator " << communicator << endl
+            << "    parent   : " << parentCommunicator_[communicator] << endl
+            << "    myProcNo : " << myProcNo_[communicator] << endl
+            << endl;
+    }
+
+    if (doPstream && parRun())
+    {
+        freePstreamCommunicator(communicator);
+    }
+    myProcNo_[communicator] = -1;
+    //procIDs_[communicator].clear();
+    parentCommunicator_[communicator] = -1;
+    linearCommunication_[communicator].clear();
+    treeCommunication_[communicator].clear();
+
+    freeComms_.push(communicator);
+}
+
+
+void Foam::Pstream::freeCommunicators(const bool doPstream)
+{
+    forAll (myProcNo_, communicator)
+    {
+        if (myProcNo_[communicator] != -1)
+        {
+            freeCommunicator(communicator, doPstream);
+        }
+    }
+}
+
+
+int Foam::Pstream::baseProcNo(const label myComm, const int myProcID)
+{
+    int procID = myProcID;
+    label comm = myComm;
+
+    while (parent(comm) != -1)
+    {
+        const List<int>& parentRanks = Pstream::procID(comm);
+        procID = parentRanks[procID];
+        comm = Pstream::parent(comm);
+    }
+
+    return procID;
+}
+
+
+Foam::label Foam::Pstream::procNo(const label myComm, const int baseProcID)
+{
+    const List<int>& parentRanks = procID(myComm);
+    label parentComm = parent(myComm);
+
+    if (parentComm == -1)
+    {
+        return findIndex(parentRanks, baseProcID);
+    }
+    else
+    {
+        label parentRank = procNo(parentComm, baseProcID);
+        return findIndex(parentRanks, parentRank);
+    }
+}
+
+
+Foam::label Foam::Pstream::procNo
+(
+    const label myComm,
+    const label currentComm,
+    const int currentProcID
+)
+{
+    label physProcID = Pstream::baseProcNo(currentComm, currentProcID);
+    return procNo(myComm, physProcID);
+}
+
+
+
+
+// * * * * * * * * * * * * * * * Member Functions  * * * * * * * * * * * * * //
+
+
 // NOTE:
 // valid parallel options vary between implementations, but flag common ones.
 // if they are not removed by MPI_Init(), the subsequent argument processing
@@ -233,7 +564,6 @@ void Foam::Pstream::addValidParOptions(HashTable<string>& validParOptions)
     validParOptions.insert("p4wd", "directory");
     validParOptions.insert("p4amslave", "");
     validParOptions.insert("p4yourname", "hostname");
-    validParOptions.insert("GAMMANP", "number of instances");
     validParOptions.insert("machinefile", "machine file");
 }
 
@@ -244,24 +574,25 @@ bool Foam::Pstream::init(int& argc, char**& argv)
 
     int numprocs;
     MPI_Comm_size(MPI_COMM_WORLD, &numprocs);
-    MPI_Comm_rank(MPI_COMM_WORLD, &myProcNo_);
+    int myRank;
+    MPI_Comm_rank(MPI_COMM_WORLD, &myRank);
+
+    if (debug)
+    {
+        Pout<< "Pstream::init : initialised with numProcs:" << numprocs
+            << " myRank:" << myRank << endl;
+    }
 
     if (numprocs <= 1)
     {
         FatalErrorIn("Pstream::init(int& argc, char**& argv)")
-            << "bool Pstream::init(int& argc, char**& argv) : "
+            << "bool IPstream::init(int& argc, char**& argv) : "
                "attempt to run parallel on 1 processor"
             << Foam::abort(FatalError);
     }
 
-    procIDs_.setSize(numprocs);
-
-    forAll(procIDs_, procNo)
-    {
-        procIDs_[procNo] = procNo;
-    }
-
-    setParRun();
+    // Initialise parallel structure
+    setParRun(numprocs);
 
 #   ifndef SGIMPI
     string bufferSizeName = getEnv("MPI_BUFFER_SIZE");
@@ -284,15 +615,12 @@ bool Foam::Pstream::init(int& argc, char**& argv)
     }
 #   endif
 
-    int processorNameLen;
-    char processorName[MPI_MAX_PROCESSOR_NAME];
-
-    MPI_Get_processor_name(processorName, &processorNameLen);
-
-    //signal(SIGABRT, stop);
-
-    // Now that nprocs is known construct communication tables.
-    initCommunicationSchedule();
+    //int processorNameLen;
+    //char processorName[MPI_MAX_PROCESSOR_NAME];
+    //
+    //MPI_Get_processor_name(processorName, &processorNameLen);
+    //processorName[processorNameLen] = '\0';
+    //Pout<< "Processor name:" << processorName << endl;
 
     return true;
 }
@@ -300,6 +628,11 @@ bool Foam::Pstream::init(int& argc, char**& argv)
 
 void Foam::Pstream::exit(int errnum)
 {
+    if (debug)
+    {
+        Pout<< "Pstream::exit." << endl;
+    }
+
 #   ifndef SGIMPI
     int size;
     char* buff;
@@ -307,6 +640,28 @@ void Foam::Pstream::exit(int errnum)
     delete[] buff;
 #   endif
 
+    if (PstreamGlobals::outstandingRequests_.size())
+    {
+        label n = PstreamGlobals::outstandingRequests_.size();
+        PstreamGlobals::outstandingRequests_.clear();
+
+        WarningIn("Pstream::exit(int)")
+            << "There are still " << n << " outstanding MPI_Requests." << endl
+            << "This means that your code exited before doing a"
+            << " Pstream::waitRequests()." << endl
+            << "This should not happen for a normal code exit."
+            << endl;
+    }
+
+    // Clean mpi communicators
+    forAll (myProcNo_, communicator)
+    {
+        if (myProcNo_[communicator] != -1)
+        {
+            freePstreamCommunicator(communicator);
+        }
+    }
+
     if (errnum == 0)
     {
         MPI_Finalize();
@@ -325,306 +680,370 @@ void Foam::Pstream::abort()
 }
 
 
-void Foam::reduce(scalar& Value, const sumOp<scalar>& bop)
+void Foam::reduce
+(
+    scalar& Value,
+    const sumOp<scalar>& bop,
+    const int tag,
+    const label comm
+)
 {
-    if (!Pstream::parRun())
+    if (Pstream::warnComm != -1 && comm != Pstream::warnComm)
     {
-        return;
+        Pout<< "** reducing:" << Value << " with comm:" << comm
+            << " warnComm:" << Pstream::warnComm
+            << endl;
+        error::printStack(Pout);
     }
 
-    if (Pstream::nProcs() <= Pstream::nProcsSimpleSum())
+    allReduce(Value, 1, MPI_SCALAR, MPI_SUM, bop, tag, comm);
+}
+
+
+void Foam::reduce
+(
+    scalar& Value,
+    const minOp<scalar>& bop,
+    const int tag,
+    const label comm
+)
+{
+    if (Pstream::warnComm != -1 && comm != Pstream::warnComm)
     {
-        if (Pstream::master())
-        {
-            for
-            (
-                int slave=Pstream::firstSlave();
-                slave<=Pstream::lastSlave();
-                slave++
-            )
-            {
-                scalar value;
-
-                if
-                (
-                    MPI_Recv
-                    (
-                        &value,
-                        1,
-                        MPI_SCALAR,
-                        Pstream::procID(slave),
-                        Pstream::msgType(),
-                        MPI_COMM_WORLD,
-                        MPI_STATUS_IGNORE
-                    )
-                )
-                {
-                    FatalErrorIn
-                    (
-                        "reduce(scalar& Value, const sumOp<scalar>& sumOp)"
-                    )   << "MPI_Recv failed"
-                        << Foam::abort(FatalError);
-                }
-
-                Value = bop(Value, value);
-            }
-        }
-        else
-        {
-            if
-            (
-                MPI_Send
-                (
-                    &Value,
-                    1,
-                    MPI_SCALAR,
-                    Pstream::procID(Pstream::masterNo()),
-                    Pstream::msgType(),
-                    MPI_COMM_WORLD
-                )
-            )
-            {
-                FatalErrorIn
-                (
-                    "reduce(scalar& Value, const sumOp<scalar>& sumOp)"
-                )   << "MPI_Send failed"
-                    << Foam::abort(FatalError);
-            }
-        }
+        Pout<< "** reducing:" << Value << " with comm:" << comm
+            << " warnComm:" << Pstream::warnComm
+            << endl;
+        error::printStack(Pout);
+    }
+    allReduce(Value, 1, MPI_SCALAR, MPI_MIN, bop, tag, comm);
+}
 
 
-        if (Pstream::master())
-        {
-            for
+void Foam::reduce
+(
+    vector2D& Value,
+    const sumOp<vector2D>& bop,
+    const int tag,
+    const label comm
+)
+{
+    if (Pstream::warnComm != -1 && comm != Pstream::warnComm)
+    {
+        Pout<< "** reducing:" << Value << " with comm:" << comm
+            << " warnComm:" << Pstream::warnComm
+            << endl;
+        error::printStack(Pout);
+    }
+    allReduce(Value, 2, MPI_SCALAR, MPI_SUM, bop, tag, comm);
+}
+
+
+void Foam::sumReduce
+(
+    scalar& Value,
+    label& Count,
+    const int tag,
+    const label comm
+)
+{
+    if (Pstream::warnComm != -1 && comm != Pstream::warnComm)
+    {
+        Pout<< "** reducing:" << Value << " with comm:" << comm
+            << " warnComm:" << Pstream::warnComm
+            << endl;
+        error::printStack(Pout);
+    }
+    vector2D twoScalars(Value, scalar(Count));
+    reduce(twoScalars, sumOp<vector2D>(), tag, comm);
+
+    Value = twoScalars.x();
+    Count = twoScalars.y();
+}
+
+
+void Foam::reduce
+(
+    scalar& Value,
+    const sumOp<scalar>& bop,
+    const int tag,
+    const label comm,
+    label& requestID
+)
+{
+#ifdef MPIX_COMM_TYPE_SHARED
+    // Assume mpich2 with non-blocking collectives extensions. Once mpi3
+    // is available this will change.
+    MPI_Request request;
+    scalar v = Value;
+    MPIX_Ireduce
+    (
+        &v,
+        &Value,
+        1,
+        MPI_SCALAR,
+        MPI_SUM,
+        0,              // root
+        PstreamGlobals::MPICommunicators_[comm],
+        &request
+    );
+
+    requestID = PstreamGlobals::outstandingRequests_.size();
+    PstreamGlobals::outstandingRequests_.append(request);
+
+    if (debug)
+    {
+        Pout<< "Pstream::allocateRequest for non-blocking reduce"
+            << " : request:" << requestID
+            << endl;
+    }
+#else
+    // Non-blocking not yet implemented in mpi
+    reduce(Value, bop, tag, comm);
+
+    requestID = -1;
+#endif
+}
+
+
+Foam::label Foam::Pstream::nRequests()
+{
+    return PstreamGlobals::outstandingRequests_.size();
+}
+
+
+void Foam::Pstream::resetRequests(const label i)
+{
+    if (i < PstreamGlobals::outstandingRequests_.size())
+    {
+        PstreamGlobals::outstandingRequests_.setSize(i);
+    }
+}
+
+
+void Foam::Pstream::waitRequests(const label start)
+{
+    if (debug)
+    {
+        Pout<< "Pstream::waitRequests : starting wait for "
+            << PstreamGlobals::outstandingRequests_.size()-start
+            << " outstanding requests starting at " << start << endl;
+    }
+
+    if (PstreamGlobals::outstandingRequests_.size())
+    {
+        SubList<MPI_Request> waitRequests
+        (
+            PstreamGlobals::outstandingRequests_,
+            PstreamGlobals::outstandingRequests_.size() - start,
+            start
+        );
+
+        if
+        (
+            MPI_Waitall
             (
-                int slave=Pstream::firstSlave();
-                slave<=Pstream::lastSlave();
-                slave++
+                waitRequests.size(),
+                waitRequests.begin(),
+                MPI_STATUSES_IGNORE
             )
-            {
-                if
-                (
-                    MPI_Send
-                    (
-                        &Value,
-                        1,
-                        MPI_SCALAR,
-                        Pstream::procID(slave),
-                        Pstream::msgType(),
-                        MPI_COMM_WORLD
-                    )
-                )
-                {
-                    FatalErrorIn
-                    (
-                        "reduce(scalar& Value, const sumOp<scalar>& sumOp)"
-                    )   << "MPI_Send failed"
-                        << Foam::abort(FatalError);
-                }
-            }
-        }
-        else
+        )
         {
-            if
+            FatalErrorIn
             (
-                MPI_Recv
-                (
-                    &Value,
-                    1,
-                    MPI_SCALAR,
-                    Pstream::procID(Pstream::masterNo()),
-                    Pstream::msgType(),
-                    MPI_COMM_WORLD,
-                    MPI_STATUS_IGNORE
-                )
-            )
-            {
-                FatalErrorIn
-                (
-                    "reduce(scalar& Value, const sumOp<scalar>& sumOp)"
-                )   << "MPI_Recv failed"
-                    << Foam::abort(FatalError);
-            }
+                "Pstream::waitRequests()"
+            )   << "MPI_Waitall returned with error" << Foam::endl;
         }
+
+        resetRequests(start);
+    }
+
+    if (debug)
+    {
+        Pout<< "Pstream::waitRequests : finished wait." << endl;
+    }
+}
+
+
+void Foam::Pstream::waitRequest(const label i)
+{
+    if (debug)
+    {
+        Pout<< "Pstream::waitRequest : starting wait for request:" << i
+            << endl;
+    }
+
+    if (i >= PstreamGlobals::outstandingRequests_.size())
+    {
+        FatalErrorIn
+        (
+            "Pstream::waitRequest(const label)"
+        )   << "There are " << PstreamGlobals::outstandingRequests_.size()
+            << " outstanding send requests and you are asking for i=" << i
+            << nl
+            << "Maybe you are mixing blocking/non-blocking comms?"
+            << Foam::abort(FatalError);
+    }
+
+    if
+    (
+        MPI_Wait
+        (
+           &PstreamGlobals::outstandingRequests_[i],
+            MPI_STATUS_IGNORE
+        )
+    )
+    {
+        FatalErrorIn
+        (
+            "Pstream::waitRequest()"
+        )   << "MPI_Wait returned with error" << Foam::endl;
+    }
+
+    if (debug)
+    {
+        Pout<< "Pstream::waitRequest : finished wait for request:" << i
+            << endl;
+    }
+}
+
+
+bool Foam::Pstream::finishedRequest(const label i)
+{
+    if (debug)
+    {
+        Pout<< "Pstream::finishedRequest : checking request:" << i
+            << endl;
+    }
+
+    if (i >= PstreamGlobals::outstandingRequests_.size())
+    {
+        FatalErrorIn
+        (
+            "Pstream::finishedRequest(const label)"
+        )   << "There are " << PstreamGlobals::outstandingRequests_.size()
+            << " outstanding send requests and you are asking for i=" << i
+            << nl
+            << "Maybe you are mixing blocking/non-blocking comms?"
+            << Foam::abort(FatalError);
+    }
+
+    int flag;
+    MPI_Test
+    (
+       &PstreamGlobals::outstandingRequests_[i],
+       &flag,
+        MPI_STATUS_IGNORE
+    );
+
+    if (debug)
+    {
+        Pout<< "Pstream::finishedRequest : finished request:" << i
+            << endl;
+    }
+
+    return flag != 0;
+}
+
+
+int Foam::Pstream::allocateTag(const char* s)
+{
+    int tag;
+    if (PstreamGlobals::freedTags_.size())
+    {
+        tag = PstreamGlobals::freedTags_.remove();
     }
     else
     {
-        scalar sum;
-        MPI_Allreduce(&Value, &sum, 1, MPI_SCALAR, MPI_SUM, MPI_COMM_WORLD);
-        Value = sum;
-
-        /*
-        int myProcNo = Pstream::myProcNo();
-        int nProcs = Pstream::nProcs();
-
-        //
-        // receive from children
-        //
-        int level = 1;
-        int thisLevelOffset = 2;
-        int childLevelOffset = thisLevelOffset/2;
-        int childProcId = 0;
-
-        while
-        (
-            (childLevelOffset < nProcs)
-         && (myProcNo % thisLevelOffset) == 0
-        )
-        {
-            childProcId = myProcNo + childLevelOffset;
-
-            scalar value;
-
-            if (childProcId < nProcs)
-            {
-                if
-                (
-                    MPI_Recv
-                    (
-                        &value,
-                        1,
-                        MPI_SCALAR,
-                        Pstream::procID(childProcId),
-                        Pstream::msgType(),
-                        MPI_COMM_WORLD,
-                        MPI_STATUS_IGNORE
-                    )
-                )
-                {
-                    FatalErrorIn
-                    (
-                        "reduce(scalar& Value, const sumOp<scalar>& sumOp)"
-                    )   << "MPI_Recv failed"
-                        << Foam::abort(FatalError);
-                }
-
-                Value = bop(Value, value);
-            }
-
-            level++;
-            thisLevelOffset <<= 1;
-            childLevelOffset = thisLevelOffset/2;
-        }
-
-        //
-        // send and receive from parent
-        //
-        if (!Pstream::master())
-        {
-            int parentId = myProcNo - (myProcNo % thisLevelOffset);
-
-            if
-            (
-                MPI_Send
-                (
-                    &Value,
-                    1,
-                    MPI_SCALAR,
-                    Pstream::procID(parentId),
-                    Pstream::msgType(),
-                    MPI_COMM_WORLD
-                )
-            )
-            {
-                FatalErrorIn
-                (
-                    "reduce(scalar& Value, const sumOp<scalar>& sumOp)"
-                )   << "MPI_Send failed"
-                    << Foam::abort(FatalError);
-            }
-
-            if
-            (
-                MPI_Recv
-                (
-                    &Value,
-                    1,
-                    MPI_SCALAR,
-                    Pstream::procID(parentId),
-                    Pstream::msgType(),
-                    MPI_COMM_WORLD,
-                    MPI_STATUS_IGNORE
-                )
-            )
-            {
-                FatalErrorIn
-                (
-                    "reduce(scalar& Value, const sumOp<scalar>& sumOp)"
-                )   << "MPI_Recv failed"
-                    << Foam::abort(FatalError);
-            }
-        }
-
-
-        //
-        // distribute to my children
-        //
-        level--;
-        thisLevelOffset >>= 1;
-        childLevelOffset = thisLevelOffset/2;
-
-        while (level > 0)
-        {
-            childProcId = myProcNo + childLevelOffset;
-
-            if (childProcId < nProcs)
-            {
-                if
-                (
-                    MPI_Send
-                    (
-                        &Value,
-                        1,
-                        MPI_SCALAR,
-                        Pstream::procID(childProcId),
-                        Pstream::msgType(),
-                        MPI_COMM_WORLD
-                    )
-                )
-                {
-                    FatalErrorIn
-                    (
-                        "reduce(scalar& Value, const sumOp<scalar>& sumOp)"
-                    )   << "MPI_Send failed"
-                        << Foam::abort(FatalError);
-                }
-            }
-
-            level--;
-            thisLevelOffset >>= 1;
-            childLevelOffset = thisLevelOffset/2;
-        }
-        */
+        tag = PstreamGlobals::nTags_++;
     }
+
+    if (debug)
+    {
+        Pout<< "Pstream::allocateTag " << s
+            << " : tag:" << tag
+            << endl;
+    }
+
+    return tag;
+}
+
+
+int Foam::Pstream::allocateTag(const word& s)
+{
+    int tag;
+    if (PstreamGlobals::freedTags_.size())
+    {
+        tag = PstreamGlobals::freedTags_.remove();
+    }
+    else
+    {
+        tag = PstreamGlobals::nTags_++;
+    }
+
+    if (debug)
+    {
+        Pout<< "Pstream::allocateTag " << s
+            << " : tag:" << tag
+            << endl;
+    }
+
+    return tag;
+}
+
+
+void Foam::Pstream::freeTag(const char* s, const int tag)
+{
+    if (debug)
+    {
+        Pout<< "Pstream::freeTag " << s << " tag:" << tag << endl;
+    }
+    PstreamGlobals::freedTags_.append(tag);
+}
+
+
+void Foam::Pstream::freeTag(const word& s, const int tag)
+{
+    if (debug)
+    {
+        Pout<< "Pstream::freeTag " << s << " tag:" << tag << endl;
+    }
+    PstreamGlobals::freedTags_.append(tag);
 }
 
 
 // * * * * * * * * * * * * * * Static Data Members * * * * * * * * * * * * * //
 
-// Initialise my process number to 0 (the master)
-int Foam::Pstream::myProcNo_(0);
-
-
 // By default this is not a parallel run
 bool Foam::Pstream::parRun_(false);
 
+// Free communicators
+Foam::LIFOStack<Foam::label> Foam::Pstream::freeComms_;
+
+// My processor number
+Foam::DynamicList<int> Foam::Pstream::myProcNo_(10);
 
 // List of process IDs
-Foam::List<int> Foam::Pstream::procIDs_(1, 0);
+Foam::DynamicList<Foam::List<int> > Foam::Pstream::procIDs_(10);
 
+// Parent communicator
+Foam::DynamicList<Foam::label> Foam::Pstream::parentCommunicator_(10);
 
 // Standard transfer message type
 const int Foam::Pstream::msgType_(1);
 
-
 // Linear communication schedule
-Foam::List<Foam::Pstream::commsStruct> Foam::Pstream::linearCommunication_(0);
-
+Foam::DynamicList<Foam::List<Foam::Pstream::commsStruct> >
+Foam::Pstream::linearCommunication_(10);
 
 // Multi level communication schedule
-Foam::List<Foam::Pstream::commsStruct> Foam::Pstream::treeCommunication_(0);
+Foam::DynamicList<Foam::List<Foam::Pstream::commsStruct> >
+Foam::Pstream::treeCommunication_(10);
 
 
+// Allocate a serial communicator. This gets overwritten in parallel mode
+// (by Pstream::setParRun())
+Foam::Pstream::communicator serialComm(-1, Foam::labelList(1, 0), false);
+
 // Number of processors at which the reduce algorithm changes from linear to
 // tree
 const Foam::debug::optimisationSwitch
@@ -635,15 +1054,30 @@ Foam::Pstream::nProcsSimpleSum
 );
 
 
-const Foam::debug::optimisationSwitch
+Foam::debug::optimisationSwitch
 Foam::Pstream::defaultCommsType
 (
     "commsType",
-//     "nonBlocking",
+    "nonBlocking",
 //     "scheduled",
-    "blocking",
+//     "blocking",
     "blocking, nonBlocking, scheduled"
 );
 
 
+const Foam::debug::optimisationSwitch
+Foam::Pstream::nPollProcInterfaces
+(
+    "nPollProcInterfaces",
+    0
+);
+
+// Default communicator
+Foam::label Foam::Pstream::worldComm(0);
+
+
+// Warn for use of any communicator
+Foam::label Foam::Pstream::warnComm(-1);
+
+
 // ************************************************************************* //
diff --git a/src/foam/db/IOstreams/Pstreams/Pstream.H b/src/foam/db/IOstreams/Pstreams/Pstream.H
index f9033988a..2f44eca57 100644
--- a/src/foam/db/IOstreams/Pstreams/Pstream.H
+++ b/src/foam/db/IOstreams/Pstreams/Pstream.H
@@ -273,11 +273,6 @@ public:
 
     // Static data
 
-        //- Should compact transfer be used in which floats replace doubles
-        //  reducing the bandwidth requirement at the expense of some loss
-        //  in accuracy
-        static const debug::optimisationSwitch floatTransfer;
-
         //- Number of processors at which the sum algorithm changes from linear
         //  to tree
         static const debug::optimisationSwitch nProcsSimpleSum;

From 927b4bb37c812bcd96045d335f0790c1755ac6dd Mon Sep 17 00:00:00 2001
From: Hrvoje Jasak <h.jasak@wikki.co.uk>
Date: Fri, 23 Sep 2016 10:19:27 +0100
Subject: [PATCH 057/130] Changed clock type in block matrix coarsening

---
 .../BlockMatrixAgglomeration.C                | 32 +++++++++----------
 .../BlockMatrixAgglomeration.H                |  4 +--
 2 files changed, 18 insertions(+), 18 deletions(-)

diff --git a/src/foam/matrices/blockLduMatrix/BlockAMG/BlockMatrixCoarsening/BlockMatrixAgglomeration/BlockMatrixAgglomeration.C b/src/foam/matrices/blockLduMatrix/BlockAMG/BlockMatrixCoarsening/BlockMatrixAgglomeration/BlockMatrixAgglomeration.C
index b42f757af..4bd5a29b1 100644
--- a/src/foam/matrices/blockLduMatrix/BlockAMG/BlockMatrixCoarsening/BlockMatrixAgglomeration/BlockMatrixAgglomeration.C
+++ b/src/foam/matrices/blockLduMatrix/BlockAMG/BlockMatrixCoarsening/BlockMatrixAgglomeration/BlockMatrixAgglomeration.C
@@ -143,7 +143,7 @@ void Foam::BlockMatrixAgglomeration<Type>::calcAgglomeration()
         // Reset agglomeration index array
         agglomIndex_ = -1;
     }
-    Info<< "End prepare addressing: " << lTime_.elapsedClockTime() << endl;
+    Info<< "End prepare addressing: " << lTime_.elapsedCpuTime() << endl;
 
 
     // Calculate agglomeration
@@ -179,7 +179,7 @@ void Foam::BlockMatrixAgglomeration<Type>::calcAgglomeration()
 
         return;
     }
-    Info<< "End coeff mag: " << lTime_.elapsedClockTime() << endl;
+    Info<< "End coeff mag: " << lTime_.elapsedCpuTime() << endl;
 
     labelList sizeOfGroups(nRows, 0);
 
@@ -235,7 +235,7 @@ void Foam::BlockMatrixAgglomeration<Type>::calcAgglomeration()
             }
         }
     }
-    Info<< "End weak connections: " << lTime_.elapsedClockTime() << endl;
+    Info<< "End weak connections: " << lTime_.elapsedCpuTime() << endl;
 
     // Go through the off-diagonal and create clusters, marking the child array
     label indexUngrouped, indexGrouped;
@@ -343,7 +343,7 @@ void Foam::BlockMatrixAgglomeration<Type>::calcAgglomeration()
             }
         }
     }
-    Info<< "End agglom index: " << lTime_.elapsedClockTime() << endl;
+    Info<< "End agglom index: " << lTime_.elapsedCpuTime() << endl;
 
     // The decision on parallel agglomeration needs to be made for the
     // whole gang of processes; otherwise I may end up with a different
@@ -373,7 +373,7 @@ void Foam::BlockMatrixAgglomeration<Type>::calcAgglomeration()
             Pout << ".  Rejected" << endl;
         }
     }
-    Info<< "End calcAgglomeration: " << lTime_.elapsedClockTime() << endl;
+    Info<< "End calcAgglomeration: " << lTime_.elapsedCpuTime() << endl;
 }
 
 
@@ -385,7 +385,7 @@ void Foam::BlockMatrixAgglomeration<Type>::restrictDiag
 ) const
 {
     typedef CoeffField<Type> TypeCoeffField;
-    Info<< "Start restrictDiag: " << lTime_.elapsedClockTime() << endl;
+    Info<< "Start restrictDiag: " << lTime_.elapsedCpuTime() << endl;
     if
     (
         Coeff.activeType() == blockCoeffBase::SQUARE
@@ -460,7 +460,7 @@ void Foam::BlockMatrixAgglomeration<Type>::restrictDiag
         )   << "Problem in coeff type morphing"
             << abort(FatalError);
     }
-    Info<< "End restrictDiag: " << lTime_.elapsedClockTime() << endl;
+    Info<< "End restrictDiag: " << lTime_.elapsedCpuTime() << endl;
 }
 
 
@@ -477,7 +477,7 @@ void Foam::BlockMatrixAgglomeration<Type>::agglomerateCoeffs
 {
     // Does the matrix have solo equations
     bool soloEqns = nSolo_ > 0;
-    Info<< "Start agglomerateCoeffs symm: " << lTime_.elapsedClockTime() << endl;
+    Info<< "Start agglomerateCoeffs symm: " << lTime_.elapsedCpuTime() << endl;
     // Get addressing
     const unallocLabelList& upperAddr = matrix_.lduAddr().upperAddr();
     const unallocLabelList& lowerAddr = matrix_.lduAddr().lowerAddr();
@@ -510,7 +510,7 @@ void Foam::BlockMatrixAgglomeration<Type>::agglomerateCoeffs
               + activeFineUpperTranspose[fineCoeffI];
         }
     }
-    Info<< "End agglomerateCoeffs symm: " << lTime_.elapsedClockTime() << endl;
+    Info<< "End agglomerateCoeffs symm: " << lTime_.elapsedCpuTime() << endl;
 }
 
 
@@ -528,7 +528,7 @@ void Foam::BlockMatrixAgglomeration<Type>::agglomerateCoeffs
 {
     // Does the matrix have solo equations
     bool soloEqns = nSolo_ > 0;
-    Info<< "Start agglomerateCoeffs asymm: " << lTime_.elapsedClockTime() << endl;
+    Info<< "Start agglomerateCoeffs asymm: " << lTime_.elapsedCpuTime() << endl;
     // Get addressing
     const unallocLabelList& upperAddr = matrix_.lduAddr().upperAddr();
     const unallocLabelList& lowerAddr = matrix_.lduAddr().lowerAddr();
@@ -560,7 +560,7 @@ void Foam::BlockMatrixAgglomeration<Type>::agglomerateCoeffs
               + activeFineLower[fineCoeffI];
         }
     }
-    Info<< "End agglomerateCoeffs asymm: " << lTime_.elapsedClockTime() << endl;
+    Info<< "End agglomerateCoeffs asymm: " << lTime_.elapsedCpuTime() << endl;
 }
 
 
@@ -689,7 +689,7 @@ Foam::BlockMatrixAgglomeration<Type>::restrictMatrix() const
     //    insert the coeffs in the upper order. At the same time, collect the
     //    owner and neighbour addressing.
     // 4) Agglomerate the diagonal by summing up the fine diagonal
-    Info<< "Start restrictMatrix: " << lTime_.elapsedClockTime() << endl;
+    Info<< "Start restrictMatrix: " << lTime_.elapsedCpuTime() << endl;
 
     // Get addressing
     const unallocLabelList& upperAddr = matrix_.lduAddr().upperAddr();
@@ -864,7 +864,7 @@ Foam::BlockMatrixAgglomeration<Type>::restrictMatrix() const
     blockNbrsData.setSize(0);
     initCoarseNeighb.setSize(0);
     coarseCoeffMap.setSize(0);
-    Info<< "End restrictMatrix core: " << lTime_.elapsedClockTime() << endl;
+    Info<< "End restrictMatrix core: " << lTime_.elapsedCpuTime() << endl;
 
 
     // Create coarse-level coupled interfaces
@@ -970,7 +970,7 @@ Foam::BlockMatrixAgglomeration<Type>::restrictMatrix() const
             coarseInterfaceAddr[intI] = coarseInterface.faceCells();
         }
     }
-    Info<< "End restrictMatrix interfaces: " << lTime_.elapsedClockTime() << endl;
+    Info<< "End restrictMatrix interfaces: " << lTime_.elapsedCpuTime() << endl;
 
     // Add interfaces
     coarseAddrPtr->addInterfaces
@@ -1047,7 +1047,7 @@ Foam::BlockMatrixAgglomeration<Type>::restrictMatrix() const
     }
 
     // Matrix restriction done!
-    Info<< "End restrictMatrix: " << lTime_.elapsedClockTime() << endl;
+    Info<< "End restrictMatrix: " << lTime_.elapsedCpuTime() << endl;
 
     typedef CoeffField<Type> TypeCoeffField;
 
@@ -1244,7 +1244,7 @@ Foam::BlockMatrixAgglomeration<Type>::restrictMatrix() const
                 << abort(FatalError);
         }
     }
-    Info<< "End restrictMatrix agglomerateCoeffs: " << lTime_.elapsedClockTime() << endl;
+    Info<< "End restrictMatrix agglomerateCoeffs: " << lTime_.elapsedCpuTime() << endl;
 
     // Create and return BlockAMGLevel
     return autoPtr<BlockAMGLevel<Type> >
diff --git a/src/foam/matrices/blockLduMatrix/BlockAMG/BlockMatrixCoarsening/BlockMatrixAgglomeration/BlockMatrixAgglomeration.H b/src/foam/matrices/blockLduMatrix/BlockAMG/BlockMatrixCoarsening/BlockMatrixAgglomeration/BlockMatrixAgglomeration.H
index 65ac5eb8f..0cd3e0d1a 100644
--- a/src/foam/matrices/blockLduMatrix/BlockAMG/BlockMatrixCoarsening/BlockMatrixAgglomeration/BlockMatrixAgglomeration.H
+++ b/src/foam/matrices/blockLduMatrix/BlockAMG/BlockMatrixCoarsening/BlockMatrixAgglomeration/BlockMatrixAgglomeration.H
@@ -43,7 +43,7 @@ SourceFiles
 #include "BlockCoeffNorm.H"
 #include "BlockCoeff.H"
 #include "tolerancesSwitch.H"
-#include "clock.H"
+#include "cpuTime.H"
 
 // * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
 
@@ -85,7 +85,7 @@ class BlockMatrixAgglomeration
         //- Can a coarse level be constructed?
         bool coarsen_;
 
-        clock lTime_;
+        cpuTime lTime_;
 
 
     // Private Member Functions

From e067357bd02a92527cc4823f1d8127cc6dc5b028 Mon Sep 17 00:00:00 2001
From: Hrvoje Jasak <h.jasak@wikki.co.uk>
Date: Fri, 23 Sep 2016 15:24:42 +0100
Subject: [PATCH 058/130] Performance measure

---
 .../BlockMatrixAgglomeration.C                 |  5 +++--
 .../ggiAMGInterface/ggiAMGInterface.C          | 18 +++++++++++-------
 .../ggiAMGInterface/ggiAMGInterface.H          |  4 ++++
 3 files changed, 18 insertions(+), 9 deletions(-)

diff --git a/src/foam/matrices/blockLduMatrix/BlockAMG/BlockMatrixCoarsening/BlockMatrixAgglomeration/BlockMatrixAgglomeration.C b/src/foam/matrices/blockLduMatrix/BlockAMG/BlockMatrixCoarsening/BlockMatrixAgglomeration/BlockMatrixAgglomeration.C
index 4bd5a29b1..9ce0ddf16 100644
--- a/src/foam/matrices/blockLduMatrix/BlockAMG/BlockMatrixCoarsening/BlockMatrixAgglomeration/BlockMatrixAgglomeration.C
+++ b/src/foam/matrices/blockLduMatrix/BlockAMG/BlockMatrixCoarsening/BlockMatrixAgglomeration/BlockMatrixAgglomeration.C
@@ -957,9 +957,10 @@ Foam::BlockMatrixAgglomeration<Type>::restrictMatrix() const
                     fineInterfaceAddr[intI]
                 ).ptr()
             );
+            Info<< " Interface " << intI << ": " << lTime_.elapsedCpuTime() << endl;
         }
     }
-
+    Info<< "End restrictMatrix interfaces: " << lTime_.elapsedCpuTime() << endl;
     forAll (interfaceFields, intI)
     {
         if (interfaceFields.set(intI))
@@ -970,7 +971,7 @@ Foam::BlockMatrixAgglomeration<Type>::restrictMatrix() const
             coarseInterfaceAddr[intI] = coarseInterface.faceCells();
         }
     }
-    Info<< "End restrictMatrix interfaces: " << lTime_.elapsedCpuTime() << endl;
+    Info<< "End restrictMatrix interface fields: " << lTime_.elapsedCpuTime() << endl;
 
     // Add interfaces
     coarseAddrPtr->addInterfaces
diff --git a/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/ggiAMGInterface/ggiAMGInterface.C b/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/ggiAMGInterface/ggiAMGInterface.C
index 9ad831eb4..a8b654c94 100644
--- a/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/ggiAMGInterface/ggiAMGInterface.C
+++ b/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/ggiAMGInterface/ggiAMGInterface.C
@@ -71,7 +71,7 @@ void Foam::ggiAMGInterface::initFastReduce() const
 
         return;
     }
-
+    Info<< "Start initFastReduce " << lTime_.elapsedCpuTime() << endl;
     // From here on, work on processors within the communicator
     // HJ, 20/Sep/2016
 
@@ -215,6 +215,7 @@ void Foam::ggiAMGInterface::initFastReduce() const
     // Map will return the object of the size of remote zone
     // HJ, 9/May/2016
     mapPtr_ = new mapDistribute(zoneSize(), sendMap, constructMap);
+    Info<< "End initFastReduce " << lTime_.elapsedCpuTime() << endl;
 }
 
 
@@ -236,7 +237,8 @@ Foam::ggiAMGInterface::ggiAMGInterface
     procMasterFaces_(),
     comm_(fineGgiInterface_.comm()),
     tag_(fineGgiInterface_.tag()),
-    mapPtr_(NULL)
+    mapPtr_(NULL),
+    lTime_()
 {
     // New algorithm will assemble local clusters on the master side and
     // create zone ordering by collecting all faces (coarse pairs) from proc0,
@@ -244,7 +246,7 @@ Foam::ggiAMGInterface::ggiAMGInterface
     // each processor only to perform the analysis on locally created coarse
     // faces
     // HJ, 13/Jun/2016
-
+    Info<< "Start ggiAMGInterface constructor " << lTime_.elapsedCpuTime() << endl;
     // Note: local addressing contains only local faces
     const labelList& fineZa =  fineGgiInterface_.zoneAddressing();
 
@@ -264,7 +266,7 @@ Foam::ggiAMGInterface::ggiAMGInterface
             neighbourExpandAddressing
         );
     }
-
+    Info<< "ggiAMGInterface expandToAddr1 " << lTime_.elapsedCpuTime() << endl;
     // Create addressing for neighbour processors.  Note: expandAddrToZone will
     // expand the addressing to zone size.  HJ, 13/Jun/2016
     labelField neighbourExpandProc
@@ -283,7 +285,7 @@ Foam::ggiAMGInterface::ggiAMGInterface
             neighbourExpandProc
         );
     }
-
+    Info<< "ggiAMGInterface expandToAddr2 " << lTime_.elapsedCpuTime() << endl;
     // Note: neighbourExpandAddressing and neighbourExpandProc
     // will be filled with NaNs for faces which are not local
 
@@ -709,7 +711,7 @@ Foam::ggiAMGInterface::ggiAMGInterface
     fineAddressing_.setSize(nAgglomPairs, -1);
     restrictAddressing_.setSize(nAgglomPairs, -1);
     restrictWeights_.setSize(nAgglomPairs);
-
+    Info<< "ggiAMGInterface end Addr " << lTime_.elapsedCpuTime() << endl;
     // In order to assemble the coarse global face zone, find out
     // how many faces have been created on each processor.
     // Note that masters and slaves both count faces so we will
@@ -724,7 +726,7 @@ Foam::ggiAMGInterface::ggiAMGInterface
     // This needs to be handled separately in the initFastReduce
     // HJ, 20/Sep/2016
     reduce(nCoarseFacesPerProc, sumOp<labelList>(), tag(), comm());
-
+    Info<< "ggiAMGInterface end reduce 1 " << lTime_.elapsedCpuTime() << endl;
     // Coarse global face zone is assembled by adding all faces from proc0,
     // followed by all faces from proc1 etc.
     // Therefore, on procN, my master offset
@@ -921,6 +923,7 @@ Foam::ggiAMGInterface::ggiAMGInterface
                 }
             }
         }
+        Info<< "ggiAMGInterface end agglom master " << lTime_.elapsedCpuTime() << endl;
     }
     // Agglomerate slave
     else
@@ -1042,6 +1045,7 @@ Foam::ggiAMGInterface::ggiAMGInterface
                 nProcFaces++;
             }
         }
+        Info<< "ggiAMGInterface end agglom slave " << lTime_.elapsedCpuTime() << endl;
     }
 }
 
diff --git a/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/ggiAMGInterface/ggiAMGInterface.H b/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/ggiAMGInterface/ggiAMGInterface.H
index e18fe220f..6917ebcef 100644
--- a/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/ggiAMGInterface/ggiAMGInterface.H
+++ b/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/ggiAMGInterface/ggiAMGInterface.H
@@ -42,6 +42,8 @@ SourceFiles
 #include "ggiLduInterface.H"
 #include "mapDistribute.H"
 
+#include "cpuTime.H"
+
 // * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
 
 namespace Foam
@@ -86,6 +88,8 @@ class ggiAMGInterface
             //- Map-distribute comms tool
             mutable mapDistribute* mapPtr_;
 
+            cpuTime lTime_;
+
 
     // Private Member Functions
 

From 4306c963652c6d5ebcf4f721004b2367b2911d7f Mon Sep 17 00:00:00 2001
From: Hrvoje Jasak <h.jasak@wikki.co.uk>
Date: Fri, 23 Sep 2016 18:38:12 +0100
Subject: [PATCH 059/130] Formatting

---
 .../matrices/blockLduMatrix/BlockAMG/BlockAMGCycle.H |  5 ++---
 .../blockLduMatrix/BlockAMG/coarseBlockAMGLevel.H    | 12 ++++++------
 .../BlockGaussSeidelPrecon/BlockGaussSeidelPrecon.H  |  7 +++----
 3 files changed, 11 insertions(+), 13 deletions(-)

diff --git a/src/foam/matrices/blockLduMatrix/BlockAMG/BlockAMGCycle.H b/src/foam/matrices/blockLduMatrix/BlockAMG/BlockAMGCycle.H
index 207a2b589..56da581f5 100644
--- a/src/foam/matrices/blockLduMatrix/BlockAMG/BlockAMGCycle.H
+++ b/src/foam/matrices/blockLduMatrix/BlockAMG/BlockAMGCycle.H
@@ -128,9 +128,8 @@ public:
         BlockAMGCycle(autoPtr<BlockAMGLevel<Type> > levelPtr);
 
 
-    // Destructor
-
-        virtual ~BlockAMGCycle();
+    //- Destructor
+    virtual ~BlockAMGCycle();
 
 
     // Member Functions
diff --git a/src/foam/matrices/blockLduMatrix/BlockAMG/coarseBlockAMGLevel.H b/src/foam/matrices/blockLduMatrix/BlockAMG/coarseBlockAMGLevel.H
index d788a1af6..a2d6f4711 100644
--- a/src/foam/matrices/blockLduMatrix/BlockAMG/coarseBlockAMGLevel.H
+++ b/src/foam/matrices/blockLduMatrix/BlockAMG/coarseBlockAMGLevel.H
@@ -119,18 +119,18 @@ public:
 
     // Member Functions
 
-        //- Return reference to x
-        virtual Field<Type>& x();
-
-        //- Return reference to b
-        virtual Field<Type>& b();
-
         //- Return reference to dictionary
         const dictionary& dict() const
         {
             return dict_;
         }
 
+        //- Return reference to x
+        virtual Field<Type>& x();
+
+        //- Return reference to b
+        virtual Field<Type>& b();
+
         //- Calculate residual
         virtual void residual
         (
diff --git a/src/foam/matrices/blockLduMatrix/BlockLduPrecons/BlockGaussSeidelPrecon/BlockGaussSeidelPrecon.H b/src/foam/matrices/blockLduMatrix/BlockLduPrecons/BlockGaussSeidelPrecon/BlockGaussSeidelPrecon.H
index 74383d0f2..df85068b3 100644
--- a/src/foam/matrices/blockLduMatrix/BlockLduPrecons/BlockGaussSeidelPrecon/BlockGaussSeidelPrecon.H
+++ b/src/foam/matrices/blockLduMatrix/BlockLduPrecons/BlockGaussSeidelPrecon/BlockGaussSeidelPrecon.H
@@ -152,10 +152,9 @@ public:
         );
 
 
-    // Destructor
-
-        virtual ~BlockGaussSeidelPrecon()
-        {}
+    //- Destructor
+    virtual ~BlockGaussSeidelPrecon()
+    {}
 
 
     // Member Functions

From 19ab01acb86bf6eaf8a6fbce63990c67ae0efe82 Mon Sep 17 00:00:00 2001
From: Hrvoje Jasak <h.jasak@wikki.co.uk>
Date: Fri, 23 Sep 2016 18:41:03 +0100
Subject: [PATCH 060/130] Added template specialisation for reduce of bool

---
 src/foam/db/IOstreams/Pstreams/Pstream.C      | 107 +++++++++++++++++-
 .../db/IOstreams/Pstreams/PstreamReduceOps.H  |  31 ++++-
 2 files changed, 133 insertions(+), 5 deletions(-)

diff --git a/src/foam/db/IOstreams/Pstreams/Pstream.C b/src/foam/db/IOstreams/Pstreams/Pstream.C
index e74c72f6e..e87e9836d 100644
--- a/src/foam/db/IOstreams/Pstreams/Pstream.C
+++ b/src/foam/db/IOstreams/Pstreams/Pstream.C
@@ -682,8 +682,8 @@ void Foam::Pstream::abort()
 
 void Foam::reduce
 (
-    scalar& Value,
-    const sumOp<scalar>& bop,
+    bool& Value,
+    const andOp<bool>& bop,
     const int tag,
     const label comm
 )
@@ -696,7 +696,67 @@ void Foam::reduce
         error::printStack(Pout);
     }
 
-    allReduce(Value, 1, MPI_SCALAR, MPI_SUM, bop, tag, comm);
+    // Note: C++ bool is a type separate from C and cannot be cast
+    // For safety and compatibility with compilers, convert bool to int
+    // to comply with MPI types.  HJ, 23/Sep/2016
+
+    int intBool = 0;
+
+    if (Value)
+    {
+        intBool = 1;
+    }
+
+    allReduce(intBool, 1, MPI_INT, MPI_LAND, bop, tag, comm);
+
+    if (intBool > 0)
+    {
+        Value = true;
+    }
+    else
+    {
+        Value = false;
+    }
+}
+
+
+void Foam::reduce
+(
+    bool& Value,
+    const orOp<bool>& bop,
+    const int tag,
+    const label comm
+)
+{
+    if (Pstream::warnComm != -1 && comm != Pstream::warnComm)
+    {
+        Pout<< "** reducing:" << Value << " with comm:" << comm
+            << " warnComm:" << Pstream::warnComm
+            << endl;
+        error::printStack(Pout);
+    }
+
+    // Note: C++ bool is a type separate from C and cannot be cast
+    // For safety and compatibility with compilers, convert bool to int
+    // to comply with MPI types.  HJ, 23/Sep/2016
+
+    int intBool = 0;
+
+    if (Value)
+    {
+        intBool = 1;
+    }
+
+    allReduce(intBool, 1, MPI_INT, MPI_LOR, bop, tag, comm);
+
+    if (intBool > 0)
+    {
+        Value = true;
+    }
+    else
+    {
+        Value = false;
+    }
 }
 
 
@@ -719,6 +779,45 @@ void Foam::reduce
 }
 
 
+void Foam::reduce
+(
+    scalar& Value,
+    const maxOp<scalar>& bop,
+    const int tag,
+    const label comm
+)
+{
+    if (Pstream::warnComm != -1 && comm != Pstream::warnComm)
+    {
+        Pout<< "** reducing:" << Value << " with comm:" << comm
+            << " warnComm:" << Pstream::warnComm
+            << endl;
+        error::printStack(Pout);
+    }
+    allReduce(Value, 1, MPI_SCALAR, MPI_MAX, bop, tag, comm);
+}
+
+
+void Foam::reduce
+(
+    scalar& Value,
+    const sumOp<scalar>& bop,
+    const int tag,
+    const label comm
+)
+{
+    if (Pstream::warnComm != -1 && comm != Pstream::warnComm)
+    {
+        Pout<< "** reducing:" << Value << " with comm:" << comm
+            << " warnComm:" << Pstream::warnComm
+            << endl;
+        error::printStack(Pout);
+    }
+
+    allReduce(Value, 1, MPI_SCALAR, MPI_SUM, bop, tag, comm);
+}
+
+
 void Foam::reduce
 (
     vector2D& Value,
@@ -1050,7 +1149,7 @@ const Foam::debug::optimisationSwitch
 Foam::Pstream::nProcsSimpleSum
 (
     "nProcsSimpleSum",
-    16
+    0
 );
 
 
diff --git a/src/foam/db/IOstreams/Pstreams/PstreamReduceOps.H b/src/foam/db/IOstreams/Pstreams/PstreamReduceOps.H
index f92ed0bca..65970bf88 100644
--- a/src/foam/db/IOstreams/Pstreams/PstreamReduceOps.H
+++ b/src/foam/db/IOstreams/Pstreams/PstreamReduceOps.H
@@ -139,7 +139,28 @@ void reduce
 }
 
 
-// Insist there are specialisations for the common reductions of scalar(s)
+// Insist there are specialisations for the common reductions of bool
+// See implementation notes on casting of a C++ bool into MPI_INT in
+// Pstream.C  HJ, 23/Sep/2016
+
+void reduce
+(
+    bool& Value,
+    const andOp<bool>& bop,
+    const int tag = Pstream::msgType(),
+    const label comm = Pstream::worldComm
+);
+
+void reduce
+(
+    bool& Value,
+    const orOp<bool>& bop,
+    const int tag = Pstream::msgType(),
+    const label comm = Pstream::worldComm
+);
+
+// Insist there are specialisations for the common reductions of scalars
+
 void reduce
 (
     scalar& Value,
@@ -156,6 +177,14 @@ void reduce
     const label comm = Pstream::worldComm
 );
 
+void reduce
+(
+    scalar& Value,
+    const maxOp<scalar>& bop,
+    const int tag = Pstream::msgType(),
+    const label comm = Pstream::worldComm
+);
+
 void reduce
 (
     vector2D& Value,

From 74933ca82b3932265848f325661d7beafffb20fa Mon Sep 17 00:00:00 2001
From: Hrvoje Jasak <h.jasak@wikki.co.uk>
Date: Fri, 23 Sep 2016 18:41:40 +0100
Subject: [PATCH 061/130] Comments and formatting

---
 .../BlockMatrixAgglomeration/BlockMatrixAgglomeration.C      | 2 ++
 .../BlockMatrixAgglomeration/BlockMatrixAgglomeration.H      | 5 ++---
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/src/foam/matrices/blockLduMatrix/BlockAMG/BlockMatrixCoarsening/BlockMatrixAgglomeration/BlockMatrixAgglomeration.C b/src/foam/matrices/blockLduMatrix/BlockAMG/BlockMatrixCoarsening/BlockMatrixAgglomeration/BlockMatrixAgglomeration.C
index 9ce0ddf16..fa578df20 100644
--- a/src/foam/matrices/blockLduMatrix/BlockAMG/BlockMatrixCoarsening/BlockMatrixAgglomeration/BlockMatrixAgglomeration.C
+++ b/src/foam/matrices/blockLduMatrix/BlockAMG/BlockMatrixCoarsening/BlockMatrixAgglomeration/BlockMatrixAgglomeration.C
@@ -349,6 +349,8 @@ void Foam::BlockMatrixAgglomeration<Type>::calcAgglomeration()
     // whole gang of processes; otherwise I may end up with a different
     // number of agglomeration levels on different processors.
 
+    // If the number of coarse equations is les than minimum and
+    // if the matrix has reduced in size by at least 1/3, coarsen
     if
     (
         nCoarseEqns_ > BlockMatrixCoarsening<Type>::minCoarseEqns()
diff --git a/src/foam/matrices/blockLduMatrix/BlockAMG/BlockMatrixCoarsening/BlockMatrixAgglomeration/BlockMatrixAgglomeration.H b/src/foam/matrices/blockLduMatrix/BlockAMG/BlockMatrixCoarsening/BlockMatrixAgglomeration/BlockMatrixAgglomeration.H
index 0cd3e0d1a..237c57b55 100644
--- a/src/foam/matrices/blockLduMatrix/BlockAMG/BlockMatrixCoarsening/BlockMatrixAgglomeration/BlockMatrixAgglomeration.H
+++ b/src/foam/matrices/blockLduMatrix/BlockAMG/BlockMatrixCoarsening/BlockMatrixAgglomeration/BlockMatrixAgglomeration.H
@@ -164,9 +164,8 @@ public:
         );
 
 
-    // Destructor
-
-        virtual ~BlockMatrixAgglomeration();
+    //- Destructor
+    virtual ~BlockMatrixAgglomeration();
 
 
     // Member Functions

From 0c1fa2853e75c26fd513dcad0827db7924d945b8 Mon Sep 17 00:00:00 2001
From: Hrvoje Jasak <h.jasak@wikki.co.uk>
Date: Fri, 23 Sep 2016 18:42:20 +0100
Subject: [PATCH 062/130] Bugfix: IPstream and OPstream on local comm

---
 .../ggiAMGInterface/ggiAMGInterface.C         | 32 +++++++++++++++----
 1 file changed, 25 insertions(+), 7 deletions(-)

diff --git a/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/ggiAMGInterface/ggiAMGInterface.C b/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/ggiAMGInterface/ggiAMGInterface.C
index a8b654c94..7abf2c555 100644
--- a/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/ggiAMGInterface/ggiAMGInterface.C
+++ b/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/ggiAMGInterface/ggiAMGInterface.C
@@ -246,7 +246,10 @@ Foam::ggiAMGInterface::ggiAMGInterface
     // each processor only to perform the analysis on locally created coarse
     // faces
     // HJ, 13/Jun/2016
-    Info<< "Start ggiAMGInterface constructor " << lTime_.elapsedCpuTime() << endl;
+    Info<< "Start ggiAMGInterface constructor for size "
+        << fineGgiInterface_.interfaceSize() << ": " 
+        << lTime_.elapsedCpuTime() << endl;
+
     // Note: local addressing contains only local faces
     const labelList& fineZa =  fineGgiInterface_.zoneAddressing();
 
@@ -266,7 +269,7 @@ Foam::ggiAMGInterface::ggiAMGInterface
             neighbourExpandAddressing
         );
     }
-    Info<< "ggiAMGInterface expandToAddr1 " << lTime_.elapsedCpuTime() << endl;
+
     // Create addressing for neighbour processors.  Note: expandAddrToZone will
     // expand the addressing to zone size.  HJ, 13/Jun/2016
     labelField neighbourExpandProc
@@ -285,7 +288,7 @@ Foam::ggiAMGInterface::ggiAMGInterface
             neighbourExpandProc
         );
     }
-    Info<< "ggiAMGInterface expandToAddr2 " << lTime_.elapsedCpuTime() << endl;
+
     // Note: neighbourExpandAddressing and neighbourExpandProc
     // will be filled with NaNs for faces which are not local
 
@@ -711,7 +714,7 @@ Foam::ggiAMGInterface::ggiAMGInterface
     fineAddressing_.setSize(nAgglomPairs, -1);
     restrictAddressing_.setSize(nAgglomPairs, -1);
     restrictWeights_.setSize(nAgglomPairs);
-    Info<< "ggiAMGInterface end Addr " << lTime_.elapsedCpuTime() << endl;
+
     // In order to assemble the coarse global face zone, find out
     // how many faces have been created on each processor.
     // Note that masters and slaves both count faces so we will
@@ -726,7 +729,7 @@ Foam::ggiAMGInterface::ggiAMGInterface
     // This needs to be handled separately in the initFastReduce
     // HJ, 20/Sep/2016
     reduce(nCoarseFacesPerProc, sumOp<labelList>(), tag(), comm());
-    Info<< "ggiAMGInterface end reduce 1 " << lTime_.elapsedCpuTime() << endl;
+
     // Coarse global face zone is assembled by adding all faces from proc0,
     // followed by all faces from proc1 etc.
     // Therefore, on procN, my master offset
@@ -904,12 +907,26 @@ Foam::ggiAMGInterface::ggiAMGInterface
 
                 if (Pstream::myProcNo() == sendProc)
                 {
-                    OPstream toNbr(Pstream::scheduled, recvProc);
+                    OPstream toNbr
+                    (
+                        Pstream::scheduled,
+                        recvProc,
+                        0,
+                        tag(),
+                        comm()
+                    );
                     toNbr << labelList(procMasterFacesLL[recvProc]);
                 }
                 else if (Pstream::myProcNo() == recvProc)
                 {
-                    IPstream fromNbr(Pstream::scheduled, sendProc);
+                    IPstream fromNbr
+                    (
+                        Pstream::scheduled,
+                        sendProc,
+                        0,
+                        tag(),
+                        comm()
+                    );
 
                     procMasterFaces_[sendProc] = labelList(fromNbr);
                 }
@@ -1047,6 +1064,7 @@ Foam::ggiAMGInterface::ggiAMGInterface
         }
         Info<< "ggiAMGInterface end agglom slave " << lTime_.elapsedCpuTime() << endl;
     }
+    Info<< "End ggiAMGInterface constructor " << lTime_.elapsedCpuTime() << endl;
 }
 
 

From 62cf3774f901e472d96e4f34cb747f735bddcfed Mon Sep 17 00:00:00 2001
From: Hrvoje Jasak <h.jasak@wikki.co.uk>
Date: Fri, 23 Sep 2016 18:49:44 +0100
Subject: [PATCH 063/130] Bugfix: define Pstream procNo with comm

---
 .../solvers/AMG/interfaces/ggiAMGInterface/ggiAMGInterface.C  | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/ggiAMGInterface/ggiAMGInterface.C b/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/ggiAMGInterface/ggiAMGInterface.C
index 7abf2c555..b0efacf91 100644
--- a/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/ggiAMGInterface/ggiAMGInterface.C
+++ b/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/ggiAMGInterface/ggiAMGInterface.C
@@ -910,7 +910,7 @@ Foam::ggiAMGInterface::ggiAMGInterface
                     OPstream toNbr
                     (
                         Pstream::scheduled,
-                        recvProc,
+                        Pstream::procNo(comm(), recvProc),
                         0,
                         tag(),
                         comm()
@@ -922,7 +922,7 @@ Foam::ggiAMGInterface::ggiAMGInterface
                     IPstream fromNbr
                     (
                         Pstream::scheduled,
-                        sendProc,
+                        Pstream::procNo(comm(), sendProc),
                         0,
                         tag(),
                         comm()

From 65eff6644a57187fdc56097b827afca3845ade29 Mon Sep 17 00:00:00 2001
From: Hrvoje Jasak <h.jasak@wikki.co.uk>
Date: Tue, 4 Oct 2016 13:54:20 +0100
Subject: [PATCH 064/130] GGI AMG coarsening with dynamic lists

---
 .../ggiAMGInterface/ggiAMGInterface.C         | 324 ++++++++----------
 1 file changed, 136 insertions(+), 188 deletions(-)

diff --git a/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/ggiAMGInterface/ggiAMGInterface.C b/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/ggiAMGInterface/ggiAMGInterface.C
index b0efacf91..6aa850bca 100644
--- a/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/ggiAMGInterface/ggiAMGInterface.C
+++ b/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/ggiAMGInterface/ggiAMGInterface.C
@@ -71,7 +71,7 @@ void Foam::ggiAMGInterface::initFastReduce() const
 
         return;
     }
-    Info<< "Start initFastReduce " << lTime_.elapsedCpuTime() << endl;
+    if (Pstream::myProcNo() == 7) Pout<< "Start initFastReduce " << lTime_.elapsedCpuTime() << endl;
     // From here on, work on processors within the communicator
     // HJ, 20/Sep/2016
 
@@ -215,7 +215,7 @@ void Foam::ggiAMGInterface::initFastReduce() const
     // Map will return the object of the size of remote zone
     // HJ, 9/May/2016
     mapPtr_ = new mapDistribute(zoneSize(), sendMap, constructMap);
-    Info<< "End initFastReduce " << lTime_.elapsedCpuTime() << endl;
+    if (Pstream::myProcNo() == 7) Pout<< "End initFastReduce " << lTime_.elapsedCpuTime() << endl;
 }
 
 
@@ -246,8 +246,8 @@ Foam::ggiAMGInterface::ggiAMGInterface
     // each processor only to perform the analysis on locally created coarse
     // faces
     // HJ, 13/Jun/2016
-    Info<< "Start ggiAMGInterface constructor for size "
-        << fineGgiInterface_.interfaceSize() << ": " 
+    if (Pstream::myProcNo() == 7) Pout<< "Start ggiAMGInterface constructor for size "
+        << fineGgiInterface_.interfaceSize() << ": "
         << lTime_.elapsedCpuTime() << endl;
 
     // Note: local addressing contains only local faces
@@ -300,34 +300,36 @@ Foam::ggiAMGInterface::ggiAMGInterface
 
     // Coded neighbour index. Note: using long int to simplify encoding
     // HJ, 1/Aug/2016
-    HashTable<SLList<long>, long, Hash<long> > neighboursTable
+    HashTable<DynamicList<long, 4>, long, Hash<long> > neighboursTable
     (
         Foam::max(128, fineGgiInterface_.interfaceSize()/4)
     );
 
     // Neignbour processor index
-    HashTable<SLList<label>, label, Hash<label> > nbrsProcTable
+    HashTable<DynamicList<label, 4>, label, Hash<label> > nbrsProcTable
     (
         Foam::max(128, fineGgiInterface_.interfaceSize()/4)
     );
 
     // Neighbour face-faces addressing for a face with split neighbours
-    HashTable<SLList<SLList<label> >, label, Hash<label> > faceFaceTable
+    HashTable<DynamicList<DynamicList<label, 4>, 4>, label, Hash<label> >
+    faceFaceTable
     (
         Foam::max(128, fineGgiInterface_.interfaceSize()/4)
     );
 
-    HashTable<SLList<SLList<label> >, label, Hash<label> > faceFaceNbrTable
+    HashTable<DynamicList<DynamicList<label, 4>, 4>, label, Hash<label> >
+    faceFaceNbrTable
     (
         Foam::max(128, fineGgiInterface_.interfaceSize()/4)
     );
 
     // Neighbour face-faces weights for a face with split neighbours
-    HashTable<SLList<SLList<scalar> >, label, Hash<label> >
-        faceFaceWeightsTable
-        (
-            Foam::max(128, fineGgiInterface_.interfaceSize()/4)
-        );
+    HashTable<DynamicList<DynamicList<scalar, 4>, 4>, label, Hash<label> >
+    faceFaceWeightsTable
+    (
+        Foam::max(128, fineGgiInterface_.interfaceSize()/4)
+    );
 
     // Count the number of coarse faces
     label nCoarseFaces = 0;
@@ -338,6 +340,7 @@ Foam::ggiAMGInterface::ggiAMGInterface
     // On the fine level, addressing is made in a labelListList
     if (fineGgiInterface_.fineLevel())
     {
+        if (Pstream::myProcNo() == 7) Pout<< "fineGgiInterface start: " << lTime_.elapsedCpuTime() << endl;
         // This addressing defines how to interpolate for all zone faces
         // across the interface
         const labelListList& fineAddr = fineGgiInterface_.addressing();
@@ -412,58 +415,33 @@ Foam::ggiAMGInterface::ggiAMGInterface
                     // Check all current neighbours to see if the current
                     // slave already exists.  If so, add the coefficient.
 
-                    SLList<long>& curNbrs =
+                    DynamicList<long, 4>& curNbrs =
                         neighboursTable.find(curMaster)();
 
-                    SLList<label>& curNbrsProc =
+                    DynamicList<label, 4>& curNbrsProc =
                         nbrsProcTable.find(curMaster)();
 
-                    SLList<SLList<label> >& curFaceFaces =
+                    DynamicList<DynamicList<label, 4>, 4>& curFaceFaces =
                         faceFaceTable.find(curMaster)();
 
-                    SLList<SLList<label> >& curFaceFaceNbrs =
+                    DynamicList<DynamicList<label, 4>, 4>& curFaceFaceNbrs =
                         faceFaceNbrTable.find(curMaster)();
 
-                    SLList<SLList<scalar> >& curFaceWeights =
+                    DynamicList<DynamicList<scalar, 4>, 4>& curFaceWeights =
                         faceFaceWeightsTable.find(curMaster)();
 
                     // Search for coded neighbour
                     bool nbrFound = false;
 
-                    SLList<long>::iterator nbrsIter = curNbrs.begin();
-
-                    SLList<label>::iterator nbrsProcIter =
-                        curNbrsProc.begin();
-
-                    SLList<SLList<label> >::iterator faceFacesIter =
-                        curFaceFaces.begin();
-
-                    SLList<SLList<label> >::iterator faceFaceNbrsIter =
-                        curFaceFaceNbrs.begin();
-
-                    SLList<SLList<scalar> >::iterator faceFaceWeightsIter =
-                    curFaceWeights.begin();
-
-                    for
-                    (
-                        ;
-                        nbrsIter != curNbrs.end()
-                     && nbrsProcIter != curNbrsProc.end()
-                     && faceFacesIter != curFaceFaces.end()
-                     && faceFaceNbrsIter != curFaceFaceNbrs.end()
-                     && faceFaceWeightsIter != curFaceWeights.end();
-                        ++nbrsIter, ++nbrsProcIter,
-                        ++faceFacesIter, ++faceFaceNbrsIter,
-                        ++faceFaceWeightsIter
-                    )
+                    forAll (curNbrs, curNbrI)
                     {
                         // Check neighbour slave
-                        if (nbrsIter() == curSlave)
+                        if (curNbrs[curNbrI] == curSlave)
                         {
                             nbrFound = true;
-                            faceFacesIter().append(ffI);
-                            faceFaceNbrsIter().append(nbrI);
-                            faceFaceWeightsIter().append(curNW);
+                            curFaceFaces[curNbrI].append(ffI);
+                            curFaceFaceNbrs[curNbrI].append(nbrI);
+                            curFaceWeights[curNbrI].append(curNW);
 
                             // New agglomeration pair found in already
                             // existing pair
@@ -477,9 +455,18 @@ Foam::ggiAMGInterface::ggiAMGInterface
                     {
                         curNbrs.append(curSlave);
                         curNbrsProc.append(curSlaveProc);
-                        curFaceFaces.append(SLList<label>(ffI));
-                        curFaceFaceNbrs.append(SLList<label>(nbrI));
-                        curFaceWeights.append(SLList<scalar>(curNW));
+
+                        DynamicList<label, 4> newFF;
+                        newFF.append(ffI);
+                        curFaceFaces.append(newFF);
+
+                        DynamicList<label, 4> newFNbr;
+                        newFNbr.append(nbrI);
+                        curFaceFaceNbrs.append(newFNbr);
+
+                        DynamicList<scalar, 4> newFW;
+                        newFW.append(curNW);
+                        curFaceWeights.append(newFW);
 
                         // New coarse face created for an existing master
                         nCoarseFaces++;
@@ -491,34 +478,47 @@ Foam::ggiAMGInterface::ggiAMGInterface
                     // This master has got no neighbours yet.
                     // Add a neighbour, proc and a coefficient as a
                     // new list, thus creating a new face
+                    DynamicList<long, 4> newNbrs;
+                    newNbrs.append(curSlave);
                     neighboursTable.insert
                     (
                         curMaster,
-                        SLList<long>(curSlave)
+                        newNbrs
                     );
 
+                    DynamicList<label, 4> newNbrsProc;
+                    newNbrsProc.append(curSlaveProc);
                     nbrsProcTable.insert
                     (
                         curMaster,
-                        SLList<label>(curSlaveProc)
+                        newNbrsProc
                     );
 
+                    DynamicList<DynamicList<label, 4>, 4> newFF;
+                    newFF.append(DynamicList<label, 4>());
+                    newFF[0].append(ffI);
                     faceFaceTable.insert
                     (
                         curMaster,
-                        SLList<SLList<label> >(SLList<label>(ffI))
+                        newFF
                     );
 
+                    DynamicList<DynamicList<label, 4>, 4> newFNbr;
+                    newFNbr.append(DynamicList<label, 4>());
+                    newFNbr[0].append(nbrI);
                     faceFaceNbrTable.insert
                     (
                         curMaster,
-                        SLList<SLList<label> >(SLList<label>(nbrI))
+                        newFNbr
                     );
 
+                    DynamicList<DynamicList<scalar, 4>, 4> newFFWeights;
+                    newFFWeights.append(DynamicList<scalar, 4>());
+                    newFFWeights[0].append(curNW);
                     faceFaceWeightsTable.insert
                     (
                         curMaster,
-                        SLList<SLList<scalar> >(SLList<scalar>(curNW))
+                        newFFWeights
                     );
 
                     // New coarse face created for a new master
@@ -527,12 +527,13 @@ Foam::ggiAMGInterface::ggiAMGInterface
                 }
             } // end for all current neighbours
         } // end for all fine faces
+        if (Pstream::myProcNo() == 7) Pout<< "fineGgiInterface end: " << lTime_.elapsedCpuTime() << endl;
     }
     else
     {
         // Coarse level, addressing is stored in faceCells
         // This addressing defines which faces from zone are local
-
+        if (Pstream::myProcNo() == 7) Pout<< "coarseGgiInterface start: " << lTime_.elapsedCpuTime() << endl;
         // Perform analysis only for local faces
         // HJ, 22/Jun/2016
         forAll (fineZa, fineZaI)
@@ -592,56 +593,35 @@ Foam::ggiAMGInterface::ggiAMGInterface
                 // already exists and if so, add the fine face
                 // to the agglomeration.
 
-                SLList<long>& curNbrs = neighboursTable.find(curMaster)();
+                DynamicList<long, 4>& curNbrs =
+                    neighboursTable.find(curMaster)();
 
-                SLList<label>& curNbrsProc =
+                DynamicList<label, 4>& curNbrsProc =
                     nbrsProcTable.find(curMaster)();
 
-                SLList<SLList<label> >& curFaceFaces =
+                DynamicList<DynamicList<label, 4>, 4>& curFaceFaces =
                     faceFaceTable.find(curMaster)();
 
-                SLList<SLList<label> >& curFaceFaceNbrs =
+                DynamicList<DynamicList<label, 4>, 4>& curFaceFaceNbrs =
                     faceFaceNbrTable.find(curMaster)();
 
-                SLList<SLList<scalar> >& curFaceWeights =
+                DynamicList<DynamicList<scalar, 4>, 4>& curFaceWeights =
                     faceFaceWeightsTable.find(curMaster)();
 
+                // Search for coded neighbour
                 bool nbrFound = false;
 
-                SLList<long>::iterator nbrsIter = curNbrs.begin();
-
-                SLList<label>::iterator nbrsProcIter =
-                    curNbrsProc.begin();
-
-                SLList<SLList<label> >::iterator faceFacesIter =
-                    curFaceFaces.begin();
-
-                SLList<SLList<label> >::iterator faceFaceNbrsIter =
-                    curFaceFaceNbrs.begin();
-
-                SLList<SLList<scalar> >::iterator faceFaceWeightsIter =
-                    curFaceWeights.begin();
-
-                for
-                (
-                    ;
-                    nbrsIter != curNbrs.end()
-                 && faceFacesIter != curFaceFaces.end()
-                 && faceFaceNbrsIter != curFaceFaceNbrs.end()
-                 && faceFaceWeightsIter != curFaceWeights.end();
-                    ++nbrsIter, ++nbrsProcIter,
-                    ++faceFacesIter, ++faceFaceNbrsIter, ++faceFaceWeightsIter
-                )
+                forAll (curNbrs, curNbrI)
                 {
                     // Check neighbour slave
-                    if (nbrsIter() == curSlave)
+                    if (curNbrs[curNbrI] == curSlave)
                     {
                         nbrFound = true;
-                        faceFacesIter().append(ffI);
+                        curFaceFaces[curNbrI].append(ffI);
                         // Add dummy nbr
-                        faceFaceNbrsIter().append(0);
+                        curFaceFaceNbrs[curNbrI].append(0);
                         // Add dummy weight
-                        faceFaceWeightsIter().append(1.0);
+                        curFaceWeights[curNbrI].append(1.0);
 
                         // New agglomeration pair found in already
                         // existing pair
@@ -655,11 +635,20 @@ Foam::ggiAMGInterface::ggiAMGInterface
                 {
                     curNbrs.append(curSlave);
                     curNbrsProc.append(curSlaveProc);
-                    curFaceFaces.append(SLList<label>(ffI));
+
+                    DynamicList<label, 4> newFF;
+                    newFF.append(ffI);
+                    curFaceFaces.append(newFF);
+
                     // Add dummy nbr
-                    curFaceFaceNbrs.append(SLList<label>(0));
+                    DynamicList<label, 4> newFNbr;
+                    newFNbr.append(0);
+                    curFaceFaceNbrs.append(newFNbr);
+
                     // Add dummy weight
-                    curFaceWeights.append(SLList<scalar>(1.0));
+                    DynamicList<scalar, 4> newFW;
+                    newFW.append(1.0);
+                    curFaceWeights.append(newFW);
 
                     // New coarse face created for an existing master
                     nCoarseFaces++;
@@ -670,36 +659,49 @@ Foam::ggiAMGInterface::ggiAMGInterface
             {
                 // This master has got no neighbours yet.  Add a neighbour
                 // and a coefficient, thus creating a new face
+                DynamicList<long, 4> newNbrs;
+                newNbrs.append(curSlave);
                 neighboursTable.insert
                 (
                     curMaster,
-                    SLList<long>(curSlave)
+                    newNbrs
                 );
 
+                DynamicList<label, 4> newNbrsProc;
+                newNbrsProc.append(curSlaveProc);
                 nbrsProcTable.insert
                 (
                     curMaster,
-                    SLList<label>(curSlaveProc)
+                    newNbrsProc
                 );
 
+                DynamicList<DynamicList<label, 4>, 4> newFF;
+                newFF.append(DynamicList<label, 4>());
+                newFF[0].append(ffI);
                 faceFaceTable.insert
                 (
                     curMaster,
-                    SLList<SLList<label> >(SLList<label>(ffI))
+                    newFF
                 );
 
                 // Add dummy nbr
+                DynamicList<DynamicList<label, 4>, 4> newFNbr;
+                newFNbr.append(DynamicList<label, 4>());
+                newFNbr[0].append(0);
                 faceFaceNbrTable.insert
                 (
                     curMaster,
-                    SLList<SLList<label> >(SLList<label>(0))
+                    newFNbr
                 );
 
                 // Add dummy weight
+                DynamicList<DynamicList<scalar, 4>, 4> newFFWeights;
+                newFFWeights.append(DynamicList<scalar, 4>());
+                newFFWeights[0].append(1.0);
                 faceFaceWeightsTable.insert
                 (
                     curMaster,
-                    SLList<SLList<scalar> >(SLList<scalar>(1.0))
+                    newFFWeights
                 );
 
                 // New coarse face created for a new master
@@ -707,12 +709,13 @@ Foam::ggiAMGInterface::ggiAMGInterface
                 nAgglomPairs++;
             }
         } // end for all fine faces
+        if (Pstream::myProcNo() == 7) Pout<< "coarseGgiInterface end: " << lTime_.elapsedCpuTime() << endl;
     } // end of else in fine level (coarse level)
 
     // Since only local faces are analysed, lists can now be resized
-    faceCells_.setSize(nCoarseFaces, -1);
-    fineAddressing_.setSize(nAgglomPairs, -1);
-    restrictAddressing_.setSize(nAgglomPairs, -1);
+    faceCells_.setSize(nCoarseFaces);
+    fineAddressing_.setSize(nAgglomPairs);
+    restrictAddressing_.setSize(nAgglomPairs);
     restrictWeights_.setSize(nAgglomPairs);
 
     // In order to assemble the coarse global face zone, find out
@@ -769,6 +772,7 @@ Foam::ggiAMGInterface::ggiAMGInterface
     // the local zone is created.  HJ, 1/Aug/2016
     if (master())
     {
+        if (Pstream::myProcNo() == 7) Pout<< "ggiAMGInterface start agglom master: " << lTime_.elapsedCpuTime() << endl;
         // Note:
         // When I am agglomerating the master, faces are stacked up in order
         // but on the slave side, all I know is the master cluster index and
@@ -779,7 +783,7 @@ Foam::ggiAMGInterface::ggiAMGInterface
 
         // For each new global face created on master proc,
         // record its index under the slave proc array
-        List<SLList<label> > procMasterFacesLL(Pstream::nProcs());
+        List<DynamicList<label> > procMasterFacesLL(Pstream::nProcs());
 
         // Note: zone addressing will be assembled only for local clusters
         // using the coarseGlobalFaceOffset
@@ -799,49 +803,27 @@ Foam::ggiAMGInterface::ggiAMGInterface
         // On master side, the owner addressing is stored in table of contents
         forAll (contents, masterI)
         {
-            SLList<long>& curNbrs =
+            DynamicList<long, 4>& curNbrs =
                 neighboursTable.find(contents[masterI])();
 
-            SLList<label>& curNbrsProc =
+            DynamicList<label, 4>& curNbrsProc =
                 nbrsProcTable.find(contents[masterI])();
 
-            SLList<SLList<label> >& curFaceFaces =
+            DynamicList<DynamicList<label, 4>, 4>& curFaceFaces =
                 faceFaceTable.find(contents[masterI])();
 
-            SLList<SLList<scalar> >& curFaceWeights =
+            DynamicList<DynamicList<scalar, 4>, 4>& curFaceWeights =
                 faceFaceWeightsTable.find(contents[masterI])();
 
-            SLList<long>::iterator nbrsIter = curNbrs.begin();
-
-            SLList<label>::iterator nbrsProcIter =
-                curNbrsProc.begin();
-
-            SLList<SLList<label> >::iterator faceFacesIter =
-                curFaceFaces.begin();
-
-            SLList<SLList<scalar> >::iterator faceFaceWeightsIter =
-                curFaceWeights.begin();
-
-            for
-            (
-                ;
-                nbrsIter != curNbrs.end()
-             && nbrsProcIter != curNbrsProc.end()
-             && faceFacesIter != curFaceFaces.end()
-             && faceFaceWeightsIter != curFaceWeights.end();
-                ++nbrsIter, ++nbrsProcIter,
-                ++faceFacesIter, ++faceFaceWeightsIter
-            )
+            forAll (curNbrs, curNbrI)
             {
                 // Check if master is on local processor: no longer needed,
                 // as only local processor is being searched.  HJ, 13/Jun/2016
 
                 // Get faces and weights
-                SLList<label>::iterator facesIter =
-                    faceFacesIter().begin();
+                DynamicList<label, 4>& facesIter = curFaceFaces[curNbrI];
 
-                SLList<scalar>::iterator weightsIter =
-                    faceFaceWeightsIter().begin();
+                DynamicList<scalar, 4>& weightsIter = curFaceWeights[curNbrI];
 
                 // Record that this face belongs locally
                 // Use offset to indicate its position in the list
@@ -853,28 +835,22 @@ Foam::ggiAMGInterface::ggiAMGInterface
                     contents[masterI] - procOffset*Pstream::myProcNo();
 
                 // Record global processor face
-                procMasterFacesLL[nbrsProcIter()].append
+                procMasterFacesLL[curNbrsProc[curNbrI]].append
                 (
                     nProcFaces + coarseGlobalFaceOffset
                 );
 
                 // Collect agglomeration data
-                for
-                (
-                    ;
-                    facesIter != faceFacesIter().end()
-                 && weightsIter != faceFaceWeightsIter().end();
-                    ++facesIter, ++weightsIter
-                )
+                forAll (facesIter, facesIterI)
                 {
-                    fineAddressing_[nAgglomPairs] = facesIter();
+                    fineAddressing_[nAgglomPairs] = facesIter[facesIterI];
 
                     // Master processor zone face is calculated from
                     // global offset
                     restrictAddressing_[nAgglomPairs] =
                         nProcFaces + coarseGlobalFaceOffset;
 
-                    restrictWeights_[nAgglomPairs] = weightsIter();
+                    restrictWeights_[nAgglomPairs] = weightsIter[facesIterI];
                     nAgglomPairs++;
                 }
 
@@ -940,7 +916,7 @@ Foam::ggiAMGInterface::ggiAMGInterface
                 }
             }
         }
-        Info<< "ggiAMGInterface end agglom master " << lTime_.elapsedCpuTime() << endl;
+        if (Pstream::myProcNo() == 7) Pout<< "ggiAMGInterface end agglom master " << lTime_.elapsedCpuTime() << endl;
     }
     // Agglomerate slave
     else
@@ -967,54 +943,32 @@ Foam::ggiAMGInterface::ggiAMGInterface
 
         // Count how many global faces are used for each proc on the other side
         labelList npmf(Pstream::nProcs(), 0);
-
+        if (Pstream::myProcNo() == 7) Pout<< "ggiAMGInterface start agglom slave: " << lTime_.elapsedCpuTime() << endl;
         // On slave side, the owner addressing is stored in linked lists
         forAll (contents, masterI)
         {
-            SLList<long>& curNbrs = neighboursTable.find(contents[masterI])();
+            DynamicList<long, 4>& curNbrs =
+                neighboursTable.find(contents[masterI])();
 
-            SLList<SLList<label> >& curFaceFaces =
+            DynamicList<DynamicList<label, 4>, 4>& curFaceFaces =
                 faceFaceTable.find(contents[masterI])();
 
-            SLList<SLList<label> >& curFaceFaceNbrs =
+            DynamicList<DynamicList<label, 4>, 4>& curFaceFaceNbrs =
                 faceFaceNbrTable.find(contents[masterI])();
 
-            SLList<SLList<scalar> >& curFaceWeights =
+            DynamicList<DynamicList<scalar, 4>, 4>& curFaceWeights =
                 faceFaceWeightsTable.find(contents[masterI])();
 
-            SLList<long>::iterator nbrsIter = curNbrs.begin();
-
-            SLList<SLList<label> >::iterator faceFacesIter =
-                curFaceFaces.begin();
-
-            SLList<SLList<label> >::iterator faceFaceNbrsIter =
-                curFaceFaceNbrs.begin();
-
-            SLList<SLList<scalar> >::iterator faceFaceWeightsIter =
-                curFaceWeights.begin();
-
-            for
-            (
-                ;
-                nbrsIter != curNbrs.end()
-             && faceFacesIter != curFaceFaces.end()
-             && faceFaceNbrsIter != curFaceFaceNbrs.end()
-             && faceFaceWeightsIter != curFaceWeights.end();
-                ++nbrsIter, ++faceFacesIter, ++faceFaceNbrsIter,
-                ++faceFaceWeightsIter
-            )
+            forAll (curNbrs, curNbrI)
             {
                 // Check if the face is on local processor: no longer needed,
                 // as only local processor is being searched.  HJ, 13/Jun/2016
 
-                SLList<label>::iterator facesIter =
-                    faceFacesIter().begin();
+                DynamicList<label, 4>& facesIter = curFaceFaces[curNbrI];
 
-                SLList<label>::iterator faceNbrsIter =
-                    faceFaceNbrsIter().begin();
+                DynamicList<label, 4>& faceNbrsIter = curFaceFaceNbrs[curNbrI];
 
-                SLList<scalar>::iterator weightsIter =
-                    faceFaceWeightsIter().begin();
+                DynamicList<scalar, 4>& weightsIter = curFaceWeights[curNbrI];
 
                 // Find neighbour proc index from the first face
                 // on the other side
@@ -1026,11 +980,11 @@ Foam::ggiAMGInterface::ggiAMGInterface
                         fineGgiInterface_.addressing();
 
                     nbrProc = neighbourExpandProc
-                        [fineAddr[facesIter()][faceNbrsIter()]];
+                        [fineAddr[facesIter[0]][faceNbrsIter[0]]];
                 }
                 else
                 {
-                    nbrProc = neighbourExpandProc[facesIter()];
+                    nbrProc = neighbourExpandProc[facesIter[0]];
                 }
 
                 // Read coarse face index
@@ -1042,19 +996,13 @@ Foam::ggiAMGInterface::ggiAMGInterface
 
                 zoneAddressing_[nProcFaces] = coarseFace;
                 faceCells_[nProcFaces] =
-                    nbrsIter() - procOffset*Pstream::myProcNo();
+                   curNbrs[curNbrI] - procOffset*Pstream::myProcNo();
 
-                for
-                (
-                    ;
-                    facesIter != faceFacesIter().end()
-                 && weightsIter != faceFaceWeightsIter().end();
-                    ++facesIter, ++weightsIter
-                )
+                forAll (facesIter, facesIterI)
                 {
-                    fineAddressing_[nAgglomPairs] = facesIter();
+                    fineAddressing_[nAgglomPairs] = facesIter[facesIterI];
                     restrictAddressing_[nAgglomPairs] = coarseFace;
-                    restrictWeights_[nAgglomPairs] = weightsIter();
+                    restrictWeights_[nAgglomPairs] = weightsIter[facesIterI];
 
                     nAgglomPairs++;
                 }
@@ -1062,9 +1010,9 @@ Foam::ggiAMGInterface::ggiAMGInterface
                 nProcFaces++;
             }
         }
-        Info<< "ggiAMGInterface end agglom slave " << lTime_.elapsedCpuTime() << endl;
+        if (Pstream::myProcNo() == 7) Pout<< "ggiAMGInterface end agglom slave " << lTime_.elapsedCpuTime() << endl;
     }
-    Info<< "End ggiAMGInterface constructor " << lTime_.elapsedCpuTime() << endl;
+    if (Pstream::myProcNo() == 7) Pout<< "End ggiAMGInterface constructor " << lTime_.elapsedCpuTime() << endl;
 }
 
 

From 011492fea10cc96db3c0cc1c3d8eb2af6aabc036 Mon Sep 17 00:00:00 2001
From: Hrvoje Jasak <h.jasak@wikki.co.uk>
Date: Tue, 4 Oct 2016 13:54:35 +0100
Subject: [PATCH 065/130] Bugfix: invalid message tag

---
 .../meshes/polyMesh/polyPatches/constraint/ggi/ggiPolyPatch.C | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/foam/meshes/polyMesh/polyPatches/constraint/ggi/ggiPolyPatch.C b/src/foam/meshes/polyMesh/polyPatches/constraint/ggi/ggiPolyPatch.C
index 621039612..6f949c1cc 100644
--- a/src/foam/meshes/polyMesh/polyPatches/constraint/ggi/ggiPolyPatch.C
+++ b/src/foam/meshes/polyMesh/polyPatches/constraint/ggi/ggiPolyPatch.C
@@ -658,7 +658,7 @@ Foam::ggiPolyPatch::ggiPolyPatch
     reconFaceCellCentresPtr_(NULL),
     localParallelPtr_(NULL),
     comm_(Pstream::worldComm),
-    tag_(-1),
+    tag_(Pstream::msgType()),
     mapPtr_(NULL)
 {}
 
@@ -684,7 +684,7 @@ Foam::ggiPolyPatch::ggiPolyPatch
     reconFaceCellCentresPtr_(NULL),
     localParallelPtr_(NULL),
     comm_(Pstream::worldComm),
-    tag_(-1),
+    tag_(Pstream::msgType()),
     mapPtr_(NULL)
 {
     if (dict.found("quickReject"))

From d7af6267c9c10e3e09575fc97b64277ef8d1ca89 Mon Sep 17 00:00:00 2001
From: Hrvoje Jasak <h.jasak@wikki.co.uk>
Date: Tue, 4 Oct 2016 13:59:34 +0100
Subject: [PATCH 066/130] Change Info

---
 .../ggiAMGInterface/ggiAMGInterface.C         | 24 +++++++++----------
 1 file changed, 12 insertions(+), 12 deletions(-)

diff --git a/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/ggiAMGInterface/ggiAMGInterface.C b/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/ggiAMGInterface/ggiAMGInterface.C
index 6aa850bca..95e389e84 100644
--- a/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/ggiAMGInterface/ggiAMGInterface.C
+++ b/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/ggiAMGInterface/ggiAMGInterface.C
@@ -71,7 +71,7 @@ void Foam::ggiAMGInterface::initFastReduce() const
 
         return;
     }
-    if (Pstream::myProcNo() == 7) Pout<< "Start initFastReduce " << lTime_.elapsedCpuTime() << endl;
+    Info<< "Start initFastReduce " << lTime_.elapsedCpuTime() << endl;
     // From here on, work on processors within the communicator
     // HJ, 20/Sep/2016
 
@@ -215,7 +215,7 @@ void Foam::ggiAMGInterface::initFastReduce() const
     // Map will return the object of the size of remote zone
     // HJ, 9/May/2016
     mapPtr_ = new mapDistribute(zoneSize(), sendMap, constructMap);
-    if (Pstream::myProcNo() == 7) Pout<< "End initFastReduce " << lTime_.elapsedCpuTime() << endl;
+    Info<< "End initFastReduce " << lTime_.elapsedCpuTime() << endl;
 }
 
 
@@ -246,7 +246,7 @@ Foam::ggiAMGInterface::ggiAMGInterface
     // each processor only to perform the analysis on locally created coarse
     // faces
     // HJ, 13/Jun/2016
-    if (Pstream::myProcNo() == 7) Pout<< "Start ggiAMGInterface constructor for size "
+    Info<< "Start ggiAMGInterface constructor for size "
         << fineGgiInterface_.interfaceSize() << ": "
         << lTime_.elapsedCpuTime() << endl;
 
@@ -340,7 +340,7 @@ Foam::ggiAMGInterface::ggiAMGInterface
     // On the fine level, addressing is made in a labelListList
     if (fineGgiInterface_.fineLevel())
     {
-        if (Pstream::myProcNo() == 7) Pout<< "fineGgiInterface start: " << lTime_.elapsedCpuTime() << endl;
+        Info<< "fineGgiInterface start: " << lTime_.elapsedCpuTime() << endl;
         // This addressing defines how to interpolate for all zone faces
         // across the interface
         const labelListList& fineAddr = fineGgiInterface_.addressing();
@@ -527,13 +527,13 @@ Foam::ggiAMGInterface::ggiAMGInterface
                 }
             } // end for all current neighbours
         } // end for all fine faces
-        if (Pstream::myProcNo() == 7) Pout<< "fineGgiInterface end: " << lTime_.elapsedCpuTime() << endl;
+        Info<< "fineGgiInterface end: " << lTime_.elapsedCpuTime() << endl;
     }
     else
     {
         // Coarse level, addressing is stored in faceCells
         // This addressing defines which faces from zone are local
-        if (Pstream::myProcNo() == 7) Pout<< "coarseGgiInterface start: " << lTime_.elapsedCpuTime() << endl;
+        Info<< "coarseGgiInterface start: " << lTime_.elapsedCpuTime() << endl;
         // Perform analysis only for local faces
         // HJ, 22/Jun/2016
         forAll (fineZa, fineZaI)
@@ -709,7 +709,7 @@ Foam::ggiAMGInterface::ggiAMGInterface
                 nAgglomPairs++;
             }
         } // end for all fine faces
-        if (Pstream::myProcNo() == 7) Pout<< "coarseGgiInterface end: " << lTime_.elapsedCpuTime() << endl;
+        Info<< "coarseGgiInterface end: " << lTime_.elapsedCpuTime() << endl;
     } // end of else in fine level (coarse level)
 
     // Since only local faces are analysed, lists can now be resized
@@ -772,7 +772,7 @@ Foam::ggiAMGInterface::ggiAMGInterface
     // the local zone is created.  HJ, 1/Aug/2016
     if (master())
     {
-        if (Pstream::myProcNo() == 7) Pout<< "ggiAMGInterface start agglom master: " << lTime_.elapsedCpuTime() << endl;
+        Info<< "ggiAMGInterface start agglom master: " << lTime_.elapsedCpuTime() << endl;
         // Note:
         // When I am agglomerating the master, faces are stacked up in order
         // but on the slave side, all I know is the master cluster index and
@@ -916,7 +916,7 @@ Foam::ggiAMGInterface::ggiAMGInterface
                 }
             }
         }
-        if (Pstream::myProcNo() == 7) Pout<< "ggiAMGInterface end agglom master " << lTime_.elapsedCpuTime() << endl;
+        Info<< "ggiAMGInterface end agglom master " << lTime_.elapsedCpuTime() << endl;
     }
     // Agglomerate slave
     else
@@ -943,7 +943,7 @@ Foam::ggiAMGInterface::ggiAMGInterface
 
         // Count how many global faces are used for each proc on the other side
         labelList npmf(Pstream::nProcs(), 0);
-        if (Pstream::myProcNo() == 7) Pout<< "ggiAMGInterface start agglom slave: " << lTime_.elapsedCpuTime() << endl;
+        Info<< "ggiAMGInterface start agglom slave: " << lTime_.elapsedCpuTime() << endl;
         // On slave side, the owner addressing is stored in linked lists
         forAll (contents, masterI)
         {
@@ -1010,9 +1010,9 @@ Foam::ggiAMGInterface::ggiAMGInterface
                 nProcFaces++;
             }
         }
-        if (Pstream::myProcNo() == 7) Pout<< "ggiAMGInterface end agglom slave " << lTime_.elapsedCpuTime() << endl;
+        Info<< "ggiAMGInterface end agglom slave " << lTime_.elapsedCpuTime() << endl;
     }
-    if (Pstream::myProcNo() == 7) Pout<< "End ggiAMGInterface constructor " << lTime_.elapsedCpuTime() << endl;
+    Info<< "End ggiAMGInterface constructor " << lTime_.elapsedCpuTime() << endl;
 }
 
 

From 327be98074ffe9c05877d8d543f4b09db95bdf41 Mon Sep 17 00:00:00 2001
From: Hrvoje Jasak <h.jasak@wikki.co.uk>
Date: Thu, 6 Oct 2016 12:48:55 +0100
Subject: [PATCH 067/130] Bugfix: use optimised comm

---
 src/lduSolvers/amg/fineAmgLevel.C | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/lduSolvers/amg/fineAmgLevel.C b/src/lduSolvers/amg/fineAmgLevel.C
index d84129676..fc7908c5e 100644
--- a/src/lduSolvers/amg/fineAmgLevel.C
+++ b/src/lduSolvers/amg/fineAmgLevel.C
@@ -257,8 +257,8 @@ void Foam::fineAmgLevel::scaleX
         scalingFactorDenom += x[i]*Ax[i];
     }
 
-    vector scalingVector(scalingFactorNum, scalingFactorDenom, 0);
-    reduce(scalingVector, sumOp<vector>());
+    vector2D scalingVector(scalingFactorNum, scalingFactorDenom);
+    reduce(scalingVector, sumOp<vector2D>());
 
     // Scale x
     if

From b7b1edcf95ed9203681e01ff7f1945ca8b15b2a1 Mon Sep 17 00:00:00 2001
From: Hrvoje Jasak <h.jasak@wikki.co.uk>
Date: Thu, 6 Oct 2016 12:49:25 +0100
Subject: [PATCH 068/130] Improved parallel comms by using MPI allreduce
 directly on primitive types

---
 src/foam/Make/files                           |   1 +
 src/foam/db/IOstreams/Pstreams/Pstream.C      | 236 ------------
 .../db/IOstreams/Pstreams/PstreamReduceOps.C  | 357 ++++++++++++++++++
 .../db/IOstreams/Pstreams/PstreamReduceOps.H  |  55 +++
 .../IOstreams/Pstreams/allReduceTemplates.C   | 292 +++++++-------
 .../ggiAMGInterface/ggiAMGInterface.C         |  38 +-
 src/foam/primitives/ints/label/label.H        |   9 +
 7 files changed, 591 insertions(+), 397 deletions(-)
 create mode 100644 src/foam/db/IOstreams/Pstreams/PstreamReduceOps.C

diff --git a/src/foam/Make/files b/src/foam/Make/files
index d506e7a3f..f0f928e7d 100644
--- a/src/foam/Make/files
+++ b/src/foam/Make/files
@@ -141,6 +141,7 @@ $(StringStreams)/StringStreamsPrint.C
 
 Pstreams = $(Streams)/Pstreams
 $(Pstreams)/Pstream.C
+$(Pstreams)/PstreamReduceOps.C
 $(Pstreams)/PstreamCommsStruct.C
 $(Pstreams)/PstreamGlobals.C
 $(Pstreams)/IPstream.C
diff --git a/src/foam/db/IOstreams/Pstreams/Pstream.C b/src/foam/db/IOstreams/Pstreams/Pstream.C
index e87e9836d..cb78615d0 100644
--- a/src/foam/db/IOstreams/Pstreams/Pstream.C
+++ b/src/foam/db/IOstreams/Pstreams/Pstream.C
@@ -32,21 +32,11 @@ License
 #include "OSspecific.H"
 #include "PstreamGlobals.H"
 #include "SubList.H"
-#include "allReduce.H"
 
 #include <cstring>
 #include <cstdlib>
 #include <csignal>
 
-#if defined(WM_SP)
-#   define MPI_SCALAR MPI_FLOAT
-#elif defined(WM_DP)
-#   define MPI_SCALAR MPI_DOUBLE
-#elif defined(WM_LDP)
-#   define MPI_SCALAR MPI_LONG_DOUBLE
-#endif
-
-
 // * * * * * * * * * * * * * * Static Data Members * * * * * * * * * * * * * //
 
 defineTypeNameAndDebug(Foam::Pstream, 0);
@@ -548,8 +538,6 @@ Foam::label Foam::Pstream::procNo
 }
 
 
-
-
 // * * * * * * * * * * * * * * * Member Functions  * * * * * * * * * * * * * //
 
 
@@ -680,230 +668,6 @@ void Foam::Pstream::abort()
 }
 
 
-void Foam::reduce
-(
-    bool& Value,
-    const andOp<bool>& bop,
-    const int tag,
-    const label comm
-)
-{
-    if (Pstream::warnComm != -1 && comm != Pstream::warnComm)
-    {
-        Pout<< "** reducing:" << Value << " with comm:" << comm
-            << " warnComm:" << Pstream::warnComm
-            << endl;
-        error::printStack(Pout);
-    }
-
-    // Note: C++ bool is a type separate from C and cannot be cast
-    // For safety and compatibility with compilers, convert bool to int
-    // to comply with MPI types.  HJ, 23/Sep/2016
-
-    int intBool = 0;
-
-    if (Value)
-    {
-        intBool = 1;
-    }
-
-    allReduce(intBool, 1, MPI_INT, MPI_LAND, bop, tag, comm);
-
-    if (intBool > 0)
-    {
-        Value = true;
-    }
-    else
-    {
-        Value = false;
-    }
-}
-
-
-void Foam::reduce
-(
-    bool& Value,
-    const orOp<bool>& bop,
-    const int tag,
-    const label comm
-)
-{
-    if (Pstream::warnComm != -1 && comm != Pstream::warnComm)
-    {
-        Pout<< "** reducing:" << Value << " with comm:" << comm
-            << " warnComm:" << Pstream::warnComm
-            << endl;
-        error::printStack(Pout);
-    }
-
-    // Note: C++ bool is a type separate from C and cannot be cast
-    // For safety and compatibility with compilers, convert bool to int
-    // to comply with MPI types.  HJ, 23/Sep/2016
-
-    int intBool = 0;
-
-    if (Value)
-    {
-        intBool = 1;
-    }
-
-    allReduce(intBool, 1, MPI_INT, MPI_LOR, bop, tag, comm);
-
-    if (intBool > 0)
-    {
-        Value = true;
-    }
-    else
-    {
-        Value = false;
-    }
-}
-
-
-void Foam::reduce
-(
-    scalar& Value,
-    const minOp<scalar>& bop,
-    const int tag,
-    const label comm
-)
-{
-    if (Pstream::warnComm != -1 && comm != Pstream::warnComm)
-    {
-        Pout<< "** reducing:" << Value << " with comm:" << comm
-            << " warnComm:" << Pstream::warnComm
-            << endl;
-        error::printStack(Pout);
-    }
-    allReduce(Value, 1, MPI_SCALAR, MPI_MIN, bop, tag, comm);
-}
-
-
-void Foam::reduce
-(
-    scalar& Value,
-    const maxOp<scalar>& bop,
-    const int tag,
-    const label comm
-)
-{
-    if (Pstream::warnComm != -1 && comm != Pstream::warnComm)
-    {
-        Pout<< "** reducing:" << Value << " with comm:" << comm
-            << " warnComm:" << Pstream::warnComm
-            << endl;
-        error::printStack(Pout);
-    }
-    allReduce(Value, 1, MPI_SCALAR, MPI_MAX, bop, tag, comm);
-}
-
-
-void Foam::reduce
-(
-    scalar& Value,
-    const sumOp<scalar>& bop,
-    const int tag,
-    const label comm
-)
-{
-    if (Pstream::warnComm != -1 && comm != Pstream::warnComm)
-    {
-        Pout<< "** reducing:" << Value << " with comm:" << comm
-            << " warnComm:" << Pstream::warnComm
-            << endl;
-        error::printStack(Pout);
-    }
-
-    allReduce(Value, 1, MPI_SCALAR, MPI_SUM, bop, tag, comm);
-}
-
-
-void Foam::reduce
-(
-    vector2D& Value,
-    const sumOp<vector2D>& bop,
-    const int tag,
-    const label comm
-)
-{
-    if (Pstream::warnComm != -1 && comm != Pstream::warnComm)
-    {
-        Pout<< "** reducing:" << Value << " with comm:" << comm
-            << " warnComm:" << Pstream::warnComm
-            << endl;
-        error::printStack(Pout);
-    }
-    allReduce(Value, 2, MPI_SCALAR, MPI_SUM, bop, tag, comm);
-}
-
-
-void Foam::sumReduce
-(
-    scalar& Value,
-    label& Count,
-    const int tag,
-    const label comm
-)
-{
-    if (Pstream::warnComm != -1 && comm != Pstream::warnComm)
-    {
-        Pout<< "** reducing:" << Value << " with comm:" << comm
-            << " warnComm:" << Pstream::warnComm
-            << endl;
-        error::printStack(Pout);
-    }
-    vector2D twoScalars(Value, scalar(Count));
-    reduce(twoScalars, sumOp<vector2D>(), tag, comm);
-
-    Value = twoScalars.x();
-    Count = twoScalars.y();
-}
-
-
-void Foam::reduce
-(
-    scalar& Value,
-    const sumOp<scalar>& bop,
-    const int tag,
-    const label comm,
-    label& requestID
-)
-{
-#ifdef MPIX_COMM_TYPE_SHARED
-    // Assume mpich2 with non-blocking collectives extensions. Once mpi3
-    // is available this will change.
-    MPI_Request request;
-    scalar v = Value;
-    MPIX_Ireduce
-    (
-        &v,
-        &Value,
-        1,
-        MPI_SCALAR,
-        MPI_SUM,
-        0,              // root
-        PstreamGlobals::MPICommunicators_[comm],
-        &request
-    );
-
-    requestID = PstreamGlobals::outstandingRequests_.size();
-    PstreamGlobals::outstandingRequests_.append(request);
-
-    if (debug)
-    {
-        Pout<< "Pstream::allocateRequest for non-blocking reduce"
-            << " : request:" << requestID
-            << endl;
-    }
-#else
-    // Non-blocking not yet implemented in mpi
-    reduce(Value, bop, tag, comm);
-
-    requestID = -1;
-#endif
-}
-
-
 Foam::label Foam::Pstream::nRequests()
 {
     return PstreamGlobals::outstandingRequests_.size();
diff --git a/src/foam/db/IOstreams/Pstreams/PstreamReduceOps.C b/src/foam/db/IOstreams/Pstreams/PstreamReduceOps.C
new file mode 100644
index 000000000..1ccf5ed27
--- /dev/null
+++ b/src/foam/db/IOstreams/Pstreams/PstreamReduceOps.C
@@ -0,0 +1,357 @@
+/*---------------------------------------------------------------------------*\
+  =========                 |
+  \\      /  F ield         | foam-extend: Open Source CFD
+   \\    /   O peration     | Version:     4.0
+    \\  /    A nd           | Web:         http://www.foam-extend.org
+     \\/     M anipulation  | For copyright notice see file Copyright
+-------------------------------------------------------------------------------
+License
+    This file is part of foam-extend.
+
+    foam-extend is free software: you can redistribute it and/or modify it
+    under the terms of the GNU General Public License as published by the
+    Free Software Foundation, either version 3 of the License, or (at your
+    option) any later version.
+
+    foam-extend is distributed in the hope that it will be useful, but
+    WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+    General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with foam-extend.  If not, see <http://www.gnu.org/licenses/>.
+
+\*---------------------------------------------------------------------------*/
+
+#include "mpi.h"
+
+#include "label.H"
+#include "Pstream.H"
+#include "PstreamReduceOps.H"
+#include "allReduce.H"
+
+// Check type of label for use in MPI calls
+#if defined(WM_INT)
+#   define MPI_LABEL MPI_INT
+#elif defined(WM_LONG)
+#   define MPI_LABEL MPI_LONG
+#elif defined(WM_LLONG)
+#   define MPI_LABEL MPI_LONG_LONG
+#endif
+
+// Check type of scalar for use in MPI calls
+#if defined(WM_SP)
+#   define MPI_SCALAR MPI_FLOAT
+#elif defined(WM_DP)
+#   define MPI_SCALAR MPI_DOUBLE
+#elif defined(WM_LDP)
+#   define MPI_SCALAR MPI_LONG_DOUBLE
+#endif
+
+
+// * * * * * * * * * * * * * * * Member Functions  * * * * * * * * * * * * * //
+
+// Optimizing template specialisations using MPI_REDUCE
+void Foam::reduce
+(
+    bool& Value,
+    const andOp<bool>& bop,
+    const int tag,
+    const label comm
+)
+{
+    if (Pstream::warnComm != -1 && comm != Pstream::warnComm)
+    {
+        Pout<< "** reducing:" << Value << " with comm:" << comm
+            << " warnComm:" << Pstream::warnComm
+            << endl;
+        error::printStack(Pout);
+    }
+
+    // Note: C++ bool is a type separate from C and cannot be cast
+    // For safety and compatibility with compilers, convert bool to int
+    // to comply with MPI types.  HJ, 23/Sep/2016
+
+    int intBool = 0;
+
+    if (Value)
+    {
+        intBool = 1;
+    }
+
+    allReduce(intBool, 1, MPI_INT, MPI_LAND, bop, tag, comm);
+
+    if (intBool > 0)
+    {
+        Value = true;
+    }
+    else
+    {
+        Value = false;
+    }
+}
+
+
+void Foam::reduce
+(
+    bool& Value,
+    const orOp<bool>& bop,
+    const int tag,
+    const label comm
+)
+{
+    if (Pstream::warnComm != -1 && comm != Pstream::warnComm)
+    {
+        Pout<< "** reducing:" << Value << " with comm:" << comm
+            << " warnComm:" << Pstream::warnComm
+            << endl;
+        error::printStack(Pout);
+    }
+
+    // Note: C++ bool is a type separate from C and cannot be cast
+    // For safety and compatibility with compilers, convert bool to int
+    // to comply with MPI types.  HJ, 23/Sep/2016
+
+    int intBool = 0;
+
+    if (Value)
+    {
+        intBool = 1;
+    }
+
+    allReduce(intBool, 1, MPI_INT, MPI_LOR, bop, tag, comm);
+
+    if (intBool > 0)
+    {
+        Value = true;
+    }
+    else
+    {
+        Value = false;
+    }
+}
+
+
+void Foam::reduce
+(
+    label& Value,
+    const minOp<label>& bop,
+    const int tag,
+    const label comm
+)
+{
+    if (Pstream::warnComm != -1 && comm != Pstream::warnComm)
+    {
+        Pout<< "** reducing:" << Value << " with comm:" << comm
+            << " warnComm:" << Pstream::warnComm
+            << endl;
+        error::printStack(Pout);
+    }
+    allReduce(Value, 1, MPI_LABEL, MPI_MIN, bop, tag, comm);
+}
+
+
+void Foam::reduce
+(
+    label& Value,
+    const maxOp<label>& bop,
+    const int tag,
+    const label comm
+)
+{
+    if (Pstream::warnComm != -1 && comm != Pstream::warnComm)
+    {
+        Pout<< "** reducing:" << Value << " with comm:" << comm
+            << " warnComm:" << Pstream::warnComm
+            << endl;
+        error::printStack(Pout);
+    }
+    allReduce(Value, 1, MPI_LABEL, MPI_MAX, bop, tag, comm);
+}
+
+
+void Foam::reduce
+(
+    label& Value,
+    const sumOp<label>& bop,
+    const int tag,
+    const label comm
+)
+{
+    if (Pstream::warnComm != -1 && comm != Pstream::warnComm)
+    {
+        Pout<< "** reducing:" << Value << " with comm:" << comm
+            << " warnComm:" << Pstream::warnComm
+            << endl;
+        error::printStack(Pout);
+    }
+
+    allReduce(Value, 1, MPI_LABEL, MPI_SUM, bop, tag, comm);
+}
+
+
+void Foam::reduce
+(
+    scalar& Value,
+    const minOp<scalar>& bop,
+    const int tag,
+    const label comm
+)
+{
+    if (Pstream::warnComm != -1 && comm != Pstream::warnComm)
+    {
+        Pout<< "** reducing:" << Value << " with comm:" << comm
+            << " warnComm:" << Pstream::warnComm
+            << endl;
+        error::printStack(Pout);
+    }
+    allReduce(Value, 1, MPI_SCALAR, MPI_MIN, bop, tag, comm);
+}
+
+
+void Foam::reduce
+(
+    scalar& Value,
+    const maxOp<scalar>& bop,
+    const int tag,
+    const label comm
+)
+{
+    if (Pstream::warnComm != -1 && comm != Pstream::warnComm)
+    {
+        Pout<< "** reducing:" << Value << " with comm:" << comm
+            << " warnComm:" << Pstream::warnComm
+            << endl;
+        error::printStack(Pout);
+    }
+    allReduce(Value, 1, MPI_SCALAR, MPI_MAX, bop, tag, comm);
+}
+
+
+void Foam::reduce
+(
+    scalar& Value,
+    const sumOp<scalar>& bop,
+    const int tag,
+    const label comm
+)
+{
+    if (Pstream::warnComm != -1 && comm != Pstream::warnComm)
+    {
+        Pout<< "** reducing:" << Value << " with comm:" << comm
+            << " warnComm:" << Pstream::warnComm
+            << endl;
+        error::printStack(Pout);
+    }
+
+    allReduce(Value, 1, MPI_SCALAR, MPI_SUM, bop, tag, comm);
+}
+
+
+void Foam::reduce
+(
+    UList<label>& Value,
+    const sumOp<UList<label> >& bop,
+    const int tag,
+    const label comm
+)
+{
+    if (Pstream::warnComm != -1 && comm != Pstream::warnComm)
+    {
+        Pout<< "** reducing:" << Value << " with comm:" << comm
+            << " warnComm:" << Pstream::warnComm
+            << endl;
+        error::printStack(Pout);
+    }
+
+    allReduce(*Value.begin(), Value.size(), MPI_LABEL, MPI_SUM, bop, tag, comm);
+}
+
+
+void Foam::reduce
+(
+    vector2D& Value,
+    const sumOp<vector2D>& bop,
+    const int tag,
+    const label comm
+)
+{
+    if (Pstream::warnComm != -1 && comm != Pstream::warnComm)
+    {
+        Pout<< "** reducing:" << Value << " with comm:" << comm
+            << " warnComm:" << Pstream::warnComm
+            << endl;
+        error::printStack(Pout);
+    }
+    allReduce(Value, 2, MPI_SCALAR, MPI_SUM, bop, tag, comm);
+}
+
+
+void Foam::sumReduce
+(
+    scalar& Value,
+    label& Count,
+    const int tag,
+    const label comm
+)
+{
+    if (Pstream::warnComm != -1 && comm != Pstream::warnComm)
+    {
+        Pout<< "** reducing:" << Value << " with comm:" << comm
+            << " warnComm:" << Pstream::warnComm
+            << endl;
+        error::printStack(Pout);
+    }
+    vector2D twoScalars(Value, scalar(Count));
+    reduce(twoScalars, sumOp<vector2D>(), tag, comm);
+
+    Value = twoScalars.x();
+    Count = twoScalars.y();
+}
+
+
+void Foam::reduce
+(
+    scalar& Value,
+    const sumOp<scalar>& bop,
+    const int tag,
+    const label comm,
+    label& requestID
+)
+{
+#ifdef MPIX_COMM_TYPE_SHARED
+    // Assume mpich2 with non-blocking collectives extensions. Once mpi3
+    // is available this will change.
+    MPI_Request request;
+    scalar v = Value;
+    MPIX_Ireduce
+    (
+        &v,
+        &Value,
+        1,
+        MPI_SCALAR,
+        MPI_SUM,
+        0,              // root
+        PstreamGlobals::MPICommunicators_[comm],
+        &request
+    );
+
+    requestID = PstreamGlobals::outstandingRequests_.size();
+    PstreamGlobals::outstandingRequests_.append(request);
+
+    if (debug)
+    {
+        Pout<< "Pstream::allocateRequest for non-blocking reduce"
+            << " : request:" << requestID
+            << endl;
+    }
+#else
+    // Non-blocking not yet implemented in mpi
+    reduce(Value, bop, tag, comm);
+
+    requestID = -1;
+#endif
+}
+
+
+// ************************************************************************* //
diff --git a/src/foam/db/IOstreams/Pstreams/PstreamReduceOps.H b/src/foam/db/IOstreams/Pstreams/PstreamReduceOps.H
index 65970bf88..d81dc8487 100644
--- a/src/foam/db/IOstreams/Pstreams/PstreamReduceOps.H
+++ b/src/foam/db/IOstreams/Pstreams/PstreamReduceOps.H
@@ -159,6 +159,33 @@ void reduce
     const label comm = Pstream::worldComm
 );
 
+
+// Insist there are specialisations for the common reductions of labels
+
+void reduce
+(
+    label& Value,
+    const sumOp<label>& bop,
+    const int tag = Pstream::msgType(),
+    const label comm = Pstream::worldComm
+);
+
+void reduce
+(
+    label& Value,
+    const minOp<label>& bop,
+    const int tag = Pstream::msgType(),
+    const label comm = Pstream::worldComm
+);
+
+void reduce
+(
+    label& Value,
+    const maxOp<label>& bop,
+    const int tag = Pstream::msgType(),
+    const label comm = Pstream::worldComm
+);
+
 // Insist there are specialisations for the common reductions of scalars
 
 void reduce
@@ -211,6 +238,34 @@ void reduce
 );
 
 
+// Insist there are specialisations for the common reductions of
+// lists of labels
+
+void reduce
+(
+    UList<label>& Value,
+    const sumOp<UList<label> >& bop,
+    const int tag = Pstream::msgType(),
+    const label comm = Pstream::worldComm
+);
+
+void reduce
+(
+    UList<label>& Value,
+    const minOp<UList<label> >& bop,
+    const int tag = Pstream::msgType(),
+    const label comm = Pstream::worldComm
+);
+
+void reduce
+(
+    UList<label>& Value,
+    const maxOp<UList<label> >& bop,
+    const int tag = Pstream::msgType(),
+    const label comm = Pstream::worldComm
+);
+
+
 // * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
 
 } // End namespace Foam
diff --git a/src/foam/db/IOstreams/Pstreams/allReduceTemplates.C b/src/foam/db/IOstreams/Pstreams/allReduceTemplates.C
index 13b6eddfc..89a2597e9 100644
--- a/src/foam/db/IOstreams/Pstreams/allReduceTemplates.C
+++ b/src/foam/db/IOstreams/Pstreams/allReduceTemplates.C
@@ -44,155 +44,155 @@ void Foam::allReduce
         return;
     }
 
-    if (Pstream::nProcs(comm) <= Pstream::nProcsSimpleSum)
-    {
-        if (Pstream::master(comm))
-        {
-            for
-            (
-                int slave = Pstream::firstSlave();
-                slave <= Pstream::lastSlave(comm);
-                slave++
-            )
-            {
-                Type value;
+//     if (Pstream::nProcs(comm) <= Pstream::nProcsSimpleSum)
+//     {
+//         if (Pstream::master(comm))
+//         {
+//             for
+//             (
+//                 int slave = Pstream::firstSlave();
+//                 slave <= Pstream::lastSlave(comm);
+//                 slave++
+//             )
+//             {
+//                 Type value;
 
-                if
-                (
-                    MPI_Recv
-                    (
-                        &value,
-                        MPICount,
-                        MPIType,
-                        slave,
-                        tag,
-                        PstreamGlobals::MPICommunicators_[comm],
-                        MPI_STATUS_IGNORE
-                    )
-                )
-                {
-                    FatalErrorIn
-                    (
-                        "void Foam::allReduce\n"
-                        "(\n"
-                        "    Type&,\n"
-                        "    int,\n"
-                        "    MPI_Datatype,\n"
-                        "    MPI_Op,\n"
-                        "    const BinaryOp&,\n"
-                        "    const int\n"
-                        ")\n"
-                    )   << "MPI_Recv failed"
-                        << Foam::abort(FatalError);
-                }
+//                 if
+//                 (
+//                     MPI_Recv
+//                     (
+//                         &value,
+//                         MPICount,
+//                         MPIType,
+//                         slave,
+//                         tag,
+//                         PstreamGlobals::MPICommunicators_[comm],
+//                         MPI_STATUS_IGNORE
+//                     )
+//                 )
+//                 {
+//                     FatalErrorIn
+//                     (
+//                         "void Foam::allReduce\n"
+//                         "(\n"
+//                         "    Type&,\n"
+//                         "    int,\n"
+//                         "    MPI_Datatype,\n"
+//                         "    MPI_Op,\n"
+//                         "    const BinaryOp&,\n"
+//                         "    const int\n"
+//                         ")\n"
+//                     )   << "MPI_Recv failed"
+//                         << Foam::abort(FatalError);
+//                 }
 
-                Value = bop(Value, value);
-            }
-        }
-        else
-        {
-            if
-            (
-                MPI_Send
-                (
-                    &Value,
-                    MPICount,
-                    MPIType,
-                    Pstream::masterNo(),
-                    tag,
-                    PstreamGlobals::MPICommunicators_[comm]
-                )
-            )
-            {
-                FatalErrorIn
-                (
-                    "void Foam::allReduce\n"
-                    "(\n"
-                    "    Type&,\n"
-                    "    int,\n"
-                    "    MPI_Datatype,\n"
-                    "    MPI_Op,\n"
-                    "    const BinaryOp&,\n"
-                    "    const int\n"
-                    ")\n"
-                )   << "MPI_Send failed"
-                    << Foam::abort(FatalError);
-            }
-        }
+//                 Value = bop(Value, value);
+//             }
+//         }
+//         else
+//         {
+//             if
+//             (
+//                 MPI_Send
+//                 (
+//                     &Value,
+//                     MPICount,
+//                     MPIType,
+//                     Pstream::masterNo(),
+//                     tag,
+//                     PstreamGlobals::MPICommunicators_[comm]
+//                 )
+//             )
+//             {
+//                 FatalErrorIn
+//                 (
+//                     "void Foam::allReduce\n"
+//                     "(\n"
+//                     "    Type&,\n"
+//                     "    int,\n"
+//                     "    MPI_Datatype,\n"
+//                     "    MPI_Op,\n"
+//                     "    const BinaryOp&,\n"
+//                     "    const int\n"
+//                     ")\n"
+//                 )   << "MPI_Send failed"
+//                     << Foam::abort(FatalError);
+//             }
+//         }
 
 
-        if (Pstream::master(comm))
-        {
-            for
-            (
-                int slave = Pstream::firstSlave();
-                slave <= Pstream::lastSlave(comm);
-                slave++
-            )
-            {
-                if
-                (
-                    MPI_Send
-                    (
-                        &Value,
-                        MPICount,
-                        MPIType,
-                        slave,
-                        tag,
-                        PstreamGlobals::MPICommunicators_[comm]
-                    )
-                )
-                {
-                    FatalErrorIn
-                    (
-                        "void Foam::allReduce\n"
-                        "(\n"
-                        "    Type&,\n"
-                        "    int,\n"
-                        "    MPI_Datatype,\n"
-                        "    MPI_Op,\n"
-                        "    const BinaryOp&,\n"
-                        "    const int\n"
-                        ")\n"
-                    )   << "MPI_Send failed"
-                        << Foam::abort(FatalError);
-                }
-            }
-        }
-        else
-        {
-            if
-            (
-                MPI_Recv
-                (
-                    &Value,
-                    MPICount,
-                    MPIType,
-                    Pstream::masterNo(),
-                    tag,
-                    PstreamGlobals::MPICommunicators_[comm],
-                    MPI_STATUS_IGNORE
-                )
-            )
-            {
-                FatalErrorIn
-                (
-                    "void Foam::allReduce\n"
-                    "(\n"
-                    "    Type&,\n"
-                    "    int,\n"
-                    "    MPI_Datatype,\n"
-                    "    MPI_Op,\n"
-                    "    const BinaryOp&,\n"
-                    "    const int\n"
-                    ")\n"
-                )   << "MPI_Recv failed"
-                    << Foam::abort(FatalError);
-            }
-        }
-    }
-    else
-    {
+//         if (Pstream::master(comm))
+//         {
+//             for
+//             (
+//                 int slave = Pstream::firstSlave();
+//                 slave <= Pstream::lastSlave(comm);
+//                 slave++
+//             )
+//             {
+//                 if
+//                 (
+//                     MPI_Send
+//                     (
+//                         &Value,
+//                         MPICount,
+//                         MPIType,
+//                         slave,
+//                         tag,
+//                         PstreamGlobals::MPICommunicators_[comm]
+//                     )
+//                 )
+//                 {
+//                     FatalErrorIn
+//                     (
+//                         "void Foam::allReduce\n"
+//                         "(\n"
+//                         "    Type&,\n"
+//                         "    int,\n"
+//                         "    MPI_Datatype,\n"
+//                         "    MPI_Op,\n"
+//                         "    const BinaryOp&,\n"
+//                         "    const int\n"
+//                         ")\n"
+//                     )   << "MPI_Send failed"
+//                         << Foam::abort(FatalError);
+//                 }
+//             }
+//         }
+//         else
+//         {
+//             if
+//             (
+//                 MPI_Recv
+//                 (
+//                     &Value,
+//                     MPICount,
+//                     MPIType,
+//                     Pstream::masterNo(),
+//                     tag,
+//                     PstreamGlobals::MPICommunicators_[comm],
+//                     MPI_STATUS_IGNORE
+//                 )
+//             )
+//             {
+//                 FatalErrorIn
+//                 (
+//                     "void Foam::allReduce\n"
+//                     "(\n"
+//                     "    Type&,\n"
+//                     "    int,\n"
+//                     "    MPI_Datatype,\n"
+//                     "    MPI_Op,\n"
+//                     "    const BinaryOp&,\n"
+//                     "    const int\n"
+//                     ")\n"
+//                 )   << "MPI_Recv failed"
+//                     << Foam::abort(FatalError);
+//             }
+//         }
+//     }
+//     else
+//     {
         Type sum;
 
         MPI_Allreduce
@@ -206,7 +206,7 @@ void Foam::allReduce
         );
 
         Value = sum;
-    }
+//     }
 }
 
 
diff --git a/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/ggiAMGInterface/ggiAMGInterface.C b/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/ggiAMGInterface/ggiAMGInterface.C
index 95e389e84..cd7dd75a1 100644
--- a/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/ggiAMGInterface/ggiAMGInterface.C
+++ b/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/ggiAMGInterface/ggiAMGInterface.C
@@ -97,7 +97,7 @@ void Foam::ggiAMGInterface::initFastReduce() const
     // Note: reduce with a comm will only be present on processors containing
     // master or slave faces.  Other processors created a dummy map above
     // HJ, 20/Sep/2016
-    reduce(zoneProcID, maxOp<labelField>(), tag(), comm());
+    reduce(zoneProcID, maxOp<UList<label> >(), tag(), comm());
 
     // Find out where my zone data is coming from
     labelList nRecv(Pstream::nProcs(), 0);
@@ -340,7 +340,6 @@ Foam::ggiAMGInterface::ggiAMGInterface
     // On the fine level, addressing is made in a labelListList
     if (fineGgiInterface_.fineLevel())
     {
-        Info<< "fineGgiInterface start: " << lTime_.elapsedCpuTime() << endl;
         // This addressing defines how to interpolate for all zone faces
         // across the interface
         const labelListList& fineAddr = fineGgiInterface_.addressing();
@@ -348,6 +347,10 @@ Foam::ggiAMGInterface::ggiAMGInterface
 
         // Perform analysis only for local faces
         // HJ, 22/Jun/2016
+
+        label curMasterProc, curSlaveProc;
+        long curMaster, curSlave;
+
         forAll (fineZa, fineZaI)
         {
             // Get the local face (from zone) to analyse
@@ -358,10 +361,10 @@ Foam::ggiAMGInterface::ggiAMGInterface
 
             forAll (curFineNbrs, nbrI)
             {
-                long curMaster = -1;
-                label curMasterProc = -1;
-                long curSlave = -1;
-                label curSlaveProc = -1;
+                curMaster = -1;
+                curMasterProc = -1;
+                curSlave = -1;
+                curSlaveProc = -1;
 
                 // Note.  Signalling in global clustering requires
                 // me to recognise clustering from separate
@@ -527,24 +530,27 @@ Foam::ggiAMGInterface::ggiAMGInterface
                 }
             } // end for all current neighbours
         } // end for all fine faces
-        Info<< "fineGgiInterface end: " << lTime_.elapsedCpuTime() << endl;
     }
     else
     {
         // Coarse level, addressing is stored in faceCells
         // This addressing defines which faces from zone are local
-        Info<< "coarseGgiInterface start: " << lTime_.elapsedCpuTime() << endl;
+
         // Perform analysis only for local faces
         // HJ, 22/Jun/2016
+
+        label curMasterProc, curSlaveProc;
+        long curMaster, curSlave;
+
         forAll (fineZa, fineZaI)
         {
             // Get the local face (from zone) to analyse
             const label ffI = fineZa[fineZaI];
 
-            long curMaster = -1;
-            label curMasterProc = -1;
-            long curSlave = -1;
-            label curSlaveProc = -1;
+            curMaster = -1;
+            curMasterProc = -1;
+            curSlave = -1;
+            curSlaveProc = -1;
 
             // Note.  Signalling in global clustering requires
             // me to recognise clustering from separate
@@ -709,7 +715,6 @@ Foam::ggiAMGInterface::ggiAMGInterface
                 nAgglomPairs++;
             }
         } // end for all fine faces
-        Info<< "coarseGgiInterface end: " << lTime_.elapsedCpuTime() << endl;
     } // end of else in fine level (coarse level)
 
     // Since only local faces are analysed, lists can now be resized
@@ -721,7 +726,7 @@ Foam::ggiAMGInterface::ggiAMGInterface
     // In order to assemble the coarse global face zone, find out
     // how many faces have been created on each processor.
     // Note that masters and slaves both count faces so we will
-    //  only ask master sizes to count
+    // only ask master sizes to count
     labelList nCoarseFacesPerProc(Pstream::nProcs(), 0);
 
     nCoarseFacesPerProc[Pstream::myProcNo()] = nCoarseFaces;
@@ -731,7 +736,10 @@ Foam::ggiAMGInterface::ggiAMGInterface
     // contact with the GGI interface will have zero zone size.
     // This needs to be handled separately in the initFastReduce
     // HJ, 20/Sep/2016
-    reduce(nCoarseFacesPerProc, sumOp<labelList>(), tag(), comm());
+
+    // Optimised comm: Wait for info from previous processor, add your
+    // number of coarse faces and pass to next processor
+    reduce(nCoarseFacesPerProc, sumOp<UList<label> >(), tag(), comm());
 
     // Coarse global face zone is assembled by adding all faces from proc0,
     // followed by all faces from proc1 etc.
diff --git a/src/foam/primitives/ints/label/label.H b/src/foam/primitives/ints/label/label.H
index bdbfee903..89de13388 100644
--- a/src/foam/primitives/ints/label/label.H
+++ b/src/foam/primitives/ints/label/label.H
@@ -55,6 +55,9 @@ Description
 # undef  FOAM_LABEL_MAX
 # define FOAM_LABEL_MAX INT_MAX
 
+// Define identifier for label type for int
+# define WM_INT
+
 # include "int.H"
 # include "long.H"
 # include "longLong.H"
@@ -80,6 +83,9 @@ namespace Foam
 # undef  FOAM_LABEL_MAX
 # define FOAM_LABEL_MAX LONG_MAX
 
+// Define identifier for label type for long
+# define WM_LONG
+
 # include "int.H"
 # include "long.H"
 
@@ -105,6 +111,9 @@ namespace Foam
 # undef  FOAM_LABEL_MAX
 # define FOAM_LABEL_MAX LLONG_MAX
 
+// Define identifier for label type for long long
+# define WM_LLONG
+
 # include "int.H"
 # include "long.H"
 # include "longLong.H"

From 52e99137eb3063f7eab67a6a036373580d196d9c Mon Sep 17 00:00:00 2001
From: Hrvoje Jasak <h.jasak@wikki.co.uk>
Date: Sat, 8 Oct 2016 19:05:14 +0100
Subject: [PATCH 069/130] Optimised global reduce for labelList

---
 .../db/IOstreams/Pstreams/PstreamReduceOps.C  |  88 ++++++++-
 .../db/IOstreams/Pstreams/PstreamReduceOps.H  |  15 +-
 .../IOstreams/Pstreams/allReduceTemplates.C   | 173 ++----------------
 3 files changed, 106 insertions(+), 170 deletions(-)

diff --git a/src/foam/db/IOstreams/Pstreams/PstreamReduceOps.C b/src/foam/db/IOstreams/Pstreams/PstreamReduceOps.C
index 1ccf5ed27..08bbd77fa 100644
--- a/src/foam/db/IOstreams/Pstreams/PstreamReduceOps.C
+++ b/src/foam/db/IOstreams/Pstreams/PstreamReduceOps.C
@@ -250,8 +250,8 @@ void Foam::reduce
 
 void Foam::reduce
 (
-    UList<label>& Value,
-    const sumOp<UList<label> >& bop,
+    List<label>& Value,
+    const sumOp<List<label> >& bop,
     const int tag,
     const label comm
 )
@@ -264,7 +264,89 @@ void Foam::reduce
         error::printStack(Pout);
     }
 
-    allReduce(*Value.begin(), Value.size(), MPI_LABEL, MPI_SUM, bop, tag, comm);
+    // Make a copy of the Value in the send buffer so that Value can be
+    // used for receive.  HJ, 8/Oct/2016
+    labelList send(Value);
+
+    int MPISize = Value.size();
+
+    MPI_Allreduce
+    (
+        send.begin(),
+        Value.begin(),
+        MPISize,
+        MPI_LABEL,
+        MPI_SUM,
+        PstreamGlobals::MPICommunicators_[comm]
+    );
+}
+
+
+void Foam::reduce
+(
+    List<label>& Value,
+    const minOp<List<label> >& bop,
+    const int tag,
+    const label comm
+)
+{
+    if (Pstream::warnComm != -1 && comm != Pstream::warnComm)
+    {
+        Pout<< "** reducing:" << Value << " with comm:" << comm
+            << " warnComm:" << Pstream::warnComm
+            << endl;
+        error::printStack(Pout);
+    }
+
+    // Make a copy of the Value in the send buffer so that Value can be
+    // used for receive.  HJ, 8/Oct/2016
+    labelList send(Value);
+
+    int MPISize = Value.size();
+
+    MPI_Allreduce
+    (
+        send.begin(),
+        Value.begin(),
+        MPISize,
+        MPI_LABEL,
+        MPI_MIN,
+        PstreamGlobals::MPICommunicators_[comm]
+    );
+}
+
+
+void Foam::reduce
+(
+    List<label>& Value,
+    const maxOp<List<label> >& bop,
+    const int tag,
+    const label comm
+)
+{
+    if (Pstream::warnComm != -1 && comm != Pstream::warnComm)
+    {
+        Pout<< "** reducing:" << Value << " with comm:" << comm
+            << " warnComm:" << Pstream::warnComm
+            << endl;
+        error::printStack(Pout);
+    }
+
+    // Make a copy of the Value in the send buffer so that Value can be
+    // used for receive.  HJ, 8/Oct/2016
+    labelList send(Value);
+
+    int MPISize = Value.size();
+
+    MPI_Allreduce
+    (
+        send.begin(),
+        Value.begin(),
+        MPISize,
+        MPI_LABEL,
+        MPI_MAX,
+        PstreamGlobals::MPICommunicators_[comm]
+    );
 }
 
 
diff --git a/src/foam/db/IOstreams/Pstreams/PstreamReduceOps.H b/src/foam/db/IOstreams/Pstreams/PstreamReduceOps.H
index d81dc8487..e61a74a14 100644
--- a/src/foam/db/IOstreams/Pstreams/PstreamReduceOps.H
+++ b/src/foam/db/IOstreams/Pstreams/PstreamReduceOps.H
@@ -239,28 +239,29 @@ void reduce
 
 
 // Insist there are specialisations for the common reductions of
-// lists of labels
+// lists of labels.  Note: template function specialisation must be the
+// exact match on argument types.  HJ, 8/Oct/2016
 
 void reduce
 (
-    UList<label>& Value,
-    const sumOp<UList<label> >& bop,
+    List<label>& Value,
+    const sumOp<List<label> >& bop,
     const int tag = Pstream::msgType(),
     const label comm = Pstream::worldComm
 );
 
 void reduce
 (
-    UList<label>& Value,
-    const minOp<UList<label> >& bop,
+    List<label>& Value,
+    const minOp<List<label> >& bop,
     const int tag = Pstream::msgType(),
     const label comm = Pstream::worldComm
 );
 
 void reduce
 (
-    UList<label>& Value,
-    const maxOp<UList<label> >& bop,
+    List<label>& Value,
+    const maxOp<List<label> >& bop,
     const int tag = Pstream::msgType(),
     const label comm = Pstream::worldComm
 );
diff --git a/src/foam/db/IOstreams/Pstreams/allReduceTemplates.C b/src/foam/db/IOstreams/Pstreams/allReduceTemplates.C
index 89a2597e9..608a26dd7 100644
--- a/src/foam/db/IOstreams/Pstreams/allReduceTemplates.C
+++ b/src/foam/db/IOstreams/Pstreams/allReduceTemplates.C
@@ -44,169 +44,22 @@ void Foam::allReduce
         return;
     }
 
-//     if (Pstream::nProcs(comm) <= Pstream::nProcsSimpleSum)
-//     {
-//         if (Pstream::master(comm))
-//         {
-//             for
-//             (
-//                 int slave = Pstream::firstSlave();
-//                 slave <= Pstream::lastSlave(comm);
-//                 slave++
-//             )
-//             {
-//                 Type value;
+    // Removed send-received loop: use Allreduce instead.
+    // HJ, 8/Oct/2016
 
-//                 if
-//                 (
-//                     MPI_Recv
-//                     (
-//                         &value,
-//                         MPICount,
-//                         MPIType,
-//                         slave,
-//                         tag,
-//                         PstreamGlobals::MPICommunicators_[comm],
-//                         MPI_STATUS_IGNORE
-//                     )
-//                 )
-//                 {
-//                     FatalErrorIn
-//                     (
-//                         "void Foam::allReduce\n"
-//                         "(\n"
-//                         "    Type&,\n"
-//                         "    int,\n"
-//                         "    MPI_Datatype,\n"
-//                         "    MPI_Op,\n"
-//                         "    const BinaryOp&,\n"
-//                         "    const int\n"
-//                         ")\n"
-//                     )   << "MPI_Recv failed"
-//                         << Foam::abort(FatalError);
-//                 }
+    Type sum;
 
-//                 Value = bop(Value, value);
-//             }
-//         }
-//         else
-//         {
-//             if
-//             (
-//                 MPI_Send
-//                 (
-//                     &Value,
-//                     MPICount,
-//                     MPIType,
-//                     Pstream::masterNo(),
-//                     tag,
-//                     PstreamGlobals::MPICommunicators_[comm]
-//                 )
-//             )
-//             {
-//                 FatalErrorIn
-//                 (
-//                     "void Foam::allReduce\n"
-//                     "(\n"
-//                     "    Type&,\n"
-//                     "    int,\n"
-//                     "    MPI_Datatype,\n"
-//                     "    MPI_Op,\n"
-//                     "    const BinaryOp&,\n"
-//                     "    const int\n"
-//                     ")\n"
-//                 )   << "MPI_Send failed"
-//                     << Foam::abort(FatalError);
-//             }
-//         }
+    MPI_Allreduce
+    (
+        &Value,
+        &sum,
+        MPICount,
+        MPIType,
+        MPIOp,
+        PstreamGlobals::MPICommunicators_[comm]
+    );
 
-
-//         if (Pstream::master(comm))
-//         {
-//             for
-//             (
-//                 int slave = Pstream::firstSlave();
-//                 slave <= Pstream::lastSlave(comm);
-//                 slave++
-//             )
-//             {
-//                 if
-//                 (
-//                     MPI_Send
-//                     (
-//                         &Value,
-//                         MPICount,
-//                         MPIType,
-//                         slave,
-//                         tag,
-//                         PstreamGlobals::MPICommunicators_[comm]
-//                     )
-//                 )
-//                 {
-//                     FatalErrorIn
-//                     (
-//                         "void Foam::allReduce\n"
-//                         "(\n"
-//                         "    Type&,\n"
-//                         "    int,\n"
-//                         "    MPI_Datatype,\n"
-//                         "    MPI_Op,\n"
-//                         "    const BinaryOp&,\n"
-//                         "    const int\n"
-//                         ")\n"
-//                     )   << "MPI_Send failed"
-//                         << Foam::abort(FatalError);
-//                 }
-//             }
-//         }
-//         else
-//         {
-//             if
-//             (
-//                 MPI_Recv
-//                 (
-//                     &Value,
-//                     MPICount,
-//                     MPIType,
-//                     Pstream::masterNo(),
-//                     tag,
-//                     PstreamGlobals::MPICommunicators_[comm],
-//                     MPI_STATUS_IGNORE
-//                 )
-//             )
-//             {
-//                 FatalErrorIn
-//                 (
-//                     "void Foam::allReduce\n"
-//                     "(\n"
-//                     "    Type&,\n"
-//                     "    int,\n"
-//                     "    MPI_Datatype,\n"
-//                     "    MPI_Op,\n"
-//                     "    const BinaryOp&,\n"
-//                     "    const int\n"
-//                     ")\n"
-//                 )   << "MPI_Recv failed"
-//                     << Foam::abort(FatalError);
-//             }
-//         }
-//     }
-//     else
-//     {
-        Type sum;
-
-        MPI_Allreduce
-        (
-            &Value,
-            &sum,
-            MPICount,
-            MPIType,
-            MPIOp,
-            PstreamGlobals::MPICommunicators_[comm]
-        );
-
-        Value = sum;
-//     }
+    Value = sum;
 }
 
 

From 0febbe591925ee4eab0bbbe11de8a6a87fbb2ba4 Mon Sep 17 00:00:00 2001
From: Hrvoje Jasak <h.jasak@wikki.co.uk>
Date: Sat, 8 Oct 2016 19:05:25 +0100
Subject: [PATCH 070/130] Use optimised global reduce for labelList

---
 .../AMG/interfaces/ggiAMGInterface/ggiAMGInterface.C        | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/ggiAMGInterface/ggiAMGInterface.C b/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/ggiAMGInterface/ggiAMGInterface.C
index cd7dd75a1..32c935c5f 100644
--- a/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/ggiAMGInterface/ggiAMGInterface.C
+++ b/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/ggiAMGInterface/ggiAMGInterface.C
@@ -97,7 +97,7 @@ void Foam::ggiAMGInterface::initFastReduce() const
     // Note: reduce with a comm will only be present on processors containing
     // master or slave faces.  Other processors created a dummy map above
     // HJ, 20/Sep/2016
-    reduce(zoneProcID, maxOp<UList<label> >(), tag(), comm());
+    reduce(zoneProcID, maxOp<labelList>(), tag(), comm());
 
     // Find out where my zone data is coming from
     labelList nRecv(Pstream::nProcs(), 0);
@@ -739,8 +739,8 @@ Foam::ggiAMGInterface::ggiAMGInterface
 
     // Optimised comm: Wait for info from previous processor, add your
     // number of coarse faces and pass to next processor
-    reduce(nCoarseFacesPerProc, sumOp<UList<label> >(), tag(), comm());
-
+    reduce(nCoarseFacesPerProc, sumOp<List<label> >(), tag(), comm());
+    Pout<< "nCoarseFacesPerProc: " << nCoarseFacesPerProc << endl;
     // Coarse global face zone is assembled by adding all faces from proc0,
     // followed by all faces from proc1 etc.
     // Therefore, on procN, my master offset

From 54f5bdac4d29b5d8d46b2aeeb98ac95afbe2478b Mon Sep 17 00:00:00 2001
From: Hrvoje Jasak <h.jasak@wikki.co.uk>
Date: Sun, 9 Oct 2016 19:30:04 +0100
Subject: [PATCH 071/130] Temporary fix: global communicator

---
 src/foam/db/IOstreams/Pstreams/PstreamReduceOps.C | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/src/foam/db/IOstreams/Pstreams/PstreamReduceOps.C b/src/foam/db/IOstreams/Pstreams/PstreamReduceOps.C
index 08bbd77fa..b4cb12584 100644
--- a/src/foam/db/IOstreams/Pstreams/PstreamReduceOps.C
+++ b/src/foam/db/IOstreams/Pstreams/PstreamReduceOps.C
@@ -269,7 +269,8 @@ void Foam::reduce
     labelList send(Value);
 
     int MPISize = Value.size();
-
+    Pout<< "comm: " << comm << " size: "
+        << PstreamGlobals::MPICommunicators_.size() << endl;
     MPI_Allreduce
     (
         send.begin(),
@@ -277,7 +278,8 @@ void Foam::reduce
         MPISize,
         MPI_LABEL,
         MPI_SUM,
-        PstreamGlobals::MPICommunicators_[comm]
+        PstreamGlobals::MPICommunicators_[Pstream::worldComm]
+//         PstreamGlobals::MPICommunicators_[comm]
     );
 }
 

From 4d780b2594b4eceadc810e3cf6e5cc3f0915c976 Mon Sep 17 00:00:00 2001
From: Hrvoje Jasak <h.jasak@wikki.co.uk>
Date: Mon, 10 Oct 2016 10:55:42 +0100
Subject: [PATCH 072/130] Comms on Allreduce: explicitly excluding processors
 outside of comm

---
 .../db/IOstreams/Pstreams/PstreamReduceOps.C  | 34 +++++++++++++++----
 .../IOstreams/Pstreams/allReduceTemplates.C   |  6 ++++
 2 files changed, 34 insertions(+), 6 deletions(-)

diff --git a/src/foam/db/IOstreams/Pstreams/PstreamReduceOps.C b/src/foam/db/IOstreams/Pstreams/PstreamReduceOps.C
index b4cb12584..8e2711faf 100644
--- a/src/foam/db/IOstreams/Pstreams/PstreamReduceOps.C
+++ b/src/foam/db/IOstreams/Pstreams/PstreamReduceOps.C
@@ -147,6 +147,7 @@ void Foam::reduce
             << endl;
         error::printStack(Pout);
     }
+
     allReduce(Value, 1, MPI_LABEL, MPI_MIN, bop, tag, comm);
 }
 
@@ -166,6 +167,7 @@ void Foam::reduce
             << endl;
         error::printStack(Pout);
     }
+
     allReduce(Value, 1, MPI_LABEL, MPI_MAX, bop, tag, comm);
 }
 
@@ -205,6 +207,7 @@ void Foam::reduce
             << endl;
         error::printStack(Pout);
     }
+
     allReduce(Value, 1, MPI_SCALAR, MPI_MIN, bop, tag, comm);
 }
 
@@ -224,6 +227,7 @@ void Foam::reduce
             << endl;
         error::printStack(Pout);
     }
+
     allReduce(Value, 1, MPI_SCALAR, MPI_MAX, bop, tag, comm);
 }
 
@@ -264,13 +268,18 @@ void Foam::reduce
         error::printStack(Pout);
     }
 
+    // Skip processors that are not in the communicator
+    if (Pstream::myProcNo(comm) == -1)
+    {
+        return;
+    }
+
     // Make a copy of the Value in the send buffer so that Value can be
     // used for receive.  HJ, 8/Oct/2016
     labelList send(Value);
 
     int MPISize = Value.size();
-    Pout<< "comm: " << comm << " size: "
-        << PstreamGlobals::MPICommunicators_.size() << endl;
+    Pout<< "In labelList reduce, sumOp" << endl;
     MPI_Allreduce
     (
         send.begin(),
@@ -278,8 +287,7 @@ void Foam::reduce
         MPISize,
         MPI_LABEL,
         MPI_SUM,
-        PstreamGlobals::MPICommunicators_[Pstream::worldComm]
-//         PstreamGlobals::MPICommunicators_[comm]
+        PstreamGlobals::MPICommunicators_[comm]
     );
 }
 
@@ -300,12 +308,18 @@ void Foam::reduce
         error::printStack(Pout);
     }
 
+    // Skip processors that are not in the communicator
+    if (Pstream::myProcNo(comm) == -1)
+    {
+        return;
+    }
+
     // Make a copy of the Value in the send buffer so that Value can be
     // used for receive.  HJ, 8/Oct/2016
     labelList send(Value);
 
     int MPISize = Value.size();
-
+    Pout<< "In labelList reduce, minOp" << endl;
     MPI_Allreduce
     (
         send.begin(),
@@ -334,12 +348,18 @@ void Foam::reduce
         error::printStack(Pout);
     }
 
+    // Skip processors that are not in the communicator
+    if (Pstream::myProcNo(comm) == -1)
+    {
+        return;
+    }
+
     // Make a copy of the Value in the send buffer so that Value can be
     // used for receive.  HJ, 8/Oct/2016
     labelList send(Value);
 
     int MPISize = Value.size();
-
+    Pout<< "In labelList reduce, maxOp" << endl;
     MPI_Allreduce
     (
         send.begin(),
@@ -367,6 +387,7 @@ void Foam::reduce
             << endl;
         error::printStack(Pout);
     }
+
     allReduce(Value, 2, MPI_SCALAR, MPI_SUM, bop, tag, comm);
 }
 
@@ -386,6 +407,7 @@ void Foam::sumReduce
             << endl;
         error::printStack(Pout);
     }
+
     vector2D twoScalars(Value, scalar(Count));
     reduce(twoScalars, sumOp<vector2D>(), tag, comm);
 
diff --git a/src/foam/db/IOstreams/Pstreams/allReduceTemplates.C b/src/foam/db/IOstreams/Pstreams/allReduceTemplates.C
index 608a26dd7..31117b950 100644
--- a/src/foam/db/IOstreams/Pstreams/allReduceTemplates.C
+++ b/src/foam/db/IOstreams/Pstreams/allReduceTemplates.C
@@ -47,6 +47,12 @@ void Foam::allReduce
     // Removed send-received loop: use Allreduce instead.
     // HJ, 8/Oct/2016
 
+    // Skip processors that are not in the communicator
+    if (Pstream::myProcNo(comm) == -1)
+    {
+        return;
+    }
+
     Type sum;
 
     MPI_Allreduce

From 87e936b3cc74c90d74545a1f6cca9aad146fb783 Mon Sep 17 00:00:00 2001
From: Hrvoje Jasak <h.jasak@wikki.co.uk>
Date: Mon, 10 Oct 2016 10:56:02 +0100
Subject: [PATCH 073/130] Bugfix: direct type match for optimised reduce

---
 .../solvers/AMG/interfaces/ggiAMGInterface/ggiAMGInterface.C  | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/ggiAMGInterface/ggiAMGInterface.C b/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/ggiAMGInterface/ggiAMGInterface.C
index 32c935c5f..d314e3720 100644
--- a/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/ggiAMGInterface/ggiAMGInterface.C
+++ b/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/ggiAMGInterface/ggiAMGInterface.C
@@ -87,7 +87,7 @@ void Foam::ggiAMGInterface::initFastReduce() const
 
     // Make a zone-sized field and fill it in with proc markings for processor
     // that holds and requires the data
-    labelField zoneProcID(zoneSize(), -1);
+    labelList zoneProcID(zoneSize(), -1);
 
     forAll (za, zaI)
     {
@@ -740,7 +740,7 @@ Foam::ggiAMGInterface::ggiAMGInterface
     // Optimised comm: Wait for info from previous processor, add your
     // number of coarse faces and pass to next processor
     reduce(nCoarseFacesPerProc, sumOp<List<label> >(), tag(), comm());
-    Pout<< "nCoarseFacesPerProc: " << nCoarseFacesPerProc << endl;
+
     // Coarse global face zone is assembled by adding all faces from proc0,
     // followed by all faces from proc1 etc.
     // Therefore, on procN, my master offset

From 03bb8c8467362afb019c1633feae0ffe2c432588 Mon Sep 17 00:00:00 2001
From: Hrvoje Jasak <h.jasak@wikki.co.uk>
Date: Mon, 10 Oct 2016 11:00:44 +0100
Subject: [PATCH 074/130] Removed debug messages

---
 src/foam/db/IOstreams/Pstreams/PstreamReduceOps.C | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/foam/db/IOstreams/Pstreams/PstreamReduceOps.C b/src/foam/db/IOstreams/Pstreams/PstreamReduceOps.C
index 8e2711faf..f78bd0b38 100644
--- a/src/foam/db/IOstreams/Pstreams/PstreamReduceOps.C
+++ b/src/foam/db/IOstreams/Pstreams/PstreamReduceOps.C
@@ -279,7 +279,7 @@ void Foam::reduce
     labelList send(Value);
 
     int MPISize = Value.size();
-    Pout<< "In labelList reduce, sumOp" << endl;
+
     MPI_Allreduce
     (
         send.begin(),
@@ -319,7 +319,7 @@ void Foam::reduce
     labelList send(Value);
 
     int MPISize = Value.size();
-    Pout<< "In labelList reduce, minOp" << endl;
+
     MPI_Allreduce
     (
         send.begin(),
@@ -359,7 +359,7 @@ void Foam::reduce
     labelList send(Value);
 
     int MPISize = Value.size();
-    Pout<< "In labelList reduce, maxOp" << endl;
+
     MPI_Allreduce
     (
         send.begin(),

From e7e64a8134df2f1fbd9b29acfedffeb145157388 Mon Sep 17 00:00:00 2001
From: Hrvoje Jasak <h.jasak@wikki.co.uk>
Date: Tue, 11 Oct 2016 10:03:25 +0100
Subject: [PATCH 075/130] Bugfix: remove locking: if processor does not touch
 GGI, skip it

---
 .../AMG/interfaces/ggiAMGInterface/ggiAMGInterface.C        | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/ggiAMGInterface/ggiAMGInterface.C b/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/ggiAMGInterface/ggiAMGInterface.C
index d314e3720..44e9ee8dd 100644
--- a/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/ggiAMGInterface/ggiAMGInterface.C
+++ b/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/ggiAMGInterface/ggiAMGInterface.C
@@ -250,6 +250,12 @@ Foam::ggiAMGInterface::ggiAMGInterface
         << fineGgiInterface_.interfaceSize() << ": "
         << lTime_.elapsedCpuTime() << endl;
 
+    // If the processor is not in the GGI comm, escape.  HJ, 10/Oct/2016
+    if (Pstream::myProcNo(comm()) == -1)
+    {
+        return;
+    }
+
     // Note: local addressing contains only local faces
     const labelList& fineZa =  fineGgiInterface_.zoneAddressing();
 

From a2180b7a17788e22bb386c2ce415a1aecabc4954 Mon Sep 17 00:00:00 2001
From: Hrvoje Jasak <h.jasak@wikki.co.uk>
Date: Tue, 11 Oct 2016 10:03:50 +0100
Subject: [PATCH 076/130] Improved checking on optimised reduce with comms

---
 .../db/IOstreams/Pstreams/PstreamReduceOps.C  | 333 +++++++++++++++---
 1 file changed, 291 insertions(+), 42 deletions(-)

diff --git a/src/foam/db/IOstreams/Pstreams/PstreamReduceOps.C b/src/foam/db/IOstreams/Pstreams/PstreamReduceOps.C
index f78bd0b38..28d93e559 100644
--- a/src/foam/db/IOstreams/Pstreams/PstreamReduceOps.C
+++ b/src/foam/db/IOstreams/Pstreams/PstreamReduceOps.C
@@ -68,6 +68,25 @@ void Foam::reduce
         error::printStack(Pout);
     }
 
+#   ifdef FULLDEBUG
+    // Check for processors that are not in the communicator
+    if (Pstream::myProcNo(comm) == -1)
+    {
+        FatalErrorIn
+        (
+            "void Foam::reduce\n"
+            "(\n"
+            "    bool& Value,\n"
+            "    const andOp<bool>& bop,\n"
+            "    const int tag,\n"
+            "    const label comm\n"
+            ")"
+        )   << "Reduce called on the processor which is not a member "
+            << "of comm.  This is not allowed"
+            << abort(FatalError);
+    }
+#   endif
+
     // Note: C++ bool is a type separate from C and cannot be cast
     // For safety and compatibility with compilers, convert bool to int
     // to comply with MPI types.  HJ, 23/Sep/2016
@@ -108,6 +127,25 @@ void Foam::reduce
         error::printStack(Pout);
     }
 
+#   ifdef FULLDEBUG
+    // Check for processors that are not in the communicator
+    if (Pstream::myProcNo(comm) == -1)
+    {
+        FatalErrorIn
+        (
+            "void Foam::reduce\n"
+            "(\n"
+            "    bool& Value,\n"
+            "    const orOp<bool>& bop,\n"
+            "    const int tag,\n"
+            "    const label comm\n"
+            ")"
+        )   << "Reduce called on the processor which is not a member "
+            << "of comm.  This is not allowed"
+            << abort(FatalError);
+    }
+#   endif
+
     // Note: C++ bool is a type separate from C and cannot be cast
     // For safety and compatibility with compilers, convert bool to int
     // to comply with MPI types.  HJ, 23/Sep/2016
@@ -148,6 +186,25 @@ void Foam::reduce
         error::printStack(Pout);
     }
 
+#   ifdef FULLDEBUG
+    // Check for processors that are not in the communicator
+    if (Pstream::myProcNo(comm) == -1)
+    {
+        FatalErrorIn
+        (
+            "void Foam::reduce\n"
+            "(\n"
+            "    label& Value,\n"
+            "    const minOp<label>& bop,\n"
+            "    const int tag,\n"
+            "    const label comm\n"
+            ")"
+        )   << "Reduce called on the processor which is not a member "
+            << "of comm.  This is not allowed"
+            << abort(FatalError);
+    }
+#   endif
+
     allReduce(Value, 1, MPI_LABEL, MPI_MIN, bop, tag, comm);
 }
 
@@ -168,6 +225,25 @@ void Foam::reduce
         error::printStack(Pout);
     }
 
+#   ifdef FULLDEBUG
+    // Check for processors that are not in the communicator
+    if (Pstream::myProcNo(comm) == -1)
+    {
+        FatalErrorIn
+        (
+            "void Foam::reduce\n"
+            "(\n"
+            "    label& Value,\n"
+            "    const maxOp<label>& bop,\n"
+            "    const int tag,\n"
+            "    const label comm\n"
+            ")"
+        )   << "Reduce called on the processor which is not a member "
+            << "of comm.  This is not allowed"
+            << abort(FatalError);
+    }
+#   endif
+
     allReduce(Value, 1, MPI_LABEL, MPI_MAX, bop, tag, comm);
 }
 
@@ -188,6 +264,25 @@ void Foam::reduce
         error::printStack(Pout);
     }
 
+#   ifdef FULLDEBUG
+    // Check for processors that are not in the communicator
+    if (Pstream::myProcNo(comm) == -1)
+    {
+        FatalErrorIn
+        (
+            "void Foam::reduce\n"
+            "(\n"
+            "    label& Value,\n"
+            "    const sumOp<label>& bop,\n"
+            "    const int tag,\n"
+            "    const label comm\n"
+            ")"
+        )   << "Reduce called on the processor which is not a member "
+            << "of comm.  This is not allowed"
+            << abort(FatalError);
+    }
+#   endif
+
     allReduce(Value, 1, MPI_LABEL, MPI_SUM, bop, tag, comm);
 }
 
@@ -208,6 +303,25 @@ void Foam::reduce
         error::printStack(Pout);
     }
 
+#   ifdef FULLDEBUG
+    // Check for processors that are not in the communicator
+    if (Pstream::myProcNo(comm) == -1)
+    {
+        FatalErrorIn
+        (
+            "void Foam::reduce\n"
+            "(\n"
+            "    scalar& Value,\n"
+            "    const minOp<scalar>& bop,\n"
+            "    const int tag,\n"
+            "    const label comm\n"
+            ")"
+        )   << "Reduce called on the processor which is not a member "
+            << "of comm.  This is not allowed"
+            << abort(FatalError);
+    }
+#   endif
+
     allReduce(Value, 1, MPI_SCALAR, MPI_MIN, bop, tag, comm);
 }
 
@@ -228,6 +342,25 @@ void Foam::reduce
         error::printStack(Pout);
     }
 
+#   ifdef FULLDEBUG
+    // Check for processors that are not in the communicator
+    if (Pstream::myProcNo(comm) == -1)
+    {
+        FatalErrorIn
+        (
+            "void Foam::reduce\n"
+            "(\n"
+            "    scalar& Value,\n"
+            "    const maxOp<scalar>& bop,\n"
+            "    const int tag,\n"
+            "    const label comm\n"
+            ")"
+        )   << "Reduce called on the processor which is not a member "
+            << "of comm.  This is not allowed"
+            << abort(FatalError);
+    }
+#   endif
+
     allReduce(Value, 1, MPI_SCALAR, MPI_MAX, bop, tag, comm);
 }
 
@@ -248,47 +381,26 @@ void Foam::reduce
         error::printStack(Pout);
     }
 
-    allReduce(Value, 1, MPI_SCALAR, MPI_SUM, bop, tag, comm);
-}
-
-
-void Foam::reduce
-(
-    List<label>& Value,
-    const sumOp<List<label> >& bop,
-    const int tag,
-    const label comm
-)
-{
-    if (Pstream::warnComm != -1 && comm != Pstream::warnComm)
-    {
-        Pout<< "** reducing:" << Value << " with comm:" << comm
-            << " warnComm:" << Pstream::warnComm
-            << endl;
-        error::printStack(Pout);
-    }
-
-    // Skip processors that are not in the communicator
+#   ifdef FULLDEBUG
+    // Check for processors that are not in the communicator
     if (Pstream::myProcNo(comm) == -1)
     {
-        return;
+        FatalErrorIn
+        (
+            "void Foam::reduce\n"
+            "(\n"
+            "    scalar& Value,\n"
+            "    const sumOp<scalar>& bop,\n"
+            "    const int tag,\n"
+            "    const label comm\n"
+            ")"
+        )   << "Reduce called on the processor which is not a member "
+            << "of comm.  This is not allowed"
+            << abort(FatalError);
     }
+#   endif
 
-    // Make a copy of the Value in the send buffer so that Value can be
-    // used for receive.  HJ, 8/Oct/2016
-    labelList send(Value);
-
-    int MPISize = Value.size();
-
-    MPI_Allreduce
-    (
-        send.begin(),
-        Value.begin(),
-        MPISize,
-        MPI_LABEL,
-        MPI_SUM,
-        PstreamGlobals::MPICommunicators_[comm]
-    );
+    allReduce(Value, 1, MPI_SCALAR, MPI_SUM, bop, tag, comm);
 }
 
 
@@ -308,11 +420,24 @@ void Foam::reduce
         error::printStack(Pout);
     }
 
-    // Skip processors that are not in the communicator
+#   ifdef FULLDEBUG
+    // Check for processors that are not in the communicator
     if (Pstream::myProcNo(comm) == -1)
     {
-        return;
+        FatalErrorIn
+        (
+            "void Foam::reduce\n"
+            "(\n"
+            "    List<label>& Value,\n"
+            "    const minOp<List<label> >& bop,\n"
+            "    const int tag,\n"
+            "    const label comm\n"
+            ")"
+        )   << "Reduce called on the processor which is not a member "
+            << "of comm.  This is not allowed"
+            << abort(FatalError);
     }
+#   endif
 
     // Make a copy of the Value in the send buffer so that Value can be
     // used for receive.  HJ, 8/Oct/2016
@@ -348,11 +473,77 @@ void Foam::reduce
         error::printStack(Pout);
     }
 
-    // Skip processors that are not in the communicator
+#   ifdef FULLDEBUG
+    // Check for processors that are not in the communicator
     if (Pstream::myProcNo(comm) == -1)
     {
-        return;
+        FatalErrorIn
+        (
+            "void Foam::reduce\n"
+            "(\n"
+            "    List<label>& Value,\n"
+            "    const maxOp<List<label> >& bop,\n"
+            "    const int tag,\n"
+            "    const label comm\n"
+            ")"
+        )   << "Reduce called on the processor which is not a member "
+            << "of comm.  This is not allowed"
+            << abort(FatalError);
     }
+#   endif
+
+    // Make a copy of the Value in the send buffer so that Value can be
+    // used for receive.  HJ, 8/Oct/2016
+    labelList send(Value);
+
+    int MPISize = Value.size();
+    Pout<< "In labelList reduce, maxOp" << endl;
+    MPI_Allreduce
+    (
+        send.begin(),
+        Value.begin(),
+        MPISize,
+        MPI_LABEL,
+        MPI_MAX,
+        PstreamGlobals::MPICommunicators_[comm]
+    );
+}
+
+
+void Foam::reduce
+(
+    List<label>& Value,
+    const sumOp<List<label> >& bop,
+    const int tag,
+    const label comm
+)
+{
+    if (Pstream::warnComm != -1 && comm != Pstream::warnComm)
+    {
+        Pout<< "** reducing:" << Value << " with comm:" << comm
+            << " warnComm:" << Pstream::warnComm
+            << endl;
+        error::printStack(Pout);
+    }
+
+#   ifdef FULLDEBUG
+    // Check for processors that are not in the communicator
+    if (Pstream::myProcNo(comm) == -1)
+    {
+        FatalErrorIn
+        (
+            "void Foam::reduce\n"
+            "(\n"
+            "    List<label>& Value,\n"
+            "    const sumOp<List<label> >& bop,\n"
+            "    const int tag,\n"
+            "    const label comm\n"
+            ")"
+        )   << "Reduce called on the processor which is not a member "
+            << "of comm.  This is not allowed"
+            << abort(FatalError);
+    }
+#   endif
 
     // Make a copy of the Value in the send buffer so that Value can be
     // used for receive.  HJ, 8/Oct/2016
@@ -366,7 +557,7 @@ void Foam::reduce
         Value.begin(),
         MPISize,
         MPI_LABEL,
-        MPI_MAX,
+        MPI_SUM,
         PstreamGlobals::MPICommunicators_[comm]
     );
 }
@@ -388,6 +579,25 @@ void Foam::reduce
         error::printStack(Pout);
     }
 
+#   ifdef FULLDEBUG
+    // Check for processors that are not in the communicator
+    if (Pstream::myProcNo(comm) == -1)
+    {
+        FatalErrorIn
+        (
+            "void Foam::reduce\n"
+            "(\n"
+            "    vector2D& Value,\n"
+            "    const sumOp<vector2D>& bop,\n"
+            "    const int tag,\n"
+            "    const label comm\n"
+            ")"
+        )   << "Reduce called on the processor which is not a member "
+            << "of comm.  This is not allowed"
+            << abort(FatalError);
+    }
+#   endif
+
     allReduce(Value, 2, MPI_SCALAR, MPI_SUM, bop, tag, comm);
 }
 
@@ -408,6 +618,25 @@ void Foam::sumReduce
         error::printStack(Pout);
     }
 
+#   ifdef FULLDEBUG
+    // Check for processors that are not in the communicator
+    if (Pstream::myProcNo(comm) == -1)
+    {
+        FatalErrorIn
+        (
+            "void Foam::sumReduce\n"
+            "(\n"
+            "    scalar& Value,\n"
+            "    label& Count,\n"
+            "    const int tag,\n"
+            "    const label comm\n"
+            ")"
+        )   << "Reduce called on the processor which is not a member "
+            << "of comm.  This is not allowed"
+            << abort(FatalError);
+    }
+#   endif
+
     vector2D twoScalars(Value, scalar(Count));
     reduce(twoScalars, sumOp<vector2D>(), tag, comm);
 
@@ -425,6 +654,26 @@ void Foam::reduce
     label& requestID
 )
 {
+#   ifdef FULLDEBUG
+    // Check for processors that are not in the communicator
+    if (Pstream::myProcNo(comm) == -1)
+    {
+        FatalErrorIn
+        (
+            "void Foam::reduce\n"
+            "(\n"
+            "    scalar& Value,\n"
+            "    const sumOp<scalar>& bop,\n"
+            "    const int tag,\n"
+            "    const label comm,\n"
+            "    label& requestID\n"
+            ")"
+        )   << "Reduce called on the processor which is not a member "
+            << "of comm.  This is not allowed"
+            << abort(FatalError);
+    }
+#   endif
+
 #ifdef MPIX_COMM_TYPE_SHARED
     // Assume mpich2 with non-blocking collectives extensions. Once mpi3
     // is available this will change.

From 9b1d737fd99be93022af277baf2183f864017cc2 Mon Sep 17 00:00:00 2001
From: Hrvoje Jasak <h.jasak@wikki.co.uk>
Date: Tue, 18 Oct 2016 10:51:17 +0100
Subject: [PATCH 077/130] Better debug for allReduce with comms

---
 .../IOstreams/Pstreams/allReduceTemplates.C   | 20 +++++++++++++++++--
 1 file changed, 18 insertions(+), 2 deletions(-)

diff --git a/src/foam/db/IOstreams/Pstreams/allReduceTemplates.C b/src/foam/db/IOstreams/Pstreams/allReduceTemplates.C
index 31117b950..0d95f05f8 100644
--- a/src/foam/db/IOstreams/Pstreams/allReduceTemplates.C
+++ b/src/foam/db/IOstreams/Pstreams/allReduceTemplates.C
@@ -47,11 +47,27 @@ void Foam::allReduce
     // Removed send-received loop: use Allreduce instead.
     // HJ, 8/Oct/2016
 
-    // Skip processors that are not in the communicator
+#   ifdef FULLDEBUG
+    // Check for processors that are not in the communicator
     if (Pstream::myProcNo(comm) == -1)
     {
-        return;
+        FatalErrorIn
+        (
+            "void Foam::allReduce\n"
+            "(\n"
+            "    Type& Value,\n"
+            "    int MPICount,\n"
+            "    MPI_Datatype MPIType,\n"
+            "    MPI_Op MPIOp,\n"
+            "    const BinaryOp& bop,\n"
+            "    const int tag,\n"
+            "    const label comm\n"
+            ")"
+        )   << "Reduce called on the processor which is not a member "
+            << "of comm.  This is not allowed"
+            << abort(FatalError);
     }
+#   endif
 
     Type sum;
 

From 4ecd13081a94a3c12d9c78e74cbd464b9657a7ee Mon Sep 17 00:00:00 2001
From: Hrvoje Jasak <h.jasak@wikki.co.uk>
Date: Tue, 18 Oct 2016 11:15:12 +0100
Subject: [PATCH 078/130] Removed Info message

---
 src/foam/db/IOstreams/Pstreams/PstreamReduceOps.C | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/foam/db/IOstreams/Pstreams/PstreamReduceOps.C b/src/foam/db/IOstreams/Pstreams/PstreamReduceOps.C
index 28d93e559..4b58153b1 100644
--- a/src/foam/db/IOstreams/Pstreams/PstreamReduceOps.C
+++ b/src/foam/db/IOstreams/Pstreams/PstreamReduceOps.C
@@ -497,7 +497,7 @@ void Foam::reduce
     labelList send(Value);
 
     int MPISize = Value.size();
-    Pout<< "In labelList reduce, maxOp" << endl;
+
     MPI_Allreduce
     (
         send.begin(),

From 149a3fc2af16e757ac3d8e48460dc79c0defe364 Mon Sep 17 00:00:00 2001
From: Hrvoje Jasak <h.jasak@wikki.co.uk>
Date: Tue, 18 Oct 2016 11:22:08 +0100
Subject: [PATCH 079/130] Checking performance with different comms types

---
 .../mapPolyMesh/mapDistribute/mapDistribute.C | 43 +++++++++++++++----
 1 file changed, 34 insertions(+), 9 deletions(-)

diff --git a/src/foam/meshes/polyMesh/mapPolyMesh/mapDistribute/mapDistribute.C b/src/foam/meshes/polyMesh/mapPolyMesh/mapDistribute/mapDistribute.C
index e8142cbd7..59c83fe51 100644
--- a/src/foam/meshes/polyMesh/mapPolyMesh/mapDistribute/mapDistribute.C
+++ b/src/foam/meshes/polyMesh/mapPolyMesh/mapDistribute/mapDistribute.C
@@ -70,12 +70,20 @@ Foam::List<Foam::labelPair> Foam::mapDistribute::schedule
         // Receive and merge
         for
         (
-            int slave=Pstream::firstSlave();
-            slave<=Pstream::lastSlave();
+            int slave = Pstream::firstSlave();
+            slave <= Pstream::lastSlave();
             slave++
         )
         {
-            IPstream fromSlave(Pstream::scheduled, slave, 0, tag);
+            IPstream fromSlave
+            (
+                Pstream::defaultComms(),
+//                Pstream::scheduled,
+                slave,
+                0,
+                tag
+            );
+
             List<labelPair> nbrData(fromSlave);
 
             forAll (nbrData, i)
@@ -83,7 +91,7 @@ Foam::List<Foam::labelPair> Foam::mapDistribute::schedule
                 if (findIndex(allComms, nbrData[i]) == -1)
                 {
                     label sz = allComms.size();
-                    allComms.setSize(sz+1);
+                    allComms.setSize(sz + 1);
                     allComms[sz] = nbrData[i];
                 }
             }
@@ -91,25 +99,42 @@ Foam::List<Foam::labelPair> Foam::mapDistribute::schedule
         // Send back
         for
         (
-            int slave=Pstream::firstSlave();
-            slave<=Pstream::lastSlave();
+            int slave = Pstream::firstSlave();
+            slave <= Pstream::lastSlave();
             slave++
         )
         {
-            OPstream toSlave(Pstream::scheduled, slave, 0, tag);
+            OPstream toSlave
+            (
+                Pstream::defaultComms(),
+//                Pstream::scheduled,
+                slave,
+                0,
+                tag
+            );
+
             toSlave << allComms;
         }
     }
     else
     {
         {
-            OPstream toMaster(Pstream::scheduled, Pstream::masterNo(), 0, tag);
+            OPstream toMaster
+            (
+                Pstream::defaultComms(),
+//                Pstream::scheduled,
+                Pstream::masterNo(),
+                0,
+                tag
+            );
+
             toMaster << allComms;
         }
         {
             IPstream fromMaster
             (
-                Pstream::scheduled,
+                Pstream::defaultComms(),
+//                Pstream::scheduled,
                 Pstream::masterNo(),
                 0,
                 tag

From e9758a867deea5d6078c215c8561f379cd8e47f8 Mon Sep 17 00:00:00 2001
From: Hrvoje Jasak <h.jasak@wikki.co.uk>
Date: Tue, 18 Oct 2016 13:20:12 +0100
Subject: [PATCH 080/130] Use comms in ordered Pstream communication for
 ggiAMGInterface

---
 .../ggiAMGInterface/ggiAMGInterface.C         | 23 ++++++++++++++++---
 1 file changed, 20 insertions(+), 3 deletions(-)

diff --git a/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/ggiAMGInterface/ggiAMGInterface.C b/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/ggiAMGInterface/ggiAMGInterface.C
index 44e9ee8dd..3a6adf3cd 100644
--- a/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/ggiAMGInterface/ggiAMGInterface.C
+++ b/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/ggiAMGInterface/ggiAMGInterface.C
@@ -250,12 +250,32 @@ Foam::ggiAMGInterface::ggiAMGInterface
         << fineGgiInterface_.interfaceSize() << ": "
         << lTime_.elapsedCpuTime() << endl;
 
+    // Initialise fine map
+    // Note: the mapDistribute contains a waitRequests call which cannot
+    // be limited to a comm.  Therefore, all processors need to calculate
+    // the mapDistribute schedule before escaping the constructor,
+    // even if there are no ggi faces available.
+    // HJ, 18/Oct/2016
+    fineGgiInterface_.map().schedule();
+
     // If the processor is not in the GGI comm, escape.  HJ, 10/Oct/2016
     if (Pstream::myProcNo(comm()) == -1)
     {
+        // Processor does not have either master or shadow faces
+        // Set all sizes to zero
+        zoneSize_ = 0;
+        zoneAddressing_.clear();
+        fineAddressing_.clear();
+        restrictAddressing_.clear();
+        restrictWeights_.clear();
+
         return;
     }
 
+    // Continuing only with interfaces within the GGI comm only.
+    // Note: on interfaces without the GGI comm, zone size will be zero
+    // HJ, 11/Oct/2016
+
     // Note: local addressing contains only local faces
     const labelList& fineZa =  fineGgiInterface_.zoneAddressing();
 
@@ -742,9 +762,6 @@ Foam::ggiAMGInterface::ggiAMGInterface
     // contact with the GGI interface will have zero zone size.
     // This needs to be handled separately in the initFastReduce
     // HJ, 20/Sep/2016
-
-    // Optimised comm: Wait for info from previous processor, add your
-    // number of coarse faces and pass to next processor
     reduce(nCoarseFacesPerProc, sumOp<List<label> >(), tag(), comm());
 
     // Coarse global face zone is assembled by adding all faces from proc0,

From 1ef446246124316331dfba5fdf07febb4ddb05e2 Mon Sep 17 00:00:00 2001
From: Hrvoje Jasak <h.jasak@wikki.co.uk>
Date: Wed, 19 Oct 2016 16:46:36 +0100
Subject: [PATCH 081/130] Bugfix: reduce in serial does nothing

---
 .../db/IOstreams/Pstreams/PstreamReduceOps.C  | 20 +++++++++++++++++++
 1 file changed, 20 insertions(+)

diff --git a/src/foam/db/IOstreams/Pstreams/PstreamReduceOps.C b/src/foam/db/IOstreams/Pstreams/PstreamReduceOps.C
index 4b58153b1..b0a7afc0b 100644
--- a/src/foam/db/IOstreams/Pstreams/PstreamReduceOps.C
+++ b/src/foam/db/IOstreams/Pstreams/PstreamReduceOps.C
@@ -412,6 +412,11 @@ void Foam::reduce
     const label comm
 )
 {
+    if (!Pstream::parRun())
+    {
+        return;
+    }
+
     if (Pstream::warnComm != -1 && comm != Pstream::warnComm)
     {
         Pout<< "** reducing:" << Value << " with comm:" << comm
@@ -465,6 +470,11 @@ void Foam::reduce
     const label comm
 )
 {
+    if (!Pstream::parRun())
+    {
+        return;
+    }
+
     if (Pstream::warnComm != -1 && comm != Pstream::warnComm)
     {
         Pout<< "** reducing:" << Value << " with comm:" << comm
@@ -518,6 +528,11 @@ void Foam::reduce
     const label comm
 )
 {
+    if (!Pstream::parRun())
+    {
+        return;
+    }
+
     if (Pstream::warnComm != -1 && comm != Pstream::warnComm)
     {
         Pout<< "** reducing:" << Value << " with comm:" << comm
@@ -654,6 +669,11 @@ void Foam::reduce
     label& requestID
 )
 {
+    if (!Pstream::parRun())
+    {
+        return;
+    }
+
 #   ifdef FULLDEBUG
     // Check for processors that are not in the communicator
     if (Pstream::myProcNo(comm) == -1)

From db04b34cff284c88adb8507a1cedded48fc720a4 Mon Sep 17 00:00:00 2001
From: Hrvoje Jasak <h.jasak@wikki.co.uk>
Date: Wed, 19 Oct 2016 16:47:30 +0100
Subject: [PATCH 082/130] Bugfix: serial execution does not require mapping

---
 .../ggiAMGInterface/ggiAMGInterface.C           | 17 ++++++-----------
 1 file changed, 6 insertions(+), 11 deletions(-)

diff --git a/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/ggiAMGInterface/ggiAMGInterface.C b/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/ggiAMGInterface/ggiAMGInterface.C
index 3a6adf3cd..b217d64e4 100644
--- a/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/ggiAMGInterface/ggiAMGInterface.C
+++ b/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/ggiAMGInterface/ggiAMGInterface.C
@@ -71,7 +71,7 @@ void Foam::ggiAMGInterface::initFastReduce() const
 
         return;
     }
-    Info<< "Start initFastReduce " << lTime_.elapsedCpuTime() << endl;
+
     // From here on, work on processors within the communicator
     // HJ, 20/Sep/2016
 
@@ -215,7 +215,6 @@ void Foam::ggiAMGInterface::initFastReduce() const
     // Map will return the object of the size of remote zone
     // HJ, 9/May/2016
     mapPtr_ = new mapDistribute(zoneSize(), sendMap, constructMap);
-    Info<< "End initFastReduce " << lTime_.elapsedCpuTime() << endl;
 }
 
 
@@ -246,9 +245,6 @@ Foam::ggiAMGInterface::ggiAMGInterface
     // each processor only to perform the analysis on locally created coarse
     // faces
     // HJ, 13/Jun/2016
-    Info<< "Start ggiAMGInterface constructor for size "
-        << fineGgiInterface_.interfaceSize() << ": "
-        << lTime_.elapsedCpuTime() << endl;
 
     // Initialise fine map
     // Note: the mapDistribute contains a waitRequests call which cannot
@@ -256,7 +252,10 @@ Foam::ggiAMGInterface::ggiAMGInterface
     // the mapDistribute schedule before escaping the constructor,
     // even if there are no ggi faces available.
     // HJ, 18/Oct/2016
-    fineGgiInterface_.map().schedule();
+    if (Pstream::parRun())
+    {
+        fineGgiInterface_.map().schedule();
+    }
 
     // If the processor is not in the GGI comm, escape.  HJ, 10/Oct/2016
     if (Pstream::myProcNo(comm()) == -1)
@@ -803,7 +802,6 @@ Foam::ggiAMGInterface::ggiAMGInterface
     // the local zone is created.  HJ, 1/Aug/2016
     if (master())
     {
-        Info<< "ggiAMGInterface start agglom master: " << lTime_.elapsedCpuTime() << endl;
         // Note:
         // When I am agglomerating the master, faces are stacked up in order
         // but on the slave side, all I know is the master cluster index and
@@ -947,7 +945,6 @@ Foam::ggiAMGInterface::ggiAMGInterface
                 }
             }
         }
-        Info<< "ggiAMGInterface end agglom master " << lTime_.elapsedCpuTime() << endl;
     }
     // Agglomerate slave
     else
@@ -974,7 +971,7 @@ Foam::ggiAMGInterface::ggiAMGInterface
 
         // Count how many global faces are used for each proc on the other side
         labelList npmf(Pstream::nProcs(), 0);
-        Info<< "ggiAMGInterface start agglom slave: " << lTime_.elapsedCpuTime() << endl;
+
         // On slave side, the owner addressing is stored in linked lists
         forAll (contents, masterI)
         {
@@ -1041,9 +1038,7 @@ Foam::ggiAMGInterface::ggiAMGInterface
                 nProcFaces++;
             }
         }
-        Info<< "ggiAMGInterface end agglom slave " << lTime_.elapsedCpuTime() << endl;
     }
-    Info<< "End ggiAMGInterface constructor " << lTime_.elapsedCpuTime() << endl;
 }
 
 

From 5e9baf85c699420034b0d0fb75d58e87b3ce2779 Mon Sep 17 00:00:00 2001
From: Hrvoje Jasak <h.jasak@wikki.co.uk>
Date: Wed, 19 Oct 2016 16:47:55 +0100
Subject: [PATCH 083/130] Info on debug: allocation of comms

---
 .../polyMesh/polyPatches/constraint/ggi/ggiPolyPatch.C   | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/src/foam/meshes/polyMesh/polyPatches/constraint/ggi/ggiPolyPatch.C b/src/foam/meshes/polyMesh/polyPatches/constraint/ggi/ggiPolyPatch.C
index 6f949c1cc..c9f6bc334 100644
--- a/src/foam/meshes/polyMesh/polyPatches/constraint/ggi/ggiPolyPatch.C
+++ b/src/foam/meshes/polyMesh/polyPatches/constraint/ggi/ggiPolyPatch.C
@@ -385,9 +385,12 @@ void Foam::ggiPolyPatch::calcLocalParallel() const
             ggiCommProcs
         );
 
-        Info<< "Allocating communicator for GGI patch " << name()
-            << " with " << ggiCommProcs << ": " << comm_
-            << endl;
+        if (debug && Pstream::parRun())
+        {
+            Info<< "Allocating communicator for GGI patch " << name()
+                << " with " << ggiCommProcs << ": " << comm_
+                << endl;
+        }
     }
     else
     {

From efb22e56fd29e11e7825bdc7752c46b6a59fd92c Mon Sep 17 00:00:00 2001
From: Hrvoje Jasak <h.jasak@wikki.co.uk>
Date: Wed, 19 Oct 2016 16:48:18 +0100
Subject: [PATCH 084/130] Testing: remove barrier

---
 .../meshes/polyMesh/mapPolyMesh/mapDistribute/mapDistribute.C  | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/src/foam/meshes/polyMesh/mapPolyMesh/mapDistribute/mapDistribute.C b/src/foam/meshes/polyMesh/mapPolyMesh/mapDistribute/mapDistribute.C
index 59c83fe51..e21511762 100644
--- a/src/foam/meshes/polyMesh/mapPolyMesh/mapDistribute/mapDistribute.C
+++ b/src/foam/meshes/polyMesh/mapPolyMesh/mapDistribute/mapDistribute.C
@@ -420,8 +420,7 @@ void Foam::mapDistribute::compact(const boolList& elemIsUsed, const int tag)
 
 
         // Wait for all to finish
-
-        Pstream::waitRequests(startOfRequests);
+//         Pstream::waitRequests(startOfRequests);
 
 
         // Compact out all submap entries that are referring to unused elements

From 1460d200879b6eb63b4df17afd516314c9b6fafd Mon Sep 17 00:00:00 2001
From: Hrvoje Jasak <h.jasak@wikki.co.uk>
Date: Tue, 25 Oct 2016 11:15:33 +0100
Subject: [PATCH 085/130] Barrier messaging

---
 .../mapPolyMesh/mapDistribute/mapDistribute.C     | 15 ++++++---------
 .../mapDistribute/mapDistributeTemplates.C        |  9 ++++++---
 2 files changed, 12 insertions(+), 12 deletions(-)

diff --git a/src/foam/meshes/polyMesh/mapPolyMesh/mapDistribute/mapDistribute.C b/src/foam/meshes/polyMesh/mapPolyMesh/mapDistribute/mapDistribute.C
index e21511762..a4840f6d8 100644
--- a/src/foam/meshes/polyMesh/mapPolyMesh/mapDistribute/mapDistribute.C
+++ b/src/foam/meshes/polyMesh/mapPolyMesh/mapDistribute/mapDistribute.C
@@ -77,8 +77,7 @@ Foam::List<Foam::labelPair> Foam::mapDistribute::schedule
         {
             IPstream fromSlave
             (
-                Pstream::defaultComms(),
-//                Pstream::scheduled,
+                Pstream::scheduled,
                 slave,
                 0,
                 tag
@@ -106,8 +105,7 @@ Foam::List<Foam::labelPair> Foam::mapDistribute::schedule
         {
             OPstream toSlave
             (
-                Pstream::defaultComms(),
-//                Pstream::scheduled,
+                Pstream::scheduled,
                 slave,
                 0,
                 tag
@@ -121,8 +119,7 @@ Foam::List<Foam::labelPair> Foam::mapDistribute::schedule
         {
             OPstream toMaster
             (
-                Pstream::defaultComms(),
-//                Pstream::scheduled,
+                Pstream::scheduled,
                 Pstream::masterNo(),
                 0,
                 tag
@@ -133,8 +130,7 @@ Foam::List<Foam::labelPair> Foam::mapDistribute::schedule
         {
             IPstream fromMaster
             (
-                Pstream::defaultComms(),
-//                Pstream::scheduled,
+                Pstream::scheduled,
                 Pstream::masterNo(),
                 0,
                 tag
@@ -420,7 +416,8 @@ void Foam::mapDistribute::compact(const boolList& elemIsUsed, const int tag)
 
 
         // Wait for all to finish
-//         Pstream::waitRequests(startOfRequests);
+        Pout<< "BARRIER in compact" << endl;
+        Pstream::waitRequests(startOfRequests);
 
 
         // Compact out all submap entries that are referring to unused elements
diff --git a/src/foam/meshes/polyMesh/mapPolyMesh/mapDistribute/mapDistributeTemplates.C b/src/foam/meshes/polyMesh/mapPolyMesh/mapDistribute/mapDistributeTemplates.C
index 43aa235a2..cec2f0fc0 100644
--- a/src/foam/meshes/polyMesh/mapPolyMesh/mapDistribute/mapDistributeTemplates.C
+++ b/src/foam/meshes/polyMesh/mapPolyMesh/mapDistribute/mapDistributeTemplates.C
@@ -237,6 +237,7 @@ void Foam::mapDistribute::distribute
             }
 
             // Block ourselves, waiting only for the current comms
+            Pout<< "TEMPLATE BARRIER in distribute, non-blocking non-contiguous" << endl;
             Pstream::waitRequests(nOutstanding);
 
             // Consume
@@ -258,7 +259,7 @@ void Foam::mapDistribute::distribute
                 }
             }
         }
-        else
+        else // contiguous data
         {
             // Set up sends to neighbours
 
@@ -343,7 +344,7 @@ void Foam::mapDistribute::distribute
 
 
             // Wait for all to finish
-
+            Pout<< "TEMPLATE BARRIER in distribute, non-blocking, contiguous" << endl;
             Pstream::waitRequests(nOutstanding);
 
 
@@ -584,6 +585,7 @@ void Foam::mapDistribute::distribute
             }
 
             // Block ourselves, waiting only for the current comms
+            Pout<< "TEMPLATE BARRIER in distribute, non-blocking non-contiguous 2" << endl;
             Pstream::waitRequests(nOutstanding);
 
             // Consume
@@ -605,7 +607,7 @@ void Foam::mapDistribute::distribute
                 }
             }
         }
-        else
+        else // Contiguous data
         {
             // Set up sends to neighbours
 
@@ -689,6 +691,7 @@ void Foam::mapDistribute::distribute
 
 
             // Wait for all to finish
+            Pout<< "TEMPLATE BARRIER in distribute, non-blocking contiguous 2" << endl;
             Pstream::waitRequests(nOutstanding);
 
 

From 5d63832d6ec5a488c69b8545d8e55a3acb3f3568 Mon Sep 17 00:00:00 2001
From: Hrvoje Jasak <h.jasak@wikki.co.uk>
Date: Tue, 25 Oct 2016 11:15:51 +0100
Subject: [PATCH 086/130] Pstream output data only if debug > 1

---
 src/foam/db/IOstreams/Pstreams/gatherScatterList.C | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/foam/db/IOstreams/Pstreams/gatherScatterList.C b/src/foam/db/IOstreams/Pstreams/gatherScatterList.C
index 8f872e384..b5185961a 100644
--- a/src/foam/db/IOstreams/Pstreams/gatherScatterList.C
+++ b/src/foam/db/IOstreams/Pstreams/gatherScatterList.C
@@ -294,7 +294,7 @@ void Pstream::scatterList
                     label leafID = notBelowLeaves[leafI];
                     fromAbove >> Values[leafID];
 
-                    if (debug)
+                    if (debug > 1)
                     {
                         Pout<< " received through "
                             << myComm.above() << " data for:" << leafID
@@ -342,7 +342,7 @@ void Pstream::scatterList
                     label leafID = notBelowLeaves[leafI];
                     toBelow << Values[leafID];
 
-                    if (debug)
+                    if (debug > 1)
                     {
                         Pout<< " sent through "
                             << belowID << " data for:" << leafID

From e806141209ea6eeacc49d2b12b488a299e60f693 Mon Sep 17 00:00:00 2001
From: Hrvoje Jasak <h.jasak@wikki.co.uk>
Date: Tue, 25 Oct 2016 11:16:04 +0100
Subject: [PATCH 087/130] UIPstream comment fix

---
 src/foam/db/IOstreams/Pstreams/IPread.C | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/foam/db/IOstreams/Pstreams/IPread.C b/src/foam/db/IOstreams/Pstreams/IPread.C
index 69142cc4a..abaddcef7 100644
--- a/src/foam/db/IOstreams/Pstreams/IPread.C
+++ b/src/foam/db/IOstreams/Pstreams/IPread.C
@@ -108,7 +108,7 @@ Foam::label Foam::IPstream::read
 {
     if (debug)
     {
-        Pout<< "UIPstream::read : starting read from:" << fromProcNo
+        Pout<< "IPstream::read : starting read from:" << fromProcNo
             << " tag:" << tag << " comm:" << comm
             << " wanted size:" << label(bufSize)
             << " commsType:" << Pstream::commsTypeNames[commsType]
@@ -117,7 +117,7 @@ Foam::label Foam::IPstream::read
 
     if (Pstream::warnComm != -1 && comm != Pstream::warnComm)
     {
-        Pout<< "UIPstream::read : starting read from:" << fromProcNo
+        Pout<< "IPstream::read : starting read from:" << fromProcNo
             << " tag:" << tag << " comm:" << comm
             << " wanted size:" << label(bufSize)
             << " commsType:" << Pstream::commsTypeNames[commsType]
@@ -204,7 +204,7 @@ Foam::label Foam::IPstream::read
 
         if (debug)
         {
-            Pout<< "UIPstream::read : started read from:" << fromProcNo
+            Pout<< "IPstream::read : started read from:" << fromProcNo
                 << " tag:" << tag << " read size:" << label(bufSize)
                 << " commsType:" << Pstream::commsTypeNames[commsType]
                 << " request:" << PstreamGlobals::outstandingRequests_.size()

From cf5230e54da4fd0c74a5c3f8dc066359f75ccafa Mon Sep 17 00:00:00 2001
From: Hrvoje Jasak <h.jasak@wikki.co.uk>
Date: Tue, 25 Oct 2016 11:17:46 +0100
Subject: [PATCH 088/130] Better handling of flex versions

---
 .../utilities/mesh/conversion/ansysToFoam/ansysToFoam.L   | 8 ++++----
 .../conversion/fluent3DMeshToElmer/fluent3DMeshToElmer.L  | 8 ++++----
 .../conversion/fluent3DMeshToFoam/fluent3DMeshToFoam.L    | 2 +-
 .../mesh/conversion/fluentMeshToFoam/fluentMeshToFoam.L   | 2 +-
 .../utilities/mesh/conversion/gambitToFoam/gambitToFoam.L | 8 ++++----
 .../triSurface/triSurface/interfaces/STL/readSTLASCII.L   | 8 ++++----
 src/surfMesh/surfaceFormats/stl/STLsurfaceFormatASCII.L   | 2 +-
 .../chemistryReaders/chemkinReader/chemkinLexer.L         | 2 +-
 8 files changed, 20 insertions(+), 20 deletions(-)

diff --git a/applications/utilities/mesh/conversion/ansysToFoam/ansysToFoam.L b/applications/utilities/mesh/conversion/ansysToFoam/ansysToFoam.L
index fb6cd4557..85c8e4199 100644
--- a/applications/utilities/mesh/conversion/ansysToFoam/ansysToFoam.L
+++ b/applications/utilities/mesh/conversion/ansysToFoam/ansysToFoam.L
@@ -77,11 +77,11 @@ label currentTypei = -1;
 // Dummy yywrap to keep yylex happy at compile time.
 // It is called by yylex but is not used as the mechanism to change file.
 // See <<EOF>>
-// #if YY_FLEX_SUBMINOR_VERSION < 34
-// extern "C" int yywrap()
-// #else
+#if YY_FLEX_MINOR_VERSION < 6 && YY_FLEX_SUBMINOR_VERSION < 34
+extern "C" int yywrap()
+#else
 int yyFlexLexer::yywrap()
-// #endif
+#endif
 {
     return 1;
 }
diff --git a/applications/utilities/mesh/conversion/fluent3DMeshToElmer/fluent3DMeshToElmer.L b/applications/utilities/mesh/conversion/fluent3DMeshToElmer/fluent3DMeshToElmer.L
index 153b917bf..da5115660 100644
--- a/applications/utilities/mesh/conversion/fluent3DMeshToElmer/fluent3DMeshToElmer.L
+++ b/applications/utilities/mesh/conversion/fluent3DMeshToElmer/fluent3DMeshToElmer.L
@@ -125,11 +125,11 @@ void uniquify(word& name, HashSet<word>& patchNames)
 // Dummy yywrap to keep yylex happy at compile time.
 // It is called by yylex but is not used as the mechanism to change file.
 // See <<EOF>>
-// #if YY_FLEX_SUBMINOR_VERSION < 34
-// extern "C" int yywrap()
-// #else
+#if YY_FLEX_MINOR_VERSION < 6 && YY_FLEX_SUBMINOR_VERSION < 34
+extern "C" int yywrap()
+#else
 int yyFlexLexer::yywrap()
-// #endif
+#endif
 {
     return 1;
 }
diff --git a/applications/utilities/mesh/conversion/fluent3DMeshToFoam/fluent3DMeshToFoam.L b/applications/utilities/mesh/conversion/fluent3DMeshToFoam/fluent3DMeshToFoam.L
index 4a6eaad20..766b26dd7 100644
--- a/applications/utilities/mesh/conversion/fluent3DMeshToFoam/fluent3DMeshToFoam.L
+++ b/applications/utilities/mesh/conversion/fluent3DMeshToFoam/fluent3DMeshToFoam.L
@@ -124,7 +124,7 @@ void uniquify(word& name, HashSet<word>& patchNames)
 // Dummy yywrap to keep yylex happy at compile time.
 // It is called by yylex but is not used as the mechanism to change file.
 // See <<EOF>>
-#if YY_FLEX_SUBMINOR_VERSION < 34
+#if YY_FLEX_MINOR_VERSION < 6 && YY_FLEX_SUBMINOR_VERSION < 34
 extern "C" int yywrap()
 #else
 int yyFlexLexer::yywrap()
diff --git a/applications/utilities/mesh/conversion/fluentMeshToFoam/fluentMeshToFoam.L b/applications/utilities/mesh/conversion/fluentMeshToFoam/fluentMeshToFoam.L
index 30ed08386..934bdd18c 100644
--- a/applications/utilities/mesh/conversion/fluentMeshToFoam/fluentMeshToFoam.L
+++ b/applications/utilities/mesh/conversion/fluentMeshToFoam/fluentMeshToFoam.L
@@ -103,7 +103,7 @@ wordList patchNameIDs(maxZoneID);
 // Dummy yywrap to keep yylex happy at compile time.
 // It is called by yylex but is not used as the mechanism to change file.
 // See <<EOF>>
-#if YY_FLEX_SUBMINOR_VERSION < 34
+#if YY_FLEX_MINOR_VERSION < 6 && YY_FLEX_SUBMINOR_VERSION < 34
 extern "C" int yywrap()
 #else
 int yyFlexLexer::yywrap()
diff --git a/applications/utilities/mesh/conversion/gambitToFoam/gambitToFoam.L b/applications/utilities/mesh/conversion/gambitToFoam/gambitToFoam.L
index 513d21ec0..fcb70223d 100644
--- a/applications/utilities/mesh/conversion/gambitToFoam/gambitToFoam.L
+++ b/applications/utilities/mesh/conversion/gambitToFoam/gambitToFoam.L
@@ -81,11 +81,11 @@ label nValuesForPatchFaces = 0;
 // Dummy yywrap to keep yylex happy at compile time.
 // It is called by yylex but is not used as the mechanism to change file.
 // See <<EOF>>
-// #if YY_FLEX_SUBMINOR_VERSION < 34
-// extern "C" int yywrap()
-// #else
+#if YY_FLEX_MINOR_VERSION < 6 && YY_FLEX_SUBMINOR_VERSION < 34
+extern "C" int yywrap()
+#else
 int yyFlexLexer::yywrap()
-// #endif
+#endif
 {
     return 1;
 }
diff --git a/src/meshTools/triSurface/triSurface/interfaces/STL/readSTLASCII.L b/src/meshTools/triSurface/triSurface/interfaces/STL/readSTLASCII.L
index 0792dd56e..a63fc7c04 100644
--- a/src/meshTools/triSurface/triSurface/interfaces/STL/readSTLASCII.L
+++ b/src/meshTools/triSurface/triSurface/interfaces/STL/readSTLASCII.L
@@ -53,11 +53,11 @@ int yyFlexLexer::yylex()
 // It is called by yylex but is not used as the mechanism to change file.
 // See <<EOF>>
 //! @cond dummy
-// #if YY_FLEX_SUBMINOR_VERSION < 34
-// extern "C" int yywrap()
-// #else
+#if YY_FLEX_MINOR_VERSION < 6 && YY_FLEX_SUBMINOR_VERSION < 34
+extern "C" int yywrap()
+#else
 int yyFlexLexer::yywrap()
-// #endif
+#endif
 {
     return 1;
 }
diff --git a/src/surfMesh/surfaceFormats/stl/STLsurfaceFormatASCII.L b/src/surfMesh/surfaceFormats/stl/STLsurfaceFormatASCII.L
index ecb74e7c1..e392ef8f6 100644
--- a/src/surfMesh/surfaceFormats/stl/STLsurfaceFormatASCII.L
+++ b/src/surfMesh/surfaceFormats/stl/STLsurfaceFormatASCII.L
@@ -50,7 +50,7 @@ int yyFlexLexer::yylex()
 // It is called by yylex but is not used as the mechanism to change file.
 // See <<EOF>>
 //! @cond dummy
-#if YY_FLEX_SUBMINOR_VERSION < 34
+#if YY_FLEX_MINOR_VERSION < 6 && YY_FLEX_SUBMINOR_VERSION < 34
 extern "C" int yywrap()
 #else
 int yyFlexLexer::yywrap()
diff --git a/src/thermophysicalModels/reactionThermo/chemistryReaders/chemkinReader/chemkinLexer.L b/src/thermophysicalModels/reactionThermo/chemistryReaders/chemkinReader/chemkinLexer.L
index 466d8e017..23dfca728 100644
--- a/src/thermophysicalModels/reactionThermo/chemistryReaders/chemkinReader/chemkinLexer.L
+++ b/src/thermophysicalModels/reactionThermo/chemistryReaders/chemkinReader/chemkinLexer.L
@@ -54,7 +54,7 @@ int yyFlexLexer::yylex()
 // It is called by yylex but is not used as the mechanism to change file.
 // See <<EOF>>
 //! @cond dummy
-#if YY_FLEX_SUBMINOR_VERSION < 34
+#if YY_FLEX_MINOR_VERSION < 6 && YY_FLEX_SUBMINOR_VERSION < 34
 extern "C" int yywrap()
 #else
 int yyFlexLexer::yywrap()

From e971799c47e8ed1d9728d40316a92581cd9f756b Mon Sep 17 00:00:00 2001
From: Hrvoje Jasak <h.jasak@wikki.co.uk>
Date: Tue, 25 Oct 2016 11:17:57 +0100
Subject: [PATCH 089/130] Removed x permission

---
 wmake/rules/linux64Gcc/mplibINTELMPI | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 mode change 100755 => 100644 wmake/rules/linux64Gcc/mplibINTELMPI

diff --git a/wmake/rules/linux64Gcc/mplibINTELMPI b/wmake/rules/linux64Gcc/mplibINTELMPI
old mode 100755
new mode 100644

From f4f6e9248ff7d57fc5772c4a602cd5b248d80a71 Mon Sep 17 00:00:00 2001
From: Hrvoje Jasak <h.jasak@wikki.co.uk>
Date: Tue, 25 Oct 2016 11:18:55 +0100
Subject: [PATCH 090/130] Indentation

---
 etc/controlDict-EXAMPLE | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/etc/controlDict-EXAMPLE b/etc/controlDict-EXAMPLE
index e70f4285f..011913358 100644
--- a/etc/controlDict-EXAMPLE
+++ b/etc/controlDict-EXAMPLE
@@ -943,7 +943,7 @@ Tolerances
     slidingEdgeEndCutoffTol         0.0001;
     slidingEdgeCoPlanarTol          0.8;
 
-     GGIAreaErrorTol                 1e-8;
+    GGIAreaErrorTol                 1e-8;
 }
 
 DimensionedConstants

From 585900828804848d39d5ea19a7dbdce6a1a890e6 Mon Sep 17 00:00:00 2001
From: Hrvoje Jasak <h.jasak@wikki.co.uk>
Date: Tue, 25 Oct 2016 11:19:18 +0100
Subject: [PATCH 091/130] Comment

---
 src/finiteVolume/cfdTools/general/findRefCell/findRefCell.C | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/finiteVolume/cfdTools/general/findRefCell/findRefCell.C b/src/finiteVolume/cfdTools/general/findRefCell/findRefCell.C
index 25f33159b..a0dca512b 100644
--- a/src/finiteVolume/cfdTools/general/findRefCell/findRefCell.C
+++ b/src/finiteVolume/cfdTools/general/findRefCell/findRefCell.C
@@ -62,7 +62,7 @@ void Foam::setRefCell
                          ")",
                         dict
                     )   << "Illegal master cellID " << refCelli
-                        << ". Should be 0.." << field.mesh().nCells()
+                        << ". Should be 0." << field.mesh().nCells()
                         << exit(FatalIOError);
                 }
             }

From 71e45fa3815fdd6f5a51de0c7880041f117e9b88 Mon Sep 17 00:00:00 2001
From: Hrvoje Jasak <h.jasak@wikki.co.uk>
Date: Mon, 28 Nov 2016 12:08:33 +0000
Subject: [PATCH 092/130] Formatting and removal of debug statements

---
 .../polyMesh/mapPolyMesh/mapDistribute/mapDistribute.C   | 5 ++---
 .../mapPolyMesh/mapDistribute/mapDistributeTemplates.C   | 9 ---------
 2 files changed, 2 insertions(+), 12 deletions(-)

diff --git a/src/foam/meshes/polyMesh/mapPolyMesh/mapDistribute/mapDistribute.C b/src/foam/meshes/polyMesh/mapPolyMesh/mapDistribute/mapDistribute.C
index a4840f6d8..2e37ed6f1 100644
--- a/src/foam/meshes/polyMesh/mapPolyMesh/mapDistribute/mapDistribute.C
+++ b/src/foam/meshes/polyMesh/mapPolyMesh/mapDistribute/mapDistribute.C
@@ -188,6 +188,7 @@ const Foam::List<Foam::labelPair>& Foam::mapDistribute::schedule() const
             )
         );
     }
+
     return schedulePtr_();
 }
 
@@ -414,12 +415,9 @@ void Foam::mapDistribute::compact(const boolList& elemIsUsed, const int tag)
             }
         }
 
-
         // Wait for all to finish
-        Pout<< "BARRIER in compact" << endl;
         Pstream::waitRequests(startOfRequests);
 
-
         // Compact out all submap entries that are referring to unused elements
         for (label domain = 0; domain < Pstream::nProcs(); domain++)
         {
@@ -479,6 +477,7 @@ void Foam::mapDistribute::compact(const boolList& elemIsUsed, const int tag)
     constructSize_ = maxConstructIndex + 1;
 
     // Clear the schedule (note:not necessary if nothing changed)
+    Pout<< "CLEAR SCHEDULE POINTER" << endl;
     schedulePtr_.clear();
 }
 
diff --git a/src/foam/meshes/polyMesh/mapPolyMesh/mapDistribute/mapDistributeTemplates.C b/src/foam/meshes/polyMesh/mapPolyMesh/mapDistribute/mapDistributeTemplates.C
index cec2f0fc0..5a5d11719 100644
--- a/src/foam/meshes/polyMesh/mapPolyMesh/mapDistribute/mapDistributeTemplates.C
+++ b/src/foam/meshes/polyMesh/mapPolyMesh/mapDistribute/mapDistributeTemplates.C
@@ -237,7 +237,6 @@ void Foam::mapDistribute::distribute
             }
 
             // Block ourselves, waiting only for the current comms
-            Pout<< "TEMPLATE BARRIER in distribute, non-blocking non-contiguous" << endl;
             Pstream::waitRequests(nOutstanding);
 
             // Consume
@@ -344,12 +343,9 @@ void Foam::mapDistribute::distribute
 
 
             // Wait for all to finish
-            Pout<< "TEMPLATE BARRIER in distribute, non-blocking, contiguous" << endl;
             Pstream::waitRequests(nOutstanding);
 
-
             // Collect neighbour fields
-
             for (label domain = 0; domain < Pstream::nProcs(); domain++)
             {
                 const labelList& map = constructMap[domain];
@@ -585,7 +581,6 @@ void Foam::mapDistribute::distribute
             }
 
             // Block ourselves, waiting only for the current comms
-            Pout<< "TEMPLATE BARRIER in distribute, non-blocking non-contiguous 2" << endl;
             Pstream::waitRequests(nOutstanding);
 
             // Consume
@@ -689,14 +684,10 @@ void Foam::mapDistribute::distribute
                 }
             }
 
-
             // Wait for all to finish
-            Pout<< "TEMPLATE BARRIER in distribute, non-blocking contiguous 2" << endl;
             Pstream::waitRequests(nOutstanding);
 
-
             // Collect neighbour fields
-
             for (label domain = 0; domain < Pstream::nProcs(); domain++)
             {
                 const labelList& map = constructMap[domain];

From 727c4d67fb34edbae52d88c8726d17bce3cb741c Mon Sep 17 00:00:00 2001
From: Hrvoje Jasak <h.jasak@wikki.co.uk>
Date: Mon, 28 Nov 2016 16:55:00 +0000
Subject: [PATCH 093/130] Improved non-blocking comms

---
 .../processor/processorFvPatchField.C         | 392 +++++++++++++++---
 .../processor/processorFvPatchField.H         |  21 +-
 ...tedInletOutletVelocityFvPatchVectorField.C |  26 +-
 ...eDirectedInletVelocityFvPatchVectorField.C |  25 +-
 ...ureInletOutletVelocityFvPatchVectorField.C |   1 -
 src/finiteVolume/fvMesh/fvMesh.C              |  18 +
 src/finiteVolume/fvMesh/fvMesh.H              |   1 +
 .../GGIInterpolationWeights.C                 |  22 +-
 .../GGIBlockAMGInterfaceField.C               |   3 -
 .../GGIBlockAMGInterfaceField.H               |   9 -
 .../ProcessorBlockAMGInterfaceField.C         | 107 ++++-
 .../ProcessorBlockAMGInterfaceField.H         |  28 +-
 .../BlockMatrixAgglomeration.C                |  23 +-
 .../BlockLduInterfaceField.H                  |   7 +-
 .../GGIBlockLduInterfaceField.C               |   4 +-
 .../GGIBlockLduInterfaceField.H               |   3 -
 .../MixingPlaneBlockLduInterfaceField.C       |   4 +-
 .../MixingPlaneBlockLduInterfaceField.H       |   3 -
 .../OverlapGGIBlockLduInterfaceField.C        |   4 +-
 .../OverlapGGIBlockLduInterfaceField.H        |   3 -
 .../ProcessorBlockLduInterfaceField.C         |   4 +-
 .../ProcessorBlockLduInterfaceField.H         |   3 -
 .../BlockLduMatrixUpdateInterfaces.C          |  68 ++-
 .../processorLduInterfaceTemplates.C          |   4 +-
 .../lduMatrixUpdateMatrixInterfaces.C         |  69 ++-
 .../processorAMGInterfaceField.C              | 102 ++++-
 .../processorAMGInterfaceField.H              |  21 +
 27 files changed, 794 insertions(+), 181 deletions(-)

diff --git a/src/finiteVolume/fields/fvPatchFields/constraint/processor/processorFvPatchField.C b/src/finiteVolume/fields/fvPatchFields/constraint/processor/processorFvPatchField.C
index e6ce38d34..ddbbaa98d 100644
--- a/src/finiteVolume/fields/fvPatchFields/constraint/processor/processorFvPatchField.C
+++ b/src/finiteVolume/fields/fvPatchFields/constraint/processor/processorFvPatchField.C
@@ -31,27 +31,28 @@ License
 #include "coeffFields.H"
 #include "demandDrivenData.H"
 
-// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
-
-namespace Foam
-{
-
 // * * * * * * * * * * * * * * * * Constructors * * * * * * * * * * * * * * //
 
 template<class Type>
-processorFvPatchField<Type>::processorFvPatchField
+Foam::processorFvPatchField<Type>::processorFvPatchField
 (
     const fvPatch& p,
     const DimensionedField<Type, volMesh>& iF
 )
 :
     coupledFvPatchField<Type>(p, iF),
-    procPatch_(refCast<const processorFvPatch>(p))
+    procPatch_(refCast<const processorFvPatch>(p)),
+    outstandingSendRequest_(-1),
+    outstandingRecvRequest_(-1),
+    sendBuf_(0),
+    receiveBuf_(0),
+    scalarSendBuf_(0),
+    scalarReceiveBuf_(0)
 {}
 
 
 template<class Type>
-processorFvPatchField<Type>::processorFvPatchField
+Foam::processorFvPatchField<Type>::processorFvPatchField
 (
     const fvPatch& p,
     const DimensionedField<Type, volMesh>& iF,
@@ -59,13 +60,18 @@ processorFvPatchField<Type>::processorFvPatchField
 )
 :
     coupledFvPatchField<Type>(p, iF, f),
-    procPatch_(refCast<const processorFvPatch>(p))
+    procPatch_(refCast<const processorFvPatch>(p)),
+    outstandingSendRequest_(-1),
+    outstandingRecvRequest_(-1),
+    sendBuf_(0),
+    receiveBuf_(0),
+    scalarSendBuf_(0),
+    scalarReceiveBuf_(0)
 {}
 
 
-// Construct by mapping given processorFvPatchField<Type>
 template<class Type>
-processorFvPatchField<Type>::processorFvPatchField
+Foam::processorFvPatchField<Type>::processorFvPatchField
 (
     const processorFvPatchField<Type>& ptf,
     const fvPatch& p,
@@ -74,9 +80,15 @@ processorFvPatchField<Type>::processorFvPatchField
 )
 :
     coupledFvPatchField<Type>(ptf, p, iF, mapper),
-    procPatch_(refCast<const processorFvPatch>(p))
+    procPatch_(refCast<const processorFvPatch>(p)),
+    outstandingSendRequest_(-1),
+    outstandingRecvRequest_(-1),
+    sendBuf_(0),
+    receiveBuf_(0),
+    scalarSendBuf_(0),
+    scalarReceiveBuf_(0)
 {
-    if (!isType<processorFvPatch>(this->patch()))
+    if (!isA<processorFvPatch>(this->patch()))
     {
         FatalErrorIn
         (
@@ -98,7 +110,7 @@ processorFvPatchField<Type>::processorFvPatchField
 
 
 template<class Type>
-processorFvPatchField<Type>::processorFvPatchField
+Foam::processorFvPatchField<Type>::processorFvPatchField
 (
     const fvPatch& p,
     const DimensionedField<Type, volMesh>& iF,
@@ -106,9 +118,15 @@ processorFvPatchField<Type>::processorFvPatchField
 )
 :
     coupledFvPatchField<Type>(p, iF, dict),
-    procPatch_(refCast<const processorFvPatch>(p))
+    procPatch_(refCast<const processorFvPatch>(p)),
+    outstandingSendRequest_(-1),
+    outstandingRecvRequest_(-1),
+    sendBuf_(0),
+    receiveBuf_(0),
+    scalarSendBuf_(0),
+    scalarReceiveBuf_(0)
 {
-    if (!isType<processorFvPatch>(p))
+    if (!isA<processorFvPatch>(p))
     {
         FatalIOErrorIn
         (
@@ -130,42 +148,79 @@ processorFvPatchField<Type>::processorFvPatchField
 
 
 template<class Type>
-processorFvPatchField<Type>::processorFvPatchField
+Foam::processorFvPatchField<Type>::processorFvPatchField
 (
     const processorFvPatchField<Type>& ptf
 )
 :
     processorLduInterfaceField(),
     coupledFvPatchField<Type>(ptf),
-    procPatch_(refCast<const processorFvPatch>(ptf.patch()))
+    procPatch_(refCast<const processorFvPatch>(ptf.patch())),
+    outstandingSendRequest_(-1),
+    outstandingRecvRequest_(-1),
+    sendBuf_(0),
+    receiveBuf_(0),
+    scalarSendBuf_(0),
+    scalarReceiveBuf_(0)
 {}
 
 
 template<class Type>
-processorFvPatchField<Type>::processorFvPatchField
+Foam::processorFvPatchField<Type>::processorFvPatchField
 (
     const processorFvPatchField<Type>& ptf,
     const DimensionedField<Type, volMesh>& iF
 )
 :
     coupledFvPatchField<Type>(ptf, iF),
-    procPatch_(refCast<const processorFvPatch>(ptf.patch()))
-{}
+    procPatch_(refCast<const processorFvPatch>(ptf.patch())),
+    outstandingSendRequest_(-1),
+    outstandingRecvRequest_(-1),
+    sendBuf_(0),
+    receiveBuf_(0),
+    scalarSendBuf_(0),
+    scalarReceiveBuf_(0)
+{
+    if (debug && !ptf.ready())
+    {
+        FatalErrorIn
+        (
+            "processorFvPatchField<Type>::processorFvPatchField\n"
+            "(\n"
+            "    const processorFvPatchField<Type>& ptf,\n"
+            "    const DimensionedField<Type, volMesh>& iF\n"
+            ")"
+        )   << "On patch " << procPatch_.name() << " outstanding request."
+            << abort(FatalError);
+    }
+}
 
 
 // * * * * * * * * * * * * * * * * * Destructor  * * * * * * * * * * * * * * //
 
 template<class Type>
-processorFvPatchField<Type>::~processorFvPatchField()
+Foam::processorFvPatchField<Type>::~processorFvPatchField()
 {}
 
 
 // * * * * * * * * * * * * * * * Member Functions  * * * * * * * * * * * * * //
 
 template<class Type>
-tmp<Field<Type> > processorFvPatchField<Type>::patchNeighbourField() const
+Foam::tmp<Foam::Field<Type> >
+Foam::processorFvPatchField<Type>::patchNeighbourField() const
 {
-    // Warning: returning own patch field, which on update stores
+    if (debug && !this->ready())
+    {
+        FatalErrorIn
+        (
+            "tmp<Field<Type> >"
+            "processorFvPatchField<Type>::patchNeighbourField() const"
+        )   << "On patch " << procPatch_.name()
+            << " outstanding request."
+            << abort(FatalError);
+    }
+
+    // Warning: returning own patch field, which only after update stores
     // actual neighbour data
     // HJ, 14/May/2009
     return *this;
@@ -173,27 +228,79 @@ tmp<Field<Type> > processorFvPatchField<Type>::patchNeighbourField() const
 
 
 template<class Type>
-void processorFvPatchField<Type>::initEvaluate
+void Foam::processorFvPatchField<Type>::initEvaluate
 (
     const Pstream::commsTypes commsType
 )
 {
     if (Pstream::parRun())
     {
-        procPatch_.send(commsType, this->patchInternalField()());
+        // Collect data into send buffer
+        sendBuf_ = this->patchInternalField();
+
+        if (commsType == Pstream::nonBlocking)
+        {
+            // Fast path. Receive into *this
+
+            outstandingRecvRequest_ = Pstream::nRequests();
+
+            IPstream::read
+            (
+                Pstream::nonBlocking,
+                procPatch_.neighbProcNo(),
+                reinterpret_cast<char*>(this->begin()),
+                this->byteSize(),
+                procPatch_.tag(),
+                procPatch_.comm()
+            );
+
+            outstandingSendRequest_ = Pstream::nRequests();
+
+            OPstream::write
+            (
+                Pstream::nonBlocking,
+                procPatch_.neighbProcNo(),
+                reinterpret_cast<const char*>(sendBuf_.begin()),
+                this->byteSize(),
+                procPatch_.tag(),
+                procPatch_.comm()
+            );
+        }
+        else
+        {
+            procPatch_.send(commsType, sendBuf_);
+        }
     }
 }
 
 
 template<class Type>
-void processorFvPatchField<Type>::evaluate
+void Foam::processorFvPatchField<Type>::evaluate
 (
     const Pstream::commsTypes commsType
 )
 {
     if (Pstream::parRun())
     {
-        procPatch_.receive<Type>(commsType, *this);
+        if (commsType == Pstream::nonBlocking)
+        {
+            // Fast path. Received into *this
+
+            if
+            (
+                outstandingRecvRequest_ >= 0
+             && outstandingRecvRequest_ < Pstream::nRequests()
+            )
+            {
+                Pstream::waitRequest(outstandingRecvRequest_);
+            }
+            outstandingSendRequest_ = -1;
+            outstandingRecvRequest_ = -1;
+        }
+        else
+        {
+            procPatch_.receive<Type>(commsType, *this);
+        }
 
         if (doTransform())
         {
@@ -204,14 +311,15 @@ void processorFvPatchField<Type>::evaluate
 
 
 template<class Type>
-tmp<Field<Type> > processorFvPatchField<Type>::snGrad() const
+Foam::tmp<Foam::Field<Type> >
+Foam::processorFvPatchField<Type>::snGrad() const
 {
     return this->patch().deltaCoeffs()*(*this - this->patchInternalField());
 }
 
 
 template<class Type>
-void processorFvPatchField<Type>::initInterfaceMatrixUpdate
+void Foam::processorFvPatchField<Type>::initInterfaceMatrixUpdate
 (
     const scalarField& psiInternal,
     scalarField&,
@@ -222,16 +330,68 @@ void processorFvPatchField<Type>::initInterfaceMatrixUpdate
     const bool switchToLhs
 ) const
 {
-    procPatch_.send
-    (
-        commsType,
-        this->patch().patchInternalField(psiInternal)()
-    );
+    scalarSendBuf_ = this->patch().patchInternalField(psiInternal);
+
+    if (commsType == Pstream::nonBlocking)
+    {
+        // Fast path.
+        if (debug && !this->ready())
+        {
+            FatalErrorIn
+            (
+                "void processorFvPatchField<Type>::initInterfaceMatrixUpdate\n"
+                "(\n"
+                "    const scalarField& psiInternal,\n"
+                "    scalarField&,\n"
+                "    const lduMatrix&,\n"
+                "    const scalarField&,\n"
+                "    const direction,\n"
+                "    const Pstream::commsTypes commsType,\n"
+                "    const bool switchToLhs\n"
+                ") const"
+            )   << "On patch " << procPatch_.name()
+                << " outstanding request."
+                << abort(FatalError);
+        }
+
+        // Check buffer size
+        scalarReceiveBuf_.setSize(this->size());
+
+        outstandingRecvRequest_ = Pstream::nRequests();
+
+        IPstream::read
+        (
+            Pstream::nonBlocking,
+            procPatch_.neighbProcNo(),
+            reinterpret_cast<char*>(scalarReceiveBuf_.begin()),
+            scalarReceiveBuf_.byteSize(),
+            procPatch_.tag(),
+            procPatch_.comm()
+        );
+
+        outstandingSendRequest_ = Pstream::nRequests();
+
+        OPstream::write
+        (
+            Pstream::nonBlocking,
+            procPatch_.neighbProcNo(),
+            reinterpret_cast<const char*>(scalarSendBuf_.begin()),
+            scalarSendBuf_.byteSize(),
+            procPatch_.tag(),
+            procPatch_.comm()
+        );
+    }
+    else
+    {
+        procPatch_.send(commsType, scalarSendBuf_);
+    }
+
+    const_cast<processorFvPatchField<Type>&>(*this).updatedMatrix() = false;
 }
 
 
 template<class Type>
-void processorFvPatchField<Type>::updateInterfaceMatrix
+void Foam::processorFvPatchField<Type>::updateInterfaceMatrix
 (
     const scalarField&,
     scalarField& result,
@@ -242,36 +402,65 @@ void processorFvPatchField<Type>::updateInterfaceMatrix
     const bool switchToLhs
 ) const
 {
-    scalarField pnf
-    (
-        procPatch_.receive<scalar>(commsType, this->size())()
-    );
+    if (this->updatedMatrix())
+    {
+        return;
+    }
+
+    if (commsType == Pstream::nonBlocking)
+    {
+        // Fast path.
+        if
+        (
+            outstandingRecvRequest_ >= 0
+         && outstandingRecvRequest_ < Pstream::nRequests()
+        )
+        {
+            Pstream::waitRequest(outstandingRecvRequest_);
+        }
+
+        // Recv finished so assume sending finished as well.
+        outstandingSendRequest_ = -1;
+        outstandingRecvRequest_ = -1;
+    }
+    else
+    {
+        // Check size
+        scalarReceiveBuf_.setSize(this->size());
+
+        procPatch_.receive<scalar>(commsType, scalarReceiveBuf_);
+    }
+
+    // The data is now in scalarReceiveBuf_ for both cases
 
     // Transform according to the transformation tensor
-    transformCoupleField(pnf, cmpt);
+    transformCoupleField(scalarReceiveBuf_, cmpt);
 
     // Multiply the field by coefficients and add into the result
+
     const unallocLabelList& faceCells = this->patch().faceCells();
 
     if (switchToLhs)
     {
         forAll(faceCells, elemI)
         {
-            result[faceCells[elemI]] += coeffs[elemI]*pnf[elemI];
+            result[faceCells[elemI]] += coeffs[elemI]*scalarReceiveBuf_[elemI];
         }
     }
     else
     {
         forAll(faceCells, elemI)
         {
-            result[faceCells[elemI]] -= coeffs[elemI]*pnf[elemI];
+            result[faceCells[elemI]] -= coeffs[elemI]*scalarReceiveBuf_[elemI];
         }
     }
+
+    const_cast<processorFvPatchField<Type>&>(*this).updatedMatrix() = true;
 }
 
 
 template<class Type>
-void processorFvPatchField<Type>::initInterfaceMatrixUpdate
+void Foam::processorFvPatchField<Type>::initInterfaceMatrixUpdate
 (
     const Field<Type>& psiInternal,
     Field<Type>&,
@@ -281,16 +470,71 @@ void processorFvPatchField<Type>::initInterfaceMatrixUpdate
     const bool switchToLhs
 ) const
 {
-    procPatch_.send
-    (
-        commsType,
-        this->patch().patchInternalField(psiInternal)()
-    );
+    sendBuf_ = this->patch().patchInternalField(psiInternal);
+
+    if (commsType == Pstream::nonBlocking)
+    {
+        // Fast path.
+        if (debug && !this->ready())
+        {
+            FatalErrorIn
+            (
+                "void processorFvPatchField<Type>::initInterfaceMatrixUpdate\n"
+                "(\n"
+                "    const Field<Type>& psiInternal,\n"
+                "    Field<Type>&,\n"
+                "    const BlockLduMatrix<Type>&,\n"
+                "    const CoeffField<Type>&,\n"
+                "    const Pstream::commsTypes commsType,\n"
+                "    const bool switchToLhs\n"
+                ") const"
+            )   << "On patch " << procPatch_.name()
+                << " outstanding request."
+                << abort(FatalError);
+        }
+
+        // Check buffer size
+        receiveBuf_.setSize(this->size());
+
+        outstandingRecvRequest_ = Pstream::nRequests();
+
+        IPstream::read
+        (
+            Pstream::nonBlocking,
+            procPatch_.neighbProcNo(),
+            reinterpret_cast<char*>(receiveBuf_.begin()),
+            receiveBuf_.byteSize(),
+            procPatch_.tag(),
+            procPatch_.comm()
+        );
+
+        outstandingSendRequest_ = Pstream::nRequests();
+
+        OPstream::write
+        (
+            Pstream::nonBlocking,
+            procPatch_.neighbProcNo(),
+            reinterpret_cast<const char*>(sendBuf_.begin()),
+            sendBuf_.byteSize(),
+            procPatch_.tag(),
+            procPatch_.comm()
+        );
+    }
+    else
+    {
+        procPatch_.send
+        (
+            commsType,
+            this->patch().patchInternalField(psiInternal)()
+        );
+    }
+
+    const_cast<processorFvPatchField<Type>&>(*this).updatedMatrix() = false;
 }
 
 
 template<class Type>
-void processorFvPatchField<Type>::updateInterfaceMatrix
+void Foam::processorFvPatchField<Type>::updateInterfaceMatrix
 (
     const Field<Type>& psiInternal,
     Field<Type>& result,
@@ -300,14 +544,40 @@ void processorFvPatchField<Type>::updateInterfaceMatrix
     const bool switchToLhs
 ) const
 {
-    Field<Type> pnf
-    (
-        procPatch_.receive<Type>(commsType, this->size())()
-    );
+    if (this->updatedMatrix())
+    {
+        return;
+    }
 
-    // Multiply neighbour field with coeffs and re-use pnf for result
+    if (commsType == Pstream::nonBlocking)
+    {
+        // Fast path.
+        if
+        (
+            outstandingRecvRequest_ >= 0
+         && outstandingRecvRequest_ < Pstream::nRequests()
+        )
+        {
+            Pstream::waitRequest(outstandingRecvRequest_);
+        }
+
+        // Recv finished so assume sending finished as well.
+        outstandingSendRequest_ = -1;
+        outstandingRecvRequest_ = -1;
+    }
+    else
+    {
+        // Check size
+        receiveBuf_.setSize(this->size());
+
+        procPatch_.receive<Type>(commsType, receiveBuf_);
+    }
+
+    // The data is now in receiveBuf_ for both cases
+
+    // Multiply neighbour field with coeffs and re-use buffer for result
     // of multiplication
-    multiply(pnf, coeffs, pnf);
+    multiply(receiveBuf_, coeffs, receiveBuf_);
 
     const unallocLabelList& faceCells = this->patch().faceCells();
 
@@ -315,21 +585,19 @@ void processorFvPatchField<Type>::updateInterfaceMatrix
     {
         forAll(faceCells, elemI)
         {
-            result[faceCells[elemI]] += pnf[elemI];
+            result[faceCells[elemI]] += receiveBuf_[elemI];
         }
     }
     else
     {
         forAll(faceCells, elemI)
         {
-            result[faceCells[elemI]] -= pnf[elemI];
+            result[faceCells[elemI]] -= receiveBuf_[elemI];
         }
     }
+
+    const_cast<processorFvPatchField<Type>&>(*this).updatedMatrix() = true;
 }
 
 
-// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
-
-} // End namespace Foam
-
 // ************************************************************************* //
diff --git a/src/finiteVolume/fields/fvPatchFields/constraint/processor/processorFvPatchField.H b/src/finiteVolume/fields/fvPatchFields/constraint/processor/processorFvPatchField.H
index b3711d6ab..6f9ed9f96 100644
--- a/src/finiteVolume/fields/fvPatchFields/constraint/processor/processorFvPatchField.H
+++ b/src/finiteVolume/fields/fvPatchFields/constraint/processor/processorFvPatchField.H
@@ -69,6 +69,25 @@ class processorFvPatchField
         //- Local reference cast into the processor patch
         const processorFvPatch& procPatch_;
 
+        // Sending and receiving
+
+            //- Outstanding request
+            mutable label outstandingSendRequest_;
+
+            //- Outstanding request
+            mutable label outstandingRecvRequest_;
+
+            //- Send buffer.
+            mutable Field<Type> sendBuf_;
+
+            //- Receive buffer.
+            mutable Field<Type> receiveBuf_;
+
+            //- Scalar send buffer
+            mutable Field<scalar> scalarSendBuf_;
+
+            //- Scalar receive buffer
+            mutable Field<scalar> scalarReceiveBuf_;
 
 public:
 
@@ -143,7 +162,7 @@ public:
 
 
     //- Destructor
-    ~processorFvPatchField();
+    virtual ~processorFvPatchField();
 
 
     // Member functions
diff --git a/src/finiteVolume/fields/fvPatchFields/derived/pressureDirectedInletOutletVelocity/pressureDirectedInletOutletVelocityFvPatchVectorField.C b/src/finiteVolume/fields/fvPatchFields/derived/pressureDirectedInletOutletVelocity/pressureDirectedInletOutletVelocityFvPatchVectorField.C
index 6d9425903..83b9e1c8b 100644
--- a/src/finiteVolume/fields/fvPatchFields/derived/pressureDirectedInletOutletVelocity/pressureDirectedInletOutletVelocityFvPatchVectorField.C
+++ b/src/finiteVolume/fields/fvPatchFields/derived/pressureDirectedInletOutletVelocity/pressureDirectedInletOutletVelocityFvPatchVectorField.C
@@ -46,7 +46,7 @@ pressureDirectedInletOutletVelocityFvPatchVectorField
     mixedFvPatchVectorField(p, iF),
     phiName_("phi"),
     rhoName_("rho"),
-    inletDir_(p.size())
+    inletDir_(p.size(), vector(1, 0, 0))
 {
     refValue() = *this;
     refGrad() = vector::zero;
@@ -84,6 +84,30 @@ pressureDirectedInletOutletVelocityFvPatchVectorField
     inletDir_("inletDirection", dict, p.size())
 {
     fvPatchVectorField::operator=(vectorField("value", dict, p.size()));
+
+    if (!inletDir_.empty())
+    {
+        if (min(mag(inletDir_)) < SMALL)
+        {
+            FatalErrorIn
+            (
+                "pressureDirectedInletOutletVelocityFvPatchVectorField::\n"
+                "pressureDirectedInletOutletVelocityFvPatchVectorField\n"
+                "(\n"
+                "    const fvPatch& p,\n"
+                "    const DimensionedField<vector, volMesh>& iF,\n"
+                "    const dictionary& dict\n"
+                ")"
+            )    << "Badly defined inlet direction for field "
+                 << this->dimensionedInternalField().name()
+                 << " and patch " << this->patch().name()
+                 << abort(FatalError);
+        }
+    }       
+
+    // Normalise to obtain the flow direction
+    inletDir_ /= (mag(inletDir_) + SMALL);
+
     refValue() = *this;
     refGrad() = vector::zero;
     valueFraction() = 0.0;
diff --git a/src/finiteVolume/fields/fvPatchFields/derived/pressureDirectedInletVelocity/pressureDirectedInletVelocityFvPatchVectorField.C b/src/finiteVolume/fields/fvPatchFields/derived/pressureDirectedInletVelocity/pressureDirectedInletVelocityFvPatchVectorField.C
index 5b7760545..61b713b96 100644
--- a/src/finiteVolume/fields/fvPatchFields/derived/pressureDirectedInletVelocity/pressureDirectedInletVelocityFvPatchVectorField.C
+++ b/src/finiteVolume/fields/fvPatchFields/derived/pressureDirectedInletVelocity/pressureDirectedInletVelocityFvPatchVectorField.C
@@ -46,7 +46,7 @@ pressureDirectedInletVelocityFvPatchVectorField
     fixedValueFvPatchVectorField(p, iF),
     phiName_("phi"),
     rhoName_("rho"),
-    inletDir_(p.size())
+    inletDir_(p.size(), vector(1, 0, 0))
 {}
 
 
@@ -80,6 +80,29 @@ pressureDirectedInletVelocityFvPatchVectorField
     inletDir_("inletDirection", dict, p.size())
 {
     fvPatchVectorField::operator=(vectorField("value", dict, p.size()));
+
+    if (!inletDir_.empty())
+    {
+        if (min(mag(inletDir_)) < SMALL)
+        {
+            FatalErrorIn
+            (
+                "pressureDirectedInletVelocityFvPatchVectorField::\n"
+                "pressureDirectedInletVelocityFvPatchVectorField\n"
+                "(\n"
+                "    const fvPatch& p,\n"
+                "    const DimensionedField<vector, volMesh>& iF,\n"
+                "    const dictionary& dict\n"
+                ")"
+            )    << "Badly defined inlet direction for field "
+                 << this->dimensionedInternalField().name()
+                 << " and patch " << this->patch().name()
+                 << abort(FatalError);
+        }
+    }       
+
+    // Normalise to obtain the flow direction
+    inletDir_ /= (mag(inletDir_) + SMALL);
 }
 
 
diff --git a/src/finiteVolume/fields/fvPatchFields/derived/pressureInletOutletVelocity/pressureInletOutletVelocityFvPatchVectorField.C b/src/finiteVolume/fields/fvPatchFields/derived/pressureInletOutletVelocity/pressureInletOutletVelocityFvPatchVectorField.C
index 948de74ac..fddc43131 100644
--- a/src/finiteVolume/fields/fvPatchFields/derived/pressureInletOutletVelocity/pressureInletOutletVelocityFvPatchVectorField.C
+++ b/src/finiteVolume/fields/fvPatchFields/derived/pressureInletOutletVelocity/pressureInletOutletVelocityFvPatchVectorField.C
@@ -192,7 +192,6 @@ void pressureInletOutletVelocityFvPatchVectorField::updateCoeffs()
     }
 
     directionMixedFvPatchVectorField::updateCoeffs();
-    directionMixedFvPatchVectorField::evaluate();
 }
 
 
diff --git a/src/finiteVolume/fvMesh/fvMesh.C b/src/finiteVolume/fvMesh/fvMesh.C
index bd69d37e7..928b3b83b 100644
--- a/src/finiteVolume/fvMesh/fvMesh.C
+++ b/src/finiteVolume/fvMesh/fvMesh.C
@@ -556,6 +556,12 @@ void Foam::fvMesh::updateMesh(const mapPolyMesh& mpm)
     // This is a temporary solution
     surfaceInterpolation::movePoints();
 
+    // Note: deltaCoeffs cannot be left on lazy evaluation on mesh motion
+    // because tangled comms will occur when they are accessed from
+    // individual boundary conditions
+    // HJ, VV and IG, 25/Oct/2016
+    deltaCoeffs();
+
     // Function object update moved to polyMesh
     // HJ, 29/Aug/2010
 }
@@ -577,6 +583,12 @@ void Foam::fvMesh::syncUpdateMesh()
     // This is a temporary solution
     surfaceInterpolation::movePoints();
 
+    // Note: deltaCoeffs cannot be left on lazy evaluation on mesh motion
+    // because tangled comms will occur when they are accessed from
+    // individual boundary conditions
+    // HJ, VV and IG, 25/Oct/2016
+    deltaCoeffs();
+
     // Function object update moved to polyMesh
     // HJ, 29/Aug/2010
 }
@@ -681,6 +693,12 @@ Foam::tmp<Foam::scalarField> Foam::fvMesh::movePoints(const pointField& p)
     // Function object update moved to polyMesh
     // HJ, 29/Aug/2010
 
+    // Note: deltaCoeffs cannot be left on lazy evaluation on mesh motion
+    // because tangled comms will occur when they are accessed from
+    // individual boundary conditions
+    // HJ, VV and IG, 25/Oct/2016
+    deltaCoeffs();
+
     return tsweptVols;
 }
 
diff --git a/src/finiteVolume/fvMesh/fvMesh.H b/src/finiteVolume/fvMesh/fvMesh.H
index 44e7dc78c..86d083a80 100644
--- a/src/finiteVolume/fvMesh/fvMesh.H
+++ b/src/finiteVolume/fvMesh/fvMesh.H
@@ -123,6 +123,7 @@ class fvMesh
 
        // Make geometric data
 
+            //- Make face areas
             void makeSf() const;
 
             //- Make face area magnitudes
diff --git a/src/foam/interpolations/GGIInterpolation/GGIInterpolationWeights.C b/src/foam/interpolations/GGIInterpolation/GGIInterpolationWeights.C
index c22340401..48845d702 100644
--- a/src/foam/interpolations/GGIInterpolation/GGIInterpolationWeights.C
+++ b/src/foam/interpolations/GGIInterpolation/GGIInterpolationWeights.C
@@ -466,17 +466,17 @@ void GGIInterpolation<MasterPatch, SlavePatch>::calcAddressing() const
                 }
                 else
                 {
-                    WarningIn
-                    (
-                        "GGIInterpolation<MasterPatch, SlavePatch>::"
-                        "calcAddressing()"
-                    )   << "polygonIntersection is returning a "
-                        << "zero surface area between " << nl
-                        << "     Master face: " << faceMi
-                        << " and Neighbour face: " << curCMN[neighbI]
-                        << " intersection area = " << intersectionArea << nl
-                        << "Please check the two quick-check algorithms for "
-                        << "GGIInterpolation.  Something is  missing." << endl;
+//                     WarningIn
+//                     (
+//                         "GGIInterpolation<MasterPatch, SlavePatch>::"
+//                         "calcAddressing()"
+//                     )   << "polygonIntersection is returning a "
+//                         << "zero surface area between " << nl
+//                         << "     Master face: " << faceMi
+//                         << " and Neighbour face: " << curCMN[neighbI]
+//                         << " intersection area = " << intersectionArea << nl
+//                         << "Please check the two quick-check algorithms for "
+//                         << "GGIInterpolation.  Something is  missing." << endl;
                 }
             }
         }
diff --git a/src/foam/matrices/blockLduMatrix/BlockAMG/BlockAMGInterfaceFields/GGIBlockAMGInterfaceField/GGIBlockAMGInterfaceField.C b/src/foam/matrices/blockLduMatrix/BlockAMG/BlockAMGInterfaceFields/GGIBlockAMGInterfaceField/GGIBlockAMGInterfaceField.C
index e3a2621a6..b6353bdca 100644
--- a/src/foam/matrices/blockLduMatrix/BlockAMG/BlockAMGInterfaceFields/GGIBlockAMGInterfaceField/GGIBlockAMGInterfaceField.C
+++ b/src/foam/matrices/blockLduMatrix/BlockAMG/BlockAMGInterfaceFields/GGIBlockAMGInterfaceField/GGIBlockAMGInterfaceField.C
@@ -100,7 +100,6 @@ Foam::GGIBlockAMGInterfaceField<Type>::GGIBlockAMGInterfaceField
     BlockAMGInterfaceField<Type>(AMGCp, fineInterfaceField),
     ggiInterface_(refCast<const ggiAMGInterface>(AMGCp)),
     doTransform_(false),
-    rank_(),
     fieldTransferBuffer_()
 {
     // If the interface based on a patch this must be taken care specially of
@@ -113,7 +112,6 @@ Foam::GGIBlockAMGInterfaceField<Type>::GGIBlockAMGInterfaceField
             );
 
         doTransform_ = p.doTransform();
-        rank_ = p.rank();
     }
     else if (isA<ggiLduInterfaceField>(fineInterfaceField))
     {
@@ -121,7 +119,6 @@ Foam::GGIBlockAMGInterfaceField<Type>::GGIBlockAMGInterfaceField
             refCast<const ggiLduInterfaceField >(fineInterfaceField);
 
         doTransform_ = p.doTransform();
-        rank_ = p.rank();
     }
     else
     {
diff --git a/src/foam/matrices/blockLduMatrix/BlockAMG/BlockAMGInterfaceFields/GGIBlockAMGInterfaceField/GGIBlockAMGInterfaceField.H b/src/foam/matrices/blockLduMatrix/BlockAMG/BlockAMGInterfaceFields/GGIBlockAMGInterfaceField/GGIBlockAMGInterfaceField.H
index 807e82ae2..b2d861ccc 100644
--- a/src/foam/matrices/blockLduMatrix/BlockAMG/BlockAMGInterfaceFields/GGIBlockAMGInterfaceField/GGIBlockAMGInterfaceField.H
+++ b/src/foam/matrices/blockLduMatrix/BlockAMG/BlockAMGInterfaceFields/GGIBlockAMGInterfaceField/GGIBlockAMGInterfaceField.H
@@ -65,9 +65,6 @@ class GGIBlockAMGInterfaceField
         //- Is the transform required
         bool doTransform_;
 
-        //- Rank of component for transformation.
-        int rank_;
-
         //- Field transfer buffer
         mutable Field<Type> fieldTransferBuffer_;
 
@@ -199,11 +196,6 @@ public:
                 return ggiInterface_.reverseT();
             }
 
-            //- Return rank of component for transform
-            virtual int rank() const
-            {
-                return rank_;
-            }
 
         // Transfer buffer access
 
@@ -212,7 +204,6 @@ public:
             {
                 return fieldTransferBuffer_;
             }
-
 };
 
 
diff --git a/src/foam/matrices/blockLduMatrix/BlockAMG/BlockAMGInterfaceFields/ProcessorBlockAMGInterfaceField/ProcessorBlockAMGInterfaceField.C b/src/foam/matrices/blockLduMatrix/BlockAMG/BlockAMGInterfaceFields/ProcessorBlockAMGInterfaceField/ProcessorBlockAMGInterfaceField.C
index 483cbe802..a01ff436c 100644
--- a/src/foam/matrices/blockLduMatrix/BlockAMG/BlockAMGInterfaceFields/ProcessorBlockAMGInterfaceField/ProcessorBlockAMGInterfaceField.C
+++ b/src/foam/matrices/blockLduMatrix/BlockAMG/BlockAMGInterfaceFields/ProcessorBlockAMGInterfaceField/ProcessorBlockAMGInterfaceField.C
@@ -28,9 +28,6 @@ License
 #include "addToRunTimeSelectionTable.H"
 #include "lduMatrix.H"
 
-// * * * * * * * * * * * * * * Static Data Members * * * * * * * * * * * * * //
-
-
 // * * * * * * * * * * * * * * * * Constructors  * * * * * * * * * * * * * * //
 
 template<class Type>
@@ -42,7 +39,11 @@ Foam::ProcessorBlockAMGInterfaceField<Type>::ProcessorBlockAMGInterfaceField
 :
     BlockAMGInterfaceField<Type>(AMGCp, fineInterfaceField),
     procInterface_(refCast<const processorAMGInterface>(AMGCp)),
-    doTransform_(false)
+    doTransform_(false),
+    outstandingSendRequest_(-1),
+    outstandingRecvRequest_(-1),
+    sendBuf_(0),
+    receiveBuf_(0)
 {
     // If the interface based on a patch this must be taken care specially of
     if (isA<ProcessorBlockLduInterfaceField<Type> >(fineInterfaceField))
@@ -94,11 +95,51 @@ void Foam::ProcessorBlockAMGInterfaceField<Type>::initInterfaceMatrixUpdate
     const bool switchToLhs
 ) const
 {
-    procInterface_.send
-    (
-        commsType,
-        procInterface_.interfaceInternalField(psiInternal)()
-    );
+    label oldWarn = Pstream::warnComm;
+    Pstream::warnComm = comm();
+
+    sendBuf_ = procInterface_.interfaceInternalField(psiInternal);
+
+    if (commsType == Pstream::nonBlocking)
+    {
+        // Fast path.
+        receiveBuf_.setSize(sendBuf_.size());
+        outstandingRecvRequest_ = Pstream::nRequests();
+        IPstream::read
+        (
+            Pstream::nonBlocking,
+            procInterface_.neighbProcNo(),
+            reinterpret_cast<char*>(receiveBuf_.begin()),
+            receiveBuf_.byteSize(),
+            procInterface_.tag(),
+            comm()
+        );
+
+        outstandingSendRequest_ = Pstream::nRequests();
+        OPstream::write
+        (
+            Pstream::nonBlocking,
+            procInterface_.neighbProcNo(),
+            reinterpret_cast<const char*>(sendBuf_.begin()),
+            sendBuf_.byteSize(),
+            procInterface_.tag(),
+            comm()
+        );
+    }
+    else
+    {
+        procInterface_.send
+        (
+            commsType,
+            procInterface_.interfaceInternalField(psiInternal)()
+        );
+    }
+
+    // Mark as ready for update
+    const_cast<ProcessorBlockAMGInterfaceField<Type>&>(*this).updatedMatrix() =
+        false;
+
+    Pstream::warnComm = oldWarn;
 }
 
 
@@ -113,14 +154,42 @@ void Foam::ProcessorBlockAMGInterfaceField<Type>::updateInterfaceMatrix
     const bool switchToLhs
 ) const
 {
-    Field<Type> pnf
-    (
-        procInterface_.receive<Type>(commsType, this->size())
-    );
+    if (this->updatedMatrix())
+    {
+        return;
+    }
 
-    // Multiply neighbour field with coeffs and re-use pnf for result
+    if (commsType == Pstream::nonBlocking)
+    {
+        // Fast path.
+        if
+        (
+            outstandingRecvRequest_ >= 0
+         && outstandingRecvRequest_ < Pstream::nRequests()
+        )
+        {
+            Pstream::waitRequest(outstandingRecvRequest_);
+        }
+
+        // Recv finished so assume sending finished as well.
+        outstandingSendRequest_ = -1;
+        outstandingRecvRequest_ = -1;
+    }
+    else
+    {
+        // Check size
+        receiveBuf_.setSize(sendBuf_.size());
+
+        procInterface_.receive<Type>(commsType, receiveBuf_);
+    }
+
+    // The data is now in receiveBuf_ for both cases
+
+    // Transformation missing.  Is it needed?  HJ, 28/Nov/2016
+
+    // Multiply neighbour field with coeffs and re-use buffer for result
     // of multiplication
-    multiply(pnf, coeffs, pnf);
+    multiply(receiveBuf_, coeffs, receiveBuf_);
 
     const unallocLabelList& faceCells = procInterface_.faceCells();
 
@@ -128,16 +197,20 @@ void Foam::ProcessorBlockAMGInterfaceField<Type>::updateInterfaceMatrix
     {
         forAll(faceCells, elemI)
         {
-            result[faceCells[elemI]] += pnf[elemI];
+            result[faceCells[elemI]] += receiveBuf_[elemI];
         }
     }
     else
     {
         forAll(faceCells, elemI)
         {
-            result[faceCells[elemI]] -= pnf[elemI];
+            result[faceCells[elemI]] -= receiveBuf_[elemI];
         }
     }
+
+    // Mark as updated
+    const_cast<ProcessorBlockAMGInterfaceField<Type>&>(*this).updatedMatrix() =
+        true;
 }
 
 
diff --git a/src/foam/matrices/blockLduMatrix/BlockAMG/BlockAMGInterfaceFields/ProcessorBlockAMGInterfaceField/ProcessorBlockAMGInterfaceField.H b/src/foam/matrices/blockLduMatrix/BlockAMG/BlockAMGInterfaceFields/ProcessorBlockAMGInterfaceField/ProcessorBlockAMGInterfaceField.H
index f8882e42c..87170a371 100644
--- a/src/foam/matrices/blockLduMatrix/BlockAMG/BlockAMGInterfaceFields/ProcessorBlockAMGInterfaceField/ProcessorBlockAMGInterfaceField.H
+++ b/src/foam/matrices/blockLduMatrix/BlockAMG/BlockAMGInterfaceFields/ProcessorBlockAMGInterfaceField/ProcessorBlockAMGInterfaceField.H
@@ -62,8 +62,20 @@ class ProcessorBlockAMGInterfaceField
         //- Is the transform required
         bool doTransform_;
 
-        //- Rank of component for transformation
-        int rank_;
+
+        // Sending and receiving
+
+            //- Outstanding request
+            mutable label outstandingSendRequest_;
+
+            //- Outstanding request
+            mutable label outstandingRecvRequest_;
+
+            //- Send buffer.
+            mutable Field<Type> sendBuf_;
+
+            //- Receive buffer.
+            mutable Field<Type> receiveBuf_;
 
 
     // Private Member Functions
@@ -159,6 +171,12 @@ public:
 
         //- Processor interface functions
 
+            //- Return communicator used for comms
+            virtual label comm() const
+            {
+                return procInterface_.comm();
+            }
+
             //- Return processor number
             virtual int myProcNo() const
             {
@@ -182,12 +200,6 @@ public:
             {
                 return procInterface_.forwardT();
             }
-
-            //- Return rank of component for transform
-            virtual int rank() const
-            {
-                return rank_;
-            }
 };
 
 
diff --git a/src/foam/matrices/blockLduMatrix/BlockAMG/BlockMatrixCoarsening/BlockMatrixAgglomeration/BlockMatrixAgglomeration.C b/src/foam/matrices/blockLduMatrix/BlockAMG/BlockMatrixCoarsening/BlockMatrixAgglomeration/BlockMatrixAgglomeration.C
index fa578df20..4a9c13d36 100644
--- a/src/foam/matrices/blockLduMatrix/BlockAMG/BlockMatrixCoarsening/BlockMatrixAgglomeration/BlockMatrixAgglomeration.C
+++ b/src/foam/matrices/blockLduMatrix/BlockAMG/BlockMatrixCoarsening/BlockMatrixAgglomeration/BlockMatrixAgglomeration.C
@@ -74,7 +74,6 @@ void Foam::BlockMatrixAgglomeration<Type>::calcAgglomeration()
     // Create row-based addressing
 
     const label nRows = matrix_.lduAddr().size();
-    Info<< "Start, coarse size: " << nRows << endl;
 
     const unallocLabelList& upperAddr = matrix_.lduAddr().upperAddr();
     const unallocLabelList& lowerAddr = matrix_.lduAddr().lowerAddr();
@@ -143,7 +142,6 @@ void Foam::BlockMatrixAgglomeration<Type>::calcAgglomeration()
         // Reset agglomeration index array
         agglomIndex_ = -1;
     }
-    Info<< "End prepare addressing: " << lTime_.elapsedCpuTime() << endl;
 
 
     // Calculate agglomeration
@@ -179,7 +177,6 @@ void Foam::BlockMatrixAgglomeration<Type>::calcAgglomeration()
 
         return;
     }
-    Info<< "End coeff mag: " << lTime_.elapsedCpuTime() << endl;
 
     labelList sizeOfGroups(nRows, 0);
 
@@ -235,7 +232,6 @@ void Foam::BlockMatrixAgglomeration<Type>::calcAgglomeration()
             }
         }
     }
-    Info<< "End weak connections: " << lTime_.elapsedCpuTime() << endl;
 
     // Go through the off-diagonal and create clusters, marking the child array
     label indexUngrouped, indexGrouped;
@@ -343,7 +339,6 @@ void Foam::BlockMatrixAgglomeration<Type>::calcAgglomeration()
             }
         }
     }
-    Info<< "End agglom index: " << lTime_.elapsedCpuTime() << endl;
 
     // The decision on parallel agglomeration needs to be made for the
     // whole gang of processes; otherwise I may end up with a different
@@ -375,7 +370,6 @@ void Foam::BlockMatrixAgglomeration<Type>::calcAgglomeration()
             Pout << ".  Rejected" << endl;
         }
     }
-    Info<< "End calcAgglomeration: " << lTime_.elapsedCpuTime() << endl;
 }
 
 
@@ -387,7 +381,7 @@ void Foam::BlockMatrixAgglomeration<Type>::restrictDiag
 ) const
 {
     typedef CoeffField<Type> TypeCoeffField;
-    Info<< "Start restrictDiag: " << lTime_.elapsedCpuTime() << endl;
+
     if
     (
         Coeff.activeType() == blockCoeffBase::SQUARE
@@ -462,7 +456,6 @@ void Foam::BlockMatrixAgglomeration<Type>::restrictDiag
         )   << "Problem in coeff type morphing"
             << abort(FatalError);
     }
-    Info<< "End restrictDiag: " << lTime_.elapsedCpuTime() << endl;
 }
 
 
@@ -479,7 +472,7 @@ void Foam::BlockMatrixAgglomeration<Type>::agglomerateCoeffs
 {
     // Does the matrix have solo equations
     bool soloEqns = nSolo_ > 0;
-    Info<< "Start agglomerateCoeffs symm: " << lTime_.elapsedCpuTime() << endl;
+
     // Get addressing
     const unallocLabelList& upperAddr = matrix_.lduAddr().upperAddr();
     const unallocLabelList& lowerAddr = matrix_.lduAddr().lowerAddr();
@@ -512,7 +505,6 @@ void Foam::BlockMatrixAgglomeration<Type>::agglomerateCoeffs
               + activeFineUpperTranspose[fineCoeffI];
         }
     }
-    Info<< "End agglomerateCoeffs symm: " << lTime_.elapsedCpuTime() << endl;
 }
 
 
@@ -530,7 +522,7 @@ void Foam::BlockMatrixAgglomeration<Type>::agglomerateCoeffs
 {
     // Does the matrix have solo equations
     bool soloEqns = nSolo_ > 0;
-    Info<< "Start agglomerateCoeffs asymm: " << lTime_.elapsedCpuTime() << endl;
+
     // Get addressing
     const unallocLabelList& upperAddr = matrix_.lduAddr().upperAddr();
     const unallocLabelList& lowerAddr = matrix_.lduAddr().lowerAddr();
@@ -562,7 +554,6 @@ void Foam::BlockMatrixAgglomeration<Type>::agglomerateCoeffs
               + activeFineLower[fineCoeffI];
         }
     }
-    Info<< "End agglomerateCoeffs asymm: " << lTime_.elapsedCpuTime() << endl;
 }
 
 
@@ -691,7 +682,6 @@ Foam::BlockMatrixAgglomeration<Type>::restrictMatrix() const
     //    insert the coeffs in the upper order. At the same time, collect the
     //    owner and neighbour addressing.
     // 4) Agglomerate the diagonal by summing up the fine diagonal
-    Info<< "Start restrictMatrix: " << lTime_.elapsedCpuTime() << endl;
 
     // Get addressing
     const unallocLabelList& upperAddr = matrix_.lduAddr().upperAddr();
@@ -866,7 +856,6 @@ Foam::BlockMatrixAgglomeration<Type>::restrictMatrix() const
     blockNbrsData.setSize(0);
     initCoarseNeighb.setSize(0);
     coarseCoeffMap.setSize(0);
-    Info<< "End restrictMatrix core: " << lTime_.elapsedCpuTime() << endl;
 
 
     // Create coarse-level coupled interfaces
@@ -959,10 +948,9 @@ Foam::BlockMatrixAgglomeration<Type>::restrictMatrix() const
                     fineInterfaceAddr[intI]
                 ).ptr()
             );
-            Info<< " Interface " << intI << ": " << lTime_.elapsedCpuTime() << endl;
         }
     }
-    Info<< "End restrictMatrix interfaces: " << lTime_.elapsedCpuTime() << endl;
+
     forAll (interfaceFields, intI)
     {
         if (interfaceFields.set(intI))
@@ -973,7 +961,6 @@ Foam::BlockMatrixAgglomeration<Type>::restrictMatrix() const
             coarseInterfaceAddr[intI] = coarseInterface.faceCells();
         }
     }
-    Info<< "End restrictMatrix interface fields: " << lTime_.elapsedCpuTime() << endl;
 
     // Add interfaces
     coarseAddrPtr->addInterfaces
@@ -1050,7 +1037,6 @@ Foam::BlockMatrixAgglomeration<Type>::restrictMatrix() const
     }
 
     // Matrix restriction done!
-    Info<< "End restrictMatrix: " << lTime_.elapsedCpuTime() << endl;
 
     typedef CoeffField<Type> TypeCoeffField;
 
@@ -1247,7 +1233,6 @@ Foam::BlockMatrixAgglomeration<Type>::restrictMatrix() const
                 << abort(FatalError);
         }
     }
-    Info<< "End restrictMatrix agglomerateCoeffs: " << lTime_.elapsedCpuTime() << endl;
 
     // Create and return BlockAMGLevel
     return autoPtr<BlockAMGLevel<Type> >
diff --git a/src/foam/matrices/blockLduMatrix/BlockLduMatrix/BlockLduInterfaceFields/BlockLduInterfaceField/BlockLduInterfaceField.H b/src/foam/matrices/blockLduMatrix/BlockLduMatrix/BlockLduInterfaceFields/BlockLduInterfaceField/BlockLduInterfaceField.H
index 157afb8e2..1b1a03708 100644
--- a/src/foam/matrices/blockLduMatrix/BlockLduMatrix/BlockLduInterfaceFields/BlockLduInterfaceField/BlockLduInterfaceField.H
+++ b/src/foam/matrices/blockLduMatrix/BlockLduMatrix/BlockLduInterfaceFields/BlockLduInterfaceField/BlockLduInterfaceField.H
@@ -85,10 +85,9 @@ public:
         {}
 
 
-    // Destructor
-
-        virtual ~BlockLduInterfaceField()
-        {}
+    //- Destructor
+    virtual ~BlockLduInterfaceField()
+    {}
 
 
     // Member Functions
diff --git a/src/foam/matrices/blockLduMatrix/BlockLduMatrix/BlockLduInterfaceFields/GGIBlockLduInterfaceField/GGIBlockLduInterfaceField.C b/src/foam/matrices/blockLduMatrix/BlockLduMatrix/BlockLduInterfaceFields/GGIBlockLduInterfaceField/GGIBlockLduInterfaceField.C
index b4dac3f00..f3f4b13aa 100644
--- a/src/foam/matrices/blockLduMatrix/BlockLduMatrix/BlockLduInterfaceFields/GGIBlockLduInterfaceField/GGIBlockLduInterfaceField.C
+++ b/src/foam/matrices/blockLduMatrix/BlockLduMatrix/BlockLduInterfaceFields/GGIBlockLduInterfaceField/GGIBlockLduInterfaceField.C
@@ -48,11 +48,11 @@ void Foam::GGIBlockLduInterfaceField<Type>::transformCoupleField
     {
         if (forwardT().size() == 1)
         {
-            f *= pow(diag(forwardT()[0]).component(cmpt), rank());
+            f *= pow(diag(forwardT()[0]).component(cmpt), pTraits<Type>::rank);
         }
         else
         {
-            f *= pow(diag(forwardT())().component(cmpt), rank());
+            f *= pow(diag(forwardT())().component(cmpt), pTraits<Type>::rank);
         }
     }
 }
diff --git a/src/foam/matrices/blockLduMatrix/BlockLduMatrix/BlockLduInterfaceFields/GGIBlockLduInterfaceField/GGIBlockLduInterfaceField.H b/src/foam/matrices/blockLduMatrix/BlockLduMatrix/BlockLduInterfaceFields/GGIBlockLduInterfaceField/GGIBlockLduInterfaceField.H
index 262af796b..ae6f79db1 100644
--- a/src/foam/matrices/blockLduMatrix/BlockLduMatrix/BlockLduInterfaceFields/GGIBlockLduInterfaceField/GGIBlockLduInterfaceField.H
+++ b/src/foam/matrices/blockLduMatrix/BlockLduMatrix/BlockLduInterfaceFields/GGIBlockLduInterfaceField/GGIBlockLduInterfaceField.H
@@ -81,9 +81,6 @@ public:
             //- Return face transformation tensor
             virtual const tensorField& forwardT() const = 0;
 
-            //- Return rank of component for transform
-            virtual int rank() const = 0;
-
 
         //- Transform given patch component field
         void transformCoupleField
diff --git a/src/foam/matrices/blockLduMatrix/BlockLduMatrix/BlockLduInterfaceFields/MixingPlaneBlockLduInterfaceField/MixingPlaneBlockLduInterfaceField.C b/src/foam/matrices/blockLduMatrix/BlockLduMatrix/BlockLduInterfaceFields/MixingPlaneBlockLduInterfaceField/MixingPlaneBlockLduInterfaceField.C
index 3dba3fd84..6c5b53daa 100644
--- a/src/foam/matrices/blockLduMatrix/BlockLduMatrix/BlockLduInterfaceFields/MixingPlaneBlockLduInterfaceField/MixingPlaneBlockLduInterfaceField.C
+++ b/src/foam/matrices/blockLduMatrix/BlockLduMatrix/BlockLduInterfaceFields/MixingPlaneBlockLduInterfaceField/MixingPlaneBlockLduInterfaceField.C
@@ -49,11 +49,11 @@ void Foam::MixingPlaneBlockLduInterfaceField<Type>::transformCoupleField
     {
         if (forwardT().size() == 1)
         {
-            f *= pow(diag(forwardT()[0]).component(cmpt), rank());
+            f *= pow(diag(forwardT()[0]).component(cmpt), pTraits<Type>::rank);
         }
         else
         {
-            f *= pow(diag(forwardT())().component(cmpt), rank());
+            f *= pow(diag(forwardT())().component(cmpt), pTraits<Type>::rank);
         }
     }
 }
diff --git a/src/foam/matrices/blockLduMatrix/BlockLduMatrix/BlockLduInterfaceFields/MixingPlaneBlockLduInterfaceField/MixingPlaneBlockLduInterfaceField.H b/src/foam/matrices/blockLduMatrix/BlockLduMatrix/BlockLduInterfaceFields/MixingPlaneBlockLduInterfaceField/MixingPlaneBlockLduInterfaceField.H
index 0f23e8add..0cd135e17 100644
--- a/src/foam/matrices/blockLduMatrix/BlockLduMatrix/BlockLduInterfaceFields/MixingPlaneBlockLduInterfaceField/MixingPlaneBlockLduInterfaceField.H
+++ b/src/foam/matrices/blockLduMatrix/BlockLduMatrix/BlockLduInterfaceFields/MixingPlaneBlockLduInterfaceField/MixingPlaneBlockLduInterfaceField.H
@@ -81,9 +81,6 @@ public:
             //- Return face transformation tensor
             virtual const tensorField& forwardT() const = 0;
 
-            //- Return rank of component for transform
-            virtual int rank() const = 0;
-
 
         //- Transform given patch component field
         void transformCoupleField
diff --git a/src/foam/matrices/blockLduMatrix/BlockLduMatrix/BlockLduInterfaceFields/OverlapGGIBlockLduInterfaceField/OverlapGGIBlockLduInterfaceField.C b/src/foam/matrices/blockLduMatrix/BlockLduMatrix/BlockLduInterfaceFields/OverlapGGIBlockLduInterfaceField/OverlapGGIBlockLduInterfaceField.C
index 1dd2d710b..504028708 100644
--- a/src/foam/matrices/blockLduMatrix/BlockLduMatrix/BlockLduInterfaceFields/OverlapGGIBlockLduInterfaceField/OverlapGGIBlockLduInterfaceField.C
+++ b/src/foam/matrices/blockLduMatrix/BlockLduMatrix/BlockLduInterfaceFields/OverlapGGIBlockLduInterfaceField/OverlapGGIBlockLduInterfaceField.C
@@ -49,11 +49,11 @@ void Foam::OverlapGGIBlockLduInterfaceField<Type>::transformCoupleField
     {
         if (forwardT().size() == 1)
         {
-            f *= pow(diag(forwardT()[0]).component(cmpt), rank());
+            f *= pow(diag(forwardT()[0]).component(cmpt), pTraits<Type>::rank);
         }
         else
         {
-            f *= pow(diag(forwardT())().component(cmpt), rank());
+            f *= pow(diag(forwardT())().component(cmpt), pTraits<Type>::rank);
         }
     }
 }
diff --git a/src/foam/matrices/blockLduMatrix/BlockLduMatrix/BlockLduInterfaceFields/OverlapGGIBlockLduInterfaceField/OverlapGGIBlockLduInterfaceField.H b/src/foam/matrices/blockLduMatrix/BlockLduMatrix/BlockLduInterfaceFields/OverlapGGIBlockLduInterfaceField/OverlapGGIBlockLduInterfaceField.H
index 1a54b74f6..3db9f4f0e 100644
--- a/src/foam/matrices/blockLduMatrix/BlockLduMatrix/BlockLduInterfaceFields/OverlapGGIBlockLduInterfaceField/OverlapGGIBlockLduInterfaceField.H
+++ b/src/foam/matrices/blockLduMatrix/BlockLduMatrix/BlockLduInterfaceFields/OverlapGGIBlockLduInterfaceField/OverlapGGIBlockLduInterfaceField.H
@@ -81,9 +81,6 @@ public:
             //- Return face transformation tensor
             virtual const tensorField& forwardT() const = 0;
 
-            //- Return rank of component for transform
-            virtual int rank() const = 0;
-
 
         //- Transform given patch component field
         void transformCoupleField
diff --git a/src/foam/matrices/blockLduMatrix/BlockLduMatrix/BlockLduInterfaceFields/ProcessorBlockLduInterfaceField/ProcessorBlockLduInterfaceField.C b/src/foam/matrices/blockLduMatrix/BlockLduMatrix/BlockLduInterfaceFields/ProcessorBlockLduInterfaceField/ProcessorBlockLduInterfaceField.C
index dbaa36fe0..787b52e58 100644
--- a/src/foam/matrices/blockLduMatrix/BlockLduMatrix/BlockLduInterfaceFields/ProcessorBlockLduInterfaceField/ProcessorBlockLduInterfaceField.C
+++ b/src/foam/matrices/blockLduMatrix/BlockLduMatrix/BlockLduInterfaceFields/ProcessorBlockLduInterfaceField/ProcessorBlockLduInterfaceField.C
@@ -48,11 +48,11 @@ void Foam::ProcessorBlockLduInterfaceField<Type>::transformCoupleField
     {
         if (forwardT().size() == 1)
         {
-            f *= pow(diag(forwardT()[0]).component(cmpt), rank());
+            f *= pow(diag(forwardT()[0]).component(cmpt), pTraits<Type>::rank);
         }
         else
         {
-            f *= pow(diag(forwardT())().component(cmpt), rank());
+            f *= pow(diag(forwardT())().component(cmpt), pTraits<Type>::rank);
         }
     }
 }
diff --git a/src/foam/matrices/blockLduMatrix/BlockLduMatrix/BlockLduInterfaceFields/ProcessorBlockLduInterfaceField/ProcessorBlockLduInterfaceField.H b/src/foam/matrices/blockLduMatrix/BlockLduMatrix/BlockLduInterfaceFields/ProcessorBlockLduInterfaceField/ProcessorBlockLduInterfaceField.H
index 1f05c9303..3fc594aad 100644
--- a/src/foam/matrices/blockLduMatrix/BlockLduMatrix/BlockLduInterfaceFields/ProcessorBlockLduInterfaceField/ProcessorBlockLduInterfaceField.H
+++ b/src/foam/matrices/blockLduMatrix/BlockLduMatrix/BlockLduInterfaceFields/ProcessorBlockLduInterfaceField/ProcessorBlockLduInterfaceField.H
@@ -87,9 +87,6 @@ public:
             //- Return face transformation tensor
             virtual const tensorField& forwardT() const = 0;
 
-            //- Return rank of component for transform
-            virtual int rank() const = 0;
-
 
         //- Transform given patch component field
         void transformCoupleField
diff --git a/src/foam/matrices/blockLduMatrix/BlockLduMatrix/BlockLduMatrixUpdateInterfaces.C b/src/foam/matrices/blockLduMatrix/BlockLduMatrix/BlockLduMatrixUpdateInterfaces.C
index ecf9c836b..901dda54b 100644
--- a/src/foam/matrices/blockLduMatrix/BlockLduMatrix/BlockLduMatrixUpdateInterfaces.C
+++ b/src/foam/matrices/blockLduMatrix/BlockLduMatrix/BlockLduMatrixUpdateInterfaces.C
@@ -127,13 +127,73 @@ void Foam::BlockLduMatrix<Type>::updateInterfaces
     }
     else if (Pstream::defaultComms() == Pstream::nonBlocking)
     {
-        // Block until all sends/receives have been finished
-        IPstream::waitRequests();
-        OPstream::waitRequests();
+        // Try and consume interfaces as they become available
+        bool allUpdated = false;
 
+        for (label i = 0; i < Pstream::nPollProcInterfaces; i++)
+        {
+            allUpdated = true;
+
+            forAll (interfaces_, interfaceI)
+            {
+                if (interfaces_.set(interfaceI))
+                {
+                    if (!interfaces_[interfaceI].updatedMatrix())
+                    {
+                        if (interfaces_[interfaceI].ready())
+                        {
+                            interfaces_[interfaceI].updateInterfaceMatrix
+                            (
+                                psi,
+                                result,
+                                *this,
+                                interfaceCoeffs[interfaceI],
+                                Pstream::defaultComms(),
+                                switchToLhs
+                            );
+                        }
+                        else
+                        {
+                            allUpdated = false;
+                        }
+                    }
+                }
+            }
+
+            if (allUpdated)
+            {
+                break;
+            }
+        }
+
+        // Block for everything
+        if (Pstream::parRun())
+        {
+            if (allUpdated)
+            {
+                // All received. Just remove all storage of requests
+                // Note that we don't know what starting number of requests
+                // was before start of sends and receives (since set from
+                // initMatrixInterfaces) so set to 0 and lose any in-flight
+                // requests.
+                Pstream::resetRequests(0);
+            }
+            else
+            {
+                // Block for all requests and remove storage
+                IPstream::waitRequests();
+                OPstream::waitRequests();
+            }
+        }
+
+        // Consume
         forAll (interfaces_, interfaceI)
         {
-            if (interfaces_.set(interfaceI))
+            if
+            (
+                interfaces_.set(interfaceI)
+            && !interfaces_[interfaceI].updatedMatrix()
+            )
             {
                 interfaces_[interfaceI].updateInterfaceMatrix
                 (
diff --git a/src/foam/matrices/lduMatrix/lduAddressing/lduInterfaces/processorLduInterface/processorLduInterfaceTemplates.C b/src/foam/matrices/lduMatrix/lduAddressing/lduInterfaces/processorLduInterface/processorLduInterfaceTemplates.C
index 2a471f2af..a4f4d612c 100644
--- a/src/foam/matrices/lduMatrix/lduAddressing/lduInterfaces/processorLduInterface/processorLduInterfaceTemplates.C
+++ b/src/foam/matrices/lduMatrix/lduAddressing/lduInterfaces/processorLduInterface/processorLduInterfaceTemplates.C
@@ -52,14 +52,14 @@ void Foam::processorLduInterface::send
     }
     else if (commsType == Pstream::nonBlocking)
     {
-        resizeBuf(receiveBuf_, f.size()*sizeof(Type));
+        resizeBuf(receiveBuf_, nBytes);
 
         IPstream::read
         (
             commsType,
             neighbProcNo(),
             receiveBuf_.begin(),
-            receiveBuf_.size(),
+            nBytes,
             tag(),
             comm()
         );
diff --git a/src/foam/matrices/lduMatrix/lduMatrix/lduMatrixUpdateMatrixInterfaces.C b/src/foam/matrices/lduMatrix/lduMatrix/lduMatrixUpdateMatrixInterfaces.C
index 36876469e..39995364b 100644
--- a/src/foam/matrices/lduMatrix/lduMatrix/lduMatrixUpdateMatrixInterfaces.C
+++ b/src/foam/matrices/lduMatrix/lduMatrix/lduMatrixUpdateMatrixInterfaces.C
@@ -129,15 +129,74 @@ void Foam::lduMatrix::updateMatrixInterfaces
     }
     else if (Pstream::defaultComms() == Pstream::nonBlocking)
     {
-        // FOAM-3.1 implementation
+        // Try and consume interfaces as they become available
+        bool allUpdated = false;
 
-        // Block until all sends/receives have been finished
-        IPstream::waitRequests();
-        OPstream::waitRequests();
+        for (label i = 0; i < Pstream::nPollProcInterfaces; i++)
+        {
+            allUpdated = true;
 
+            forAll (interfaces, interfaceI)
+            {
+                if (interfaces.set(interfaceI))
+                {
+                    if (!interfaces[interfaceI].updatedMatrix())
+                    {
+                        if (interfaces[interfaceI].ready())
+                        {
+                            interfaces[interfaceI].updateInterfaceMatrix
+                            (
+                                psiif,
+                                result,
+                                *this,
+                                coupleCoeffs[interfaceI],
+                                cmpt,
+                                Pstream::defaultComms(),
+                                switchToLhs
+                            );
+                        }
+                        else
+                        {
+                            allUpdated = false;
+                        }
+                    }
+                }
+            }
+
+            if (allUpdated)
+            {
+                break;
+            }
+        }
+
+        // Block for everything
+        if (Pstream::parRun())
+        {
+            if (allUpdated)
+            {
+                // All received. Just remove all storage of requests
+                // Note that we don't know what starting number of requests
+                // was before start of sends and receives (since set from
+                // initMatrixInterfaces) so set to 0 and lose any in-flight
+                // requests.
+                Pstream::resetRequests(0);
+            }
+            else
+            {
+                // Block for all requests and remove storage
+                IPstream::waitRequests();
+                OPstream::waitRequests();
+            }
+        }
+
+        // Consume
         forAll (interfaces, interfaceI)
         {
-            if (interfaces.set(interfaceI))
+            if
+            (
+                interfaces.set(interfaceI)
+            && !interfaces[interfaceI].updatedMatrix()
+            )
             {
                 interfaces[interfaceI].updateInterfaceMatrix
                 (
diff --git a/src/foam/matrices/lduMatrix/solvers/AMG/interfaceFields/processorAMGInterfaceField/processorAMGInterfaceField.C b/src/foam/matrices/lduMatrix/solvers/AMG/interfaceFields/processorAMGInterfaceField/processorAMGInterfaceField.C
index e87076154..dcd6e44d6 100644
--- a/src/foam/matrices/lduMatrix/solvers/AMG/interfaceFields/processorAMGInterfaceField/processorAMGInterfaceField.C
+++ b/src/foam/matrices/lduMatrix/solvers/AMG/interfaceFields/processorAMGInterfaceField/processorAMGInterfaceField.C
@@ -52,7 +52,11 @@ Foam::processorAMGInterfaceField::processorAMGInterfaceField
     AMGInterfaceField(AMGCp, fineInterfaceField),
     procInterface_(refCast<const processorAMGInterface>(AMGCp)),
     doTransform_(false),
-    rank_(0)
+    rank_(0),
+    outstandingSendRequest_(-1),
+    outstandingRecvRequest_(-1),
+    scalarSendBuf_(0),
+    scalarReceiveBuf_(0)
 {
     const processorLduInterfaceField& p =
         refCast<const processorLduInterfaceField>(fineInterfaceField);
@@ -81,11 +85,50 @@ void Foam::processorAMGInterfaceField::initInterfaceMatrixUpdate
     const bool switchToLhs
 ) const
 {
-    procInterface_.send
-    (
-        commsType,
-        procInterface_.interfaceInternalField(psiInternal)()
-    );
+    label oldWarn = Pstream::warnComm;
+    Pstream::warnComm = comm();
+
+    scalarSendBuf_ = procInterface_.interfaceInternalField(psiInternal);
+
+    if (commsType == Pstream::nonBlocking)
+    {
+        // Fast path.
+        scalarReceiveBuf_.setSize(scalarSendBuf_.size());
+        outstandingRecvRequest_ = Pstream::nRequests();
+        IPstream::read
+        (
+            Pstream::nonBlocking,
+            procInterface_.neighbProcNo(),
+            reinterpret_cast<char*>(scalarReceiveBuf_.begin()),
+            scalarReceiveBuf_.byteSize(),
+            procInterface_.tag(),
+            comm()
+        );
+
+        outstandingSendRequest_ = Pstream::nRequests();
+        OPstream::write
+        (
+            Pstream::nonBlocking,
+            procInterface_.neighbProcNo(),
+            reinterpret_cast<const char*>(scalarSendBuf_.begin()),
+            scalarSendBuf_.byteSize(),
+            procInterface_.tag(),
+            comm()
+        );
+    }
+    else
+    {
+        procInterface_.send
+        (
+            commsType,
+            procInterface_.interfaceInternalField(psiInternal)()
+        );
+    }
+
+    // Mark as ready for update
+    const_cast<processorAMGInterfaceField&>(*this).updatedMatrix() = false;
+
+    Pstream::warnComm = oldWarn;
 }
 
 
@@ -100,11 +143,41 @@ void Foam::processorAMGInterfaceField::updateInterfaceMatrix
     const bool switchToLhs
 ) const
 {
-    scalarField pnf
-    (
-        procInterface_.receive<scalar>(commsType, coeffs.size())
-    );
-    transformCoupleField(pnf, cmpt);
+    if (this->updatedMatrix())
+    {
+        return;
+    }
+
+    if (commsType == Pstream::nonBlocking)
+    {
+        // Fast path.
+        if
+        (
+            outstandingRecvRequest_ >= 0
+         && outstandingRecvRequest_ < Pstream::nRequests()
+        )
+        {
+            Pstream::waitRequest(outstandingRecvRequest_);
+        }
+
+        // Recv finished so assume sending finished as well.
+        outstandingSendRequest_ = -1;
+        outstandingRecvRequest_ = -1;
+    }
+    else
+    {
+        // Check size
+        scalarReceiveBuf_.setSize(scalarSendBuf_.size());
+
+        procInterface_.receive<scalar>(commsType, scalarReceiveBuf_);
+    }
+
+    // The data is now in scalarReceiveBuf_ for both cases
+
+    // Transform according to the transformation tensor
+    transformCoupleField(scalarReceiveBuf_, cmpt);
+
+    // Multiply the field by coefficients and add into the result
 
     const unallocLabelList& faceCells = procInterface_.faceCells();
 
@@ -112,16 +185,19 @@ void Foam::processorAMGInterfaceField::updateInterfaceMatrix
     {
         forAll(faceCells, elemI)
         {
-            result[faceCells[elemI]] += coeffs[elemI]*pnf[elemI];
+            result[faceCells[elemI]] += coeffs[elemI]*scalarReceiveBuf_[elemI];
         }
     }
     else
     {
         forAll(faceCells, elemI)
         {
-            result[faceCells[elemI]] -= coeffs[elemI]*pnf[elemI];
+            result[faceCells[elemI]] -= coeffs[elemI]*scalarReceiveBuf_[elemI];
         }
     }
+
+    // Mark as updated
+    const_cast<processorAMGInterfaceField&>(*this).updatedMatrix() = true;
 }
 
 
diff --git a/src/foam/matrices/lduMatrix/solvers/AMG/interfaceFields/processorAMGInterfaceField/processorAMGInterfaceField.H b/src/foam/matrices/lduMatrix/solvers/AMG/interfaceFields/processorAMGInterfaceField/processorAMGInterfaceField.H
index 4b39d0445..b0eadacd7 100644
--- a/src/foam/matrices/lduMatrix/solvers/AMG/interfaceFields/processorAMGInterfaceField/processorAMGInterfaceField.H
+++ b/src/foam/matrices/lduMatrix/solvers/AMG/interfaceFields/processorAMGInterfaceField/processorAMGInterfaceField.H
@@ -65,6 +65,21 @@ class processorAMGInterfaceField
         int rank_;
 
 
+        // Sending and receiving
+
+            //- Outstanding request
+            mutable label outstandingSendRequest_;
+
+            //- Outstanding request
+            mutable label outstandingRecvRequest_;
+
+            //- Scalar send buffer
+            mutable Field<scalar> scalarSendBuf_;
+
+            //- Scalar receive buffer
+            mutable Field<scalar> scalarReceiveBuf_;
+
+
     // Private Member Functions
 
         //- Disallow default bitwise copy construct
@@ -144,6 +159,12 @@ public:
 
         //- Processor interface functions
 
+            //- Return communicator used for comms
+            virtual label comm() const
+            {
+                return procInterface_.comm();
+            }
+
             //- Return processor number
             virtual int myProcNo() const
             {

From a91cb63d8863539918e06a470b41237e3a259d54 Mon Sep 17 00:00:00 2001
From: Hrvoje Jasak <h.jasak@wikki.co.uk>
Date: Tue, 29 Nov 2016 12:44:48 +0000
Subject: [PATCH 094/130] Bugfix: processor polling in non-blocking comms

---
 .../processor/processorFvPatchField.C         |  36 +++++
 .../processor/processorFvPatchField.H         |   3 +
 .../processor/processorFvPatchScalarField.C   | 124 ++++++------------
 .../processor/processorFvPatchScalarField.H   |  38 +-----
 src/foam/db/IOstreams/Pstreams/Pstream.C      |   2 -
 5 files changed, 85 insertions(+), 118 deletions(-)

diff --git a/src/finiteVolume/fields/fvPatchFields/constraint/processor/processorFvPatchField.C b/src/finiteVolume/fields/fvPatchFields/constraint/processor/processorFvPatchField.C
index ddbbaa98d..85fabfec5 100644
--- a/src/finiteVolume/fields/fvPatchFields/constraint/processor/processorFvPatchField.C
+++ b/src/finiteVolume/fields/fvPatchFields/constraint/processor/processorFvPatchField.C
@@ -600,4 +600,40 @@ void Foam::processorFvPatchField<Type>::updateInterfaceMatrix
 }
 
 
+template<class Type>
+bool Foam::processorFvPatchField<Type>::ready() const
+{
+    if
+    (
+        outstandingSendRequest_ >= 0
+     && outstandingSendRequest_ < Pstream::nRequests()
+    )
+    {
+        bool finished = Pstream::finishedRequest(outstandingSendRequest_);
+        if (!finished)
+        {
+            return false;
+        }
+    }
+    outstandingSendRequest_ = -1;
+
+    if
+    (
+        outstandingRecvRequest_ >= 0
+     && outstandingRecvRequest_ < Pstream::nRequests()
+    )
+    {
+        bool finished = Pstream::finishedRequest(outstandingRecvRequest_);
+
+        if (!finished)
+        {
+            return false;
+        }
+    }
+    outstandingRecvRequest_ = -1;
+
+    return true;
+}
+
+
 // ************************************************************************* //
diff --git a/src/finiteVolume/fields/fvPatchFields/constraint/processor/processorFvPatchField.H b/src/finiteVolume/fields/fvPatchFields/constraint/processor/processorFvPatchField.H
index 6f9ed9f96..05972d2ae 100644
--- a/src/finiteVolume/fields/fvPatchFields/constraint/processor/processorFvPatchField.H
+++ b/src/finiteVolume/fields/fvPatchFields/constraint/processor/processorFvPatchField.H
@@ -285,6 +285,9 @@ public:
 
             // Communication support
 
+                //- Is all data available
+                virtual bool ready() const;
+
                 //- Return communicator used for parallel communication
                 virtual int comm() const
                 {
diff --git a/src/finiteVolume/fields/fvPatchFields/constraint/processor/processorFvPatchScalarField.C b/src/finiteVolume/fields/fvPatchFields/constraint/processor/processorFvPatchScalarField.C
index 36e3392d9..acce48d51 100644
--- a/src/finiteVolume/fields/fvPatchFields/constraint/processor/processorFvPatchScalarField.C
+++ b/src/finiteVolume/fields/fvPatchFields/constraint/processor/processorFvPatchScalarField.C
@@ -32,26 +32,6 @@ namespace Foam
 
 // * * * * * * * * * * * * * * * Member Functions  * * * * * * * * * * * * * //
 
-template<>
-void processorFvPatchField<scalar>::initInterfaceMatrixUpdate
-(
-    const scalarField& psiInternal,
-    scalarField&,
-    const lduMatrix&,
-    const scalarField&,
-    const direction,
-    const Pstream::commsTypes commsType,
-    const bool switchToLhs
-) const
-{
-    procPatch_.send
-    (
-        commsType,
-        patch().patchInternalField(psiInternal)()
-    );
-}
-
-
 template<>
 void processorFvPatchField<scalar>::updateInterfaceMatrix
 (
@@ -64,82 +44,60 @@ void processorFvPatchField<scalar>::updateInterfaceMatrix
     const bool switchToLhs
 ) const
 {
-    scalarField pnf
-    (
-        procPatch_.receive<scalar>(commsType, this->size())()
-    );
+    if (this->updatedMatrix())
+    {
+        return;
+    }
 
-    const unallocLabelList& faceCells = patch().faceCells();
+    if (commsType == Pstream::nonBlocking)
+    {
+        // Fast path.
+        if
+        (
+            outstandingRecvRequest_ >= 0
+         && outstandingRecvRequest_ < Pstream::nRequests()
+        )
+        {
+            Pstream::waitRequest(outstandingRecvRequest_);
+        }
+
+        // Recv finished so assume sending finished as well.
+        outstandingSendRequest_ = -1;
+        outstandingRecvRequest_ = -1;
+    }
+    else
+    {
+        // Check size
+        scalarReceiveBuf_.setSize(this->size());
+
+        procPatch_.receive<scalar>(commsType, scalarReceiveBuf_);
+    }
+
+    // The data is now in scalarReceiveBuf_ for both cases
+
+    // Transform according to the transformation tensor
+    // No transform for scalar.  HJ, 29/Nov/2016
+    
+    // Multiply the field by coefficients and add into the result
+
+    const unallocLabelList& faceCells = this->patch().faceCells();
 
     if (switchToLhs)
     {
-        forAll (faceCells, facei)
+        forAll(faceCells, elemI)
         {
-            result[faceCells[facei]] += coeffs[facei]*pnf[facei];
+            result[faceCells[elemI]] += coeffs[elemI]*scalarReceiveBuf_[elemI];
         }
     }
     else
     {
-        forAll (faceCells, facei)
+        forAll(faceCells, elemI)
         {
-            result[faceCells[facei]] -= coeffs[facei]*pnf[facei];
+            result[faceCells[elemI]] -= coeffs[elemI]*scalarReceiveBuf_[elemI];
         }
     }
-}
 
-
-template<>
-void processorFvPatchField<scalar>::initInterfaceMatrixUpdate
-(
-    const scalarField& psiInternal,
-    scalarField&,
-    const BlockLduMatrix<scalar>&,
-    const CoeffField<scalar>&,
-    const Pstream::commsTypes commsType,
-    const bool switchToLhs
-) const
-{
-    procPatch_.send
-    (
-        commsType,
-        patch().patchInternalField(psiInternal)()
-    );
-}
-
-
-template<>
-void processorFvPatchField<scalar>::updateInterfaceMatrix
-(
-    const scalarField&,
-    scalarField& result,
-    const BlockLduMatrix<scalar>&,
-    const CoeffField<scalar>& coeffs,
-    const Pstream::commsTypes commsType,
-    const bool switchToLhs
-) const
-{
-    scalarField pnf
-    (
-        procPatch_.receive<scalar>(commsType, this->size())()
-    );
-
-    const unallocLabelList& faceCells = patch().faceCells();
-    const scalarField& scalarCoeffs = coeffs.asScalar();
-
-    if (switchToLhs)
-    {
-        forAll (faceCells, facei)
-        {
-            result[faceCells[facei]] += scalarCoeffs[facei]*pnf[facei];
-        }
-    }
-    else
-    {
-        forAll (faceCells, facei)
-        {
-            result[faceCells[facei]] -= scalarCoeffs[facei]*pnf[facei];
-        }
-    }
+    const_cast<processorFvPatchField<scalar>&>(*this).updatedMatrix() = true;
 }
 
 
diff --git a/src/finiteVolume/fields/fvPatchFields/constraint/processor/processorFvPatchScalarField.H b/src/finiteVolume/fields/fvPatchFields/constraint/processor/processorFvPatchScalarField.H
index 4c8e21a54..1c9d84b79 100644
--- a/src/finiteVolume/fields/fvPatchFields/constraint/processor/processorFvPatchScalarField.H
+++ b/src/finiteVolume/fields/fvPatchFields/constraint/processor/processorFvPatchScalarField.H
@@ -36,18 +36,8 @@ namespace Foam
 
 // * * * * * * * * * * * * * * * Member Functions  * * * * * * * * * * * * * //
 
-template<>
-void processorFvPatchField<scalar>::initInterfaceMatrixUpdate
-(
-    const scalarField&,
-    scalarField&,
-    const lduMatrix&,
-    const scalarField&,
-    const direction,
-    const Pstream::commsTypes commsType,
-    const bool switchToLhs
-) const;
-
+// Removed unnecessary specialisation for initInterfaceMatrixUpdate
+// HJ, 29/Nov/2016
 
 template<>
 void processorFvPatchField<scalar>::updateInterfaceMatrix
@@ -62,29 +52,11 @@ void processorFvPatchField<scalar>::updateInterfaceMatrix
 ) const;
 
 
-template<>
-void processorFvPatchField<scalar>::initInterfaceMatrixUpdate
-(
-    const scalarField& psiInternal,
-    scalarField&,
-    const BlockLduMatrix<scalar>&,
-    const CoeffField<scalar>&,
-    const Pstream::commsTypes commsType,
-    const bool switchToLhs
-) const;
+// Removed unnecessary specialisation for initInterfaceMatrixUpdate
+// and updateInterfaceMatrix for block matrices
+// HJ, 29/Nov/2016
 
 
-template<>
-void processorFvPatchField<scalar>::updateInterfaceMatrix
-(
-    const scalarField&,
-    scalarField& result,
-    const BlockLduMatrix<scalar>&,
-    const CoeffField<scalar>& coeffs,
-    const Pstream::commsTypes commsType,
-    const bool switchToLhs
-) const;
-
 // * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
 
 } // End namespace Foam
diff --git a/src/foam/db/IOstreams/Pstreams/Pstream.C b/src/foam/db/IOstreams/Pstreams/Pstream.C
index cb78615d0..a7e6dc63c 100644
--- a/src/foam/db/IOstreams/Pstreams/Pstream.C
+++ b/src/foam/db/IOstreams/Pstreams/Pstream.C
@@ -922,8 +922,6 @@ Foam::Pstream::defaultCommsType
 (
     "commsType",
     "nonBlocking",
-//     "scheduled",
-//     "blocking",
     "blocking, nonBlocking, scheduled"
 );
 

From 4d0125278f00a8a8b2588876e18456d003251b40 Mon Sep 17 00:00:00 2001
From: Hrvoje Jasak <h.jasak@wikki.co.uk>
Date: Tue, 6 Dec 2016 12:49:27 +0000
Subject: [PATCH 095/130] Finite area updates, Matthias Rauter

---
 src/finiteArea/Make/files                     |   3 +
 .../basic/transform/transformFaPatchFields.C  |   5 -
 .../constraint/wedge/wedgeFaPatchFields.C     |   5 -
 .../edgeNormalFixedValueFaPatchVectorField.C  | 155 ++++++++
 .../edgeNormalFixedValueFaPatchVectorField.H  | 173 +++++++++
 ...timeVaryingUniformFixedValueFaPatchField.C | 137 +++++++
 ...timeVaryingUniformFixedValueFaPatchField.H | 182 ++++++++++
 ...imeVaryingUniformFixedValueFaPatchFields.C |  43 +++
 ...imeVaryingUniformFixedValueFaPatchFields.H |  49 +++
 ...VaryingUniformFixedValueFaPatchFieldsFwd.H |  51 +++
 .../faPatchFields/faPatchField/faPatchField.H | 125 ++++---
 .../gaussFaDivScheme/gaussFaDivScheme.C       |   2 +-
 src/finiteArea/finiteArea/fac/fac.H           |   2 +
 src/finiteArea/finiteArea/fac/facDiv.C        |  65 +++-
 src/finiteArea/finiteArea/fac/facGrad.C       |  34 +-
 src/finiteArea/finiteArea/fac/facNDiv.C       | 333 ++++++++++++++++++
 src/finiteArea/finiteArea/fac/facNDiv.H       | 187 ++++++++++
 src/finiteArea/finiteArea/fac/facNGrad.C      | 202 +++++++++++
 src/finiteArea/finiteArea/fac/facNGrad.H      | 131 +++++++
 src/finiteArea/finiteArea/fam/fam.H           |   1 +
 src/finiteArea/finiteArea/fam/famDiv.C        |  32 +-
 src/finiteArea/finiteArea/fam/famDiv.H        |   4 +
 src/finiteArea/finiteArea/fam/famNDiv.C       | 108 ++++++
 src/finiteArea/finiteArea/fam/famNDiv.H       | 100 ++++++
 src/finiteArea/finiteArea/fam/vectorFamDiv.C  |  69 ++++
 src/finiteArea/finiteArea/fam/vectorFamDiv.H  |  74 ++++
 .../gradSchemes/gaussFaGrad/gaussFaGrad.C     |  17 +-
 .../leastSquaresFaGrad/leastSquaresFaGrad.C   |   5 +-
 28 files changed, 2196 insertions(+), 98 deletions(-)
 create mode 100644 src/finiteArea/fields/faPatchFields/derived/edgeNormalFixedValue/edgeNormalFixedValueFaPatchVectorField.C
 create mode 100644 src/finiteArea/fields/faPatchFields/derived/edgeNormalFixedValue/edgeNormalFixedValueFaPatchVectorField.H
 create mode 100644 src/finiteArea/fields/faPatchFields/derived/timeVaryingUniformFixedValue/timeVaryingUniformFixedValueFaPatchField.C
 create mode 100644 src/finiteArea/fields/faPatchFields/derived/timeVaryingUniformFixedValue/timeVaryingUniformFixedValueFaPatchField.H
 create mode 100644 src/finiteArea/fields/faPatchFields/derived/timeVaryingUniformFixedValue/timeVaryingUniformFixedValueFaPatchFields.C
 create mode 100644 src/finiteArea/fields/faPatchFields/derived/timeVaryingUniformFixedValue/timeVaryingUniformFixedValueFaPatchFields.H
 create mode 100644 src/finiteArea/fields/faPatchFields/derived/timeVaryingUniformFixedValue/timeVaryingUniformFixedValueFaPatchFieldsFwd.H
 create mode 100644 src/finiteArea/finiteArea/fac/facNDiv.C
 create mode 100644 src/finiteArea/finiteArea/fac/facNDiv.H
 create mode 100644 src/finiteArea/finiteArea/fac/facNGrad.C
 create mode 100644 src/finiteArea/finiteArea/fac/facNGrad.H
 create mode 100644 src/finiteArea/finiteArea/fam/famNDiv.C
 create mode 100644 src/finiteArea/finiteArea/fam/famNDiv.H
 create mode 100644 src/finiteArea/finiteArea/fam/vectorFamDiv.C
 create mode 100644 src/finiteArea/finiteArea/fam/vectorFamDiv.H

diff --git a/src/finiteArea/Make/files b/src/finiteArea/Make/files
index 6cd6d6288..9e8ffce86 100644
--- a/src/finiteArea/Make/files
+++ b/src/finiteArea/Make/files
@@ -48,6 +48,8 @@ derivedFaPatchFields = $(faPatchFields)/derived
 $(derivedFaPatchFields)/fixedValueOutflow/fixedValueOutflowFaPatchFields.C
 $(derivedFaPatchFields)/inletOutlet/inletOutletFaPatchFields.C
 $(derivedFaPatchFields)/slip/slipFaPatchFields.C
+$(derivedFaPatchFields)/edgeNormalFixedValue/edgeNormalFixedValueFaPatchVectorField.C
+$(derivedFaPatchFields)/timeVaryingUniformFixedValue/timeVaryingUniformFixedValueFaPatchFields.C
 
 faePatchFields = fields/faePatchFields
 $(faePatchFields)/faePatchField/faePatchFields.C
@@ -92,6 +94,7 @@ $(ddtSchemes)/backwardFaDdtScheme/backwardFaDdtSchemes.C
 $(ddtSchemes)/boundedBackwardFaDdtScheme/boundedBackwardFaDdtScheme.C
 
 divSchemes = finiteArea/divSchemes
+finiteArea/fam/vectorFamDiv.C
 $(divSchemes)/faDivScheme/faDivSchemes.C
 $(divSchemes)/gaussFaDivScheme/gaussFaDivSchemes.C
 
diff --git a/src/finiteArea/fields/faPatchFields/basic/transform/transformFaPatchFields.C b/src/finiteArea/fields/faPatchFields/basic/transform/transformFaPatchFields.C
index 2c808f7bd..24e9fbfc8 100644
--- a/src/finiteArea/fields/faPatchFields/basic/transform/transformFaPatchFields.C
+++ b/src/finiteArea/fields/faPatchFields/basic/transform/transformFaPatchFields.C
@@ -28,7 +28,6 @@ Description
 #include "faPatchFields.H"
 #include "transformFaPatchFields.H"
 #include "addToRunTimeSelectionTable.H"
-// #include "volFields.H"
 
 // * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
 
@@ -37,10 +36,6 @@ namespace Foam
 
 // * * * * * * * * * * * * * * Static Data Members * * * * * * * * * * * * * //
 
-// defineNamedTemplateTypeNameAndDebug(transformFaPatchScalarField, 0);
-// defineNamedTemplateTypeNameAndDebug(transformFaPatchVectorField, 0);
-// defineNamedTemplateTypeNameAndDebug(transformFaPatchTensorField, 0);
-
 makeFaPatchFieldsTypeName(transform);
 
 // * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
diff --git a/src/finiteArea/fields/faPatchFields/constraint/wedge/wedgeFaPatchFields.C b/src/finiteArea/fields/faPatchFields/constraint/wedge/wedgeFaPatchFields.C
index b67ab97c2..be4183c9a 100644
--- a/src/finiteArea/fields/faPatchFields/constraint/wedge/wedgeFaPatchFields.C
+++ b/src/finiteArea/fields/faPatchFields/constraint/wedge/wedgeFaPatchFields.C
@@ -28,7 +28,6 @@ Description
 #include "faPatchFields.H"
 #include "wedgeFaPatchFields.H"
 #include "addToRunTimeSelectionTable.H"
-// #include "volFields.H"
 
 // * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
 
@@ -37,10 +36,6 @@ namespace Foam
 
 // * * * * * * * * * * * * * * Static Data Members * * * * * * * * * * * * * //
 
-// defineNamedTemplateTypeNameAndDebug(wedgeFaPatchScalarField, 0);
-// defineNamedTemplateTypeNameAndDebug(wedgeFaPatchVectorField, 0);
-// defineNamedTemplateTypeNameAndDebug(wedgeFaPatchTensorField, 0);
-
 makeFaPatchFields(wedge);
 
 // * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
diff --git a/src/finiteArea/fields/faPatchFields/derived/edgeNormalFixedValue/edgeNormalFixedValueFaPatchVectorField.C b/src/finiteArea/fields/faPatchFields/derived/edgeNormalFixedValue/edgeNormalFixedValueFaPatchVectorField.C
new file mode 100644
index 000000000..bc0a93108
--- /dev/null
+++ b/src/finiteArea/fields/faPatchFields/derived/edgeNormalFixedValue/edgeNormalFixedValueFaPatchVectorField.C
@@ -0,0 +1,155 @@
+/*---------------------------------------------------------------------------*\
+  =========                 |
+  \\      /  F ield         | foam-extend: Open Source CFD
+   \\    /   O peration     | Version:     4.0
+    \\  /    A nd           | Web:         http://www.foam-extend.org
+     \\/     M anipulation  | For copyright notice see file Copyright
+-------------------------------------------------------------------------------
+License
+    This file is part of foam-extend.
+
+    foam-extend is free software: you can redistribute it and/or modify it
+    under the terms of the GNU General Public License as published by the
+    Free Software Foundation, either version 3 of the License, or (at your
+    option) any later version.
+
+    foam-extend is distributed in the hope that it will be useful, but
+    WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+    General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with foam-extend.  If not, see <http://www.gnu.org/licenses/>.
+
+\*---------------------------------------------------------------------------*/
+
+#include "edgeNormalFixedValueFaPatchVectorField.H"
+#include "addToRunTimeSelectionTable.H"
+#include "areaFields.H"
+#include "faPatchFieldMapper.H"
+
+// * * * * * * * * * * * * * * * * Constructors  * * * * * * * * * * * * * * //
+
+Foam::edgeNormalFixedValueFaPatchVectorField::
+edgeNormalFixedValueFaPatchVectorField
+(
+    const faPatch& p,
+    const DimensionedField<vector, areaMesh>& iF
+)
+:
+    fixedValueFaPatchVectorField(p, iF),
+    refValue_(p.size(), 0)
+{}
+
+
+Foam::edgeNormalFixedValueFaPatchVectorField::
+edgeNormalFixedValueFaPatchVectorField
+(
+    const edgeNormalFixedValueFaPatchVectorField& ptf,
+    const faPatch& p,
+    const DimensionedField<vector, areaMesh>& iF,
+    const faPatchFieldMapper& mapper
+)
+:
+    fixedValueFaPatchVectorField(ptf, p, iF, mapper),
+    refValue_(ptf.refValue_, mapper)
+{}
+
+
+Foam::edgeNormalFixedValueFaPatchVectorField::
+edgeNormalFixedValueFaPatchVectorField
+(
+    const faPatch& p,
+    const DimensionedField<vector, areaMesh>& iF,
+    const dictionary& dict
+)
+:
+    fixedValueFaPatchVectorField(p, iF, dict),
+    refValue_("refValue", dict, p.size())
+{}
+
+
+Foam::edgeNormalFixedValueFaPatchVectorField::
+edgeNormalFixedValueFaPatchVectorField
+(
+    const edgeNormalFixedValueFaPatchVectorField& pivpvf
+)
+:
+    fixedValueFaPatchVectorField(pivpvf),
+    refValue_(pivpvf.refValue_)
+{}
+
+
+Foam::edgeNormalFixedValueFaPatchVectorField::
+edgeNormalFixedValueFaPatchVectorField
+(
+    const edgeNormalFixedValueFaPatchVectorField& pivpvf,
+    const DimensionedField<vector, areaMesh>& iF
+)
+:
+    fixedValueFaPatchVectorField(pivpvf, iF),
+    refValue_(pivpvf.refValue_)
+{}
+
+
+// * * * * * * * * * * * * * * * Member Functions  * * * * * * * * * * * * * //
+
+void Foam::edgeNormalFixedValueFaPatchVectorField::autoMap
+(
+    const faPatchFieldMapper& m
+)
+{
+    fixedValueFaPatchVectorField::autoMap(m);
+    refValue_.autoMap(m);
+}
+
+
+void Foam::edgeNormalFixedValueFaPatchVectorField::rmap
+(
+    const faPatchVectorField& ptf,
+    const labelList& addr
+)
+{
+    fixedValueFaPatchVectorField::rmap(ptf, addr);
+
+    const edgeNormalFixedValueFaPatchVectorField& tiptf =
+        refCast<const edgeNormalFixedValueFaPatchVectorField>(ptf);
+
+    refValue_.rmap(tiptf.refValue_, addr);
+}
+
+
+void Foam::edgeNormalFixedValueFaPatchVectorField::updateCoeffs()
+{
+    if (updated())
+    {
+        return;
+    }
+
+    // Bug fix: update for moving mesh.  HJ, 15/Oct/2010
+    operator==(refValue_*patch().edgeNormals());
+}
+
+
+void Foam::edgeNormalFixedValueFaPatchVectorField::write(Ostream& os) const
+{
+    fixedValueFaPatchVectorField::write(os);
+    refValue_.writeEntry("refValue", os);
+}
+
+
+// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
+
+namespace Foam
+{
+
+makeFaPatchTypeField
+(
+    faPatchVectorField,
+    edgeNormalFixedValueFaPatchVectorField
+);
+
+} // End namespace Foam
+
+
+// ************************************************************************* //
diff --git a/src/finiteArea/fields/faPatchFields/derived/edgeNormalFixedValue/edgeNormalFixedValueFaPatchVectorField.H b/src/finiteArea/fields/faPatchFields/derived/edgeNormalFixedValue/edgeNormalFixedValueFaPatchVectorField.H
new file mode 100644
index 000000000..0cf215d73
--- /dev/null
+++ b/src/finiteArea/fields/faPatchFields/derived/edgeNormalFixedValue/edgeNormalFixedValueFaPatchVectorField.H
@@ -0,0 +1,173 @@
+/*---------------------------------------------------------------------------*\
+  =========                 |
+  \\      /  F ield         | foam-extend: Open Source CFD
+   \\    /   O peration     | Version:     4.0
+    \\  /    A nd           | Web:         http://www.foam-extend.org
+     \\/     M anipulation  | For copyright notice see file Copyright
+-------------------------------------------------------------------------------
+License
+    This file is part of foam-extend.
+
+    foam-extend is free software: you can redistribute it and/or modify it
+    under the terms of the GNU General Public License as published by the
+    Free Software Foundation, either version 3 of the License, or (at your
+    option) any later version.
+
+    foam-extend is distributed in the hope that it will be useful, but
+    WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+    General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with foam-extend.  If not, see <http://www.gnu.org/licenses/>.
+
+Class
+    Foam::edgeNormalFixedValueFaPatchVectorField
+
+Description
+    Edge normal fixed value vector field finite area boundary condition
+
+    Describes a surface normal vector boundary condition by its magnitude.
+    Note: The value is positive for outward-pointing vectors
+
+SourceFiles
+    edgeNormalFixedValueFaPatchVectorField.C
+
+\*---------------------------------------------------------------------------*/
+
+#ifndef edgeNormalFixedValueFaPatchVectorField_H
+#define edgeNormalFixedValueFaPatchVectorField_H
+
+#include "faPatchFields.H"
+#include "fixedValueFaPatchFields.H"
+
+// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
+
+namespace Foam
+{
+
+/*---------------------------------------------------------------------------*\
+                 Class edgeNormalFixedValueFaPatch Declaration
+\*---------------------------------------------------------------------------*/
+
+class edgeNormalFixedValueFaPatchVectorField
+:
+    public fixedValueFaPatchVectorField
+{
+    // Private data
+
+        //- Surface-normal velocity value
+        scalarField refValue_;
+
+
+public:
+
+    //- Runtime type information
+    TypeName("edgeNormalFixedValue");
+
+
+    // Constructors
+
+        //- Construct from patch and internal field
+        edgeNormalFixedValueFaPatchVectorField
+        (
+            const faPatch&,
+            const DimensionedField<vector, areaMesh>&
+        );
+
+        //- Construct from patch, internal field and dictionary
+        edgeNormalFixedValueFaPatchVectorField
+        (
+            const faPatch&,
+            const DimensionedField<vector, areaMesh>&,
+            const dictionary&
+        );
+
+        //- Construct by mapping given
+        //  edgeNormalFixedValueFaPatchVectorField
+        //  onto a new patch
+        edgeNormalFixedValueFaPatchVectorField
+        (
+            const edgeNormalFixedValueFaPatchVectorField&,
+            const faPatch&,
+            const DimensionedField<vector, areaMesh>&,
+            const faPatchFieldMapper&
+        );
+
+        //- Construct as copy
+        edgeNormalFixedValueFaPatchVectorField
+        (
+            const edgeNormalFixedValueFaPatchVectorField&
+        );
+
+        //- Construct and return a clone
+        virtual tmp<faPatchVectorField> clone() const
+        {
+            return tmp<faPatchVectorField>
+            (
+                new edgeNormalFixedValueFaPatchVectorField(*this)
+            );
+        }
+
+        //- Construct as copy setting internal field reference
+        edgeNormalFixedValueFaPatchVectorField
+        (
+            const edgeNormalFixedValueFaPatchVectorField&,
+            const DimensionedField<vector, areaMesh>&
+        );
+
+        //- Construct and return a clone setting internal field reference
+        virtual tmp<faPatchVectorField> clone
+        (
+            const DimensionedField<vector, areaMesh>& iF
+        ) const
+        {
+            return tmp<faPatchVectorField>
+            (
+                new edgeNormalFixedValueFaPatchVectorField
+                (
+                    *this,
+                    iF
+                )
+            );
+        }
+
+
+
+    // Member functions
+
+        // Mapping functions
+
+            //- Map (and resize as needed) from self given a mapping object
+            virtual void autoMap
+            (
+                const faPatchFieldMapper&
+            );
+
+            //- Reverse map the given faPatchField onto this faPatchField
+            virtual void rmap
+            (
+                const faPatchVectorField&,
+                const labelList&
+            );
+
+
+    // Member functions
+
+        //- Update the coefficients associated with the patch field
+        virtual void updateCoeffs();
+
+        //- Write
+        virtual void write(Ostream&) const;
+};
+
+
+// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
+
+} // End namespace Foam
+
+// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
+
+#endif
+
+// ************************************************************************* //
diff --git a/src/finiteArea/fields/faPatchFields/derived/timeVaryingUniformFixedValue/timeVaryingUniformFixedValueFaPatchField.C b/src/finiteArea/fields/faPatchFields/derived/timeVaryingUniformFixedValue/timeVaryingUniformFixedValueFaPatchField.C
new file mode 100644
index 000000000..cb2b83c26
--- /dev/null
+++ b/src/finiteArea/fields/faPatchFields/derived/timeVaryingUniformFixedValue/timeVaryingUniformFixedValueFaPatchField.C
@@ -0,0 +1,137 @@
+/*---------------------------------------------------------------------------*\
+  =========                 |
+  \\      /  F ield         | foam-extend: Open Source CFD
+   \\    /   O peration     | Version:     4.0
+    \\  /    A nd           | Web:         http://www.foam-extend.org
+     \\/     M anipulation  | For copyright notice see file Copyright
+-------------------------------------------------------------------------------
+License
+    This file is part of foam-extend.
+
+    foam-extend is free software: you can redistribute it and/or modify it
+    under the terms of the GNU General Public License as published by the
+    Free Software Foundation, either version 3 of the License, or (at your
+    option) any later version.
+
+    foam-extend is distributed in the hope that it will be useful, but
+    WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+    General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with foam-extend.  If not, see <http://www.gnu.org/licenses/>.
+
+\*---------------------------------------------------------------------------*/
+
+#include "timeVaryingUniformFixedValueFaPatchField.H"
+#include "foamTime.H"
+
+// * * * * * * * * * * * * * * * * Constructors  * * * * * * * * * * * * * * //
+
+template<class Type>
+Foam::timeVaryingUniformFixedValueFaPatchField<Type>::
+timeVaryingUniformFixedValueFaPatchField
+(
+    const faPatch& p,
+    const DimensionedField<Type, areaMesh>& iF
+)
+:
+    fixedValueFaPatchField<Type>(p, iF),
+    timeSeries_()
+{}
+
+
+template<class Type>
+Foam::timeVaryingUniformFixedValueFaPatchField<Type>::
+timeVaryingUniformFixedValueFaPatchField
+(
+    const faPatch& p,
+    const DimensionedField<Type, areaMesh>& iF,
+    const dictionary& dict
+)
+:
+    fixedValueFaPatchField<Type>(p, iF),
+    timeSeries_(dict)
+{
+   if (dict.found("value"))
+   {
+       faPatchField<Type>::operator==(Field<Type>("value", dict, p.size()));
+   }
+   else
+   {
+       updateCoeffs();
+   }
+}
+
+
+template<class Type>
+Foam::timeVaryingUniformFixedValueFaPatchField<Type>::
+timeVaryingUniformFixedValueFaPatchField
+(
+    const timeVaryingUniformFixedValueFaPatchField<Type>& ptf,
+    const faPatch& p,
+    const DimensionedField<Type, areaMesh>& iF,
+    const faPatchFieldMapper& mapper
+)
+:
+    fixedValueFaPatchField<Type>(ptf, p, iF, mapper),
+    timeSeries_(ptf.timeSeries_)
+{}
+
+
+template<class Type>
+Foam::timeVaryingUniformFixedValueFaPatchField<Type>::
+timeVaryingUniformFixedValueFaPatchField
+(
+    const timeVaryingUniformFixedValueFaPatchField<Type>& ptf
+)
+:
+    fixedValueFaPatchField<Type>(ptf),
+    timeSeries_(ptf.timeSeries_)
+{}
+
+
+template<class Type>
+Foam::timeVaryingUniformFixedValueFaPatchField<Type>::
+timeVaryingUniformFixedValueFaPatchField
+(
+    const timeVaryingUniformFixedValueFaPatchField<Type>& ptf,
+    const DimensionedField<Type, areaMesh>& iF
+)
+:
+    fixedValueFaPatchField<Type>(ptf, iF),
+    timeSeries_(ptf.timeSeries_)
+{}
+
+
+// * * * * * * * * * * * * * * * Member Functions  * * * * * * * * * * * * * //
+
+template<class Type>
+void Foam::timeVaryingUniformFixedValueFaPatchField<Type>::updateCoeffs()
+{
+    if (this->updated())
+    {
+        return;
+    }
+
+    faPatchField<Type>::operator==
+    (
+        timeSeries_(this->db().time().timeOutputValue())
+    );
+    fixedValueFaPatchField<Type>::updateCoeffs();
+}
+
+
+template<class Type>
+void Foam::timeVaryingUniformFixedValueFaPatchField<Type>::write
+(
+    Ostream& os
+) const
+{
+    faPatchField<Type>::write(os);
+    timeSeries_.write(os);
+    this->writeEntry("value", os);
+}
+
+
+// ************************************************************************* //
diff --git a/src/finiteArea/fields/faPatchFields/derived/timeVaryingUniformFixedValue/timeVaryingUniformFixedValueFaPatchField.H b/src/finiteArea/fields/faPatchFields/derived/timeVaryingUniformFixedValue/timeVaryingUniformFixedValueFaPatchField.H
new file mode 100644
index 000000000..b59f386c4
--- /dev/null
+++ b/src/finiteArea/fields/faPatchFields/derived/timeVaryingUniformFixedValue/timeVaryingUniformFixedValueFaPatchField.H
@@ -0,0 +1,182 @@
+/*---------------------------------------------------------------------------*\
+  =========                 |
+  \\      /  F ield         | foam-extend: Open Source CFD
+   \\    /   O peration     | Version:     4.0
+    \\  /    A nd           | Web:         http://www.foam-extend.org
+     \\/     M anipulation  | For copyright notice see file Copyright
+-------------------------------------------------------------------------------
+License
+    This file is part of foam-extend.
+
+    foam-extend is free software: you can redistribute it and/or modify it
+    under the terms of the GNU General Public License as published by the
+    Free Software Foundation, either version 3 of the License, or (at your
+    option) any later version.
+
+    foam-extend is distributed in the hope that it will be useful, but
+    WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+    General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with foam-extend.  If not, see <http://www.gnu.org/licenses/>.
+
+Class
+    Foam::timeVaryingUniformFixedValueFaPatchField
+
+Description
+    A time-varying form of a uniform fixed value finite area
+    boundary condition.
+
+    Example of the boundary condition specification:
+    @verbatim
+    inlet
+    {
+        type            timeVaryingUniformFixedValue;
+        fileName        "$FOAM_CASE/time-series";
+        outOfBounds     clamp;           // (error|warn|clamp|repeat)
+    }
+    @endverbatim
+
+Note
+    This class is derived directly from a fixedValue patch rather than from
+    a uniformFixedValue patch.
+
+See Also
+    Foam::interpolationTable and Foam::fixedValueFaPatchField
+
+SourceFiles
+    timeVaryingUniformFixedValueFaPatchField.C
+
+\*---------------------------------------------------------------------------*/
+
+#ifndef timeVaryingUniformFixedValueFaPatchField_H
+#define timeVaryingUniformFixedValueFaPatchField_H
+
+#include "fixedValueFaPatchField.H"
+#include "interpolationTable.H"
+
+// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
+
+namespace Foam
+{
+
+/*---------------------------------------------------------------------------*\
+           Class timeVaryingUniformFixedValueFaPatch Declaration
+\*---------------------------------------------------------------------------*/
+
+template<class Type>
+class timeVaryingUniformFixedValueFaPatchField
+:
+    public fixedValueFaPatchField<Type>
+{
+    // Private data
+
+        //- The time series being used, including the bounding treatment
+        interpolationTable<Type> timeSeries_;
+
+
+public:
+
+    //- Runtime type information
+    TypeName("timeVaryingUniformFixedValue");
+
+
+    // Constructors
+
+        //- Construct from patch and internal field
+        timeVaryingUniformFixedValueFaPatchField
+        (
+            const faPatch&,
+            const DimensionedField<Type, areaMesh>&
+        );
+
+        //- Construct from patch, internal field and dictionary
+        timeVaryingUniformFixedValueFaPatchField
+        (
+            const faPatch&,
+            const DimensionedField<Type, areaMesh>&,
+            const dictionary&
+        );
+
+        //- Construct by mapping given patch field onto a new patch
+        timeVaryingUniformFixedValueFaPatchField
+        (
+            const timeVaryingUniformFixedValueFaPatchField<Type>&,
+            const faPatch&,
+            const DimensionedField<Type, areaMesh>&,
+            const faPatchFieldMapper&
+        );
+
+        //- Construct as copy
+        timeVaryingUniformFixedValueFaPatchField
+        (
+            const timeVaryingUniformFixedValueFaPatchField<Type>&
+        );
+
+        //- Construct and return a clone
+        virtual tmp<faPatchField<Type> > clone() const
+        {
+            return tmp<faPatchField<Type> >
+            (
+                new timeVaryingUniformFixedValueFaPatchField<Type>(*this)
+            );
+        }
+
+        //- Construct as copy setting internal field reference
+        timeVaryingUniformFixedValueFaPatchField
+        (
+            const timeVaryingUniformFixedValueFaPatchField<Type>&,
+            const DimensionedField<Type, areaMesh>&
+        );
+
+        //- Construct and return a clone setting internal field reference
+        virtual tmp<faPatchField<Type> > clone
+        (
+            const DimensionedField<Type, areaMesh>& iF
+        ) const
+        {
+            return tmp<faPatchField<Type> >
+            (
+                new timeVaryingUniformFixedValueFaPatchField<Type>(*this, iF)
+            );
+        }
+
+
+    // Member functions
+
+        // Access
+
+            //- Return the time series used
+            const interpolationTable<Type>& timeSeries() const
+            {
+                return timeSeries_;
+            }
+
+
+        // Evaluation functions
+
+            //- Update the coefficients associated with the patch field
+            virtual void updateCoeffs();
+
+
+        //- Write
+        virtual void write(Ostream&) const;
+};
+
+
+// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
+
+} // End namespace Foam
+
+// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
+
+#ifdef NoRepository
+#   include "timeVaryingUniformFixedValueFaPatchField.C"
+#endif
+
+// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
+
+#endif
+
+// ************************************************************************* //
diff --git a/src/finiteArea/fields/faPatchFields/derived/timeVaryingUniformFixedValue/timeVaryingUniformFixedValueFaPatchFields.C b/src/finiteArea/fields/faPatchFields/derived/timeVaryingUniformFixedValue/timeVaryingUniformFixedValueFaPatchFields.C
new file mode 100644
index 000000000..0610f7e76
--- /dev/null
+++ b/src/finiteArea/fields/faPatchFields/derived/timeVaryingUniformFixedValue/timeVaryingUniformFixedValueFaPatchFields.C
@@ -0,0 +1,43 @@
+/*---------------------------------------------------------------------------*\
+  =========                 |
+  \\      /  F ield         | foam-extend: Open Source CFD
+   \\    /   O peration     | Version:     4.0
+    \\  /    A nd           | Web:         http://www.foam-extend.org
+     \\/     M anipulation  | For copyright notice see file Copyright
+-------------------------------------------------------------------------------
+License
+    This file is part of foam-extend.
+
+    foam-extend is free software: you can redistribute it and/or modify it
+    under the terms of the GNU General Public License as published by the
+    Free Software Foundation, either version 3 of the License, or (at your
+    option) any later version.
+
+    foam-extend is distributed in the hope that it will be useful, but
+    WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+    General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with foam-extend.  If not, see <http://www.gnu.org/licenses/>.
+
+\*---------------------------------------------------------------------------*/
+
+#include "timeVaryingUniformFixedValueFaPatchFields.H"
+#include "addToRunTimeSelectionTable.H"
+#include "areaFields.H"
+
+// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
+
+namespace Foam
+{
+
+// * * * * * * * * * * * * * * Static Data Members * * * * * * * * * * * * * //
+
+makeFaPatchFields(timeVaryingUniformFixedValue);
+
+// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
+
+} // End namespace Foam
+
+// ************************************************************************* //
diff --git a/src/finiteArea/fields/faPatchFields/derived/timeVaryingUniformFixedValue/timeVaryingUniformFixedValueFaPatchFields.H b/src/finiteArea/fields/faPatchFields/derived/timeVaryingUniformFixedValue/timeVaryingUniformFixedValueFaPatchFields.H
new file mode 100644
index 000000000..71f70dd63
--- /dev/null
+++ b/src/finiteArea/fields/faPatchFields/derived/timeVaryingUniformFixedValue/timeVaryingUniformFixedValueFaPatchFields.H
@@ -0,0 +1,49 @@
+/*---------------------------------------------------------------------------*\
+  =========                 |
+  \\      /  F ield         | foam-extend: Open Source CFD
+   \\    /   O peration     | Version:     4.0
+    \\  /    A nd           | Web:         http://www.foam-extend.org
+     \\/     M anipulation  | For copyright notice see file Copyright
+-------------------------------------------------------------------------------
+License
+    This file is part of foam-extend.
+
+    foam-extend is free software: you can redistribute it and/or modify it
+    under the terms of the GNU General Public License as published by the
+    Free Software Foundation, either version 3 of the License, or (at your
+    option) any later version.
+
+    foam-extend is distributed in the hope that it will be useful, but
+    WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+    General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with foam-extend.  If not, see <http://www.gnu.org/licenses/>.
+
+\*---------------------------------------------------------------------------*/
+
+#ifndef timeVaryingUniformFixedValueFaPatchFields_H
+#define timeVaryingUniformFixedValueFaPatchFields_H
+
+#include "timeVaryingUniformFixedValueFaPatchField.H"
+#include "fieldTypes.H"
+
+// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
+
+namespace Foam
+{
+
+// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
+
+makeFaPatchTypeFieldTypedefs(timeVaryingUniformFixedValue)
+
+// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
+
+} // End namespace Foam
+
+// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
+
+#endif
+
+// ************************************************************************* //
diff --git a/src/finiteArea/fields/faPatchFields/derived/timeVaryingUniformFixedValue/timeVaryingUniformFixedValueFaPatchFieldsFwd.H b/src/finiteArea/fields/faPatchFields/derived/timeVaryingUniformFixedValue/timeVaryingUniformFixedValueFaPatchFieldsFwd.H
new file mode 100644
index 000000000..2a8360f27
--- /dev/null
+++ b/src/finiteArea/fields/faPatchFields/derived/timeVaryingUniformFixedValue/timeVaryingUniformFixedValueFaPatchFieldsFwd.H
@@ -0,0 +1,51 @@
+/*---------------------------------------------------------------------------*\
+  =========                 |
+  \\      /  F ield         | foam-extend: Open Source CFD
+   \\    /   O peration     | Version:     4.0
+    \\  /    A nd           | Web:         http://www.foam-extend.org
+     \\/     M anipulation  | For copyright notice see file Copyright
+-------------------------------------------------------------------------------
+License
+    This file is part of foam-extend.
+
+    foam-extend is free software: you can redistribute it and/or modify it
+    under the terms of the GNU General Public License as published by the
+    Free Software Foundation, either version 3 of the License, or (at your
+    option) any later version.
+
+    foam-extend is distributed in the hope that it will be useful, but
+    WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+    General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with foam-extend.  If not, see <http://www.gnu.org/licenses/>.
+
+\*---------------------------------------------------------------------------*/
+
+#ifndef timeVaryingUniformFixedValueFaPatchFieldsFwd_H
+#define timeVaryingUniformFixedValueFaPatchFieldsFwd_H
+
+#include "faPatchField.H"
+#include "fieldTypes.H"
+
+// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
+
+namespace Foam
+{
+
+// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
+
+template<class Type> class timeVaryingUniformFixedValueFaPatchField;
+
+makeFaPatchTypeFieldTypedefs(timeVaryingUniformFixedValue);
+
+// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
+
+} // End namespace Foam
+
+// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
+
+#endif
+
+// ************************************************************************* //
diff --git a/src/finiteArea/fields/faPatchFields/faPatchField/faPatchField.H b/src/finiteArea/fields/faPatchFields/faPatchField/faPatchField.H
index dc8fefadd..018b276ac 100644
--- a/src/finiteArea/fields/faPatchFields/faPatchField/faPatchField.H
+++ b/src/finiteArea/fields/faPatchFields/faPatchField/faPatchField.H
@@ -468,60 +468,87 @@ public:
 #endif
 
 
-#define makeFaPatchTypeFieldTypeName(typePatchTypeField)                   \
-                                                                           \
-defineNamedTemplateTypeNameAndDebug(typePatchTypeField, 0);
-
-#define makeFaPatchFieldsTypeName(typePatchField)                          \
-                                                                           \
-makeFaPatchTypeFieldTypeName(typePatchField##FaPatchScalarField);          \
-makeFaPatchTypeFieldTypeName(typePatchField##FaPatchVectorField);          \
-makeFaPatchTypeFieldTypeName(typePatchField##FaPatchSphericalTensorField); \
-makeFaPatchTypeFieldTypeName(typePatchField##FaPatchSymmTensorField);      \
-makeFaPatchTypeFieldTypeName(typePatchField##FaPatchTensorField);
-
-#define makeFaPatchTypeField(PatchTypeField, typePatchTypeField)           \
-                                                                           \
-defineNamedTemplateTypeNameAndDebug(typePatchTypeField, 0);                \
-                                                                           \
-addToRunTimeSelectionTable                                                 \
-(                                                                          \
-    PatchTypeField, typePatchTypeField, patch                              \
-);                                                                         \
-                                                                           \
-addToRunTimeSelectionTable                                                 \
-(                                                                          \
-    PatchTypeField,                                                        \
-    typePatchTypeField,                                                    \
-    patchMapper                                                            \
-);                                                                         \
-                                                                           \
-addToRunTimeSelectionTable                                                 \
-(                                                                          \
-    PatchTypeField, typePatchTypeField, dictionary                         \
+#define addToFaPatchFieldRunTimeSelection(PatchTypeField, typePatchTypeField) \
+                                                                              \
+addToRunTimeSelectionTable                                                    \
+(                                                                             \
+    PatchTypeField, typePatchTypeField, patch                                 \
+);                                                                            \
+                                                                              \
+addToRunTimeSelectionTable                                                    \
+(                                                                             \
+    PatchTypeField,                                                           \
+    typePatchTypeField,                                                       \
+    patchMapper                                                               \
+);                                                                            \
+                                                                              \
+addToRunTimeSelectionTable                                                    \
+(                                                                             \
+    PatchTypeField, typePatchTypeField, dictionary                            \
 );
 
 
-#define makeFaPatchFields(type)                                            \
-                                                                           \
-makeFaPatchTypeField(faPatchScalarField, type##FaPatchScalarField);        \
-makeFaPatchTypeField(faPatchVectorField, type##FaPatchVectorField);        \
-makeFaPatchTypeField                                                       \
-(                                                                          \
-    faPatchSphericalTensorField,                                           \
-    type##FaPatchSphericalTensorField                                      \
-);                                                                         \
-makeFaPatchTypeField(faPatchSymmTensorField, type##FaPatchSymmTensorField);\
-makeFaPatchTypeField(faPatchTensorField, type##FaPatchTensorField);
+#define makeFaPatchTypeFieldTypeName(typePatchTypeField)                      \
+                                                                              \
+defineNamedTemplateTypeNameAndDebug(typePatchTypeField, 0);
 
 
-#define makeFaPatchTypeFieldTypedefs(type)                                   \
-                                                                             \
-typedef type##FaPatchField<scalar> type##FaPatchScalarField;                 \
-typedef type##FaPatchField<vector> type##FaPatchVectorField;                 \
-typedef type##FaPatchField<sphericalTensor>                                  \
-     type##FaPatchSphericalTensorField;                                      \
-typedef type##FaPatchField<symmTensor> type##FaPatchSymmTensorField;         \
+#define makeFaPatchFieldsTypeName(typePatchField)                             \
+                                                                              \
+makeFaPatchTypeFieldTypeName(typePatchField##FaPatchScalarField);             \
+makeFaPatchTypeFieldTypeName(typePatchField##FaPatchVectorField);             \
+makeFaPatchTypeFieldTypeName(typePatchField##FaPatchSphericalTensorField);    \
+makeFaPatchTypeFieldTypeName(typePatchField##FaPatchSymmTensorField);         \
+makeFaPatchTypeFieldTypeName(typePatchField##FaPatchTensorField);
+
+
+#define makeFaPatchTypeField(PatchTypeField, typePatchTypeField)              \
+                                                                              \
+defineTypeNameAndDebug(typePatchTypeField, 0);                                \
+                                                                              \
+addToFaPatchFieldRunTimeSelection                                             \
+(                                                                             \
+    PatchTypeField, typePatchTypeField                                        \
+);
+
+#define makeTemplateFaPatchTypeField(PatchTypeField, typePatchTypeField)      \
+                                                                              \
+defineNamedTemplateTypeNameAndDebug(typePatchTypeField, 0);                   \
+                                                                              \
+addToFaPatchFieldRunTimeSelection                                             \
+(                                                                             \
+    PatchTypeField, typePatchTypeField                                        \
+);
+
+
+#define makeFaPatchFields(type)                                               \
+                                                                              \
+makeTemplateFaPatchTypeField(faPatchScalarField, type##FaPatchScalarField);   \
+makeTemplateFaPatchTypeField(faPatchVectorField, type##FaPatchVectorField);   \
+makeTemplateFaPatchTypeField                                                  \
+(                                                                             \
+    faPatchSphericalTensorField,                                              \
+    type##FaPatchSphericalTensorField                                         \
+);                                                                            \
+makeTemplateFaPatchTypeField                                                  \
+(                                                                             \
+    faPatchSymmTensorField,                                                   \
+    type##FaPatchSymmTensorField                                              \
+);                                                                            \
+makeTemplateFaPatchTypeField                                                  \
+(                                                                             \
+    faPatchTensorField,                                                       \
+    type##FaPatchTensorField                                                  \
+);
+
+
+#define makeFaPatchTypeFieldTypedefs(type)                                    \
+                                                                              \
+typedef type##FaPatchField<scalar> type##FaPatchScalarField;                  \
+typedef type##FaPatchField<vector> type##FaPatchVectorField;                  \
+typedef type##FaPatchField<sphericalTensor>                                   \
+     type##FaPatchSphericalTensorField;                                       \
+typedef type##FaPatchField<symmTensor> type##FaPatchSymmTensorField;          \
 typedef type##FaPatchField<tensor> type##FaPatchTensorField;
 
 
diff --git a/src/finiteArea/finiteArea/divSchemes/gaussFaDivScheme/gaussFaDivScheme.C b/src/finiteArea/finiteArea/divSchemes/gaussFaDivScheme/gaussFaDivScheme.C
index c4d43aefc..802b543e0 100644
--- a/src/finiteArea/finiteArea/divSchemes/gaussFaDivScheme/gaussFaDivScheme.C
+++ b/src/finiteArea/finiteArea/divSchemes/gaussFaDivScheme/gaussFaDivScheme.C
@@ -62,7 +62,7 @@ gaussDivScheme<Type>::facDiv
         (
             this->mesh_.Le() & this->tinterpScheme_().interpolate(vf)
         )
-      - this->mesh_.faceCurvatures()*(this->mesh_.faceAreaNormals()&vf)
+        // Removed for consistencty.  Matthias Rauter, 6/Dec/2016
     );
 
     tDiv().rename("div(" + vf.name() + ')');
diff --git a/src/finiteArea/finiteArea/fac/fac.H b/src/finiteArea/finiteArea/fac/fac.H
index ce5e382b9..2b17490bc 100644
--- a/src/finiteArea/finiteArea/fac/fac.H
+++ b/src/finiteArea/finiteArea/fac/fac.H
@@ -43,6 +43,8 @@ Description
 #include "facAverage.H"
 #include "facLnGrad.H"
 #include "facDdt.H"
+#include "facNGrad.H"
+#include "facNDiv.H"
 
 // * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
 
diff --git a/src/finiteArea/finiteArea/fac/facDiv.C b/src/finiteArea/finiteArea/fac/facDiv.C
index cc72a13d9..e5f04f9d9 100644
--- a/src/finiteArea/finiteArea/fac/facDiv.C
+++ b/src/finiteArea/finiteArea/fac/facDiv.C
@@ -28,6 +28,7 @@ License
 #include "facEdgeIntegrate.H"
 #include "faDivScheme.H"
 #include "faConvectionScheme.H"
+#include "transformField.H"
 
 // * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
 
@@ -48,7 +49,17 @@ div
     const GeometricField<Type, faePatchField, edgeMesh>& ssf
 )
 {
-    return fac::edgeIntegrate(ssf);
+    const areaVectorField& n = ssf.mesh().faceAreaNormals();
+
+    tmp<GeometricField<Type, faPatchField, areaMesh> > tDiv =
+        fac::edgeIntegrate(ssf);
+
+    GeometricField<Type, faPatchField, areaMesh>& Div = tDiv();
+
+    Div.internalField() = transform(tensor::I - sqr(n), Div.internalField());
+    Div.correctBoundaryConditions();
+
+    return tDiv;
 }
 
 
@@ -79,10 +90,34 @@ div
     const word& name
 )
 {
-    return fa::divScheme<Type>::New
+    const areaVectorField& n = vf.mesh().faceAreaNormals();
+
+    tmp
+    <
+        GeometricField
+        <
+            typename innerProduct<vector, Type>::type,
+            faPatchField,
+            areaMesh
+        >
+    > tDiv
     (
-        vf.mesh(), vf.mesh().schemesDict().divScheme(name)
-    )().facDiv(vf);
+        fa::divScheme<Type>::New
+        (
+            vf.mesh(), vf.mesh().schemesDict().divScheme(name)
+        )().facDiv(vf)
+    );
+    GeometricField
+    <
+        typename innerProduct<vector, Type>::type,
+        faPatchField,
+        areaMesh
+    >& Div = tDiv();
+
+    Div.internalField() = transform(tensor::I - sqr(n), Div.internalField());
+    Div.correctBoundaryConditions();
+
+    return tDiv;
 }
 
 
@@ -159,12 +194,24 @@ div
     const word& name
 )
 {
-    return fa::convectionScheme<Type>::New
+    const areaVectorField& n = vf.mesh().faceAreaNormals();
+
+    tmp<GeometricField<Type, faPatchField, areaMesh> > tDiv
     (
-        vf.mesh(),
-        flux,
-        vf.mesh().schemesDict().divScheme(name)
-    )().facDiv(flux, vf);
+        fa::convectionScheme<Type>::New
+        (
+            vf.mesh(),
+            flux,
+            vf.mesh().schemesDict().divScheme(name)
+        )().facDiv(flux, vf)
+    );
+    GeometricField<Type, faPatchField, areaMesh>& Div = tDiv();
+
+    Div.internalField() = transform(tensor::I - sqr(n), Div.internalField());
+    Div.correctBoundaryConditions();
+
+    return tDiv;
+
 }
 
 
diff --git a/src/finiteArea/finiteArea/fac/facGrad.C b/src/finiteArea/finiteArea/fac/facGrad.C
index e7c3b5e88..568f4c2b6 100644
--- a/src/finiteArea/finiteArea/fac/facGrad.C
+++ b/src/finiteArea/finiteArea/fac/facGrad.C
@@ -55,7 +55,18 @@ grad
     const GeometricField<Type, faePatchField, edgeMesh>& ssf
 )
 {
-    return fac::edgeIntegrate(ssf.mesh().Sf() * ssf);
+    const areaVectorField &n = ssf.mesh().faceAreaNormals();
+    typedef typename outerProduct<vector,Type>::type GradType;
+
+    tmp<GeometricField<GradType, faPatchField, areaMesh> > tgGrad =
+        fac::edgeIntegrate(ssf.mesh().Sf()*ssf);
+
+    GeometricField<GradType, faPatchField, areaMesh>& gGrad = tgGrad();
+
+    gGrad -= (gGrad & n)*n;
+    gGrad.correctBoundaryConditions();
+
+    return tgGrad;
 }
 
 template<class Type>
@@ -95,11 +106,22 @@ grad
     const word& name
 )
 {
-    return fa::gradScheme<Type>::New
-    (
-        vf.mesh(),
-        vf.mesh().schemesDict().gradScheme(name)
-    )().grad(vf);
+    const areaVectorField &n = vf.mesh().faceAreaNormals();
+    typedef typename outerProduct<vector,Type>::type GradType;
+
+    tmp<GeometricField<GradType, faPatchField, areaMesh> > tgGrad =
+        fa::gradScheme<Type>::New
+        (
+            vf.mesh(),
+            vf.mesh().schemesDict().gradScheme(name)
+        )().grad(vf);
+
+    GeometricField<GradType, faPatchField, areaMesh>& gGrad = tgGrad();
+
+    gGrad -= (gGrad & n)*n;
+    gGrad.correctBoundaryConditions();
+
+    return tgGrad;
 }
 
 
diff --git a/src/finiteArea/finiteArea/fac/facNDiv.C b/src/finiteArea/finiteArea/fac/facNDiv.C
new file mode 100644
index 000000000..6a1953961
--- /dev/null
+++ b/src/finiteArea/finiteArea/fac/facNDiv.C
@@ -0,0 +1,333 @@
+/*---------------------------------------------------------------------------*\
+  =========                 |
+  \\      /  F ield         | foam-extend: Open Source CFD
+   \\    /   O peration     | Version:     3.2
+    \\  /    A nd           | Web:         http://www.foam-extend.org
+     \\/     M anipulation  | For copyright notice see file Copyright
+-------------------------------------------------------------------------------
+License
+    This file is part of foam-extend.
+
+    foam-extend is free software: you can redistribute it and/or modify it
+    under the terms of the GNU General Public License as published by the
+    Free Software Foundation, either version 3 of the License, or (at your
+    option) any later version.
+
+    foam-extend is distributed in the hope that it will be useful, but
+    WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+    General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with foam-extend.  If not, see <http://www.gnu.org/licenses/>.
+
+\*---------------------------------------------------------------------------*/
+
+#include "facNDiv.H"
+#include "faMesh.H"
+#include "facEdgeIntegrate.H"
+#include "faDivScheme.H"
+#include "faConvectionScheme.H"
+#include "transformField.H"
+
+// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
+
+namespace Foam
+{
+
+// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
+
+namespace fac
+{
+
+// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
+
+template<class Type>
+tmp<GeometricField<Type, faPatchField, areaMesh> >
+ndiv
+(
+    const GeometricField<Type, faePatchField, edgeMesh>& ssf
+)
+{
+    const areaVectorField &n = ssf.mesh().faceAreaNormals();
+
+    tmp<GeometricField<Type, faPatchField, areaMesh> > v = fac::edgeIntegrate(ssf);
+
+    //v.internalField() = transform(n*n, v.internalField());
+    v.internalField() = (v.internalField()&n)*n;
+
+    return v;
+}
+
+
+template<class Type>
+tmp<GeometricField<Type, faPatchField, areaMesh> >
+ndiv
+(
+    const tmp<GeometricField<Type, faePatchField, edgeMesh> >& tssf
+)
+{
+    tmp<GeometricField<Type, faPatchField, areaMesh> > Div(fac::ndiv(tssf()));
+    tssf.clear();
+    return Div;
+}
+
+
+template<class Type>
+tmp
+<
+    GeometricField
+    <
+        typename innerProduct<vector, Type>::type, faPatchField, areaMesh
+    >
+>
+ndiv
+(
+    const GeometricField<Type, faPatchField, areaMesh>& vf,
+    const word& name
+)
+{
+    const areaVectorField &n = vf.mesh().faceAreaNormals();
+
+    tmp<GeometricField<Type, faPatchField, areaMesh> > tDiv
+    (
+        fa::divScheme<Type>::New
+            (
+                vf.mesh(), vf.mesh().schemesDict().divScheme(name)
+            )().facDiv(vf)
+    );
+
+    GeometricField<Type, faPatchField, areaMesh>& Div = tDiv();
+
+    //Div.internalField() = transform(n*n, Div.internalField());
+    Div.internalField() = (Div.internalField()&n)*n;
+    return tDiv;
+}
+
+
+template<class Type>
+tmp
+<
+    GeometricField
+    <
+        typename innerProduct<vector, Type>::type, faPatchField, areaMesh
+    >
+>
+ndiv
+(
+    const tmp<GeometricField<Type, faPatchField, areaMesh> >& tvvf,
+    const word& name
+)
+{
+    typedef typename innerProduct<vector, Type>::type DivType;
+    tmp<GeometricField<DivType, faPatchField, areaMesh> > Div
+    (
+        fac::ndiv(tvvf(), name)
+    );
+    tvvf.clear();
+    return Div;
+}
+
+template<class Type>
+tmp
+<
+    GeometricField
+    <
+        typename innerProduct<vector, Type>::type, faPatchField, areaMesh
+    >
+>
+ndiv
+(
+    const GeometricField<Type, faPatchField, areaMesh>& vf
+)
+{
+    return fac::ndiv(vf, "div("+vf.name()+')');
+}
+
+
+template<class Type>
+tmp
+<
+    GeometricField
+    <
+        typename innerProduct<vector, Type>::type, faPatchField, areaMesh
+    >
+>
+ndiv
+(
+    const tmp<GeometricField<Type, faPatchField, areaMesh> >& tvvf
+)
+{
+    typedef typename innerProduct<vector, Type>::type DivType;
+    tmp<GeometricField<DivType, faPatchField, areaMesh> > Div
+    (
+        fac::ndiv(tvvf())
+    );
+
+    tvvf.clear();
+    return Div;
+}
+
+
+template<class Type>
+tmp<GeometricField<Type, faPatchField, areaMesh> >
+ndiv
+(
+    const edgeScalarField& flux,
+    const GeometricField<Type, faPatchField, areaMesh>& vf,
+    const word& name
+)
+{
+    const areaVectorField &n = vf.mesh().faceAreaNormals();
+
+    tmp<GeometricField<Type, faPatchField, areaMesh> > tDiv
+    (
+        fa::convectionScheme<Type>::New
+            (
+                vf.mesh(),
+                flux,
+                vf.mesh().schemesDict().divScheme(name)
+            )().facDiv(flux, vf)
+    );
+
+    GeometricField<Type, faPatchField, areaMesh>& Div = tDiv();
+
+    //Div.internalField() = transform(n*n, Div.internalField());
+    Div.internalField() = (Div.internalField()&n)*n;
+
+    return tDiv;
+
+}
+
+
+template<class Type>
+tmp<GeometricField<Type, faPatchField, areaMesh> >
+ndiv
+(
+    const tmp<edgeScalarField>& tflux,
+    const GeometricField<Type, faPatchField, areaMesh>& vf,
+    const word& name
+)
+{
+    tmp<GeometricField<Type, faPatchField, areaMesh> > Div
+    (
+        fac::ndiv(tflux(), vf, name)
+    );
+    tflux.clear();
+    return Div;
+}
+
+
+template<class Type>
+tmp<GeometricField<Type, faPatchField, areaMesh> >
+ndiv
+(
+    const edgeScalarField& flux,
+    const tmp<GeometricField<Type, faPatchField, areaMesh> >& tvf,
+    const word& name
+)
+{
+    tmp<GeometricField<Type, faPatchField, areaMesh> > Div
+    (
+        fac::ndiv(flux, tvf(), name)
+    );
+    tvf.clear();
+    return Div;
+}
+
+
+template<class Type>
+tmp<GeometricField<Type, faPatchField, areaMesh> >
+ndiv
+(
+    const tmp<edgeScalarField>& tflux,
+    const tmp<GeometricField<Type, faPatchField, areaMesh> >& tvf,
+    const word& name
+)
+{
+    tmp<GeometricField<Type, faPatchField, areaMesh> > Div
+    (
+        fac::ndiv(tflux(), tvf(), name)
+    );
+    tflux.clear();
+    tvf.clear();
+    return Div;
+}
+
+
+template<class Type>
+tmp<GeometricField<Type, faPatchField, areaMesh> >
+ndiv
+(
+    const edgeScalarField& flux,
+    const GeometricField<Type, faPatchField, areaMesh>& vf
+)
+{
+    return fac::ndiv
+    (
+        flux, vf, "div("+flux.name()+','+vf.name()+')'
+    );
+}
+
+
+template<class Type>
+tmp<GeometricField<Type, faPatchField, areaMesh> >
+ndiv
+(
+    const tmp<edgeScalarField>& tflux,
+    const GeometricField<Type, faPatchField, areaMesh>& vf
+)
+{
+    tmp<GeometricField<Type, faPatchField, areaMesh> > Div
+    (
+        fac::ndiv(tflux(), vf)
+    );
+    tflux.clear();
+    return Div;
+}
+
+
+template<class Type>
+tmp<GeometricField<Type, faPatchField, areaMesh> >
+ndiv
+(
+    const edgeScalarField& flux,
+    const tmp<GeometricField<Type, faPatchField, areaMesh> >& tvf
+)
+{
+    tmp<GeometricField<Type, faPatchField, areaMesh> > Div
+    (
+        fac::ndiv(flux, tvf())
+    );
+    tvf.clear();
+    return Div;
+}
+
+
+template<class Type>
+tmp<GeometricField<Type, faPatchField, areaMesh> >
+ndiv
+(
+    const tmp<edgeScalarField>& tflux,
+    const tmp<GeometricField<Type, faPatchField, areaMesh> >& tvf
+)
+{
+    tmp<GeometricField<Type, faPatchField, areaMesh> > Div
+    (
+        fac::ndiv(tflux(), tvf())
+    );
+    tflux.clear();
+    tvf.clear();
+    return Div;
+}
+
+
+// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
+
+} // End namespace fac
+
+// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
+
+} // End namespace Foam
+
+// ************************************************************************* //
diff --git a/src/finiteArea/finiteArea/fac/facNDiv.H b/src/finiteArea/finiteArea/fac/facNDiv.H
new file mode 100644
index 000000000..7aab1da79
--- /dev/null
+++ b/src/finiteArea/finiteArea/fac/facNDiv.H
@@ -0,0 +1,187 @@
+/*---------------------------------------------------------------------------*\
+  =========                 |
+  \\      /  F ield         | foam-extend: Open Source CFD
+   \\    /   O peration     | Version:     3.2
+    \\  /    A nd           | Web:         http://www.foam-extend.org
+     \\/     M anipulation  | For copyright notice see file Copyright
+-------------------------------------------------------------------------------
+License
+    This file is part of foam-extend.
+
+    foam-extend is free software: you can redistribute it and/or modify it
+    under the terms of the GNU General Public License as published by the
+    Free Software Foundation, either version 3 of the License, or (at your
+    option) any later version.
+
+    foam-extend is distributed in the hope that it will be useful, but
+    WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+    General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with foam-extend.  If not, see <http://www.gnu.org/licenses/>.
+
+Namespace
+    fac
+
+Description
+    Calculate the divergence of the given field.
+
+SourceFiles
+    facDiv.C
+
+\*---------------------------------------------------------------------------*/
+
+
+#ifndef facNDiv_H
+#define facNDiv_H
+
+#include "areaFieldsFwd.H"
+#include "edgeFieldsFwd.H"
+
+// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
+
+namespace Foam
+{
+
+/*---------------------------------------------------------------------------*\
+                     Namespace fac functions Declaration
+\*---------------------------------------------------------------------------*/
+
+namespace fac
+{
+    template<class Type>
+    tmp<GeometricField<Type, faPatchField, areaMesh> > ndiv
+    (
+        const GeometricField<Type, faePatchField, edgeMesh>&
+    );
+
+    template<class Type>
+    tmp<GeometricField<Type, faPatchField, areaMesh> > ndiv
+    (
+        const tmp<GeometricField<Type, faePatchField, edgeMesh> >&
+    );
+
+
+    template<class Type>
+    tmp
+    <
+        GeometricField
+        <typename innerProduct<vector, Type>::type, faPatchField, areaMesh>
+    > ndiv
+    (
+        const GeometricField<Type, faPatchField, areaMesh>&,
+        const word& name
+    );
+
+    template<class Type>
+    tmp
+    <
+        GeometricField
+        <typename innerProduct<vector, Type>::type, faPatchField, areaMesh>
+    > ndiv
+    (
+        const tmp<GeometricField<Type, faPatchField, areaMesh> >&,
+        const word& name
+    );
+
+
+    template<class Type>
+    tmp
+    <
+        GeometricField
+        <typename innerProduct<vector, Type>::type, faPatchField, areaMesh>
+    > ndiv
+    (
+        const GeometricField<Type, faPatchField, areaMesh>&
+    );
+
+    template<class Type>
+    tmp
+    <
+        GeometricField
+        <typename innerProduct<vector, Type>::type, faPatchField, areaMesh>
+    > ndiv
+    (
+        const tmp<GeometricField<Type, faPatchField, areaMesh> >&
+    );
+
+
+    template<class Type>
+    tmp<GeometricField<Type, faPatchField, areaMesh> > ndiv
+    (
+        const edgeScalarField&,
+        const GeometricField<Type, faPatchField, areaMesh>&,
+        const word& name
+    );
+
+    template<class Type>
+    tmp<GeometricField<Type, faPatchField, areaMesh> > ndiv
+    (
+        const tmp<edgeScalarField>&,
+        const GeometricField<Type, faPatchField, areaMesh>&,
+        const word& name
+    );
+
+    template<class Type>
+    tmp<GeometricField<Type, faPatchField, areaMesh> > ndiv
+    (
+        const edgeScalarField&,
+        const tmp<GeometricField<Type, faPatchField, areaMesh> >&,
+        const word& name
+    );
+
+    template<class Type>
+    tmp<GeometricField<Type, faPatchField, areaMesh> > ndiv
+    (
+        const tmp<edgeScalarField>&,
+        const tmp<GeometricField<Type, faPatchField, areaMesh> >&,
+        const word& name
+    );
+
+
+    template<class Type>
+    tmp<GeometricField<Type, faPatchField, areaMesh> > ndiv
+    (
+        const edgeScalarField&,
+        const GeometricField<Type, faPatchField, areaMesh>&
+    );
+
+    template<class Type>
+    tmp<GeometricField<Type, faPatchField, areaMesh> > ndiv
+    (
+        const tmp<edgeScalarField>&,
+        const GeometricField<Type, faPatchField, areaMesh>&
+    );
+
+    template<class Type>
+    tmp<GeometricField<Type, faPatchField, areaMesh> > ndiv
+    (
+        const edgeScalarField&,
+        const tmp<GeometricField<Type, faPatchField, areaMesh> >&
+    );
+
+    template<class Type>
+    tmp<GeometricField<Type, faPatchField, areaMesh> > ndiv
+    (
+        const tmp<edgeScalarField>&,
+        const tmp<GeometricField<Type, faPatchField, areaMesh> >&
+    );
+}
+
+
+// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
+
+} // End namespace Foam
+
+// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
+
+#ifdef NoRepository
+#   include "facNDiv.C"
+#endif
+
+// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
+
+#endif
+
+// ************************************************************************* //
diff --git a/src/finiteArea/finiteArea/fac/facNGrad.C b/src/finiteArea/finiteArea/fac/facNGrad.C
new file mode 100644
index 000000000..9d4d96070
--- /dev/null
+++ b/src/finiteArea/finiteArea/fac/facNGrad.C
@@ -0,0 +1,202 @@
+/*---------------------------------------------------------------------------*\
+  =========                 |
+  \\      /  F ield         | foam-extend: Open Source CFD
+   \\    /   O peration     | Version:     3.2
+    \\  /    A nd           | Web:         http://www.foam-extend.org
+     \\/     M anipulation  | For copyright notice see file Copyright
+-------------------------------------------------------------------------------
+License
+    This file is part of foam-extend.
+
+    foam-extend is free software: you can redistribute it and/or modify it
+    under the terms of the GNU General Public License as published by the
+    Free Software Foundation, either version 3 of the License, or (at your
+    option) any later version.
+
+    foam-extend is distributed in the hope that it will be useful, but
+    WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+    General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with foam-extend.  If not, see <http://www.gnu.org/licenses/>.
+
+\*---------------------------------------------------------------------------*/
+
+#include "facNGrad.H"
+#include "areaFields.H"
+#include "edgeFields.H"
+#include "facEdgeIntegrate.H"
+#include "faMesh.H"
+#include "faGradScheme.H"
+#include "transformField.H"
+
+// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
+
+namespace Foam
+{
+
+// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
+
+namespace fac
+{
+
+// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
+
+template<class Type>
+tmp
+<
+    GeometricField
+    <
+        typename outerProduct<vector, Type>::type, faPatchField, areaMesh
+    >
+>
+ngrad
+(
+    const GeometricField<Type, faePatchField, edgeMesh>& ssf
+)
+{
+    const areaVectorField &n = ssf.mesh().faceAreaNormals();
+    typedef typename outerProduct<vector,Type>::type GradType;
+
+    tmp<GeometricField<GradType, faPatchField, areaMesh> > tgGrad = fac::edgeIntegrate(ssf.mesh().Sf() * ssf);
+
+    GeometricField<GradType, faPatchField, areaMesh>& grad = tgGrad();
+
+    grad = (grad&n)*n;
+
+    return tgGrad;
+}
+
+template<class Type>
+tmp
+<
+    GeometricField
+    <
+        typename outerProduct<vector,Type>::type, faPatchField, areaMesh
+    >
+>
+ngrad
+(
+    const tmp<GeometricField<Type, faePatchField, edgeMesh> >& tssf
+)
+{
+    typedef typename outerProduct<vector, Type>::type GradType;
+    tmp<GeometricField<GradType, faPatchField, areaMesh> > Grad
+    (
+        fac::ngrad(tssf())
+    );
+    tssf.clear();
+    return Grad;
+}
+
+
+template<class Type>
+tmp
+<
+    GeometricField
+    <
+        typename outerProduct<vector,Type>::type, faPatchField, areaMesh
+    >
+>
+ngrad
+(
+    const GeometricField<Type, faPatchField, areaMesh>& vf,
+    const word& name
+)
+{
+    const areaVectorField &n = vf.mesh().faceAreaNormals();
+    typedef typename outerProduct<vector,Type>::type GradType;
+
+    tmp<GeometricField<GradType, faPatchField, areaMesh> > tgGrad = fa::gradScheme<Type>::New
+        (
+            vf.mesh(),
+            vf.mesh().schemesDict().gradScheme(name)
+        )().grad(vf);
+
+    GeometricField<GradType, faPatchField, areaMesh>& grad = tgGrad();
+
+    grad = (grad&n)*n;
+
+    return tgGrad;
+}
+
+
+template<class Type>
+tmp
+<
+    GeometricField
+    <
+        typename outerProduct<vector,Type>::type, faPatchField, areaMesh
+    >
+>
+ngrad
+(
+    const tmp<GeometricField<Type, faPatchField, areaMesh> >& tvf,
+    const word& name
+)
+{
+    tmp
+    <
+        GeometricField
+        <
+            typename outerProduct<vector, Type>::type, faPatchField, areaMesh
+        >
+    > tGrad
+    (
+        fac::ngrad(tvf(), name)
+    );
+    tvf.clear();
+    return tGrad;
+}
+
+
+template<class Type>
+tmp
+<
+    GeometricField
+    <
+        typename outerProduct<vector,Type>::type, faPatchField, areaMesh
+    >
+>
+ngrad
+(
+    const GeometricField<Type, faPatchField, areaMesh>& vf
+)
+{
+    return fac::ngrad(vf, "grad(" + vf.name() + ')');
+}
+
+
+template<class Type>
+tmp
+<
+    GeometricField
+    <
+        typename outerProduct<vector,Type>::type, faPatchField, areaMesh
+    >
+>
+ngrad
+(
+    const tmp<GeometricField<Type, faPatchField, areaMesh> >& tvf
+)
+{
+    typedef typename outerProduct<vector, Type>::type GradType;
+    tmp<GeometricField<GradType, faPatchField, areaMesh> > Grad
+    (
+        fac::ngrad(tvf())
+    );
+    tvf.clear();
+    return Grad;
+}
+
+
+// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
+
+} // End namespace fac
+
+// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
+
+} // End namespace Foam
+
+// ************************************************************************* //
diff --git a/src/finiteArea/finiteArea/fac/facNGrad.H b/src/finiteArea/finiteArea/fac/facNGrad.H
new file mode 100644
index 000000000..0375202bd
--- /dev/null
+++ b/src/finiteArea/finiteArea/fac/facNGrad.H
@@ -0,0 +1,131 @@
+/*---------------------------------------------------------------------------*\
+  =========                 |
+  \\      /  F ield         | foam-extend: Open Source CFD
+   \\    /   O peration     | Version:     3.2
+    \\  /    A nd           | Web:         http://www.foam-extend.org
+     \\/     M anipulation  | For copyright notice see file Copyright
+-------------------------------------------------------------------------------
+License
+    This file is part of foam-extend.
+
+    foam-extend is free software: you can redistribute it and/or modify it
+    under the terms of the GNU General Public License as published by the
+    Free Software Foundation, either version 3 of the License, or (at your
+    option) any later version.
+
+    foam-extend is distributed in the hope that it will be useful, but
+    WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+    General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with foam-extend.  If not, see <http://www.gnu.org/licenses/>.
+
+Namespace
+    fac
+
+Description
+    Calculate the gradient normal to the surface of the given field.
+
+SourceFiles
+    facGrad.C
+
+\*---------------------------------------------------------------------------*/
+
+
+#ifndef facNGrad_H
+#define facNGrad_H
+
+#include "areaFieldsFwd.H"
+#include "edgeFieldsFwd.H"
+
+// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
+
+namespace Foam
+{
+
+/*---------------------------------------------------------------------------*\
+                     Namespace fac functions Declaration
+\*---------------------------------------------------------------------------*/
+
+namespace fac
+{
+    template<class Type>
+    tmp
+    <
+        GeometricField
+        <typename outerProduct<vector, Type>::type, faPatchField, areaMesh>
+    > ngrad
+    (
+        const GeometricField<Type, faePatchField, edgeMesh>&
+    );
+
+    template<class Type>
+    tmp
+    <
+        GeometricField
+        <typename outerProduct<vector, Type>::type, faPatchField, areaMesh>
+    > ngrad
+    (
+        const tmp<GeometricField<Type, faePatchField, edgeMesh> >&
+    );
+
+    template<class Type>
+    tmp
+    <
+        GeometricField
+        <typename outerProduct<vector, Type>::type, faPatchField, areaMesh>
+    >ngrad
+    (
+        const GeometricField<Type, faPatchField, areaMesh>&,
+        const word& name
+    );
+
+    template<class Type>
+    tmp
+    <
+        GeometricField
+        <typename outerProduct<vector, Type>::type, faPatchField, areaMesh>
+    >ngrad
+    (
+        const tmp<GeometricField<Type, faPatchField, areaMesh> >&,
+        const word& name
+    );
+
+    template<class Type>
+    tmp
+    <
+        GeometricField
+        <typename outerProduct<vector, Type>::type, faPatchField, areaMesh>
+    >ngrad
+    (
+        const GeometricField<Type, faPatchField, areaMesh>&
+    );
+
+    template<class Type>
+    tmp
+    <
+        GeometricField
+        <typename outerProduct<vector, Type>::type, faPatchField, areaMesh>
+    >ngrad
+    (
+        const tmp<GeometricField<Type, faPatchField, areaMesh> >&
+    );
+}
+
+
+// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
+
+} // End namespace Foam
+
+// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
+
+#ifdef NoRepository
+#   include "facNGrad.C"
+#endif
+
+// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
+
+#endif
+
+// ************************************************************************* //
diff --git a/src/finiteArea/finiteArea/fam/fam.H b/src/finiteArea/finiteArea/fam/fam.H
index 65b1c6905..5e9f7ef33 100644
--- a/src/finiteArea/finiteArea/fam/fam.H
+++ b/src/finiteArea/finiteArea/fam/fam.H
@@ -43,6 +43,7 @@ SourceFiles
 #include "famDiv.H"
 #include "famLaplacian.H"
 #include "famSup.H"
+#include "famNDiv.H"
 
 // * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
 
diff --git a/src/finiteArea/finiteArea/fam/famDiv.C b/src/finiteArea/finiteArea/fam/famDiv.C
index b382b60a5..2296b224b 100644
--- a/src/finiteArea/finiteArea/fam/famDiv.C
+++ b/src/finiteArea/finiteArea/fam/famDiv.C
@@ -49,14 +49,36 @@ div
     const word& name
 )
 {
-    return fa::convectionScheme<Type>::New
+    const areaVectorField& n = vf.mesh().faceAreaNormals();
+
+    tmp<faMatrix<Type> > tM
     (
-        vf.mesh(),
-        flux,
-        vf.mesh().schemesDict().divScheme(name)
-    )().famDiv(flux, vf);
+        fa::convectionScheme<Type>::New
+        (
+            vf.mesh(),
+            flux,
+            vf.mesh().schemesDict().divScheme(name)
+        )().famDiv(flux, vf)
+    );
+    faMatrix<Type>& M = tM();
+
+    GeometricField<Type, faPatchField, areaMesh> v
+    (
+        fa::convectionScheme<Type>::New
+        (
+            vf.mesh(),
+            flux,
+            vf.mesh().schemesDict().divScheme(name)
+        )().facDiv(flux, vf)
+    );
+
+    //HJ Check if the product is from left or right.  HJ, 6/Dec/2016
+    M -= (v & n)*n;
+
+    return tM;
 }
 
+
 template<class Type>
 tmp<faMatrix<Type> >
 div
diff --git a/src/finiteArea/finiteArea/fam/famDiv.H b/src/finiteArea/finiteArea/fam/famDiv.H
index 33b92f49d..e8c4f2bb2 100644
--- a/src/finiteArea/finiteArea/fam/famDiv.H
+++ b/src/finiteArea/finiteArea/fam/famDiv.H
@@ -29,6 +29,7 @@ Description
 
 SourceFiles
     famDiv.C
+    vectorFamDiv.C
 
 \*---------------------------------------------------------------------------*/
 
@@ -89,6 +90,9 @@ namespace fam
 
 // * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
 
+// Template specialisation
+#include "vectorFamDiv.H"
+
 #ifdef NoRepository
 #   include "famDiv.C"
 #endif
diff --git a/src/finiteArea/finiteArea/fam/famNDiv.C b/src/finiteArea/finiteArea/fam/famNDiv.C
new file mode 100644
index 000000000..33c55f3fa
--- /dev/null
+++ b/src/finiteArea/finiteArea/fam/famNDiv.C
@@ -0,0 +1,108 @@
+/*---------------------------------------------------------------------------*\
+  =========                 |
+  \\      /  F ield         | foam-extend: Open Source CFD
+   \\    /   O peration     | Version:     3.2
+    \\  /    A nd           | Web:         http://www.foam-extend.org
+     \\/     M anipulation  | For copyright notice see file Copyright
+-------------------------------------------------------------------------------
+License
+    This file is part of foam-extend.
+
+    foam-extend is free software: you can redistribute it and/or modify it
+    under the terms of the GNU General Public License as published by the
+    Free Software Foundation, either version 3 of the License, or (at your
+    option) any later version.
+
+    foam-extend is distributed in the hope that it will be useful, but
+    WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+    General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with foam-extend.  If not, see <http://www.gnu.org/licenses/>.
+
+\*---------------------------------------------------------------------------*/
+
+#include "famNDiv.H"
+#include "faMesh.H"
+#include "faMatrix.H"
+#include "faConvectionScheme.H"
+
+// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
+
+namespace Foam
+{
+
+// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
+
+namespace fam
+{
+
+// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
+
+template<class Type>
+tmp<faMatrix<Type> >
+ndiv
+(
+    const edgeScalarField& flux,
+    GeometricField<Type, faPatchField, areaMesh>& vf,
+    const word& name
+)
+{
+    return fa::convectionScheme<Type>::New
+    (
+        vf.mesh(),
+        flux,
+        vf.mesh().schemesDict().divScheme(name)
+    )().famDiv(flux, vf);//TODO calculate normal
+}
+
+template<class Type>
+tmp<faMatrix<Type> >
+ndiv
+(
+    const tmp<edgeScalarField>& tflux,
+    GeometricField<Type, faPatchField, areaMesh>& vf,
+    const word& name
+)
+{
+    tmp<faMatrix<Type> > Div(fam::ndiv(tflux(), vf, name));
+    tflux.clear();
+    return Div;
+}
+
+
+template<class Type>
+tmp<faMatrix<Type> >
+ndiv
+(
+    const edgeScalarField& flux,
+    GeometricField<Type, faPatchField, areaMesh>& vf
+)
+{
+    return fam::ndiv(flux, vf, "div("+flux.name()+','+vf.name()+')');
+}
+
+template<class Type>
+tmp<faMatrix<Type> >
+ndiv
+(
+    const tmp<edgeScalarField>& tflux,
+    GeometricField<Type, faPatchField, areaMesh>& vf
+)
+{
+    tmp<faMatrix<Type> > Div(fam::ndiv(tflux(), vf));
+    tflux.clear();
+    return Div;
+}
+
+
+// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
+
+} // End namespace fam
+
+// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
+
+} // End namespace Foam
+
+// ************************************************************************* //
diff --git a/src/finiteArea/finiteArea/fam/famNDiv.H b/src/finiteArea/finiteArea/fam/famNDiv.H
new file mode 100644
index 000000000..0abf49e54
--- /dev/null
+++ b/src/finiteArea/finiteArea/fam/famNDiv.H
@@ -0,0 +1,100 @@
+/*---------------------------------------------------------------------------*\
+  =========                 |
+  \\      /  F ield         | foam-extend: Open Source CFD
+   \\    /   O peration     | Version:     3.2
+    \\  /    A nd           | Web:         http://www.foam-extend.org
+     \\/     M anipulation  | For copyright notice see file Copyright
+-------------------------------------------------------------------------------
+License
+    This file is part of foam-extend.
+
+    foam-extend is free software: you can redistribute it and/or modify it
+    under the terms of the GNU General Public License as published by the
+    Free Software Foundation, either version 3 of the License, or (at your
+    option) any later version.
+
+    foam-extend is distributed in the hope that it will be useful, but
+    WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+    General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with foam-extend.  If not, see <http://www.gnu.org/licenses/>.
+
+Namespace
+    fam
+
+Description
+    Calculate the matrix for the divergence of the given field and flux.
+
+SourceFiles
+    famDiv.C
+
+\*---------------------------------------------------------------------------*/
+
+#ifndef famNDiv_H
+#define famNDiv_H
+
+#include "areaFieldsFwd.H"
+#include "edgeFieldsFwd.H"
+#include "edgeInterpolationScheme.H"
+#include "faMatrix.H"
+
+// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
+
+namespace Foam
+{
+
+/*---------------------------------------------------------------------------*\
+                     Namespace fam functions Declaration
+\*---------------------------------------------------------------------------*/
+
+namespace fam
+{
+    template<class Type>
+    tmp<faMatrix<Type> > ndiv
+    (
+        const edgeScalarField&,
+        GeometricField<Type, faPatchField, areaMesh>&,
+        const word& name
+    );
+
+    template<class Type>
+    tmp<faMatrix<Type> > ndiv
+    (
+        const tmp<edgeScalarField>&,
+        GeometricField<Type, faPatchField, areaMesh>&,
+        const word& name
+    );
+
+    template<class Type>
+    tmp<faMatrix<Type> > ndiv
+    (
+        const edgeScalarField&,
+        GeometricField<Type, faPatchField, areaMesh>&
+    );
+
+    template<class Type>
+    tmp<faMatrix<Type> > ndiv
+    (
+        const tmp<edgeScalarField>&,
+        GeometricField<Type, faPatchField, areaMesh>&
+    );
+}
+
+
+// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
+
+} // End namespace Foam
+
+// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
+
+#ifdef NoRepository
+#   include "famNDiv.C"
+#endif
+
+// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
+
+#endif
+
+// ************************************************************************* //
diff --git a/src/finiteArea/finiteArea/fam/vectorFamDiv.C b/src/finiteArea/finiteArea/fam/vectorFamDiv.C
new file mode 100644
index 000000000..c3b941a3e
--- /dev/null
+++ b/src/finiteArea/finiteArea/fam/vectorFamDiv.C
@@ -0,0 +1,69 @@
+/*---------------------------------------------------------------------------*\
+  =========                 |
+  \\      /  F ield         | foam-extend: Open Source CFD
+   \\    /   O peration     | Version:     4.0
+    \\  /    A nd           | Web:         http://www.foam-extend.org
+     \\/     M anipulation  | For copyright notice see file Copyright
+-------------------------------------------------------------------------------
+License
+    This file is part of foam-extend.
+
+    foam-extend is free software: you can redistribute it and/or modify it
+    under the terms of the GNU General Public License as published by the
+    Free Software Foundation, either version 3 of the License, or (at your
+    option) any later version.
+
+    foam-extend is distributed in the hope that it will be useful, but
+    WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+    General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with foam-extend.  If not, see <http://www.gnu.org/licenses/>.
+
+\*---------------------------------------------------------------------------*/
+
+#include "vectorFamDiv.H"
+#include "faMesh.H"
+#include "faMatrix.H"
+#include "faConvectionScheme.H"
+
+// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
+
+namespace Foam
+{
+
+// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
+
+namespace fam
+{
+
+// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
+
+template<>
+tmp<faMatrix<scalar> >
+div
+(
+    const edgeScalarField& flux,
+    const GeometricField<scalar, faPatchField, areaMesh>& vf,
+    const word& name
+)
+{
+    return fa::convectionScheme<scalar>::New
+    (
+        vf.mesh(),
+        flux,
+        vf.mesh().schemesDict().divScheme(name)
+    )().famDiv(flux, vf);
+}
+
+
+// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
+
+} // End namespace fam
+
+// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
+
+} // End namespace Foam
+
+// ************************************************************************* //
diff --git a/src/finiteArea/finiteArea/fam/vectorFamDiv.H b/src/finiteArea/finiteArea/fam/vectorFamDiv.H
new file mode 100644
index 000000000..53b6d7eb4
--- /dev/null
+++ b/src/finiteArea/finiteArea/fam/vectorFamDiv.H
@@ -0,0 +1,74 @@
+/*---------------------------------------------------------------------------*\
+  =========                 |
+  \\      /  F ield         | foam-extend: Open Source CFD
+   \\    /   O peration     | Version:     4.0
+    \\  /    A nd           | Web:         http://www.foam-extend.org
+     \\/     M anipulation  | For copyright notice see file Copyright
+-------------------------------------------------------------------------------
+License
+    This file is part of foam-extend.
+
+    foam-extend is free software: you can redistribute it and/or modify it
+    under the terms of the GNU General Public License as published by the
+    Free Software Foundation, either version 3 of the License, or (at your
+    option) any later version.
+
+    foam-extend is distributed in the hope that it will be useful, but
+    WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+    General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with foam-extend.  If not, see <http://www.gnu.org/licenses/>.
+
+Typedef
+    Foam::fam
+
+Description
+    Specialisation of fam div for a flux.
+
+SourceFiles
+    vectorFamDiv.C
+
+\*---------------------------------------------------------------------------*/
+
+#ifndef vectorFamDiv_H
+#define vectorFamDiv_H
+
+#include "famDiv.H"
+
+// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
+
+namespace Foam
+{
+
+namespace fam
+{
+
+/*---------------------------------------------------------------------------*\
+                     Namespace fam functions Declaration
+\*---------------------------------------------------------------------------*/
+
+template<>
+tmp<faMatrix<scalar> >
+div
+(
+    const edgeScalarField& flux,
+    const GeometricField<scalar, faPatchField, areaMesh>& vf,
+    const word& name
+);
+
+    
+// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
+
+}
+
+// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
+
+} // End namespace Foam
+
+// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
+
+#endif
+
+// ************************************************************************* //
diff --git a/src/finiteArea/finiteArea/gradSchemes/gaussFaGrad/gaussFaGrad.C b/src/finiteArea/finiteArea/gradSchemes/gaussFaGrad/gaussFaGrad.C
index 6ca7b9db2..c9fef3221 100644
--- a/src/finiteArea/finiteArea/gradSchemes/gaussFaGrad/gaussFaGrad.C
+++ b/src/finiteArea/finiteArea/gradSchemes/gaussFaGrad/gaussFaGrad.C
@@ -66,12 +66,7 @@ gaussGrad<Type>::grad
 
     GeometricField<GradType, faPatchField, areaMesh>& gGrad = tgGrad();
 
-    gGrad -= vsf*fac::edgeIntegrate(vsf.mesh().Le());
-
-    // Remove component of gradient normal to surface (area)
-    const areaVectorField& n = vsf.mesh().faceAreaNormals();
-
-    gGrad -= n*(n & gGrad);
+    // Removed for consistencty.  Matthias Rauter, 6/Dec/2016
     gGrad.correctBoundaryConditions();
 
     gGrad.rename("grad(" + vsf.name() + ')');
@@ -96,15 +91,9 @@ void gaussGrad<Type>::correctBoundaryConditions
         if (!vsf.boundaryField()[patchI].coupled())
         {
             vectorField m =
-                vsf.mesh().Le().boundaryField()[patchI]
-                /vsf.mesh().magLe().boundaryField()[patchI];
+                vsf.mesh().Le().boundaryField()[patchI]/
+                vsf.mesh().magLe().boundaryField()[patchI];
 
-            // Zeljko Tukovic
-//             gGrad.boundaryField()[patchI] =
-//                 m*vsf.boundaryField()[patchI].snGrad();
-
-            //HJ Not sure: should this be a complete correction or just the
-            //   tangential part?  HJ, 24/Jul/2009
             gGrad.boundaryField()[patchI] += m*
             (
                 vsf.boundaryField()[patchI].snGrad()
diff --git a/src/finiteArea/finiteArea/gradSchemes/leastSquaresFaGrad/leastSquaresFaGrad.C b/src/finiteArea/finiteArea/gradSchemes/leastSquaresFaGrad/leastSquaresFaGrad.C
index d9796d613..a910159e3 100644
--- a/src/finiteArea/finiteArea/gradSchemes/leastSquaresFaGrad/leastSquaresFaGrad.C
+++ b/src/finiteArea/finiteArea/gradSchemes/leastSquaresFaGrad/leastSquaresFaGrad.C
@@ -138,11 +138,8 @@ leastSquaresFaGrad<Type>::grad
         }
     }
 
-
     // Remove component of gradient normal to surface (area)
-    const areaVectorField& n = vsf.mesh().faceAreaNormals();
-
-    lsGrad -= n*(n & lsGrad);
+    // Removed for consistencty.  Matthias Rauter, 6/Dec/2016
     lsGrad.correctBoundaryConditions();
 
     gaussGrad<Type>::correctBoundaryConditions(vsf, lsGrad);

From e881837bb0496efced36d19b888929136cabb681 Mon Sep 17 00:00:00 2001
From: Hrvoje Jasak <h.jasak@wikki.co.uk>
Date: Tue, 6 Dec 2016 12:50:42 +0000
Subject: [PATCH 096/130] Formatting

---
 .../fvPatchFields/constraint/processor/processorFvPatchField.C   | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/finiteVolume/fields/fvPatchFields/constraint/processor/processorFvPatchField.C b/src/finiteVolume/fields/fvPatchFields/constraint/processor/processorFvPatchField.C
index 85fabfec5..4d6d58a1d 100644
--- a/src/finiteVolume/fields/fvPatchFields/constraint/processor/processorFvPatchField.C
+++ b/src/finiteVolume/fields/fvPatchFields/constraint/processor/processorFvPatchField.C
@@ -610,6 +610,7 @@ bool Foam::processorFvPatchField<Type>::ready() const
     )
     {
         bool finished = Pstream::finishedRequest(outstandingSendRequest_);
+
         if (!finished)
         {
             return false;

From 4d19d7ec94229c68cd70d6eaaa5a1909585ee28c Mon Sep 17 00:00:00 2001
From: Hrvoje Jasak <h.jasak@wikki.co.uk>
Date: Fri, 9 Dec 2016 12:53:26 +0000
Subject: [PATCH 097/130] Bugfix: MRF boundary condition on a moving wall U is
 fixedValue

---
 tutorials/incompressible/MRFSimpleFoam/mixerGgiMRF/0/U | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tutorials/incompressible/MRFSimpleFoam/mixerGgiMRF/0/U b/tutorials/incompressible/MRFSimpleFoam/mixerGgiMRF/0/U
index 5c237bc52..ce8d03b18 100644
--- a/tutorials/incompressible/MRFSimpleFoam/mixerGgiMRF/0/U
+++ b/tutorials/incompressible/MRFSimpleFoam/mixerGgiMRF/0/U
@@ -23,7 +23,7 @@ boundaryField
 {
     impellerWall
     {
-        type            movingWallVelocity;
+        type            fixedValue;
         value           uniform (0 0 0);
     }
     baffleWall

From 3d979afaf7dbca5c317739d72b17a0b43f99f1f7 Mon Sep 17 00:00:00 2001
From: Hrvoje Jasak <h.jasak@wikki.co.uk>
Date: Mon, 9 Jan 2017 18:46:51 +0000
Subject: [PATCH 098/130] Updated block assembly of MRF and porous zones

---
 .../solvers/coupled/MRFPorousFoam/UEqn.H      | 43 ++++++++++++-------
 1 file changed, 28 insertions(+), 15 deletions(-)

diff --git a/applications/solvers/coupled/MRFPorousFoam/UEqn.H b/applications/solvers/coupled/MRFPorousFoam/UEqn.H
index 80e70f3a5..c40bb8ef2 100644
--- a/applications/solvers/coupled/MRFPorousFoam/UEqn.H
+++ b/applications/solvers/coupled/MRFPorousFoam/UEqn.H
@@ -5,6 +5,9 @@
         fvc::div(phi)
     );
 
+    // Update boundary velocity for consistency with the flux
+    mrfZones.correctBoundaryVelocity(U);
+
     // Momentum equation
     fvVectorMatrix UEqn
     (
@@ -67,28 +70,38 @@
 
         const scalarField& V = mesh.V().field();
 
-        // Note: this insertion should only happen in porous cell zones
-        // A rewrite of the porousZones class interface is needed.
+        // Note: insertion should only happen in porous cell zones
         // HJ, 14/Mar/2016
-        forAll (TUIn, cellI)
+
+        register label cellI;
+
+        forAll (pZones, pZoneI)
         {
-            const scalar& cellV = V[cellI];
+            const labelList& curZoneCells = pZones[pZoneI].zone();
 
-            const tensor& cellTU = TUIn[cellI];
+            // Loop over all cells in the zone
+            forAll (curZoneCells, zcI)
+            {
+                cellI = curZoneCells[zcI];
 
-            CoeffField<vector4>::squareType& cellDD = DD[cellI];
+                const scalar& cellV = V[cellI];
 
-            cellDD(0, 0) += cellV*cellTU.xx();
-            cellDD(0, 1) += cellV*cellTU.xy();
-            cellDD(0, 2) += cellV*cellTU.xz();
+                const tensor& cellTU = TUIn[cellI];
 
-            cellDD(1, 0) += cellV*cellTU.yx();
-            cellDD(1, 1) += cellV*cellTU.yy();
-            cellDD(2, 2) += cellV*cellTU.yz();
+                CoeffField<vector4>::squareType& cellDD = DD[cellI];
 
-            cellDD(2, 0) += cellV*cellTU.zx();
-            cellDD(2, 1) += cellV*cellTU.zy();
-            cellDD(2, 2) += cellV*cellTU.zz();
+                cellDD(0, 0) += cellV*cellTU.xx();
+                cellDD(0, 1) += cellV*cellTU.xy();
+                cellDD(0, 2) += cellV*cellTU.xz();
+
+                cellDD(1, 0) += cellV*cellTU.yx();
+                cellDD(1, 1) += cellV*cellTU.yy();
+                cellDD(2, 2) += cellV*cellTU.yz();
+
+                cellDD(2, 0) += cellV*cellTU.zx();
+                cellDD(2, 1) += cellV*cellTU.zy();
+                cellDD(2, 2) += cellV*cellTU.zz();
+            }
         }
     }
 }

From a33b0022a46eda70a6741149f3a3954853d210c2 Mon Sep 17 00:00:00 2001
From: Hrvoje Jasak <h.jasak@wikki.co.uk>
Date: Tue, 10 Jan 2017 15:42:15 +0000
Subject: [PATCH 099/130] Bugfix: Minor clean-up of etc files

---
 etc/bashrc              |  4 ++--
 etc/controlDict-EXAMPLE |  2 +-
 etc/cshrc               |  4 ++--
 etc/prefs.csh-EXAMPLE   |  3 +++
 etc/prefs.sh-EXAMPLE    |  3 +++
 etc/settings.csh        | 15 +++++----------
 etc/settings.sh         |  8 ++++----
 7 files changed, 20 insertions(+), 19 deletions(-)

diff --git a/etc/bashrc b/etc/bashrc
index de94a1b57..63a5143f3 100755
--- a/etc/bashrc
+++ b/etc/bashrc
@@ -91,11 +91,11 @@ export WM_THIRD_PARTY_DIR=$WM_PROJECT_DIR/ThirdParty
 # Use 'set' and 'unset' to adjust for your environent
 # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 # For AllMake.stage1
-export WM_THIRD_PARTY_USE_CMAKE_322=1
+#export WM_THIRD_PARTY_USE_CMAKE_322=1
 
 #
 # For AllMake.stage2
-export WM_THIRD_PARTY_USE_OPENMPI_165=1
+#export WM_THIRD_PARTY_USE_OPENMPI_165=1
 
 #
 # For AllMake.stage3
diff --git a/etc/controlDict-EXAMPLE b/etc/controlDict-EXAMPLE
index e70f4285f..011913358 100644
--- a/etc/controlDict-EXAMPLE
+++ b/etc/controlDict-EXAMPLE
@@ -943,7 +943,7 @@ Tolerances
     slidingEdgeEndCutoffTol         0.0001;
     slidingEdgeCoPlanarTol          0.8;
 
-     GGIAreaErrorTol                 1e-8;
+    GGIAreaErrorTol                 1e-8;
 }
 
 DimensionedConstants
diff --git a/etc/cshrc b/etc/cshrc
index 94875fce6..0cf6c0bc0 100644
--- a/etc/cshrc
+++ b/etc/cshrc
@@ -77,11 +77,11 @@ setenv WM_THIRD_PARTY_DIR $WM_PROJECT_DIR/ThirdParty
 # This can be overriden in prefs.sh
 # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 # For AllMake.stage1
-setenv WM_THIRD_PARTY_USE_CMAKE_322 1
+#setenv WM_THIRD_PARTY_USE_CMAKE_322 1
 
 #
 # For AllMake.stage2
-setenv WM_THIRD_PARTY_USE_OPENMPI_165 1
+#setenv WM_THIRD_PARTY_USE_OPENMPI_165 1
 
 #
 # For AllMake.stage3
diff --git a/etc/prefs.csh-EXAMPLE b/etc/prefs.csh-EXAMPLE
index 7c59e8854..16d204def 100644
--- a/etc/prefs.csh-EXAMPLE
+++ b/etc/prefs.csh-EXAMPLE
@@ -210,6 +210,9 @@ setenv WM_THIRD_PARTY_USE_OPENMPI_188 1
 #setenv WM_THIRD_PARTY_USE_OPENMPI_15 1
 #setenv WM_THIRD_PARTY_USE_OPENMPI_143 1
 #setenv WM_THIRD_PARTY_USE_OPENMPI_141 1
+#setenv WM_THIRD_PARTY_USE_MVAPICH2_22 1
+#setenv WM_THIRD_PARTY_USE_MVAPICH2_22_ConfigureAdditionalArgs '--disable-mcast'
+
 
 #
 # For AllMake.stage3
diff --git a/etc/prefs.sh-EXAMPLE b/etc/prefs.sh-EXAMPLE
index b2c490599..c8c0e0de0 100644
--- a/etc/prefs.sh-EXAMPLE
+++ b/etc/prefs.sh-EXAMPLE
@@ -220,6 +220,9 @@ export WM_THIRD_PARTY_USE_OPENMPI_188=1
 #export WM_THIRD_PARTY_USE_OPENMPI_15=1
 #export WM_THIRD_PARTY_USE_OPENMPI_143=1
 #export WM_THIRD_PARTY_USE_OPENMPI_141=1
+#export WM_THIRD_PARTY_USE_MVAPICH2_22=1
+#export WM_THIRD_PARTY_USE_MVAPICH2_22_ConfigureAdditionalArgs='--disable-mcast'
+
 
 #
 # For AllMake.stage3
diff --git a/etc/settings.csh b/etc/settings.csh
index fafa699df..d537042f7 100644
--- a/etc/settings.csh
+++ b/etc/settings.csh
@@ -300,17 +300,12 @@ case SYSTEMOPENMPI:
     breaksw
 
 case MVAPICH2:
-    set mpi_version=mvapich2
-
-    if ($?MVAPICH2_BIN_DIR != 0) then
-        if (-d "${MVAPICH2_BIN_DIR}" ) then
-        _foamAddPath $MVAPICH2_BIN_DIR
+    set mpi_version=mvapich2-2.2
+    if ($?WM_THIRD_PARTY_USE_MVAPICH2_22 != 0 && -d $WM_THIRD_PARTY_DIR/packages/mvapich2-2.2/platforms/$WM_OPTIONS ) then
+        if ($?FOAM_VERBOSE && $?prompt) then
+            echo "Using mvapich2-2.2 from the ThirdParty package: $WM_THIRD_PARTY_DIR/packages/$mpi_version"
         endif
-    else
-        set mpicc_cmd=`which mpicc`
-        setenv MVAPICH2_BIN_DIR `dirname $mpicc_cmd`
-        unset mpicc_cmd
-    endif
+        _foamSource  $WM_THIRD_PARTY_DIR/packages/$mpi_version/platforms/$WM_OPTIONS/etc/$mpi_version.csh
 
     setenv MPI_HOME `dirname $MVAPICH2_BIN_DIR`
     setenv MPI_ARCH_PATH $MPI_HOME
diff --git a/etc/settings.sh b/etc/settings.sh
index 5f2aee33e..eb59fa877 100755
--- a/etc/settings.sh
+++ b/etc/settings.sh
@@ -220,7 +220,7 @@ OPENMPI)
         then
             echo "Using openmpi-1.8.8 from the ThirdParty package: $WM_THIRD_PARTY_DIR/packages/$mpi_version"
         fi
-        _foamSource  $WM_THIRD_PARTY_DIR/packages/$mpi_version/platforms/$WM_OPTIONS/etc/$mpi_version.sh
+        _foamSource $WM_THIRD_PARTY_DIR/packages/$mpi_version/platforms/$WM_OPTIONS/etc/$mpi_version.sh
 
     elif [ ! -z $WM_THIRD_PARTY_USE_OPENMPI_165 ] && [ -e $WM_THIRD_PARTY_DIR/packages/openmpi-1.6.5/platforms/$WM_OPTIONS ]
         then
@@ -229,7 +229,7 @@ OPENMPI)
         then
             echo "Using openmpi-1.6.5 from the ThirdParty package: $WM_THIRD_PARTY_DIR/packages/$mpi_version"
         fi
-        _foamSource  $WM_THIRD_PARTY_DIR/packages/$mpi_version/platforms/$WM_OPTIONS/etc/$mpi_version.sh
+        _foamSource $WM_THIRD_PARTY_DIR/packages/$mpi_version/platforms/$WM_OPTIONS/etc/$mpi_version.sh
 
     elif [ ! -z $WM_THIRD_PARTY_USE_OPENMPI_143 ] && [ -e $WM_THIRD_PARTY_DIR/packages/openmpi-1.4.3/platforms/$WM_OPTIONS ]
         then
@@ -238,7 +238,7 @@ OPENMPI)
         then
             echo "Using openmpi-1.4.3 from the ThirdParty package: $WM_THIRD_PARTY_DIR/packages/$mpi_version"
         fi
-        _foamSource  $WM_THIRD_PARTY_DIR/packages/$mpi_version/platforms/$WM_OPTIONS/etc/$mpi_version.sh
+        _foamSource $WM_THIRD_PARTY_DIR/packages/$mpi_version/platforms/$WM_OPTIONS/etc/$mpi_version.sh
 
     elif [ ! -z $WM_THIRD_PARTY_USE_OPENMPI_15 ] && [ -e $WM_THIRD_PARTY_DIR/packages/openmpi-1.5/platforms/$WM_OPTIONS ]
         then
@@ -247,7 +247,7 @@ OPENMPI)
         then
             echo "Using openmpi-1.5 from the ThirdParty package: $WM_THIRD_PARTY_DIR/packages/$mpi_version"
         fi
-        _foamSource  $WM_THIRD_PARTY_DIR/packages/$mpi_version/platforms/$WM_OPTIONS/etc/$mpi_version.sh
+        _foamSource $WM_THIRD_PARTY_DIR/packages/$mpi_version/platforms/$WM_OPTIONS/etc/$mpi_version.sh
     fi
 
     # On Windows set mpi_version from value defined in bashrc.mingw:

From d6628ea32682e7f4dbafc98482e0a046cefdd542 Mon Sep 17 00:00:00 2001
From: Hrvoje Jasak <h.jasak@wikki.co.uk>
Date: Tue, 10 Jan 2017 18:25:44 +0000
Subject: [PATCH 100/130] Minor formatting

---
 etc/settings.csh | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/etc/settings.csh b/etc/settings.csh
index d537042f7..fd84b9d46 100644
--- a/etc/settings.csh
+++ b/etc/settings.csh
@@ -173,7 +173,7 @@ case OPENMPI:
         if ($?FOAM_VERBOSE && $?prompt) then
             echo "Using openmpi-1.8.8 from the ThirdParty package: $WM_THIRD_PARTY_DIR/packages/$mpi_version"
         endif
-        _foamSource  $WM_THIRD_PARTY_DIR/packages/$mpi_version/platforms/$WM_OPTIONS/etc/$mpi_version.csh
+        _foamSource $WM_THIRD_PARTY_DIR/packages/$mpi_version/platforms/$WM_OPTIONS/etc/$mpi_version.csh
 
     else if ($?WM_THIRD_PARTY_USE_OPENMPI_165 != 0 && -d $WM_THIRD_PARTY_DIR/packages/openmpi-1.6.5/platforms/$WM_OPTIONS ) then
         set mpi_version=openmpi-1.6.5
@@ -181,7 +181,7 @@ case OPENMPI:
         if ($?FOAM_VERBOSE && $?prompt) then
             echo "Using openmpi-1.6.5 from the ThirdParty package: $WM_THIRD_PARTY_DIR/packages/$mpi_version"
         endif
-        _foamSource  $WM_THIRD_PARTY_DIR/packages/$mpi_version/platforms/$WM_OPTIONS/etc/$mpi_version.csh
+        _foamSource $WM_THIRD_PARTY_DIR/packages/$mpi_version/platforms/$WM_OPTIONS/etc/$mpi_version.csh
 
     else if ($?WM_THIRD_PARTY_USE_OPENMPI_143 != 0 && -d $WM_THIRD_PARTY_DIR/packages/openmpi-1.4.3/platforms/$WM_OPTIONS ) then
         set mpi_version=openmpi-1.4.3
@@ -189,7 +189,7 @@ case OPENMPI:
         if ($?FOAM_VERBOSE && $?prompt) then
             echo "Using openmpi-1.4.3 from the ThirdParty package: $WM_THIRD_PARTY_DIR/packages/$mpi_version"
         endif
-        _foamSource  $WM_THIRD_PARTY_DIR/packages/$mpi_version/platforms/$WM_OPTIONS/etc/$mpi_version.csh
+        _foamSource $WM_THIRD_PARTY_DIR/packages/$mpi_version/platforms/$WM_OPTIONS/etc/$mpi_version.csh
 
     else if ($?WM_THIRD_PARTY_USE_OPENMPI_143 != 0 && -d $WM_THIRD_PARTY_DIR/packages/openmpi-1.5/platforms/$WM_OPTIONS ) then
         set mpi_version=openmpi-1.5
@@ -197,7 +197,7 @@ case OPENMPI:
         if ($?FOAM_VERBOSE && $?prompt) then
             echo "Using openmpi-1.5 from the ThirdParty package: $WM_THIRD_PARTY_DIR/packages/$mpi_version"
         endif
-        _foamSource  $WM_THIRD_PARTY_DIR/packages/$mpi_version/platforms/$WM_OPTIONS/etc/$mpi_version.csh
+        _foamSource $WM_THIRD_PARTY_DIR/packages/$mpi_version/platforms/$WM_OPTIONS/etc/$mpi_version.csh
     endif
 
     unset mpi_version
@@ -305,7 +305,7 @@ case MVAPICH2:
         if ($?FOAM_VERBOSE && $?prompt) then
             echo "Using mvapich2-2.2 from the ThirdParty package: $WM_THIRD_PARTY_DIR/packages/$mpi_version"
         endif
-        _foamSource  $WM_THIRD_PARTY_DIR/packages/$mpi_version/platforms/$WM_OPTIONS/etc/$mpi_version.csh
+        _foamSource $WM_THIRD_PARTY_DIR/packages/$mpi_version/platforms/$WM_OPTIONS/etc/$mpi_version.csh
 
     setenv MPI_HOME `dirname $MVAPICH2_BIN_DIR`
     setenv MPI_ARCH_PATH $MPI_HOME

From ce2908eac4b1da95f6619a0486493c499233da3c Mon Sep 17 00:00:00 2001
From: Hrvoje Jasak <h.jasak@wikki.co.uk>
Date: Wed, 11 Jan 2017 10:39:30 +0000
Subject: [PATCH 101/130] Bugfix: upgrade openmpi version

---
 etc/bashrc | 2 +-
 etc/cshrc  | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/etc/bashrc b/etc/bashrc
index 63a5143f3..5aae670e8 100755
--- a/etc/bashrc
+++ b/etc/bashrc
@@ -95,7 +95,7 @@ export WM_THIRD_PARTY_DIR=$WM_PROJECT_DIR/ThirdParty
 
 #
 # For AllMake.stage2
-#export WM_THIRD_PARTY_USE_OPENMPI_165=1
+export WM_THIRD_PARTY_USE_OPENMPI_188=1
 
 #
 # For AllMake.stage3
diff --git a/etc/cshrc b/etc/cshrc
index 0cf6c0bc0..6b29c5005 100644
--- a/etc/cshrc
+++ b/etc/cshrc
@@ -81,7 +81,7 @@ setenv WM_THIRD_PARTY_DIR $WM_PROJECT_DIR/ThirdParty
 
 #
 # For AllMake.stage2
-#setenv WM_THIRD_PARTY_USE_OPENMPI_165 1
+setenv WM_THIRD_PARTY_USE_OPENMPI_188 1
 
 #
 # For AllMake.stage3
@@ -152,7 +152,7 @@ if ( ! $?WM_PRECISION_OPTION ) setenv WM_PRECISION_OPTION DP
 
 # WM_COMPILE_OPTION = Opt | Debug | Prof
 if ( ! $?WM_COMPILE_OPTION ) setenv WM_COMPILE_OPTION Opt
-#setenv WM_COMPILE_OPTION Opt
+setenv WM_COMPILE_OPTION Opt
 #setenv WM_COMPILE_OPTION Debug
 
 # WM_MPLIB = SYSTEMOPENMPI | OPENMPI | MPICH | MPICH-GM | HPMPI | GAMMA | MPI | QSMPI | MVAPICH2

From 297f63cce2eddaa453f3012a90edd5f1f8db7e2d Mon Sep 17 00:00:00 2001
From: Hrvoje Jasak <h.jasak@wikki.co.uk>
Date: Wed, 11 Jan 2017 11:55:37 +0000
Subject: [PATCH 102/130] Bugfix: consistency of finite area operators

---
 src/finiteArea/finiteArea/fac/facGrad.C  |  4 +--
 src/finiteArea/finiteArea/fac/facNDiv.C  | 38 ++++++++++++++----------
 src/finiteArea/finiteArea/fac/facNGrad.C | 24 ++++++++++-----
 src/finiteArea/finiteArea/fam/famDiv.C   |  4 ++-
 src/finiteArea/finiteArea/fam/famNDiv.C  |  2 ++
 5 files changed, 45 insertions(+), 27 deletions(-)

diff --git a/src/finiteArea/finiteArea/fac/facGrad.C b/src/finiteArea/finiteArea/fac/facGrad.C
index 568f4c2b6..d5b117638 100644
--- a/src/finiteArea/finiteArea/fac/facGrad.C
+++ b/src/finiteArea/finiteArea/fac/facGrad.C
@@ -63,7 +63,7 @@ grad
 
     GeometricField<GradType, faPatchField, areaMesh>& gGrad = tgGrad();
 
-    gGrad -= (gGrad & n)*n;
+    gGrad -= n*(n & gGrad);
     gGrad.correctBoundaryConditions();
 
     return tgGrad;
@@ -118,7 +118,7 @@ grad
 
     GeometricField<GradType, faPatchField, areaMesh>& gGrad = tgGrad();
 
-    gGrad -= (gGrad & n)*n;
+    gGrad -= n*(n & gGrad);
     gGrad.correctBoundaryConditions();
 
     return tgGrad;
diff --git a/src/finiteArea/finiteArea/fac/facNDiv.C b/src/finiteArea/finiteArea/fac/facNDiv.C
index 6a1953961..67553a50a 100644
--- a/src/finiteArea/finiteArea/fac/facNDiv.C
+++ b/src/finiteArea/finiteArea/fac/facNDiv.C
@@ -49,12 +49,13 @@ ndiv
     const GeometricField<Type, faePatchField, edgeMesh>& ssf
 )
 {
-    const areaVectorField &n = ssf.mesh().faceAreaNormals();
+    const areaVectorField& n = ssf.mesh().faceAreaNormals();
 
-    tmp<GeometricField<Type, faPatchField, areaMesh> > v = fac::edgeIntegrate(ssf);
+    tmp<GeometricField<Type, faPatchField, areaMesh> > v =
+        fac::edgeIntegrate(ssf);
 
-    //v.internalField() = transform(n*n, v.internalField());
-    v.internalField() = (v.internalField()&n)*n;
+    v.internalField() = n*(n & v.internalField());
+    v.correctBoundaryConditions();
 
     return v;
 }
@@ -87,7 +88,7 @@ ndiv
     const word& name
 )
 {
-    const areaVectorField &n = vf.mesh().faceAreaNormals();
+    const areaVectorField& n = vf.mesh().faceAreaNormals();
 
     tmp<GeometricField<Type, faPatchField, areaMesh> > tDiv
     (
@@ -96,11 +97,11 @@ ndiv
                 vf.mesh(), vf.mesh().schemesDict().divScheme(name)
             )().facDiv(vf)
     );
-
     GeometricField<Type, faPatchField, areaMesh>& Div = tDiv();
 
-    //Div.internalField() = transform(n*n, Div.internalField());
-    Div.internalField() = (Div.internalField()&n)*n;
+    Div.internalField() = n*(n & Div.internalField());
+    Div.correctBoundaryConditions();
+
     return tDiv;
 }
 
@@ -125,6 +126,7 @@ ndiv
         fac::ndiv(tvvf(), name)
     );
     tvvf.clear();
+
     return Div;
 }
 
@@ -178,22 +180,22 @@ ndiv
     const word& name
 )
 {
-    const areaVectorField &n = vf.mesh().faceAreaNormals();
+    const areaVectorField& n = vf.mesh().faceAreaNormals();
 
     tmp<GeometricField<Type, faPatchField, areaMesh> > tDiv
     (
         fa::convectionScheme<Type>::New
-            (
-                vf.mesh(),
-                flux,
-                vf.mesh().schemesDict().divScheme(name)
-            )().facDiv(flux, vf)
+        (
+            vf.mesh(),
+            flux,
+            vf.mesh().schemesDict().divScheme(name)
+        )().facDiv(flux, vf)
     );
 
     GeometricField<Type, faPatchField, areaMesh>& Div = tDiv();
 
-    //Div.internalField() = transform(n*n, Div.internalField());
-    Div.internalField() = (Div.internalField()&n)*n;
+    Div.internalField() = n*(n &Div.internalField());
+    Div.correctBoundaryConditions();
 
     return tDiv;
 
@@ -214,6 +216,7 @@ ndiv
         fac::ndiv(tflux(), vf, name)
     );
     tflux.clear();
+
     return Div;
 }
 
@@ -232,6 +235,7 @@ ndiv
         fac::ndiv(flux, tvf(), name)
     );
     tvf.clear();
+
     return Div;
 }
 
@@ -251,6 +255,7 @@ ndiv
     );
     tflux.clear();
     tvf.clear();
+
     return Div;
 }
 
@@ -318,6 +323,7 @@ ndiv
     );
     tflux.clear();
     tvf.clear();
+
     return Div;
 }
 
diff --git a/src/finiteArea/finiteArea/fac/facNGrad.C b/src/finiteArea/finiteArea/fac/facNGrad.C
index 9d4d96070..8c1df7809 100644
--- a/src/finiteArea/finiteArea/fac/facNGrad.C
+++ b/src/finiteArea/finiteArea/fac/facNGrad.C
@@ -56,18 +56,21 @@ ngrad
     const GeometricField<Type, faePatchField, edgeMesh>& ssf
 )
 {
-    const areaVectorField &n = ssf.mesh().faceAreaNormals();
+    const areaVectorField& n = ssf.mesh().faceAreaNormals();
     typedef typename outerProduct<vector,Type>::type GradType;
 
-    tmp<GeometricField<GradType, faPatchField, areaMesh> > tgGrad = fac::edgeIntegrate(ssf.mesh().Sf() * ssf);
+    tmp<GeometricField<GradType, faPatchField, areaMesh> > tgGrad =
+        fac::edgeIntegrate(ssf.mesh().Sf() * ssf);
 
-    GeometricField<GradType, faPatchField, areaMesh>& grad = tgGrad();
+    GeometricField<GradType, faPatchField, areaMesh>& gGrad = tgGrad();
 
-    grad = (grad&n)*n;
+    gGrad = n*(n & gGrad);
+    gGrad.correctBoundaryConditions();
 
     return tgGrad;
 }
 
+
 template<class Type>
 tmp
 <
@@ -87,6 +90,7 @@ ngrad
         fac::ngrad(tssf())
     );
     tssf.clear();
+
     return Grad;
 }
 
@@ -105,18 +109,20 @@ ngrad
     const word& name
 )
 {
-    const areaVectorField &n = vf.mesh().faceAreaNormals();
+    const areaVectorField& n = vf.mesh().faceAreaNormals();
     typedef typename outerProduct<vector,Type>::type GradType;
 
-    tmp<GeometricField<GradType, faPatchField, areaMesh> > tgGrad = fa::gradScheme<Type>::New
+    tmp<GeometricField<GradType, faPatchField, areaMesh> > tgGrad =
+        fa::gradScheme<Type>::New
         (
             vf.mesh(),
             vf.mesh().schemesDict().gradScheme(name)
         )().grad(vf);
 
-    GeometricField<GradType, faPatchField, areaMesh>& grad = tgGrad();
+    GeometricField<GradType, faPatchField, areaMesh>& gGrad = tgGrad();
 
-    grad = (grad&n)*n;
+    gGrad = n*(n & gGrad);
+    gGrad.correctBoundaryConditions();
 
     return tgGrad;
 }
@@ -147,6 +153,7 @@ ngrad
         fac::ngrad(tvf(), name)
     );
     tvf.clear();
+
     return tGrad;
 }
 
@@ -187,6 +194,7 @@ ngrad
         fac::ngrad(tvf())
     );
     tvf.clear();
+
     return Grad;
 }
 
diff --git a/src/finiteArea/finiteArea/fam/famDiv.C b/src/finiteArea/finiteArea/fam/famDiv.C
index 2296b224b..0b2af47a7 100644
--- a/src/finiteArea/finiteArea/fam/famDiv.C
+++ b/src/finiteArea/finiteArea/fam/famDiv.C
@@ -73,7 +73,7 @@ div
     );
 
     //HJ Check if the product is from left or right.  HJ, 6/Dec/2016
-    M -= (v & n)*n;
+    M -= n*(n & v);
 
     return tM;
 }
@@ -90,6 +90,7 @@ div
 {
     tmp<faMatrix<Type> > Div(fam::div(tflux(), vf, name));
     tflux.clear();
+
     return Div;
 }
 
@@ -115,6 +116,7 @@ div
 {
     tmp<faMatrix<Type> > Div(fam::div(tflux(), vf));
     tflux.clear();
+
     return Div;
 }
 
diff --git a/src/finiteArea/finiteArea/fam/famNDiv.C b/src/finiteArea/finiteArea/fam/famNDiv.C
index 33c55f3fa..f668bb626 100644
--- a/src/finiteArea/finiteArea/fam/famNDiv.C
+++ b/src/finiteArea/finiteArea/fam/famNDiv.C
@@ -68,6 +68,7 @@ ndiv
 {
     tmp<faMatrix<Type> > Div(fam::ndiv(tflux(), vf, name));
     tflux.clear();
+
     return Div;
 }
 
@@ -93,6 +94,7 @@ ndiv
 {
     tmp<faMatrix<Type> > Div(fam::ndiv(tflux(), vf));
     tflux.clear();
+
     return Div;
 }
 

From bd5604c1000dddd0e4f24aeb24e86bc906d23f09 Mon Sep 17 00:00:00 2001
From: Hrvoje Jasak <h.jasak@wikki.co.uk>
Date: Wed, 11 Jan 2017 11:57:28 +0000
Subject: [PATCH 103/130] Clean-up and formatting

---
 .../coupled/conjugateHeatFoam/solveEnergy.H   |   9 +-
 .../coupled/pUCoupledFoam/createFields.H      | 128 +++++++++---------
 .../solvers/incompressible/simpleFoam/pEqn.H  |   4 +-
 3 files changed, 68 insertions(+), 73 deletions(-)

diff --git a/applications/solvers/coupled/conjugateHeatFoam/solveEnergy.H b/applications/solvers/coupled/conjugateHeatFoam/solveEnergy.H
index 61b5863c2..c101f7ddd 100644
--- a/applications/solvers/coupled/conjugateHeatFoam/solveEnergy.H
+++ b/applications/solvers/coupled/conjugateHeatFoam/solveEnergy.H
@@ -14,15 +14,12 @@
               + fvm::div(phi, T)
             )
           - fvm::laplacian(kappaEff, T)
-            ==
+         ==
             radiation->Ru()
           - fvm::Sp(4.0*radiation->Rp()*pow3(T), T)
           + 3.0*radiation->Rp()*pow4(T)
         );
 
-        //Done automatically in 1.6.1-ext
-        //TFluidEqn->boundaryManipulate(T.boundaryField());
-
         fvScalarMatrix* TSolidEqn = new fvScalarMatrix
         (
             fvm::ddt(rhoCpsolid, Tsolid)
@@ -30,10 +27,6 @@
           + fvm::SuSp(-solidThermo.S()/Tsolid, Tsolid)
         );
 
-        //Done automatically in 1.6.1-ext
-        //TSolidEqn->boundaryManipulate(Tsolid.boundaryField());
-
-
         // Add fluid equation
         TEqns.set(0, TFluidEqn);
 
diff --git a/applications/solvers/coupled/pUCoupledFoam/createFields.H b/applications/solvers/coupled/pUCoupledFoam/createFields.H
index 179cbd40f..a30702f03 100644
--- a/applications/solvers/coupled/pUCoupledFoam/createFields.H
+++ b/applications/solvers/coupled/pUCoupledFoam/createFields.H
@@ -1,69 +1,71 @@
-Info << "Reading field p\n" << endl;
-volScalarField p
-(
-    IOobject
+    Info << "Reading field p\n" << endl;
+    volScalarField p
     (
-        "p",
-        runTime.timeName(),
+        IOobject
+        (
+            "p",
+            runTime.timeName(),
+            mesh,
+            IOobject::MUST_READ,
+            IOobject::AUTO_WRITE
+        ),
+        mesh
+    );
+
+    Info << "Reading field U\n" << endl;
+    volVectorField U
+    (
+        IOobject
+        (
+            "U",
+            runTime.timeName(),
+            mesh,
+            IOobject::MUST_READ,
+            IOobject::AUTO_WRITE
+        ),
+        mesh
+    );
+
+#   include "createPhi.H"
+
+    singlePhaseTransportModel laminarTransport(U, phi);
+
+    autoPtr<incompressible::RASModel> turbulence
+    (
+        incompressible::RASModel::New(U, phi, laminarTransport)
+    );
+
+    // Block vector field for velocity (first entry) and pressure (second
+    // entry).
+    Info << "Creating field Up\n" << endl;
+    volVector4Field Up
+    (
+        IOobject
+        (
+            "Up",
+            runTime.timeName(),
+            mesh,
+            IOobject::NO_READ,
+            IOobject::NO_WRITE
+        ),
         mesh,
-        IOobject::MUST_READ,
-        IOobject::AUTO_WRITE
-    ),
-    mesh
-);
+        dimensionedVector4("zero", dimless, vector4::zero)
+    );
 
-Info << "Reading field U\n" << endl;
-volVectorField U
-(
-    IOobject
+    Info<< "Creating field rAU\n" << endl;
+    volScalarField rAU
     (
-        "U",
-        runTime.timeName(),
+        IOobject
+        (
+            "rAU",
+            runTime.timeName(),
+            mesh,
+            IOobject::NO_READ,
+            IOobject::NO_WRITE
+        ),
         mesh,
-        IOobject::MUST_READ,
-        IOobject::AUTO_WRITE
-    ),
-    mesh
-);
+        runTime.deltaT()
+    );
 
-#include "createPhi.H"
-
-singlePhaseTransportModel laminarTransport(U, phi);
-autoPtr<incompressible::RASModel> turbulence
-(
-    incompressible::RASModel::New(U, phi, laminarTransport)
-);
-
-// Block vector field for velocity (first entry) and pressure (second
-// entry).
-Info << "Creating field Up\n" << endl;
-volVector4Field Up
-(
-    IOobject
-    (
-       "Up",
-       runTime.timeName(),
-       mesh,
-       IOobject::NO_READ,
-       IOobject::NO_WRITE
-    ),
-    mesh,
-    dimensionedVector4("zero", dimless, vector4::zero)
-);
-
-Info<< "Creating field rAU\n" << endl;
-volScalarField rAU
-(
-    IOobject
-    (
-        "rAU",
-        runTime.timeName(),
-        mesh,
-        IOobject::NO_READ,
-        IOobject::NO_WRITE
-    ),
-    mesh,
-    runTime.deltaT()
-);
-
-mesh.schemesDict().setFluxRequired(p.name());
\ No newline at end of file
+    mesh.schemesDict().setFluxRequired(p.name());
+    mesh.schemesDict().setFluxRequired("pcorr");
diff --git a/applications/solvers/incompressible/simpleFoam/pEqn.H b/applications/solvers/incompressible/simpleFoam/pEqn.H
index aede8a3ba..7d14ec1e9 100644
--- a/applications/solvers/incompressible/simpleFoam/pEqn.H
+++ b/applications/solvers/incompressible/simpleFoam/pEqn.H
@@ -3,7 +3,7 @@
 
     // Prepare clean 1/Ap without contribution from under-relaxation
     // HJ, 26/Oct/2015
-    volScalarField rUA
+    volScalarField rAU
     (
         "(1|A(U))",
         1/HUEqn().A()
@@ -13,7 +13,7 @@
     // the flux precursor
     U.storePrevIter();
 
-    U = rUA*HUEqn().H();
+    U = rAU*HUEqn().H();
     HUEqn.clear();
 
     phi = fvc::interpolate(U) & mesh.Sf();

From 3d2d26334a62c99ce17498541357a8f22a391320 Mon Sep 17 00:00:00 2001
From: Hrvoje Jasak <h.jasak@wikki.co.uk>
Date: Wed, 11 Jan 2017 11:57:48 +0000
Subject: [PATCH 104/130] Bugfix: added ramping to MRF zones

---
 .../rhoPorousMRFPimpleFoam/pEqn.H             |   3 +
 .../steadyCompressibleMRFFoam/pEqn.H          |   3 +
 .../steadyUniversalMRFFoam/pEqn.H             |   3 +
 .../incompressible/MRFSimpleFoam/pEqn.H       |   4 +
 .../solvers/multiphase/MRFInterFoam/pEqn.H    |   3 +
 .../cfdTools/general/MRF/MRFZone.C            | 116 ++++++++++--------
 .../cfdTools/general/MRF/MRFZones.C           |  26 ++--
 .../cfdTools/general/porousMedia/porousZone.H |  17 ++-
 .../general/porousMedia/porousZones.H         |   2 +-
 9 files changed, 104 insertions(+), 73 deletions(-)

diff --git a/applications/solvers/compressible/rhoPorousMRFPimpleFoam/pEqn.H b/applications/solvers/compressible/rhoPorousMRFPimpleFoam/pEqn.H
index 920ba0e5d..18ee1a2b4 100644
--- a/applications/solvers/compressible/rhoPorousMRFPimpleFoam/pEqn.H
+++ b/applications/solvers/compressible/rhoPorousMRFPimpleFoam/pEqn.H
@@ -3,6 +3,9 @@ rho = thermo.rho();
 volScalarField rUA = 1.0/UEqn().A();
 U = rUA*UEqn().H();
 
+// Update boundary velocity for consistency with the flux
+mrfZones.correctBoundaryVelocity(U);
+
 if (pimple.nCorrPISO() <= 1)
 {
     UEqn.clear();
diff --git a/applications/solvers/compressible/steadyCompressibleMRFFoam/pEqn.H b/applications/solvers/compressible/steadyCompressibleMRFFoam/pEqn.H
index ff35bebaa..61fed64e8 100644
--- a/applications/solvers/compressible/steadyCompressibleMRFFoam/pEqn.H
+++ b/applications/solvers/compressible/steadyCompressibleMRFFoam/pEqn.H
@@ -12,6 +12,9 @@
     {
         U = rUA*UEqn.H();
 
+        // Update boundary velocity for consistency with the flux
+        mrfZones.correctBoundaryVelocity(U);
+
         // Calculate phi for boundary conditions
         phi = rhof*fvc::interpolate(U) & mesh.Sf();
 
diff --git a/applications/solvers/compressible/steadyUniversalMRFFoam/pEqn.H b/applications/solvers/compressible/steadyUniversalMRFFoam/pEqn.H
index 1d4b2dc94..bba75ac7f 100644
--- a/applications/solvers/compressible/steadyUniversalMRFFoam/pEqn.H
+++ b/applications/solvers/compressible/steadyUniversalMRFFoam/pEqn.H
@@ -12,6 +12,9 @@
     {
         U = rUA*UEqn.H();
 
+        // Update boundary velocity for consistency with the flux
+        mrfZones.correctBoundaryVelocity(U);
+
         // Calculate phi for boundary conditions
         phi = rhof*fvc::interpolate(U) & mesh.Sf();
 
diff --git a/applications/solvers/incompressible/MRFSimpleFoam/pEqn.H b/applications/solvers/incompressible/MRFSimpleFoam/pEqn.H
index 1752fc663..d5bb1e361 100644
--- a/applications/solvers/incompressible/MRFSimpleFoam/pEqn.H
+++ b/applications/solvers/incompressible/MRFSimpleFoam/pEqn.H
@@ -9,6 +9,10 @@
         1/HUEqn().A()
     );
 
+    // Update boundary velocity for consistency with the flux
+    // This is needed for a ramped MRF
+    mrfZones.correctBoundaryVelocity(U);
+
     // Store velocity under-relaxation point before using U for
     // the flux precursor
     U.storePrevIter();
diff --git a/applications/solvers/multiphase/MRFInterFoam/pEqn.H b/applications/solvers/multiphase/MRFInterFoam/pEqn.H
index 49d6dc579..eebfdfa7d 100644
--- a/applications/solvers/multiphase/MRFInterFoam/pEqn.H
+++ b/applications/solvers/multiphase/MRFInterFoam/pEqn.H
@@ -4,6 +4,9 @@
 
     U = rUA*UEqn.H();
 
+    // Update boundary velocity for consistency with the flux
+    mrfZones.correctBoundaryVelocity(U);
+
     surfaceScalarField phiU
     (
         "phiU",
diff --git a/src/finiteVolume/cfdTools/general/MRF/MRFZone.C b/src/finiteVolume/cfdTools/general/MRF/MRFZone.C
index 09dfb6e25..a26ff72ce 100644
--- a/src/finiteVolume/cfdTools/general/MRF/MRFZone.C
+++ b/src/finiteVolume/cfdTools/general/MRF/MRFZone.C
@@ -32,6 +32,8 @@ License
 #include "faceSet.H"
 #include "geometricOneField.H"
 
+using namespace Foam::mathematicalConstant;
+
 // * * * * * * * * * * * * * * Static Data Members * * * * * * * * * * * * * //
 
 defineTypeNameAndDebug(Foam::MRFZone, 0);
@@ -44,9 +46,9 @@ void Foam::MRFZone::setMRFFaces()
     const polyBoundaryMesh& patches = mesh_.boundaryMesh();
 
     // Type per face:
-    //  0:not in zone
-    //  1:moving with frame
-    //  2:other
+    //  0: not in zone
+    //  1: moving with frame
+    //  2: other
     labelList faceType(mesh_.nFaces(), 0);
 
     // Determine faces in cell zone
@@ -68,7 +70,6 @@ void Foam::MRFZone::setMRFFaces()
         }
     }
 
-
     label nZoneFaces = 0;
 
     for (label faceI = 0; faceI < mesh_.nInternalFaces(); faceI++)
@@ -83,7 +84,7 @@ void Foam::MRFZone::setMRFFaces()
 
 
     labelHashSet excludedPatches(excludedPatchLabels_);
-
+    Info<< "Excluded patches: " << excludedPatchLabels_ << endl;
     forAll (patches, patchI)
     {
         const polyPatch& pp = patches[patchI];
@@ -142,9 +143,9 @@ void Foam::MRFZone::setMRFFaces()
     {
         const polyPatch& pp = patches[patchi];
 
-        forAll (pp, patchFacei)
+        forAll (pp, patchFaceI)
         {
-            label faceI = pp.start()+patchFacei;
+            label faceI = pp.start() + patchFaceI;
 
             if (faceType[faceI] == 1)
             {
@@ -159,29 +160,29 @@ void Foam::MRFZone::setMRFFaces()
 
     includedFaces_.setSize(patches.size());
     excludedFaces_.setSize(patches.size());
-    forAll (nIncludedFaces, patchi)
+    forAll (nIncludedFaces, patchI)
     {
-        includedFaces_[patchi].setSize(nIncludedFaces[patchi]);
-        excludedFaces_[patchi].setSize(nExcludedFaces[patchi]);
+        includedFaces_[patchI].setSize(nIncludedFaces[patchI]);
+        excludedFaces_[patchI].setSize(nExcludedFaces[patchI]);
     }
     nIncludedFaces = 0;
     nExcludedFaces = 0;
 
-    forAll (patches, patchi)
+    forAll (patches, patchI)
     {
-        const polyPatch& pp = patches[patchi];
+        const polyPatch& pp = patches[patchI];
 
-        forAll (pp, patchFacei)
+        forAll (pp, patchFaceI)
         {
-            label faceI = pp.start()+patchFacei;
+            label faceI = pp.start() + patchFaceI;
 
             if (faceType[faceI] == 1)
             {
-                includedFaces_[patchi][nIncludedFaces[patchi]++] = patchFacei;
+                includedFaces_[patchI][nIncludedFaces[patchI]++] = patchFaceI;
             }
             else if (faceType[faceI] == 2)
             {
-                excludedFaces_[patchi][nExcludedFaces[patchi]++] = patchFacei;
+                excludedFaces_[patchI][nExcludedFaces[patchI]++] = patchFaceI;
             }
         }
     }
@@ -201,12 +202,13 @@ void Foam::MRFZone::setMRFFaces()
         internalFaces.write();
 
         faceSet MRFFaces(mesh_, "includedFaces", 100);
-        forAll (includedFaces_, patchi)
+
+        forAll (includedFaces_, patchI)
         {
-            forAll (includedFaces_[patchi], i)
+            forAll (includedFaces_[patchI], i)
             {
-                label patchFacei = includedFaces_[patchi][i];
-                MRFFaces.insert(patches[patchi].start()+patchFacei);
+                label patchFaceI = includedFaces_[patchI][i];
+                MRFFaces.insert(patches[patchI].start() + patchFaceI);
             }
         }
         Pout<< "Writing " << MRFFaces.size()
@@ -215,12 +217,12 @@ void Foam::MRFZone::setMRFFaces()
         MRFFaces.write();
 
         faceSet excludedFaces(mesh_, "excludedFaces", 100);
-        forAll (excludedFaces_, patchi)
+        forAll (excludedFaces_, patchI)
         {
-            forAll (excludedFaces_[patchi], i)
+            forAll (excludedFaces_[patchI], i)
             {
-                label patchFacei = excludedFaces_[patchi][i];
-                excludedFaces.insert(patches[patchi].start() + patchFacei);
+                label patchFaceI = excludedFaces_[patchI][i];
+                excludedFaces.insert(patches[patchI].start() + patchFaceI);
             }
         }
         Pout<< "Writing " << excludedFaces.size()
@@ -240,11 +242,15 @@ Foam::vector Foam::MRFZone::Omega() const
     else
     {
         // Ramping
-        Info<< "MRF Ramp: " << Foam::min(mesh_.time().value()/rampTime_, 1)
-            << endl;
+        const scalar t = mesh_.time().value();
+        const scalar ramp = sin(2*pi/(4*rampTime_)*Foam::min(rampTime_, t));
 
-        return Foam::min(mesh_.time().value()/rampTime_, 1)*
-            omega_.value()*axis_.value();
+        Info<< "MRF Ramp: " << ramp << endl;
+
+        return ramp*omega_.value()*axis_.value();
+
+        // return Foam::min(mesh_.time().value()/rampTime_, 1.0)*
+        //     omega_.value()*axis_.value();
     }
 }
 
@@ -304,7 +310,6 @@ Foam::MRFZone::MRFZone(const fvMesh& mesh, Istream& is)
         }
     }
 
-
     bool cellZoneFound = (cellZoneID_ != -1);
     reduce(cellZoneFound, orOp<bool>());
 
@@ -317,6 +322,11 @@ Foam::MRFZone::MRFZone(const fvMesh& mesh, Istream& is)
             << exit(FatalError);
     }
 
+
+    Info<< "Creating MRF for cell zone " << name_ << ". rpm = "
+        << 30*omega_.value()/mathematicalConstant::pi
+        << endl;
+
     setMRFFaces();
 }
 
@@ -384,24 +394,24 @@ void Foam::MRFZone::relativeVelocity(volVectorField& U) const
         U[celli] -= (rotVel ^ (C[celli] - origin));
     }
 
-    // Included patches
+    // Included faces
     forAll (includedFaces_, patchi)
     {
         forAll (includedFaces_[patchi], i)
         {
-            label patchFacei = includedFaces_[patchi][i];
-            U.boundaryField()[patchi][patchFacei] = vector::zero;
+            label patchFaceI = includedFaces_[patchi][i];
+            U.boundaryField()[patchi][patchFaceI] = vector::zero;
         }
     }
 
-    // Excluded patches
+    // Excluded faces
     forAll (excludedFaces_, patchi)
     {
         forAll (excludedFaces_[patchi], i)
         {
-            label patchFacei = excludedFaces_[patchi][i];
-            U.boundaryField()[patchi][patchFacei] -=
-                (rotVel ^ (C.boundaryField()[patchi][patchFacei] - origin));
+            label patchFaceI = excludedFaces_[patchi][i];
+            U.boundaryField()[patchi][patchFaceI] -=
+                (rotVel ^ (C.boundaryField()[patchi][patchFaceI] - origin));
         }
     }
 }
@@ -422,25 +432,25 @@ void Foam::MRFZone::absoluteVelocity(volVectorField& U) const
         U[celli] += (rotVel ^ (C[celli] - origin));
     }
 
-    // Included patches
+    // Included faces
     forAll (includedFaces_, patchi)
     {
         forAll (includedFaces_[patchi], i)
         {
-            label patchFacei = includedFaces_[patchi][i];
-            U.boundaryField()[patchi][patchFacei] =
-                (rotVel ^ (C.boundaryField()[patchi][patchFacei] - origin));
+            label patchFaceI = includedFaces_[patchi][i];
+            U.boundaryField()[patchi][patchFaceI] =
+                (rotVel ^ (C.boundaryField()[patchi][patchFaceI] - origin));
         }
     }
 
-    // Excluded patches
+    // Excluded faces
     forAll (excludedFaces_, patchi)
     {
         forAll (excludedFaces_[patchi], i)
         {
-            label patchFacei = excludedFaces_[patchi][i];
-            U.boundaryField()[patchi][patchFacei] +=
-                (rotVel ^ (C.boundaryField()[patchi][patchFacei] - origin));
+            label patchFaceI = excludedFaces_[patchi][i];
+            U.boundaryField()[patchi][patchFaceI] +=
+                (rotVel ^ (C.boundaryField()[patchi][patchFaceI] - origin));
         }
     }
 }
@@ -499,10 +509,10 @@ void Foam::MRFZone::meshPhi
     {
         forAll (includedFaces_[patchi], i)
         {
-            label patchFacei = includedFaces_[patchi][i];
+            label patchFaceI = includedFaces_[patchi][i];
 
-            phi.boundaryField()[patchi][patchFacei] =
-                (rotVel ^ (Cf.boundaryField()[patchi][patchFacei] - origin));
+            phi.boundaryField()[patchi][patchFaceI] =
+                (rotVel ^ (Cf.boundaryField()[patchi][patchFaceI] - origin));
         }
     }
 
@@ -511,10 +521,10 @@ void Foam::MRFZone::meshPhi
     {
         forAll (excludedFaces_[patchi], i)
         {
-            label patchFacei = excludedFaces_[patchi][i];
+            label patchFaceI = excludedFaces_[patchi][i];
 
-            phi.boundaryField()[patchi][patchFacei] =
-                (rotVel ^ (Cf.boundaryField()[patchi][patchFacei] - origin));
+            phi.boundaryField()[patchi][patchFaceI] =
+                (rotVel ^ (Cf.boundaryField()[patchi][patchFaceI] - origin));
         }
     }
 }
@@ -523,7 +533,7 @@ void Foam::MRFZone::correctBoundaryVelocity(volVectorField& U) const
 {
     const vector& origin = origin_.value();
     const vector rotVel = Omega();
-
+    Info<< "rotVel: " << rotVel << endl;
     // Included patches
     forAll (includedFaces_, patchi)
     {
@@ -533,9 +543,9 @@ void Foam::MRFZone::correctBoundaryVelocity(volVectorField& U) const
 
         forAll (includedFaces_[patchi], i)
         {
-            label patchFacei = includedFaces_[patchi][i];
+            label patchFaceI = includedFaces_[patchi][i];
 
-            pfld[patchFacei] = (rotVel ^ (patchC[patchFacei] - origin));
+            pfld[patchFaceI] = (rotVel ^ (patchC[patchFaceI] - origin));
         }
 
         U.boundaryField()[patchi] == pfld;
diff --git a/src/finiteVolume/cfdTools/general/MRF/MRFZones.C b/src/finiteVolume/cfdTools/general/MRF/MRFZones.C
index b2a6a780c..e5f29c690 100644
--- a/src/finiteVolume/cfdTools/general/MRF/MRFZones.C
+++ b/src/finiteVolume/cfdTools/general/MRF/MRFZones.C
@@ -77,7 +77,7 @@ Foam::tmp<Foam::surfaceScalarField> Foam::MRFZones::fluxCorrection() const
     );
     surfaceScalarField& MRFZonesPhiCorr = tMRFZonesPhiCorr();
 
-    forAll(*this, i)
+    forAll (*this, i)
     {
         operator[](i).relativeFlux(MRFZonesPhiCorr);
     }
@@ -106,7 +106,7 @@ Foam::tmp<Foam::surfaceVectorField> Foam::MRFZones::meshPhi() const
     );
     surfaceVectorField& MRFZonesPhiCorr = tMRFZonesPhiCorr();
 
-    forAll(*this, i)
+    forAll (*this, i)
     {
         operator[](i).meshPhi(MRFZonesPhiCorr);
     }
@@ -117,7 +117,7 @@ Foam::tmp<Foam::surfaceVectorField> Foam::MRFZones::meshPhi() const
 
 void Foam::MRFZones::addCoriolis(fvVectorMatrix& UEqn) const
 {
-    forAll(*this, i)
+    forAll (*this, i)
     {
         operator[](i).addCoriolis(UEqn);
     }
@@ -126,7 +126,7 @@ void Foam::MRFZones::addCoriolis(fvVectorMatrix& UEqn) const
 
 void Foam::MRFZones::relativeFlux(surfaceScalarField& phi) const
 {
-    forAll(*this, i)
+    forAll (*this, i)
     {
         operator[](i).relativeFlux(phi);
     }
@@ -135,7 +135,7 @@ void Foam::MRFZones::relativeFlux(surfaceScalarField& phi) const
 
 void Foam::MRFZones::absoluteFlux(surfaceScalarField& phi) const
 {
-    forAll(*this, i)
+    forAll (*this, i)
     {
         operator[](i).absoluteFlux(phi);
     }
@@ -148,7 +148,7 @@ void Foam::MRFZones::addCoriolis
     fvVectorMatrix& UEqn
 ) const
 {
-    forAll(*this, i)
+    forAll (*this, i)
     {
         operator[](i).addCoriolis(rho, UEqn);
     }
@@ -161,7 +161,7 @@ void Foam::MRFZones::relativeFlux
     surfaceScalarField& phi
 ) const
 {
-    forAll(*this, i)
+    forAll (*this, i)
     {
         operator[](i).relativeFlux(rho, phi);
     }
@@ -174,7 +174,7 @@ void Foam::MRFZones::absoluteFlux
     surfaceScalarField& phi
 ) const
 {
-    forAll(*this, i)
+    forAll (*this, i)
     {
         operator[](i).absoluteFlux(rho, phi);
     }
@@ -183,7 +183,7 @@ void Foam::MRFZones::absoluteFlux
 
 void Foam::MRFZones::relativeVelocity(volVectorField& U) const
 {
-    forAll(*this, i)
+    forAll (*this, i)
     {
         operator[](i).relativeVelocity(U);
     }
@@ -192,7 +192,7 @@ void Foam::MRFZones::relativeVelocity(volVectorField& U) const
 
 void Foam::MRFZones::absoluteVelocity(volVectorField& U) const
 {
-    forAll(*this, i)
+    forAll (*this, i)
     {
         operator[](i).absoluteVelocity(U);
     }
@@ -201,7 +201,7 @@ void Foam::MRFZones::absoluteVelocity(volVectorField& U) const
 
 void Foam::MRFZones::correctBoundaryVelocity(volVectorField& U) const
 {
-    forAll(*this, i)
+    forAll (*this, i)
     {
         operator[](i).correctBoundaryVelocity(U);
     }
@@ -237,7 +237,7 @@ Foam::tmp<Foam::volScalarField> Foam::MRFZones::Su
     tmp<volVectorField> tgradPhi = fvc::grad(phi);
     const volVectorField& gradPhi = tgradPhi();
 
-    forAll(*this, i)
+    forAll (*this, i)
     {
         operator[](i).Su(phi, gradPhi, source);
     }
@@ -275,7 +275,7 @@ Foam::tmp<Foam::volVectorField> Foam::MRFZones::Su
     tmp<volTensorField> tgradPhi = fvc::grad(phi);
     const volTensorField& gradPhi = tgradPhi();
 
-    forAll(*this, i)
+    forAll (*this, i)
     {
         operator[](i).Su(phi, gradPhi, source);
     }
diff --git a/src/finiteVolume/cfdTools/general/porousMedia/porousZone.H b/src/finiteVolume/cfdTools/general/porousMedia/porousZone.H
index 899bbf1a6..56811ceef 100644
--- a/src/finiteVolume/cfdTools/general/porousMedia/porousZone.H
+++ b/src/finiteVolume/cfdTools/general/porousMedia/porousZone.H
@@ -44,7 +44,6 @@ Description
         S = - (\mu \, d + \frac{\rho |U|}{2} \, f) U
     @f]
 
-
     Since negative Darcy/Forchheimer parameters are invalid, they can be used
     to specify a multiplier (of the max component).
 
@@ -106,15 +105,15 @@ class porousZone
         //- Coordinate system used for the zone (Cartesian)
         coordinateSystem coordSys_;
 
-        //- porosity of the zone (0 < porosity <= 1)
+        //- Porosity of the zone (0 < porosity <= 1)
         //  Placeholder for treatment of temporal terms.
         //  Currently unused.
         scalar porosity_;
 
-        //- powerLaw coefficient C0
+        //- Power Law coefficient C0
         scalar C0_;
 
-        //- powerLaw coefficient C1
+        //- Power Law coefficient C1
         scalar C1_;
 
         //- Darcy coefficient
@@ -126,7 +125,7 @@ class porousZone
 
     // Private Member Functions
 
-        //- adjust negative resistance values to be multiplier of max value
+        //- Adjust negative resistance values to be multiplier of max value
         static void adjustNegativeResistance(dimensionedVector& resist);
 
         //- Power-law resistance
@@ -247,7 +246,13 @@ public:
                 return cellZoneID_;
             }
 
-            //- dictionary values used for the porousZone
+            //- Return cellZone
+            const cellZone& zone() const
+            {
+                return mesh_.cellZones()[cellZoneID_];
+            }
+
+            //- Dictionary values used for the porousZone
             const dictionary& dict() const
             {
                 return dict_;
diff --git a/src/finiteVolume/cfdTools/general/porousMedia/porousZones.H b/src/finiteVolume/cfdTools/general/porousMedia/porousZones.H
index 354872f93..860d1c975 100644
--- a/src/finiteVolume/cfdTools/general/porousMedia/porousZones.H
+++ b/src/finiteVolume/cfdTools/general/porousMedia/porousZones.H
@@ -92,7 +92,7 @@ class porousZones
         void operator=(const porousZones&);
 
 
-        //- modify time derivative elements
+        //- Modify time derivative elements
         template<class Type>
         void modifyDdt(fvMatrix<Type>&) const;
 

From aa0342f9a17e965a76e4d32c93bd28464696a2ed Mon Sep 17 00:00:00 2001
From: Hrvoje Jasak <h.jasak@wikki.co.uk>
Date: Wed, 11 Jan 2017 12:00:45 +0000
Subject: [PATCH 105/130] Formatting of destructors

---
 .../polyMesh/polyPatches/polyPatch/polyPatch.H     |  5 ++---
 src/foam/meshes/polyMesh/zones/cellZone/cellZone.H |  3 +--
 src/foam/meshes/polyMesh/zones/faceZone/faceZone.H |  3 +--
 .../meshes/polyMesh/zones/pointZone/pointZone.H    |  3 +--
 .../field/fieldMinMax/fieldMinMax.H                |  5 ++---
 src/sampling/probes/probes.H                       |  5 ++---
 .../basic/mixtures/basicMixture/basicMixture.H     |  5 ++---
 .../basic/mixtures/pureMixture/pureMixture.H       |  5 ++---
 .../psiThermo/basicPsiThermo/basicPsiThermo.H      |  1 +
 .../hsCombustionThermo/hsCombustionThermo.H        |  6 +++++-
 .../mixtures/reactingMixture/reactingMixture.H     | 14 +++++++++-----
 .../ArrheniusReactionRate/ArrheniusReactionRate.H  |  2 +-
 .../ReversibleReaction/ReversibleReaction.H        |  2 +-
 13 files changed, 30 insertions(+), 29 deletions(-)

diff --git a/src/foam/meshes/polyMesh/polyPatches/polyPatch/polyPatch.H b/src/foam/meshes/polyMesh/polyPatches/polyPatch/polyPatch.H
index 81217ba10..bef61e41f 100644
--- a/src/foam/meshes/polyMesh/polyPatches/polyPatch/polyPatch.H
+++ b/src/foam/meshes/polyMesh/polyPatches/polyPatch/polyPatch.H
@@ -355,9 +355,8 @@ public:
         );
 
 
-    // Destructor
-
-        virtual ~polyPatch();
+    //- Destructor
+    virtual ~polyPatch();
 
 
     // Member Functions
diff --git a/src/foam/meshes/polyMesh/zones/cellZone/cellZone.H b/src/foam/meshes/polyMesh/zones/cellZone/cellZone.H
index 6a23cd1a1..4c3407b6c 100644
--- a/src/foam/meshes/polyMesh/zones/cellZone/cellZone.H
+++ b/src/foam/meshes/polyMesh/zones/cellZone/cellZone.H
@@ -207,8 +207,7 @@ public:
 
 
     //- Destructor
-
-        virtual ~cellZone();
+    virtual ~cellZone();
 
 
     // Member Functions
diff --git a/src/foam/meshes/polyMesh/zones/faceZone/faceZone.H b/src/foam/meshes/polyMesh/zones/faceZone/faceZone.H
index ae146b47e..1da0fe0a5 100644
--- a/src/foam/meshes/polyMesh/zones/faceZone/faceZone.H
+++ b/src/foam/meshes/polyMesh/zones/faceZone/faceZone.H
@@ -243,8 +243,7 @@ public:
 
 
     //- Destructor
-
-        virtual ~faceZone();
+    virtual ~faceZone();
 
 
     // Member Functions
diff --git a/src/foam/meshes/polyMesh/zones/pointZone/pointZone.H b/src/foam/meshes/polyMesh/zones/pointZone/pointZone.H
index b5da9d866..e35f1e7cf 100644
--- a/src/foam/meshes/polyMesh/zones/pointZone/pointZone.H
+++ b/src/foam/meshes/polyMesh/zones/pointZone/pointZone.H
@@ -209,8 +209,7 @@ public:
 
 
     //- Destructor
-
-        virtual ~pointZone();
+    virtual ~pointZone();
 
 
     // Member Functions
diff --git a/src/postProcessing/functionObjects/field/fieldMinMax/fieldMinMax.H b/src/postProcessing/functionObjects/field/fieldMinMax/fieldMinMax.H
index a7b11b91b..9fc92671e 100644
--- a/src/postProcessing/functionObjects/field/fieldMinMax/fieldMinMax.H
+++ b/src/postProcessing/functionObjects/field/fieldMinMax/fieldMinMax.H
@@ -139,9 +139,8 @@ public:
         );
 
 
-    // Destructor
-
-        virtual ~fieldMinMax();
+    //- Destructor
+    virtual ~fieldMinMax();
 
 
     // Member Functions
diff --git a/src/sampling/probes/probes.H b/src/sampling/probes/probes.H
index 6f598b470..eda0acf89 100644
--- a/src/sampling/probes/probes.H
+++ b/src/sampling/probes/probes.H
@@ -174,9 +174,8 @@ public:
         );
 
 
-    // Destructor
-
-        virtual ~probes();
+    //- Destructor
+    virtual ~probes();
 
 
     // Member Functions
diff --git a/src/thermophysicalModels/basic/mixtures/basicMixture/basicMixture.H b/src/thermophysicalModels/basic/mixtures/basicMixture/basicMixture.H
index e746fb8de..1710731a0 100644
--- a/src/thermophysicalModels/basic/mixtures/basicMixture/basicMixture.H
+++ b/src/thermophysicalModels/basic/mixtures/basicMixture/basicMixture.H
@@ -65,9 +65,8 @@ public:
         basicMixture(const dictionary&, const fvMesh&, const objectRegistry&);
 
 
-    // Destructor
-
-        virtual ~basicMixture();
+    //- Destructor
+    virtual ~basicMixture();
 };
 
 
diff --git a/src/thermophysicalModels/basic/mixtures/pureMixture/pureMixture.H b/src/thermophysicalModels/basic/mixtures/pureMixture/pureMixture.H
index 752635afb..f68caf0be 100644
--- a/src/thermophysicalModels/basic/mixtures/pureMixture/pureMixture.H
+++ b/src/thermophysicalModels/basic/mixtures/pureMixture/pureMixture.H
@@ -75,9 +75,8 @@ public:
         pureMixture(const dictionary&, const fvMesh&, const objectRegistry&);
 
 
-    // Destructor
-
-        ~pureMixture();
+    //- Destructor
+    ~pureMixture();
 
 
     // Member functions
diff --git a/src/thermophysicalModels/basic/psiThermo/basicPsiThermo/basicPsiThermo.H b/src/thermophysicalModels/basic/psiThermo/basicPsiThermo/basicPsiThermo.H
index 4f7428e19..2ab489b4a 100644
--- a/src/thermophysicalModels/basic/psiThermo/basicPsiThermo/basicPsiThermo.H
+++ b/src/thermophysicalModels/basic/psiThermo/basicPsiThermo/basicPsiThermo.H
@@ -95,6 +95,7 @@ public:
         return New(mesh, mesh);
     }
 
+    
     //- Destructor
     virtual ~basicPsiThermo();
 
diff --git a/src/thermophysicalModels/reactionThermo/combustionThermo/hsCombustionThermo/hsCombustionThermo.H b/src/thermophysicalModels/reactionThermo/combustionThermo/hsCombustionThermo/hsCombustionThermo.H
index 520a65756..e116c2a41 100644
--- a/src/thermophysicalModels/reactionThermo/combustionThermo/hsCombustionThermo/hsCombustionThermo.H
+++ b/src/thermophysicalModels/reactionThermo/combustionThermo/hsCombustionThermo/hsCombustionThermo.H
@@ -88,7 +88,11 @@ public:
     // Selectors
 
         //- Standard selection based on fvMesh
-        static autoPtr<hsCombustionThermo> New(const fvMesh&, const objectRegistry&);
+        static autoPtr<hsCombustionThermo> New
+        (
+            const fvMesh&,
+            const objectRegistry&
+        );
 
         static autoPtr<hsCombustionThermo> New(const fvMesh& mesh)
         {
diff --git a/src/thermophysicalModels/reactionThermo/mixtures/reactingMixture/reactingMixture.H b/src/thermophysicalModels/reactionThermo/mixtures/reactingMixture/reactingMixture.H
index 256a3129e..c6ffd66d9 100644
--- a/src/thermophysicalModels/reactionThermo/mixtures/reactingMixture/reactingMixture.H
+++ b/src/thermophysicalModels/reactionThermo/mixtures/reactingMixture/reactingMixture.H
@@ -72,13 +72,17 @@ public:
     // Constructors
 
         //- Construct from dictionary and mesh
-        reactingMixture(const dictionary&, const fvMesh&, const objectRegistry&);
+        reactingMixture
+        (
+            const dictionary&,
+            const fvMesh&,
+            const objectRegistry&
+        );
 
 
-    // Destructor
-
-        virtual ~reactingMixture()
-        {}
+    //- Destructor
+    virtual ~reactingMixture()
+    {}
 
 
     // Member functions
diff --git a/src/thermophysicalModels/specie/reaction/reactionRate/ArrheniusReactionRate/ArrheniusReactionRate.H b/src/thermophysicalModels/specie/reaction/reactionRate/ArrheniusReactionRate/ArrheniusReactionRate.H
index 3d6d61a7a..73b7c707d 100644
--- a/src/thermophysicalModels/specie/reaction/reactionRate/ArrheniusReactionRate/ArrheniusReactionRate.H
+++ b/src/thermophysicalModels/specie/reaction/reactionRate/ArrheniusReactionRate/ArrheniusReactionRate.H
@@ -44,7 +44,7 @@ namespace Foam
 {
 
 /*---------------------------------------------------------------------------*\
-                           Class ArrheniusReactionRate Declaration
+                  Class ArrheniusReactionRate Declaration
 \*---------------------------------------------------------------------------*/
 
 class ArrheniusReactionRate
diff --git a/src/thermophysicalModels/specie/reaction/reactionsTemplates/ReversibleReaction/ReversibleReaction.H b/src/thermophysicalModels/specie/reaction/reactionsTemplates/ReversibleReaction/ReversibleReaction.H
index 2afac7a3c..012770116 100644
--- a/src/thermophysicalModels/specie/reaction/reactionsTemplates/ReversibleReaction/ReversibleReaction.H
+++ b/src/thermophysicalModels/specie/reaction/reactionsTemplates/ReversibleReaction/ReversibleReaction.H
@@ -137,7 +137,7 @@ public:
                 const scalarField& c
             ) const;
 
-            //- Reverse rate constant from the given formard rate constant
+            //- Reverse rate constant from the given forward rate constant
             virtual scalar kr
             (
                 const scalar kfwd,

From 88f3579418970f4a4acfd7e87e897bfb423b360f Mon Sep 17 00:00:00 2001
From: Hrvoje Jasak <h.jasak@wikki.co.uk>
Date: Wed, 11 Jan 2017 12:01:30 +0000
Subject: [PATCH 106/130] Formatting

---
 .../mixingPlaneCheckFunctionObject.C          | 53 ++++---------------
 src/sampling/cuttingPlane/cuttingPlane.C      |  9 ++--
 .../psiThermo/basicPsiThermo/basicPsiThermo.C |  6 ++-
 .../redlichKwong/redlichKwong.C               |  4 +-
 4 files changed, 22 insertions(+), 50 deletions(-)

diff --git a/src/postProcessing/functionObjects/check/mixingPlaneCheck/mixingPlaneCheckFunctionObject.C b/src/postProcessing/functionObjects/check/mixingPlaneCheck/mixingPlaneCheckFunctionObject.C
index 4665889d5..1689c956c 100644
--- a/src/postProcessing/functionObjects/check/mixingPlaneCheck/mixingPlaneCheckFunctionObject.C
+++ b/src/postProcessing/functionObjects/check/mixingPlaneCheck/mixingPlaneCheckFunctionObject.C
@@ -87,9 +87,10 @@ bool Foam::mixingPlaneCheckFunctionObject::execute()
 
     forAll (mesh.boundaryMesh(), patchI)
     {
-        if (
-            isA<mixingPlanePolyPatch>(mesh.boundaryMesh()[patchI]) &&
-            mesh.boundaryMesh()[patchI].size()
+        if
+        (
+            isA<mixingPlanePolyPatch>(mesh.boundaryMesh()[patchI])
+          && mesh.boundaryMesh()[patchI].size()
         )
         {
             if (!visited[patchI])
@@ -202,7 +203,7 @@ bool Foam::mixingPlaneCheckFunctionObject::execute()
 
                // Old way of computing phi balance
 
-                if( mesh.foundObject<surfaceScalarField>(phiName_) )
+                if (mesh.foundObject<surfaceScalarField>(phiName_))
                 {
                     const surfaceScalarField& phi =
                         mesh.lookupObject<surfaceScalarField>(phiName_);
@@ -210,10 +211,13 @@ bool Foam::mixingPlaneCheckFunctionObject::execute()
                     // Calculate local and shadow flux
                     scalar masterPatchScaleFactor_ = 1.0;
                     scalar shadowPatchScaleFactor_ = sumMasterAreas/sumShadowAreas;
-                    scalar localFlux    = masterPatchScaleFactor_ * sum(phi.boundaryField()[patchI]);
+                    scalar localFlux =masterPatchScaleFactor_*
+                        sum(phi.boundaryField()[patchI]);
+
                     scalar localFluxMag = mag(localFlux);
 
-                    scalar shadowFlux    = shadowPatchScaleFactor_ * sum(phi.boundaryField()[shadowPatchI]);
+                    scalar shadowFlux =shadowPatchScaleFactor_*
+                        sum(phi.boundaryField()[shadowPatchI]);
 //                     scalar shadowFluxMag = mag(shadowFlux);
 
                     Info<< "Mixing plane pair "
@@ -228,43 +232,6 @@ bool Foam::mixingPlaneCheckFunctionObject::execute()
         }
     }
 
-
-//         {
-//             word patchName = phi.boundaryField()[patchI].patch().name();
-
-//             if (patchName == masterPatchName_ && !visited[patchI])
-//             {
-//                 visited[patchI] = true;
-
-//                 // Calculate local and shadow flux
-//                 scalar localFlux    = masterPatchScaleFactor_ * sum(phi.boundaryField()[patchI]);
-//                 //scalar localFluxMag = masterPatchScaleFactor_ * sumMag(phi.boundaryField()[patchI]);
-//                 scalar localFluxMag = mag(localFlux);
-
-//                 const mixingPlanePolyPatch& mixingPlanePatch =
-//                     refCast<const mixingPlanePolyPatch>
-//                     (
-//                         phi.boundaryField()[patchI].patch().patch()
-//                     );
-
-//                 const label shadowPatchI = mixingPlanePatch.shadowIndex();
-
-//                 visited[shadowPatchI] = true;
-
-//                 scalar shadowFlux    = shadowPatchScaleFactor_ * sum(phi.boundaryField()[shadowPatchI]);
-//                 //scalar shadowFluxMag = shadowPatchScaleFactor_ * sumMag(phi.boundaryField()[shadowPatchI]);
-//                 scalar shadowFluxMag = mag(shadowFlux);
-
-//                 Info<< "mixingPlane pair " << name_ << " (" << mixingPlanePatch.name() << ", " << mixingPlanePatch.shadow().name() << ") : "
-//                     << " flux: " << localFlux << " " << shadowFlux
-//                     << " : mag: " <<  localFluxMag << " " << shadowFluxMag
-//                     << " Diff = " << localFlux + shadowFlux << " or "
-//                     << mag(localFlux + shadowFlux)/(localFluxMag + SMALL)*100
-//                     << " %" << endl;
-//             }
-//         }
-//     }
-
     return true;
 }
 
diff --git a/src/sampling/cuttingPlane/cuttingPlane.C b/src/sampling/cuttingPlane/cuttingPlane.C
index ec9c26de2..ace97282e 100644
--- a/src/sampling/cuttingPlane/cuttingPlane.C
+++ b/src/sampling/cuttingPlane/cuttingPlane.C
@@ -225,7 +225,7 @@ bool Foam::cuttingPlane::walkCell
                 << "Did not find closed walk along surface of cell " << cellI
                 << " starting from edge " << startEdgeI
                 << " in " << nIter << " iterations." << nl
-                << "Collected cutPoints so far:" << faceVerts
+                << "Collected "  << faceVerts.size() << " cutPoints so far"
                 << endl;
 
             return false;
@@ -235,13 +235,14 @@ bool Foam::cuttingPlane::walkCell
 
         nIter++;
 
-        if (nIter > 1000)
+        // Note: reduce number of points in the cut before error
+        if (nIter > 100)
         {
             WarningIn("Foam::cuttingPlane::walkCell")
                 << "Did not find closed walk along surface of cell " << cellI
                 << " starting from edge " << startEdgeI
                 << " in " << nIter << " iterations." << nl
-                << "Collected cutPoints so far:" << faceVerts
+                << "Collected "  << faceVerts.size() << " cutPoints so far"
                 << endl;
             return false;
         }
@@ -258,7 +259,7 @@ bool Foam::cuttingPlane::walkCell
         WarningIn("Foam::cuttingPlane::walkCell")
             << "Did not find closed walk along surface of cell " << cellI
             << " starting from edge " << startEdgeI << nl
-            << "Collected cutPoints so far:" << faceVerts
+            << "Collected "  << faceVerts.size() << " cutPoints so far"
             << endl;
 
         return false;
diff --git a/src/thermophysicalModels/basic/psiThermo/basicPsiThermo/basicPsiThermo.C b/src/thermophysicalModels/basic/psiThermo/basicPsiThermo/basicPsiThermo.C
index 7690f4d78..802164822 100644
--- a/src/thermophysicalModels/basic/psiThermo/basicPsiThermo/basicPsiThermo.C
+++ b/src/thermophysicalModels/basic/psiThermo/basicPsiThermo/basicPsiThermo.C
@@ -36,7 +36,11 @@ namespace Foam
 
 // * * * * * * * * * * * * * * * * Constructors  * * * * * * * * * * * * * * //
 
-Foam::basicPsiThermo::basicPsiThermo(const fvMesh& mesh, const objectRegistry& obj)
+Foam::basicPsiThermo::basicPsiThermo
+(
+    const fvMesh& mesh,
+    const objectRegistry& obj
+)
 :
     basicThermo(mesh, obj)
 {}
diff --git a/src/thermophysicalModels/specie/equationOfState/cubicEquationOfState/redlichKwong/redlichKwong.C b/src/thermophysicalModels/specie/equationOfState/cubicEquationOfState/redlichKwong/redlichKwong.C
index fecb9ef6d..db4c0ec0a 100755
--- a/src/thermophysicalModels/specie/equationOfState/cubicEquationOfState/redlichKwong/redlichKwong.C
+++ b/src/thermophysicalModels/specie/equationOfState/cubicEquationOfState/redlichKwong/redlichKwong.C
@@ -66,8 +66,8 @@ Foam::redlichKwong::redlichKwong(const dictionary& dict)
     Tcrit_(readScalar(dict.subDict("equationOfState").lookup("TCritical"))),
     a_(0.42748*pow(this->RR(),2)*pow(Tcrit_,2.5)/pcrit_),
     b_(0.08664*this->RR()*Tcrit_/pcrit_),
-    b2_(pow(b_,2)),
-    b3_(pow(b_,3)),
+    b2_(pow(b_, 2)),
+    b3_(pow(b_, 3)),
     //CL: rhoMin and rhoMax are only used as boundaries for the bisection methode (see rho function)
     //CL: important: rhoMin and rhoMax are not used as boundary for the newton solver
     //CL: therefore, rho can be larger than rhoMax and smaller than rhoMin

From 4f948a2d879b93e6ab0863cc16266563bda47524 Mon Sep 17 00:00:00 2001
From: Hrvoje Jasak <h.jasak@wikki.co.uk>
Date: Wed, 11 Jan 2017 12:02:09 +0000
Subject: [PATCH 107/130] Option to add empty patch in faMesh.  Zeljko Tukovic

---
 .../finiteArea/makeFaMesh/makeFaMesh.C        | 20 +++++++++++++++++++
 1 file changed, 20 insertions(+)

diff --git a/applications/utilities/finiteArea/makeFaMesh/makeFaMesh.C b/applications/utilities/finiteArea/makeFaMesh/makeFaMesh.C
index 372e122e4..fd2f064f6 100644
--- a/applications/utilities/finiteArea/makeFaMesh/makeFaMesh.C
+++ b/applications/utilities/finiteArea/makeFaMesh/makeFaMesh.C
@@ -314,6 +314,26 @@ int main(int argc, char *argv[])
         }
     }
 
+    if (args.optionFound("addEmptyPatch"))
+    {
+        word emptyPatchName(args.optionLookup("addEmptyPatch")());
+        dictionary emptyPatchDict;
+        emptyPatchDict.add("type", "empty");
+        emptyPatchDict.add("edgeLabels", labelList());
+        emptyPatchDict.add("ngbPolyPatchIndex", -1);
+
+        faPatchLst.append
+        (
+            faPatch::New
+            (
+                emptyPatchName,
+                emptyPatchDict,
+                faPatchLst.size(),
+                areaMesh.boundary()
+            ).ptr()
+        );
+    }
+
     Info << "Add faPatches ... ";
     areaMesh.addFaPatches(List<faPatch*>(faPatchLst));
     Info << "Done" << endl;

From 183c98c71822ba17b84155cb9a4b5e2f250042a1 Mon Sep 17 00:00:00 2001
From: Hrvoje Jasak <h.jasak@wikki.co.uk>
Date: Wed, 11 Jan 2017 12:03:23 +0000
Subject: [PATCH 108/130] Bugfix: clean-up

---
 applications/solvers/combustion/rhoReactingFoam/YEqn.H        | 2 +-
 .../reconstructParMesh/reconstructParMesh.C                   | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/applications/solvers/combustion/rhoReactingFoam/YEqn.H b/applications/solvers/combustion/rhoReactingFoam/YEqn.H
index bf0e0119a..90dbdff70 100644
--- a/applications/solvers/combustion/rhoReactingFoam/YEqn.H
+++ b/applications/solvers/combustion/rhoReactingFoam/YEqn.H
@@ -13,7 +13,7 @@ tmp<fv::convectionScheme<scalar> > mvConvection
     label inertIndex = -1;
     volScalarField Yt("Yt", 0.0*Y[0]);
 
-    for (label i=0; i<Y.size(); i++)
+    for (label i = 0; i < Y.size(); i++)
     {
         if (Y[i].name() != inertSpecie)
         {
diff --git a/applications/utilities/parallelProcessing/reconstructParMesh/reconstructParMesh.C b/applications/utilities/parallelProcessing/reconstructParMesh/reconstructParMesh.C
index ed6df9075..c5a01c0c0 100644
--- a/applications/utilities/parallelProcessing/reconstructParMesh/reconstructParMesh.C
+++ b/applications/utilities/parallelProcessing/reconstructParMesh/reconstructParMesh.C
@@ -119,12 +119,12 @@ int main(int argc, char *argv[])
             << exit(FatalError);
     }
 
-    Foam::word regionName = polyMesh::defaultRegion;
+    word regionName = polyMesh::defaultRegion;
 
     if (args.optionReadIfPresent("region", regionName))
     {
         Info<< "Selecting region " << regionName << " for time = "
-            << runTime.timeName() << Foam::nl << Foam::endl;
+            << runTime.timeName() << nl << endl;
     }
 
     // Set all times on processor meshes equal to reconstructed mesh

From 59c57be5e492748cc3501f67cef7f44ed6d2708e Mon Sep 17 00:00:00 2001
From: Hrvoje Jasak <h.jasak@wikki.co.uk>
Date: Wed, 11 Jan 2017 12:03:56 +0000
Subject: [PATCH 109/130] Bugfix: Formatting

---
 src/foam/db/IOstreams/Pstreams/Pstream.C | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/src/foam/db/IOstreams/Pstreams/Pstream.C b/src/foam/db/IOstreams/Pstreams/Pstream.C
index a7e6dc63c..78c378de5 100644
--- a/src/foam/db/IOstreams/Pstreams/Pstream.C
+++ b/src/foam/db/IOstreams/Pstreams/Pstream.C
@@ -305,7 +305,7 @@ void Foam::Pstream::allocatePstreamCommunicator
         MPI_Comm_rank
         (
             PstreamGlobals::MPICommunicators_[index],
-           &myProcNo_[index]
+            &myProcNo_[index]
         );
 
         // Set the number of processes to the actual number
@@ -343,7 +343,7 @@ void Foam::Pstream::allocatePstreamCommunicator
                 MPI_Comm_rank
                 (
                     PstreamGlobals::MPICommunicators_[index],
-                   &myProcNo_[index]
+                    &myProcNo_[index]
                 )
             )
             {
@@ -751,7 +751,7 @@ void Foam::Pstream::waitRequest(const label i)
     (
         MPI_Wait
         (
-           &PstreamGlobals::outstandingRequests_[i],
+            &PstreamGlobals::outstandingRequests_[i],
             MPI_STATUS_IGNORE
         )
     )
@@ -793,8 +793,8 @@ bool Foam::Pstream::finishedRequest(const label i)
     int flag;
     MPI_Test
     (
-       &PstreamGlobals::outstandingRequests_[i],
-       &flag,
+        &PstreamGlobals::outstandingRequests_[i],
+        &flag,
         MPI_STATUS_IGNORE
     );
 

From bbcef01b581a8c721fd40c91395a9342a8ac70d6 Mon Sep 17 00:00:00 2001
From: Hrvoje Jasak <h.jasak@wikki.co.uk>
Date: Wed, 11 Jan 2017 12:04:45 +0000
Subject: [PATCH 110/130] Bugfix: pythond formatting bugfix

---
 bin/plotForces.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/bin/plotForces.py b/bin/plotForces.py
index a7d779848..ed887ca54 100755
--- a/bin/plotForces.py
+++ b/bin/plotForces.py
@@ -58,12 +58,14 @@ outForces=open('forces.dat','w')
 
 for data in zip(t,fx,fy,fz):
         outForces.write(' '.join([str(d) for d in data])+'\n')
+
 outForces.close()
 
 outMoments=open('moments.dat','w')
 
 for data in zip(t,mx,my,mz):
         outMoments.write(' '.join([str(d) for d in data])+'\n')
+
 outMoments.close()
 
 # plot forces

From 4348876544a46861ff17fea6bd8ffb7998525cb1 Mon Sep 17 00:00:00 2001
From: Hrvoje Jasak <h.jasak@wikki.co.uk>
Date: Wed, 11 Jan 2017 12:07:17 +0000
Subject: [PATCH 111/130] Added save directory to mixer tutorial

---
 .../MRFSimpleFoam/mixerGgiMRF/save/U          | 51 +++++++++++++++++++
 .../MRFSimpleFoam/mixerGgiMRF/save/p          | 49 ++++++++++++++++++
 2 files changed, 100 insertions(+)
 create mode 100644 tutorials/incompressible/MRFSimpleFoam/mixerGgiMRF/save/U
 create mode 100644 tutorials/incompressible/MRFSimpleFoam/mixerGgiMRF/save/p

diff --git a/tutorials/incompressible/MRFSimpleFoam/mixerGgiMRF/save/U b/tutorials/incompressible/MRFSimpleFoam/mixerGgiMRF/save/U
new file mode 100644
index 000000000..ce8d03b18
--- /dev/null
+++ b/tutorials/incompressible/MRFSimpleFoam/mixerGgiMRF/save/U
@@ -0,0 +1,51 @@
+/*--------------------------------*- C++ -*----------------------------------*\
+| =========                 |                                                 |
+| \\      /  F ield         | foam-extend: Open Source CFD                    |
+|  \\    /   O peration     | Version:     4.0                                |
+|   \\  /    A nd           | Web:         http://www.foam-extend.org         |
+|    \\/     M anipulation  |                                                 |
+\*---------------------------------------------------------------------------*/
+FoamFile
+{
+    version     2.0;
+    format      ascii;
+    class       volVectorField;
+    location    "0";
+    object      U;
+}
+// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
+
+dimensions      [0 1 -1 0 0 0 0];
+
+internalField   uniform (0 0 0);
+
+boundaryField
+{
+    impellerWall
+    {
+        type            fixedValue;
+        value           uniform (0 0 0);
+    }
+    baffleWall
+    {
+        type            fixedValue;
+        value           uniform (0 0 0);
+    }
+    insideSlider
+    {
+        type            ggi;
+        value           uniform (0 0 0);
+    }
+    outsideSlider
+    {
+        type            ggi;
+        value           uniform (0 0 0);
+    }
+    defaultFaces
+    {
+        type            empty;
+    }
+}
+
+
+// ************************************************************************* //
diff --git a/tutorials/incompressible/MRFSimpleFoam/mixerGgiMRF/save/p b/tutorials/incompressible/MRFSimpleFoam/mixerGgiMRF/save/p
new file mode 100644
index 000000000..6ee83768f
--- /dev/null
+++ b/tutorials/incompressible/MRFSimpleFoam/mixerGgiMRF/save/p
@@ -0,0 +1,49 @@
+/*--------------------------------*- C++ -*----------------------------------*\
+| =========                 |                                                 |
+| \\      /  F ield         | foam-extend: Open Source CFD                    |
+|  \\    /   O peration     | Version:     4.0                                |
+|   \\  /    A nd           | Web:         http://www.foam-extend.org         |
+|    \\/     M anipulation  |                                                 |
+\*---------------------------------------------------------------------------*/
+FoamFile
+{
+    version     2.0;
+    format      ascii;
+    class       volScalarField;
+    location    "0";
+    object      p;
+}
+// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
+
+dimensions      [0 2 -2 0 0 0 0];
+
+internalField   uniform 0;
+
+boundaryField
+{
+    impellerWall
+    {
+        type            zeroGradient;
+    }
+    baffleWall
+    {
+        type            zeroGradient;
+    }
+    insideSlider
+    {
+        type            ggi;
+        value           uniform 0;
+    }
+    outsideSlider
+    {
+        type            ggi;
+        value           uniform 0;
+    }
+    defaultFaces
+    {
+        type            empty;
+    }
+}
+
+
+// ************************************************************************* //

From 5a42868d78a637f0e01fc533a97bc431e9930052 Mon Sep 17 00:00:00 2001
From: Hrvoje Jasak <h.jasak@wikki.co.uk>
Date: Thu, 12 Jan 2017 15:13:58 +0000
Subject: [PATCH 112/130] Bugfix: consistent steady intertial relaxation

---
 .../steadyInertialDdtScheme.C                 | 71 ++++++++++---------
 1 file changed, 36 insertions(+), 35 deletions(-)

diff --git a/src/finiteVolume/finiteVolume/ddtSchemes/steadyInertialDdtScheme/steadyInertialDdtScheme.C b/src/finiteVolume/finiteVolume/ddtSchemes/steadyInertialDdtScheme/steadyInertialDdtScheme.C
index 496075616..ed798b4ce 100644
--- a/src/finiteVolume/finiteVolume/ddtSchemes/steadyInertialDdtScheme/steadyInertialDdtScheme.C
+++ b/src/finiteVolume/finiteVolume/ddtSchemes/steadyInertialDdtScheme/steadyInertialDdtScheme.C
@@ -239,11 +239,11 @@ steadyInertialDdtScheme<Type>::fvcDdt
                 rDeltaT.internalField()*
                 (
                     vf.internalField()
-                  - vf.oldTime().internalField()*mesh().V0()/mesh().V()
+                  - vf.prevIter().internalField()*mesh().V0()/mesh().V()
                 ),
                 rDeltaT.boundaryField()*
                 (
-                    vf.boundaryField() - vf.oldTime().boundaryField()
+                    vf.boundaryField() - vf.prevIter().boundaryField()
                 )
             )
         );
@@ -255,7 +255,7 @@ steadyInertialDdtScheme<Type>::fvcDdt
             new GeometricField<Type, fvPatchField, volMesh>
             (
                 ddtIOobject,
-                rDeltaT*(vf - vf.oldTime())
+                rDeltaT*(vf - vf.prevIter())
             )
         );
     }
@@ -291,11 +291,11 @@ steadyInertialDdtScheme<Type>::fvcDdt
                 rDeltaT.internalField()*rho.value()*
                 (
                     vf.internalField()
-                  - vf.oldTime().internalField()*mesh().V0()/mesh().V()
+                  - vf.prevIter().internalField()*mesh().V0()/mesh().V()
                 ),
                 rDeltaT.boundaryField()*rho.value()*
                 (
-                    vf.boundaryField() - vf.oldTime().boundaryField()
+                    vf.boundaryField() - vf.prevIter().boundaryField()
                 )
             )
         );
@@ -307,7 +307,7 @@ steadyInertialDdtScheme<Type>::fvcDdt
             new GeometricField<Type, fvPatchField, volMesh>
             (
                 ddtIOobject,
-                rDeltaT*rho*(vf - vf.oldTime())
+                rDeltaT*rho*(vf - vf.prevIter())
             )
         );
     }
@@ -343,14 +343,14 @@ steadyInertialDdtScheme<Type>::fvcDdt
                 rDeltaT.internalField()*
                 (
                     rho.internalField()*vf.internalField()
-                  - rho.oldTime().internalField()
-                   *vf.oldTime().internalField()*mesh().V0()/mesh().V()
+                  - rho.internalField()
+                   *vf.prevIter().internalField()*mesh().V0()/mesh().V()
                 ),
                 rDeltaT.boundaryField()*
                 (
                     rho.boundaryField()*vf.boundaryField()
-                  - rho.oldTime().boundaryField()
-                   *vf.oldTime().boundaryField()
+                  - rho.boundaryField()
+                   *vf.prevIter().boundaryField()
                 )
             )
         );
@@ -362,7 +362,7 @@ steadyInertialDdtScheme<Type>::fvcDdt
             new GeometricField<Type, fvPatchField, volMesh>
             (
                 ddtIOobject,
-                rDeltaT*(rho*vf - rho.oldTime()*vf.oldTime())
+                rDeltaT*rho*(vf - vf.prevIter())
             )
         );
     }
@@ -393,11 +393,11 @@ steadyInertialDdtScheme<Type>::fvmDdt
 
     if (mesh().moving())
     {
-        fvm.source() = rDeltaT*vf.oldTime().internalField()*mesh().V0();
+        fvm.source() = rDeltaT*vf.prevIter().internalField()*mesh().V0();
     }
     else
     {
-        fvm.source() = rDeltaT*vf.oldTime().internalField()*mesh().V();
+        fvm.source() = rDeltaT*vf.prevIter().internalField()*mesh().V();
     }
 
     return tfvm;
@@ -429,12 +429,12 @@ steadyInertialDdtScheme<Type>::fvmDdt
     if (mesh().moving())
     {
         fvm.source() = rDeltaT
-            *rho.value()*vf.oldTime().internalField()*mesh().V0();
+            *rho.value()*vf.prevIter().internalField()*mesh().V0();
     }
     else
     {
         fvm.source() = rDeltaT
-            *rho.value()*vf.oldTime().internalField()*mesh().V();
+            *rho.value()*vf.prevIter().internalField()*mesh().V();
     }
 
     return tfvm;
@@ -466,14 +466,14 @@ steadyInertialDdtScheme<Type>::fvmDdt
     if (mesh().moving())
     {
         fvm.source() = rDeltaT
-            *rho.oldTime().internalField()
-            *vf.oldTime().internalField()*mesh().V0();
+            *rho.internalField()
+            *vf.prevIter().internalField()*mesh().V0();
     }
     else
     {
         fvm.source() = rDeltaT
-            *rho.oldTime().internalField()
-            *vf.oldTime().internalField()*mesh().V();
+            *rho.internalField()
+            *vf.prevIter().internalField()*mesh().V();
     }
 
     return tfvm;
@@ -522,10 +522,10 @@ steadyInertialDdtScheme<Type>::fvcDdtPhiCorr
             new fluxFieldType
             (
                 ddtIOobject,
-                this->fvcDdtPhiCoeff(U.oldTime(), phi.oldTime())*
+                this->fvcDdtPhiCoeff(U, phi)*
                 (
-                    fvc::interpolate(rDeltaT*rA)*phi.oldTime()
-                  - (fvc::interpolate(rDeltaT*rA*U.oldTime()) & mesh().Sf())
+                    fvc::interpolate(rDeltaT*rA)*phi
+                  - (fvc::interpolate(rDeltaT*rA*U.prevIter()) & mesh().Sf())
                 )
             )
         );
@@ -583,11 +583,12 @@ steadyInertialDdtScheme<Type>::fvcDdtPhiCorr
                 new fluxFieldType
                 (
                     ddtIOobject,
-                    this->fvcDdtPhiCoeff(U.oldTime(), phi.oldTime())
+                    this->fvcDdtPhiCoeff(U, phi)
                    *(
-                        fvc::interpolate(rDeltaT*rA*rho.oldTime())*
-                        phi.oldTime()
-                      - (fvc::interpolate(rDeltaT*rA*rho.oldTime()*U.oldTime())
+                        fvc::interpolate(rDeltaT*rA*rho)*phi
+                      - (
+                            fvc::interpolate(rDeltaT*rA*rho*U.prevIter()
+                        )
                       & mesh().Sf())
                     )
                 )
@@ -606,16 +607,16 @@ steadyInertialDdtScheme<Type>::fvcDdtPhiCorr
                     ddtIOobject,
                     this->fvcDdtPhiCoeff
                     (
-                        U.oldTime(),
-                        phi.oldTime()/fvc::interpolate(rho.oldTime())
+                        U.prevIter(),
+                        phi/fvc::interpolate(rho)
                     )
                    *(
-                        fvc::interpolate(rDeltaT*rA*rho.oldTime())
-                       *phi.oldTime()/fvc::interpolate(rho.oldTime())
+                        fvc::interpolate(rDeltaT*rA*rho)
+                       *phi/fvc::interpolate(rho)
                       - (
                             fvc::interpolate
                             (
-                                rDeltaT*rA*rho.oldTime()*U.oldTime()
+                                rDeltaT*rA*rho*U.prevIter()
                             ) & mesh().Sf()
                         )
                     )
@@ -633,11 +634,11 @@ steadyInertialDdtScheme<Type>::fvcDdtPhiCorr
                 new fluxFieldType
                 (
                     ddtIOobject,
-                    this->fvcDdtPhiCoeff(rho.oldTime(), U.oldTime(), phi.oldTime())
-                   *(
-                        fvc::interpolate(rDeltaT*rA)*phi.oldTime()
+                    this->fvcDdtPhiCoeff(rho, U, phi)*
+                    (
+                        fvc::interpolate(rDeltaT*rA)*phi
                       - (
-                            fvc::interpolate(rDeltaT*rA*U.oldTime())&mesh().Sf()
+                            fvc::interpolate(rDeltaT*rA*U.prevIter()) & mesh().Sf()
                         )
                     )
                 )

From 58edb15fa974650ff6fd3a2f28ff7a350c251a76 Mon Sep 17 00:00:00 2001
From: Hrvoje Jasak <h.jasak@wikki.co.uk>
Date: Fri, 13 Jan 2017 16:08:13 +0000
Subject: [PATCH 113/130] Removed info message

---
 src/finiteVolume/cfdTools/general/MRF/MRFZone.C | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/finiteVolume/cfdTools/general/MRF/MRFZone.C b/src/finiteVolume/cfdTools/general/MRF/MRFZone.C
index a26ff72ce..0390307f2 100644
--- a/src/finiteVolume/cfdTools/general/MRF/MRFZone.C
+++ b/src/finiteVolume/cfdTools/general/MRF/MRFZone.C
@@ -533,7 +533,7 @@ void Foam::MRFZone::correctBoundaryVelocity(volVectorField& U) const
 {
     const vector& origin = origin_.value();
     const vector rotVel = Omega();
-    Info<< "rotVel: " << rotVel << endl;
+
     // Included patches
     forAll (includedFaces_, patchi)
     {

From 78a9142214076cf5533254883d777328cbc97ab1 Mon Sep 17 00:00:00 2001
From: Hrvoje Jasak <h.jasak@wikki.co.uk>
Date: Tue, 24 Jan 2017 16:44:22 +0000
Subject: [PATCH 114/130] Bugfix: MRF zone flux check

---
 .../cfdTools/general/MRF/MRFZone.C            | 60 +++++++++-------
 .../cfdTools/general/MRF/MRFZone.H            |  2 +-
 .../cfdTools/general/MRF/MRFZoneTemplates.C   | 70 +++++++++++--------
 .../cfdTools/general/MRF/MRFZones.C           | 10 +--
 .../cfdTools/general/MRF/MRFZones.H           |  4 +-
 5 files changed, 86 insertions(+), 60 deletions(-)

diff --git a/src/finiteVolume/cfdTools/general/MRF/MRFZone.C b/src/finiteVolume/cfdTools/general/MRF/MRFZone.C
index 0390307f2..9633fff7a 100644
--- a/src/finiteVolume/cfdTools/general/MRF/MRFZone.C
+++ b/src/finiteVolume/cfdTools/general/MRF/MRFZone.C
@@ -139,13 +139,15 @@ void Foam::MRFZone::setMRFFaces()
     labelList nIncludedFaces(patches.size(), 0);
     labelList nExcludedFaces(patches.size(), 0);
 
+    register label faceI;
+
     forAll (patches, patchi)
     {
         const polyPatch& pp = patches[patchi];
 
         forAll (pp, patchFaceI)
         {
-            label faceI = pp.start() + patchFaceI;
+            faceI = pp.start() + patchFaceI;
 
             if (faceType[faceI] == 1)
             {
@@ -174,7 +176,7 @@ void Foam::MRFZone::setMRFFaces()
 
         forAll (pp, patchFaceI)
         {
-            label faceI = pp.start() + patchFaceI;
+            faceI = pp.start() + patchFaceI;
 
             if (faceType[faceI] == 1)
             {
@@ -187,7 +189,6 @@ void Foam::MRFZone::setMRFFaces()
         }
     }
 
-
     if (debug)
     {
         faceSet internalFaces
@@ -489,42 +490,51 @@ void Foam::MRFZone::absoluteFlux
 
 void Foam::MRFZone::meshPhi
 (
-    surfaceVectorField& phi
+    surfaceScalarField& phi
 ) const
 {
     const surfaceVectorField& Cf = mesh_.Cf();
+    const surfaceVectorField& Sf = mesh_.Sf();
 
     const vector& origin = origin_.value();
     const vector rotVel = Omega();
 
     // Internal faces
+    const vectorField& CfIn = Cf.internalField();
+    const vectorField& SfIn = Sf.internalField();
+
+    register label faceI, patchFaceI;
+
     forAll (internalFaces_, i)
     {
-        label facei = internalFaces_[i];
-        phi[facei] = (rotVel ^ (Cf[facei] - origin));
+        faceI = internalFaces_[i];
+        phi[faceI] = SfIn[faceI] & (rotVel ^ (CfIn[faceI] - origin));
     }
 
     // Included patches
-    forAll (includedFaces_, patchi)
-    {
-        forAll (includedFaces_[patchi], i)
-        {
-            label patchFaceI = includedFaces_[patchi][i];
 
-            phi.boundaryField()[patchi][patchFaceI] =
-                (rotVel ^ (Cf.boundaryField()[patchi][patchFaceI] - origin));
+    forAll (includedFaces_, patchI)
+    {
+        forAll (includedFaces_[patchI], i)
+        {
+            patchFaceI = includedFaces_[patchI][i];
+
+            phi.boundaryField()[patchI][patchFaceI] =
+                Sf.boundaryField()[patchI][patchFaceI]
+              & (rotVel ^ (Cf.boundaryField()[patchI][patchFaceI] - origin));
         }
     }
 
     // Excluded patches
-    forAll (excludedFaces_, patchi)
+    forAll (excludedFaces_, patchI)
     {
-        forAll (excludedFaces_[patchi], i)
+        forAll (excludedFaces_[patchI], i)
         {
-            label patchFaceI = excludedFaces_[patchi][i];
+            patchFaceI = excludedFaces_[patchI][i];
 
-            phi.boundaryField()[patchi][patchFaceI] =
-                (rotVel ^ (Cf.boundaryField()[patchi][patchFaceI] - origin));
+            phi.boundaryField()[patchI][patchFaceI] =
+                Sf.boundaryField()[patchI][patchFaceI]
+              & (rotVel ^ (Cf.boundaryField()[patchI][patchFaceI] - origin));
         }
     }
 }
@@ -534,21 +544,23 @@ void Foam::MRFZone::correctBoundaryVelocity(volVectorField& U) const
     const vector& origin = origin_.value();
     const vector rotVel = Omega();
 
+    register label patchFaceI;
+
     // Included patches
-    forAll (includedFaces_, patchi)
+    forAll (includedFaces_, patchI)
     {
-        const vectorField& patchC = mesh_.Cf().boundaryField()[patchi];
+        const vectorField& patchC = mesh_.Cf().boundaryField()[patchI];
 
-        vectorField pfld(U.boundaryField()[patchi]);
+        vectorField pfld(U.boundaryField()[patchI]);
 
-        forAll (includedFaces_[patchi], i)
+        forAll (includedFaces_[patchI], i)
         {
-            label patchFaceI = includedFaces_[patchi][i];
+            patchFaceI = includedFaces_[patchI][i];
 
             pfld[patchFaceI] = (rotVel ^ (patchC[patchFaceI] - origin));
         }
 
-        U.boundaryField()[patchi] == pfld;
+        U.boundaryField()[patchI] == pfld;
     }
 }
 
diff --git a/src/finiteVolume/cfdTools/general/MRF/MRFZone.H b/src/finiteVolume/cfdTools/general/MRF/MRFZone.H
index 954bc85b1..0121eed2a 100644
--- a/src/finiteVolume/cfdTools/general/MRF/MRFZone.H
+++ b/src/finiteVolume/cfdTools/general/MRF/MRFZone.H
@@ -221,7 +221,7 @@ public:
         ) const;
 
         //- Compute the pseudo mesh phi of the MRF region
-        void meshPhi(surfaceVectorField& phi) const;
+        void meshPhi(surfaceScalarField& phi) const;
 
         //- Correct the boundary velocity for the roation of the MRF region
         void correctBoundaryVelocity(volVectorField& U) const;
diff --git a/src/finiteVolume/cfdTools/general/MRF/MRFZoneTemplates.C b/src/finiteVolume/cfdTools/general/MRF/MRFZoneTemplates.C
index 0ae07364a..a6eabcbfa 100644
--- a/src/finiteVolume/cfdTools/general/MRF/MRFZoneTemplates.C
+++ b/src/finiteVolume/cfdTools/general/MRF/MRFZoneTemplates.C
@@ -44,34 +44,42 @@ void Foam::MRFZone::relativeRhoFlux
     const vector rotVel = Omega();
 
     // Internal faces
+    const vectorField& CfIn = Cf.internalField();
+    const vectorField& SfIn = Sf.internalField();
+
+    register label faceI, patchFaceI;
+
     forAll (internalFaces_, i)
     {
-        label facei = internalFaces_[i];
-        phi[facei] -= rho[facei]*(rotVel ^ (Cf[facei] - origin)) & Sf[facei];
+        faceI = internalFaces_[i];
+
+        phi[faceI] -=
+            rho[faceI]*(rotVel ^ (CfIn[faceI] - origin)) & SfIn[faceI];
     }
 
-    // Included patches
-    forAll (includedFaces_, patchi)
+    // Included patches: reset the flux to exactly zero to avoid
+    // round-off issues
+    forAll (includedFaces_, patchI)
     {
-        forAll (includedFaces_[patchi], i)
+        forAll (includedFaces_[patchI], i)
         {
-            label patchFacei = includedFaces_[patchi][i];
+            patchFaceI = includedFaces_[patchI][i];
 
-            phi.boundaryField()[patchi][patchFacei] = 0.0;
+            phi.boundaryField()[patchI][patchFaceI] = 0.0;
         }
     }
 
     // Excluded patches
-    forAll (excludedFaces_, patchi)
+    forAll (excludedFaces_, patchI)
     {
-        forAll (excludedFaces_[patchi], i)
+        forAll (excludedFaces_[patchI], i)
         {
-            label patchFacei = excludedFaces_[patchi][i];
+            patchFaceI = excludedFaces_[patchI][i];
 
-            phi.boundaryField()[patchi][patchFacei] -=
-                rho.boundaryField()[patchi][patchFacei]
-               *(rotVel ^ (Cf.boundaryField()[patchi][patchFacei] - origin))
-              & Sf.boundaryField()[patchi][patchFacei];
+            phi.boundaryField()[patchI][patchFaceI] -=
+                rho.boundaryField()[patchI][patchFaceI]
+               *(rotVel ^ (Cf.boundaryField()[patchI][patchFaceI] - origin))
+              & Sf.boundaryField()[patchI][patchFaceI];
         }
     }
 }
@@ -91,35 +99,41 @@ void Foam::MRFZone::absoluteRhoFlux
     const vector rotVel = Omega();
 
     // Internal faces
+    const vectorField& CfIn = Cf.internalField();
+    const vectorField& SfIn = Sf.internalField();
+
+    register label faceI, patchFaceI;
+
     forAll (internalFaces_, i)
     {
-        label facei = internalFaces_[i];
-        phi[facei] += (rotVel ^ (Cf[facei] - origin)) & Sf[facei];
+        faceI = internalFaces_[i];
+        
+        phi[faceI] += (rotVel ^ (CfIn[faceI] - origin)) & SfIn[faceI];
     }
 
     // Included patches
-    forAll (includedFaces_, patchi)
+    forAll (includedFaces_, patchI)
     {
-        forAll (includedFaces_[patchi], i)
+        forAll (includedFaces_[patchI], i)
         {
-            label patchFacei = includedFaces_[patchi][i];
+            patchFaceI = includedFaces_[patchI][i];
 
-            phi.boundaryField()[patchi][patchFacei] +=
-                (rotVel ^ (Cf.boundaryField()[patchi][patchFacei] - origin))
-              & Sf.boundaryField()[patchi][patchFacei];
+            phi.boundaryField()[patchI][patchFaceI] +=
+                (rotVel ^ (Cf.boundaryField()[patchI][patchFaceI] - origin))
+              & Sf.boundaryField()[patchI][patchFaceI];
         }
     }
 
     // Excluded patches
-    forAll (excludedFaces_, patchi)
+    forAll (excludedFaces_, patchI)
     {
-        forAll (excludedFaces_[patchi], i)
+        forAll (excludedFaces_[patchI], i)
         {
-            label patchFacei = excludedFaces_[patchi][i];
+            patchFaceI = excludedFaces_[patchI][i];
 
-            phi.boundaryField()[patchi][patchFacei] +=
-                (rotVel ^ (Cf.boundaryField()[patchi][patchFacei] - origin))
-              & Sf.boundaryField()[patchi][patchFacei];
+            phi.boundaryField()[patchI][patchFaceI] +=
+                (rotVel ^ (Cf.boundaryField()[patchI][patchFaceI] - origin))
+              & Sf.boundaryField()[patchI][patchFaceI];
         }
     }
 }
diff --git a/src/finiteVolume/cfdTools/general/MRF/MRFZones.C b/src/finiteVolume/cfdTools/general/MRF/MRFZones.C
index e5f29c690..f9cb77151 100644
--- a/src/finiteVolume/cfdTools/general/MRF/MRFZones.C
+++ b/src/finiteVolume/cfdTools/general/MRF/MRFZones.C
@@ -86,11 +86,11 @@ Foam::tmp<Foam::surfaceScalarField> Foam::MRFZones::fluxCorrection() const
 }
 
 
-Foam::tmp<Foam::surfaceVectorField> Foam::MRFZones::meshPhi() const
+Foam::tmp<Foam::surfaceScalarField> Foam::MRFZones::meshPhi() const
 {
-    tmp<surfaceVectorField> tMRFZonesPhiCorr
+    tmp<surfaceScalarField> tMRFZonesPhiCorr
     (
-        new surfaceVectorField
+        new surfaceScalarField
         (
             IOobject
             (
@@ -101,10 +101,10 @@ Foam::tmp<Foam::surfaceVectorField> Foam::MRFZones::meshPhi() const
                 IOobject::NO_WRITE
             ),
             mesh_,
-            dimensionedVector("zero", dimVelocity, vector::zero)
+            dimensionedScalar("zero", dimVolume/dimTime, 0)
         )
     );
-    surfaceVectorField& MRFZonesPhiCorr = tMRFZonesPhiCorr();
+    surfaceScalarField& MRFZonesPhiCorr = tMRFZonesPhiCorr();
 
     forAll (*this, i)
     {
diff --git a/src/finiteVolume/cfdTools/general/MRF/MRFZones.H b/src/finiteVolume/cfdTools/general/MRF/MRFZones.H
index ff683525d..082fc61f2 100644
--- a/src/finiteVolume/cfdTools/general/MRF/MRFZones.H
+++ b/src/finiteVolume/cfdTools/general/MRF/MRFZones.H
@@ -81,7 +81,7 @@ public:
         tmp<surfaceScalarField> fluxCorrection() const;
 
         //- Return pseudo mesh flux
-        tmp<surfaceVectorField> meshPhi() const;
+        tmp<surfaceScalarField> meshPhi() const;
 
         // Incompressible MRF
 
@@ -134,7 +134,7 @@ public:
         void absoluteVelocity(volVectorField& Urel) const;
 
 
-        //- Correct the boundary velocity for the roation of the MRF region
+        //- Correct the boundary velocity for the rotation of the MRF region
         void correctBoundaryVelocity(volVectorField& U) const;
 
         //- Compute source term for volScalarFields

From 5ad239d7ec1d43e0c670b5a36ab694df6d62058b Mon Sep 17 00:00:00 2001
From: Hrvoje Jasak <h.jasak@wikki.co.uk>
Date: Tue, 24 Jan 2017 16:44:43 +0000
Subject: [PATCH 115/130] Bugfix: simple matrix operations

---
 .../matrices/lduMatrix/lduMatrix/lduMatrix.H  | 31 ++++++++++---------
 .../lduMatrix/lduMatrix/lduMatrixOperations.C | 12 +++++++
 2 files changed, 28 insertions(+), 15 deletions(-)

diff --git a/src/foam/matrices/lduMatrix/lduMatrix/lduMatrix.H b/src/foam/matrices/lduMatrix/lduMatrix/lduMatrix.H
index d50fc6bf4..09c9805c7 100644
--- a/src/foam/matrices/lduMatrix/lduMatrix/lduMatrix.H
+++ b/src/foam/matrices/lduMatrix/lduMatrix/lduMatrix.H
@@ -258,10 +258,9 @@ public:
             );
 
 
-        // Destructor
-
-            virtual ~solver()
-            {}
+        //- Destructor
+        virtual ~solver()
+        {}
 
 
         // Member functions
@@ -418,10 +417,9 @@ public:
             );
 
 
-        // Destructor
-
-            virtual ~smoother()
-            {}
+        //- Destructor
+        virtual ~smoother()
+        {}
 
 
         // Member functions
@@ -556,10 +554,9 @@ public:
             );
 
 
-        // Destructor
-
-            virtual ~preconditioner()
-            {}
+        //- Destructor
+        virtual ~preconditioner()
+        {}
 
 
         // Member functions
@@ -635,9 +632,8 @@ public:
         }
 
 
-    // Destructor
-
-        virtual ~lduMatrix();
+    //- Destructor
+    virtual ~lduMatrix();
 
 
     // Member functions
@@ -700,6 +696,11 @@ public:
                 return (lowerPtr_);
             }
 
+            bool empty() const
+            {
+                return (!diagPtr_ && !lowerPtr_ && !upperPtr_);
+            }
+
             bool diagonal() const
             {
                 return (diagPtr_ && !lowerPtr_ && !upperPtr_);
diff --git a/src/foam/matrices/lduMatrix/lduMatrix/lduMatrixOperations.C b/src/foam/matrices/lduMatrix/lduMatrix/lduMatrixOperations.C
index a523b6198..6497d8374 100644
--- a/src/foam/matrices/lduMatrix/lduMatrix/lduMatrixOperations.C
+++ b/src/foam/matrices/lduMatrix/lduMatrix/lduMatrixOperations.C
@@ -143,6 +143,12 @@ void Foam::lduMatrix::negate()
 
 void Foam::lduMatrix::operator+=(const lduMatrix& A)
 {
+    // Escape empty matrix
+    if (A.empty())
+    {
+        return;
+    }
+    
     if (A.diagPtr_)
     {
         diag() += A.diag();
@@ -211,6 +217,12 @@ void Foam::lduMatrix::operator+=(const lduMatrix& A)
 
 void Foam::lduMatrix::operator-=(const lduMatrix& A)
 {
+    // Escape empty matrix
+    if (A.empty())
+    {
+        return;
+    }
+    
     if (A.diagPtr_)
     {
         diag() -= A.diag();

From 98680aceaeae27b6ee4213b21ee31758131e8f52 Mon Sep 17 00:00:00 2001
From: Hrvoje Jasak <h.jasak@wikki.co.uk>
Date: Mon, 6 Feb 2017 11:29:11 +0000
Subject: [PATCH 116/130] Formatting

---
 .../solvers/coupled/conjugateHeatSimpleFoam/solveEnergy.H       | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/applications/solvers/coupled/conjugateHeatSimpleFoam/solveEnergy.H b/applications/solvers/coupled/conjugateHeatSimpleFoam/solveEnergy.H
index 833d954e8..da5d46c1d 100644
--- a/applications/solvers/coupled/conjugateHeatSimpleFoam/solveEnergy.H
+++ b/applications/solvers/coupled/conjugateHeatSimpleFoam/solveEnergy.H
@@ -14,7 +14,7 @@
               + fvm::SuSp(-fvc::div(phi), T)
             )
           - fvm::laplacian(kappaEff, T)
-            ==
+         ==
             radiation->Ru()
           - fvm::Sp(4.0*radiation->Rp()*pow3(T), T)
           + 3.0*radiation->Rp()*pow4(T)

From 71540e2164594bbee7e5b9e52c5f3eb6c13380a3 Mon Sep 17 00:00:00 2001
From: Hrvoje Jasak <h.jasak@wikki.co.uk>
Date: Mon, 6 Feb 2017 11:31:10 +0000
Subject: [PATCH 117/130] Moved conjugate heat transfer library to src

---
 .../coupled/conjugateHeatFoam/Make/options    |   4 +-
 .../conjugateHeatSimpleFoam/Make/options      |   4 +-
 src/Allwmake                                  |   2 +
 .../conjugateHeatTransfer/Make/files          |   0
 .../conjugateHeatTransfer/Make/options        |   0
 .../fieldAverageItem/fieldAverageItem.C       |   0
 .../fieldAverageItem/fieldAverageItem.H       |   0
 .../fieldAverageItem/fieldAverageItemIO.C     |   0
 .../heatFlux/heatFlux/IOFieldAverage.H        |   0
 .../heatFlux/heatFlux/detachPatches.H         |   0
 .../heatFlux/heatFlux/heatFlux.C              |   0
 .../heatFlux/heatFlux/heatFlux.H              |   0
 .../heatFlux/heatFlux/heatFluxTemplates.C     |   0
 .../heatFluxFunctionObject.C                  |   0
 .../heatFluxFunctionObject.H                  |   0
 .../adiabatic/adiabaticFvPatchScalarField.C   |   0
 .../adiabatic/adiabaticFvPatchScalarField.H   |   0
 .../chtRcTemperatureFvPatchScalarField.C      |   0
 .../chtRcTemperatureFvPatchScalarField.H      |  28 ++++-
 ...htRcThermalDiffusivityFvPatchScalarField.C | 115 +++++++-----------
 ...htRcThermalDiffusivityFvPatchScalarField.H |  16 +--
 ...lDiffusivityResistanceFvPatchScalarField.C |   0
 ...lDiffusivityResistanceFvPatchScalarField.H |   0
 ...hermalDiffusivitySlaveFvPatchScalarField.C |  57 +++++----
 ...hermalDiffusivitySlaveFvPatchScalarField.H |  11 +-
 .../chtRegionCoupleBase/chtRegionCoupleBase.C |   0
 .../chtRegionCoupleBase/chtRegionCoupleBase.H |   0
 ...olledParabolicVelocityFvPatchVectorField.C |   0
 ...olledParabolicVelocityFvPatchVectorField.H |   0
 ...tendedWallHeatTransferFvPatchScalarField.C |   0
 ...tendedWallHeatTransferFvPatchScalarField.H |   0
 .../viewFactorRadiation.C                     |   0
 .../viewFactorRadiation.H                     |   0
 .../ersConstantFlux/ersConstantFlux.C         |   0
 .../ersConstantFlux/ersConstantFlux.H         |   0
 .../ersPlaneToCylinder/ersPlaneToCylinder.C   |   0
 .../ersPlaneToCylinder/ersPlaneToCylinder.H   |   0
 .../ersPointSource/ersPointSource.C           |   0
 .../ersPointSource/ersPointSource.H           |   0
 .../ersViewFactor/ersViewFactor.C             |   0
 .../ersViewFactor/ersViewFactor.H             |   0
 .../externalRadiationSource.C                 |   0
 .../externalRadiationSource.H                 |   0
 .../newExternalRadiationSource.C              |   0
 .../fvPatchFields/fieldStorage/fieldStorage.C |   0
 .../fvPatchFields/fieldStorage/fieldStorage.H |   0
 ...allHeatFluxTemperatureFvPatchScalarField.C |   0
 ...allHeatFluxTemperatureFvPatchScalarField.H |   0
 .../constantResistanceThermalGap.C            |   0
 .../constantResistanceThermalGap.H            |   0
 .../constantThermalGap/constantThermalGap.C   |   0
 .../constantThermalGap/constantThermalGap.H   |   0
 .../thermalGaps/thermalGap/newThermalGap.C    |   0
 .../thermalGaps/thermalGap/thermalGap.C       |   0
 .../thermalGaps/thermalGap/thermalGap.H       |   0
 .../constantThermal/constantThermal.C         |   0
 .../constantThermal/constantThermal.H         |   0
 .../multiMaterialThermal.C                    |   0
 .../multiMaterialThermal.H                    |   0
 .../multiMaterialZonesThermal.C               |   0
 .../multiMaterialZonesThermal.H               |   0
 .../thermalLaws/thermalLaw/newThermalLaw.C    |   0
 .../thermalLaws/thermalLaw/thermalLaw.C       |   0
 .../thermalLaws/thermalLaw/thermalLaw.H       |   0
 .../thermalModel/thermalModel.C               |   0
 .../thermalModel/thermalModel.H               |   0
 .../constantThermalSource.C                   |   0
 .../constantThermalSource.H                   |   0
 .../thermalSource/newThermalSource.C          |   0
 .../thermalSource/thermalSource.C             |   0
 .../thermalSource/thermalSource.H             |   0
 71 files changed, 116 insertions(+), 121 deletions(-)
 rename {applications/solvers/coupled => src}/conjugateHeatTransfer/Make/files (100%)
 rename {applications/solvers/coupled => src}/conjugateHeatTransfer/Make/options (100%)
 rename {applications/solvers/coupled => src}/conjugateHeatTransfer/functionObjects/heatFlux/fieldAverageItem/fieldAverageItem.C (100%)
 rename {applications/solvers/coupled => src}/conjugateHeatTransfer/functionObjects/heatFlux/fieldAverageItem/fieldAverageItem.H (100%)
 rename {applications/solvers/coupled => src}/conjugateHeatTransfer/functionObjects/heatFlux/fieldAverageItem/fieldAverageItemIO.C (100%)
 rename {applications/solvers/coupled => src}/conjugateHeatTransfer/functionObjects/heatFlux/heatFlux/IOFieldAverage.H (100%)
 rename {applications/solvers/coupled => src}/conjugateHeatTransfer/functionObjects/heatFlux/heatFlux/detachPatches.H (100%)
 rename {applications/solvers/coupled => src}/conjugateHeatTransfer/functionObjects/heatFlux/heatFlux/heatFlux.C (100%)
 rename {applications/solvers/coupled => src}/conjugateHeatTransfer/functionObjects/heatFlux/heatFlux/heatFlux.H (100%)
 rename {applications/solvers/coupled => src}/conjugateHeatTransfer/functionObjects/heatFlux/heatFlux/heatFluxTemplates.C (100%)
 rename {applications/solvers/coupled => src}/conjugateHeatTransfer/functionObjects/heatFlux/heatFluxFunctionObject/heatFluxFunctionObject.C (100%)
 rename {applications/solvers/coupled => src}/conjugateHeatTransfer/functionObjects/heatFlux/heatFluxFunctionObject/heatFluxFunctionObject.H (100%)
 rename {applications/solvers/coupled => src}/conjugateHeatTransfer/fvPatchFields/adiabatic/adiabaticFvPatchScalarField.C (100%)
 rename {applications/solvers/coupled => src}/conjugateHeatTransfer/fvPatchFields/adiabatic/adiabaticFvPatchScalarField.H (100%)
 rename {applications/solvers/coupled => src}/conjugateHeatTransfer/fvPatchFields/chtRcTemperature/chtRcTemperatureFvPatchScalarField.C (100%)
 rename {applications/solvers/coupled => src}/conjugateHeatTransfer/fvPatchFields/chtRcTemperature/chtRcTemperatureFvPatchScalarField.H (89%)
 rename {applications/solvers/coupled => src}/conjugateHeatTransfer/fvPatchFields/chtRcThermalDiffusivity/chtRcThermalDiffusivityFvPatchScalarField.C (75%)
 rename {applications/solvers/coupled => src}/conjugateHeatTransfer/fvPatchFields/chtRcThermalDiffusivity/chtRcThermalDiffusivityFvPatchScalarField.H (93%)
 rename {applications/solvers/coupled => src}/conjugateHeatTransfer/fvPatchFields/chtRcThermalDiffusivityResistance/chtRcThermalDiffusivityResistanceFvPatchScalarField.C (100%)
 rename {applications/solvers/coupled => src}/conjugateHeatTransfer/fvPatchFields/chtRcThermalDiffusivityResistance/chtRcThermalDiffusivityResistanceFvPatchScalarField.H (100%)
 rename {applications/solvers/coupled => src}/conjugateHeatTransfer/fvPatchFields/chtRcThermalDiffusivitySlave/chtRcThermalDiffusivitySlaveFvPatchScalarField.C (77%)
 rename {applications/solvers/coupled => src}/conjugateHeatTransfer/fvPatchFields/chtRcThermalDiffusivitySlave/chtRcThermalDiffusivitySlaveFvPatchScalarField.H (93%)
 rename {applications/solvers/coupled => src}/conjugateHeatTransfer/fvPatchFields/chtRegionCoupleBase/chtRegionCoupleBase.C (100%)
 rename {applications/solvers/coupled => src}/conjugateHeatTransfer/fvPatchFields/chtRegionCoupleBase/chtRegionCoupleBase.H (100%)
 rename {applications/solvers/coupled => src}/conjugateHeatTransfer/fvPatchFields/controlledParabolicVelocity/controlledParabolicVelocityFvPatchVectorField.C (100%)
 rename {applications/solvers/coupled => src}/conjugateHeatTransfer/fvPatchFields/controlledParabolicVelocity/controlledParabolicVelocityFvPatchVectorField.H (100%)
 rename {applications/solvers/coupled => src}/conjugateHeatTransfer/fvPatchFields/extendedWallHeatTransfer/extendedWallHeatTransferFvPatchScalarField.C (100%)
 rename {applications/solvers/coupled => src}/conjugateHeatTransfer/fvPatchFields/extendedWallHeatTransfer/extendedWallHeatTransferFvPatchScalarField.H (100%)
 rename {applications/solvers/coupled => src}/conjugateHeatTransfer/fvPatchFields/extendedWallHeatTransfer/viewFactorRadiation.C (100%)
 rename {applications/solvers/coupled => src}/conjugateHeatTransfer/fvPatchFields/extendedWallHeatTransfer/viewFactorRadiation.H (100%)
 rename {applications/solvers/coupled => src}/conjugateHeatTransfer/fvPatchFields/externalRadiation/ersConstantFlux/ersConstantFlux.C (100%)
 rename {applications/solvers/coupled => src}/conjugateHeatTransfer/fvPatchFields/externalRadiation/ersConstantFlux/ersConstantFlux.H (100%)
 rename {applications/solvers/coupled => src}/conjugateHeatTransfer/fvPatchFields/externalRadiation/ersPlaneToCylinder/ersPlaneToCylinder.C (100%)
 rename {applications/solvers/coupled => src}/conjugateHeatTransfer/fvPatchFields/externalRadiation/ersPlaneToCylinder/ersPlaneToCylinder.H (100%)
 rename {applications/solvers/coupled => src}/conjugateHeatTransfer/fvPatchFields/externalRadiation/ersPointSource/ersPointSource.C (100%)
 rename {applications/solvers/coupled => src}/conjugateHeatTransfer/fvPatchFields/externalRadiation/ersPointSource/ersPointSource.H (100%)
 rename {applications/solvers/coupled => src}/conjugateHeatTransfer/fvPatchFields/externalRadiation/ersViewFactor/ersViewFactor.C (100%)
 rename {applications/solvers/coupled => src}/conjugateHeatTransfer/fvPatchFields/externalRadiation/ersViewFactor/ersViewFactor.H (100%)
 rename {applications/solvers/coupled => src}/conjugateHeatTransfer/fvPatchFields/externalRadiation/externalRadiationSource/externalRadiationSource.C (100%)
 rename {applications/solvers/coupled => src}/conjugateHeatTransfer/fvPatchFields/externalRadiation/externalRadiationSource/externalRadiationSource.H (100%)
 rename {applications/solvers/coupled => src}/conjugateHeatTransfer/fvPatchFields/externalRadiation/externalRadiationSource/newExternalRadiationSource.C (100%)
 rename {applications/solvers/coupled => src}/conjugateHeatTransfer/fvPatchFields/fieldStorage/fieldStorage.C (100%)
 rename {applications/solvers/coupled => src}/conjugateHeatTransfer/fvPatchFields/fieldStorage/fieldStorage.H (100%)
 rename {applications/solvers/coupled => src}/conjugateHeatTransfer/fvPatchFields/solidWallHeatFluxTemperature/solidWallHeatFluxTemperatureFvPatchScalarField.C (100%)
 rename {applications/solvers/coupled => src}/conjugateHeatTransfer/fvPatchFields/solidWallHeatFluxTemperature/solidWallHeatFluxTemperatureFvPatchScalarField.H (100%)
 rename {applications/solvers/coupled => src}/conjugateHeatTransfer/thermalModel/thermalGaps/constantResistanceThermalGap/constantResistanceThermalGap.C (100%)
 rename {applications/solvers/coupled => src}/conjugateHeatTransfer/thermalModel/thermalGaps/constantResistanceThermalGap/constantResistanceThermalGap.H (100%)
 rename {applications/solvers/coupled => src}/conjugateHeatTransfer/thermalModel/thermalGaps/constantThermalGap/constantThermalGap.C (100%)
 rename {applications/solvers/coupled => src}/conjugateHeatTransfer/thermalModel/thermalGaps/constantThermalGap/constantThermalGap.H (100%)
 rename {applications/solvers/coupled => src}/conjugateHeatTransfer/thermalModel/thermalGaps/thermalGap/newThermalGap.C (100%)
 rename {applications/solvers/coupled => src}/conjugateHeatTransfer/thermalModel/thermalGaps/thermalGap/thermalGap.C (100%)
 rename {applications/solvers/coupled => src}/conjugateHeatTransfer/thermalModel/thermalGaps/thermalGap/thermalGap.H (100%)
 rename {applications/solvers/coupled => src}/conjugateHeatTransfer/thermalModel/thermalLaws/constantThermal/constantThermal.C (100%)
 rename {applications/solvers/coupled => src}/conjugateHeatTransfer/thermalModel/thermalLaws/constantThermal/constantThermal.H (100%)
 rename {applications/solvers/coupled => src}/conjugateHeatTransfer/thermalModel/thermalLaws/multiMaterialThermal/multiMaterialThermal.C (100%)
 rename {applications/solvers/coupled => src}/conjugateHeatTransfer/thermalModel/thermalLaws/multiMaterialThermal/multiMaterialThermal.H (100%)
 rename {applications/solvers/coupled => src}/conjugateHeatTransfer/thermalModel/thermalLaws/multiMaterialZonesThermal/multiMaterialZonesThermal.C (100%)
 rename {applications/solvers/coupled => src}/conjugateHeatTransfer/thermalModel/thermalLaws/multiMaterialZonesThermal/multiMaterialZonesThermal.H (100%)
 rename {applications/solvers/coupled => src}/conjugateHeatTransfer/thermalModel/thermalLaws/thermalLaw/newThermalLaw.C (100%)
 rename {applications/solvers/coupled => src}/conjugateHeatTransfer/thermalModel/thermalLaws/thermalLaw/thermalLaw.C (100%)
 rename {applications/solvers/coupled => src}/conjugateHeatTransfer/thermalModel/thermalLaws/thermalLaw/thermalLaw.H (100%)
 rename {applications/solvers/coupled => src}/conjugateHeatTransfer/thermalModel/thermalModel.C (100%)
 rename {applications/solvers/coupled => src}/conjugateHeatTransfer/thermalModel/thermalModel.H (100%)
 rename {applications/solvers/coupled => src}/conjugateHeatTransfer/thermalModel/thermalSource/constantThermalSource/constantThermalSource.C (100%)
 rename {applications/solvers/coupled => src}/conjugateHeatTransfer/thermalModel/thermalSource/constantThermalSource/constantThermalSource.H (100%)
 rename {applications/solvers/coupled => src}/conjugateHeatTransfer/thermalModel/thermalSource/thermalSource/newThermalSource.C (100%)
 rename {applications/solvers/coupled => src}/conjugateHeatTransfer/thermalModel/thermalSource/thermalSource/thermalSource.C (100%)
 rename {applications/solvers/coupled => src}/conjugateHeatTransfer/thermalModel/thermalSource/thermalSource/thermalSource.H (100%)

diff --git a/applications/solvers/coupled/conjugateHeatFoam/Make/options b/applications/solvers/coupled/conjugateHeatFoam/Make/options
index 4b6d34892..3861ced2e 100644
--- a/applications/solvers/coupled/conjugateHeatFoam/Make/options
+++ b/applications/solvers/coupled/conjugateHeatFoam/Make/options
@@ -1,5 +1,4 @@
 EXE_INC = \
-    -I../conjugateHeatTransfer/lnInclude \
     -I$(LIB_SRC)/turbulenceModels \
     -I$(LIB_SRC)/turbulenceModels/incompressible/RAS/RASModel \
     -I$(LIB_SRC)/turbulenceModels/incompressible/turbulenceModel \
@@ -7,8 +6,9 @@ EXE_INC = \
     -I$(LIB_SRC)/transportModels/incompressible/singlePhaseTransportModel \
     -I$(LIB_SRC)/coupledMatrix/lnInclude \
     -I$(LIB_SRC)/finiteVolume/lnInclude \
+    -I$(LIB_SRC)/thermophysicalModels/basic/lnInclude \
     -I$(LIB_SRC)/thermophysicalModels/radiation/lnInclude \
-    -I$(LIB_SRC)/thermophysicalModels/basic/lnInclude
+    -I$(LIB_SRC)/conjugateHeatTransfer/lnInclude
 
 EXE_LIBS = \
     -lincompressibleTurbulenceModel \
diff --git a/applications/solvers/coupled/conjugateHeatSimpleFoam/Make/options b/applications/solvers/coupled/conjugateHeatSimpleFoam/Make/options
index 4b6d34892..3861ced2e 100644
--- a/applications/solvers/coupled/conjugateHeatSimpleFoam/Make/options
+++ b/applications/solvers/coupled/conjugateHeatSimpleFoam/Make/options
@@ -1,5 +1,4 @@
 EXE_INC = \
-    -I../conjugateHeatTransfer/lnInclude \
     -I$(LIB_SRC)/turbulenceModels \
     -I$(LIB_SRC)/turbulenceModels/incompressible/RAS/RASModel \
     -I$(LIB_SRC)/turbulenceModels/incompressible/turbulenceModel \
@@ -7,8 +6,9 @@ EXE_INC = \
     -I$(LIB_SRC)/transportModels/incompressible/singlePhaseTransportModel \
     -I$(LIB_SRC)/coupledMatrix/lnInclude \
     -I$(LIB_SRC)/finiteVolume/lnInclude \
+    -I$(LIB_SRC)/thermophysicalModels/basic/lnInclude \
     -I$(LIB_SRC)/thermophysicalModels/radiation/lnInclude \
-    -I$(LIB_SRC)/thermophysicalModels/basic/lnInclude
+    -I$(LIB_SRC)/conjugateHeatTransfer/lnInclude
 
 EXE_LIBS = \
     -lincompressibleTurbulenceModel \
diff --git a/src/Allwmake b/src/Allwmake
index 3b600c806..afb77baac 100755
--- a/src/Allwmake
+++ b/src/Allwmake
@@ -59,6 +59,8 @@ thermophysicalModels/Allwmake
 transportModels/Allwmake
 turbulenceModels/Allwmake
 lagrangian/Allwmake
+wmake libso conjugateHeatTransfer
+
 postProcessing/Allwmake
 conversion/Allwmake
 
diff --git a/applications/solvers/coupled/conjugateHeatTransfer/Make/files b/src/conjugateHeatTransfer/Make/files
similarity index 100%
rename from applications/solvers/coupled/conjugateHeatTransfer/Make/files
rename to src/conjugateHeatTransfer/Make/files
diff --git a/applications/solvers/coupled/conjugateHeatTransfer/Make/options b/src/conjugateHeatTransfer/Make/options
similarity index 100%
rename from applications/solvers/coupled/conjugateHeatTransfer/Make/options
rename to src/conjugateHeatTransfer/Make/options
diff --git a/applications/solvers/coupled/conjugateHeatTransfer/functionObjects/heatFlux/fieldAverageItem/fieldAverageItem.C b/src/conjugateHeatTransfer/functionObjects/heatFlux/fieldAverageItem/fieldAverageItem.C
similarity index 100%
rename from applications/solvers/coupled/conjugateHeatTransfer/functionObjects/heatFlux/fieldAverageItem/fieldAverageItem.C
rename to src/conjugateHeatTransfer/functionObjects/heatFlux/fieldAverageItem/fieldAverageItem.C
diff --git a/applications/solvers/coupled/conjugateHeatTransfer/functionObjects/heatFlux/fieldAverageItem/fieldAverageItem.H b/src/conjugateHeatTransfer/functionObjects/heatFlux/fieldAverageItem/fieldAverageItem.H
similarity index 100%
rename from applications/solvers/coupled/conjugateHeatTransfer/functionObjects/heatFlux/fieldAverageItem/fieldAverageItem.H
rename to src/conjugateHeatTransfer/functionObjects/heatFlux/fieldAverageItem/fieldAverageItem.H
diff --git a/applications/solvers/coupled/conjugateHeatTransfer/functionObjects/heatFlux/fieldAverageItem/fieldAverageItemIO.C b/src/conjugateHeatTransfer/functionObjects/heatFlux/fieldAverageItem/fieldAverageItemIO.C
similarity index 100%
rename from applications/solvers/coupled/conjugateHeatTransfer/functionObjects/heatFlux/fieldAverageItem/fieldAverageItemIO.C
rename to src/conjugateHeatTransfer/functionObjects/heatFlux/fieldAverageItem/fieldAverageItemIO.C
diff --git a/applications/solvers/coupled/conjugateHeatTransfer/functionObjects/heatFlux/heatFlux/IOFieldAverage.H b/src/conjugateHeatTransfer/functionObjects/heatFlux/heatFlux/IOFieldAverage.H
similarity index 100%
rename from applications/solvers/coupled/conjugateHeatTransfer/functionObjects/heatFlux/heatFlux/IOFieldAverage.H
rename to src/conjugateHeatTransfer/functionObjects/heatFlux/heatFlux/IOFieldAverage.H
diff --git a/applications/solvers/coupled/conjugateHeatTransfer/functionObjects/heatFlux/heatFlux/detachPatches.H b/src/conjugateHeatTransfer/functionObjects/heatFlux/heatFlux/detachPatches.H
similarity index 100%
rename from applications/solvers/coupled/conjugateHeatTransfer/functionObjects/heatFlux/heatFlux/detachPatches.H
rename to src/conjugateHeatTransfer/functionObjects/heatFlux/heatFlux/detachPatches.H
diff --git a/applications/solvers/coupled/conjugateHeatTransfer/functionObjects/heatFlux/heatFlux/heatFlux.C b/src/conjugateHeatTransfer/functionObjects/heatFlux/heatFlux/heatFlux.C
similarity index 100%
rename from applications/solvers/coupled/conjugateHeatTransfer/functionObjects/heatFlux/heatFlux/heatFlux.C
rename to src/conjugateHeatTransfer/functionObjects/heatFlux/heatFlux/heatFlux.C
diff --git a/applications/solvers/coupled/conjugateHeatTransfer/functionObjects/heatFlux/heatFlux/heatFlux.H b/src/conjugateHeatTransfer/functionObjects/heatFlux/heatFlux/heatFlux.H
similarity index 100%
rename from applications/solvers/coupled/conjugateHeatTransfer/functionObjects/heatFlux/heatFlux/heatFlux.H
rename to src/conjugateHeatTransfer/functionObjects/heatFlux/heatFlux/heatFlux.H
diff --git a/applications/solvers/coupled/conjugateHeatTransfer/functionObjects/heatFlux/heatFlux/heatFluxTemplates.C b/src/conjugateHeatTransfer/functionObjects/heatFlux/heatFlux/heatFluxTemplates.C
similarity index 100%
rename from applications/solvers/coupled/conjugateHeatTransfer/functionObjects/heatFlux/heatFlux/heatFluxTemplates.C
rename to src/conjugateHeatTransfer/functionObjects/heatFlux/heatFlux/heatFluxTemplates.C
diff --git a/applications/solvers/coupled/conjugateHeatTransfer/functionObjects/heatFlux/heatFluxFunctionObject/heatFluxFunctionObject.C b/src/conjugateHeatTransfer/functionObjects/heatFlux/heatFluxFunctionObject/heatFluxFunctionObject.C
similarity index 100%
rename from applications/solvers/coupled/conjugateHeatTransfer/functionObjects/heatFlux/heatFluxFunctionObject/heatFluxFunctionObject.C
rename to src/conjugateHeatTransfer/functionObjects/heatFlux/heatFluxFunctionObject/heatFluxFunctionObject.C
diff --git a/applications/solvers/coupled/conjugateHeatTransfer/functionObjects/heatFlux/heatFluxFunctionObject/heatFluxFunctionObject.H b/src/conjugateHeatTransfer/functionObjects/heatFlux/heatFluxFunctionObject/heatFluxFunctionObject.H
similarity index 100%
rename from applications/solvers/coupled/conjugateHeatTransfer/functionObjects/heatFlux/heatFluxFunctionObject/heatFluxFunctionObject.H
rename to src/conjugateHeatTransfer/functionObjects/heatFlux/heatFluxFunctionObject/heatFluxFunctionObject.H
diff --git a/applications/solvers/coupled/conjugateHeatTransfer/fvPatchFields/adiabatic/adiabaticFvPatchScalarField.C b/src/conjugateHeatTransfer/fvPatchFields/adiabatic/adiabaticFvPatchScalarField.C
similarity index 100%
rename from applications/solvers/coupled/conjugateHeatTransfer/fvPatchFields/adiabatic/adiabaticFvPatchScalarField.C
rename to src/conjugateHeatTransfer/fvPatchFields/adiabatic/adiabaticFvPatchScalarField.C
diff --git a/applications/solvers/coupled/conjugateHeatTransfer/fvPatchFields/adiabatic/adiabaticFvPatchScalarField.H b/src/conjugateHeatTransfer/fvPatchFields/adiabatic/adiabaticFvPatchScalarField.H
similarity index 100%
rename from applications/solvers/coupled/conjugateHeatTransfer/fvPatchFields/adiabatic/adiabaticFvPatchScalarField.H
rename to src/conjugateHeatTransfer/fvPatchFields/adiabatic/adiabaticFvPatchScalarField.H
diff --git a/applications/solvers/coupled/conjugateHeatTransfer/fvPatchFields/chtRcTemperature/chtRcTemperatureFvPatchScalarField.C b/src/conjugateHeatTransfer/fvPatchFields/chtRcTemperature/chtRcTemperatureFvPatchScalarField.C
similarity index 100%
rename from applications/solvers/coupled/conjugateHeatTransfer/fvPatchFields/chtRcTemperature/chtRcTemperatureFvPatchScalarField.C
rename to src/conjugateHeatTransfer/fvPatchFields/chtRcTemperature/chtRcTemperatureFvPatchScalarField.C
diff --git a/applications/solvers/coupled/conjugateHeatTransfer/fvPatchFields/chtRcTemperature/chtRcTemperatureFvPatchScalarField.H b/src/conjugateHeatTransfer/fvPatchFields/chtRcTemperature/chtRcTemperatureFvPatchScalarField.H
similarity index 89%
rename from applications/solvers/coupled/conjugateHeatTransfer/fvPatchFields/chtRcTemperature/chtRcTemperatureFvPatchScalarField.H
rename to src/conjugateHeatTransfer/fvPatchFields/chtRcTemperature/chtRcTemperatureFvPatchScalarField.H
index 412fa58fc..8e2801d3f 100644
--- a/applications/solvers/coupled/conjugateHeatTransfer/fvPatchFields/chtRcTemperature/chtRcTemperatureFvPatchScalarField.H
+++ b/src/conjugateHeatTransfer/fvPatchFields/chtRcTemperature/chtRcTemperatureFvPatchScalarField.H
@@ -47,7 +47,7 @@ namespace Foam
 {
 
 /*---------------------------------------------------------------------------*\
-                  Class chtRcTemperatureFvPatchScalarField Declaration
+             Class chtRcTemperatureFvPatchScalarField Declaration
 \*---------------------------------------------------------------------------*/
 
 class chtRcTemperatureFvPatchScalarField
@@ -60,7 +60,7 @@ class chtRcTemperatureFvPatchScalarField
         word kName_;
 
         //- Take radiation into account for this patch
-        bool radiation_;
+        Switch radiation_;
 
 
 public:
@@ -86,7 +86,8 @@ public:
             const dictionary&
         );
 
-        //- Construct by mapping given chtRcTemperatureFvPatchScalarField onto a new patch
+        //- Construct by mapping given chtRcTemperatureFvPatchScalarField
+        //  onto a new patch
         chtRcTemperatureFvPatchScalarField
         (
             const chtRcTemperatureFvPatchScalarField&,
@@ -128,8 +129,26 @@ public:
 
         // Access functions
 
+            //- Return kName
+            const word& kName() const
+            {
+                return kName_;
+            }
+
+            //- Return access to kName
+            word& kName()
+            {
+                return kName_;
+            }
+
             //- Return radiation
-            bool radiation() const
+            const Switch& radiation() const
+            {
+                return radiation_;
+            }
+
+            //- Return access to radiation
+            Switch& radiation()
             {
                 return radiation_;
             }
@@ -158,7 +177,6 @@ public:
 
         //- Write
         virtual void write(Ostream&) const;
-
 };
 
 
diff --git a/applications/solvers/coupled/conjugateHeatTransfer/fvPatchFields/chtRcThermalDiffusivity/chtRcThermalDiffusivityFvPatchScalarField.C b/src/conjugateHeatTransfer/fvPatchFields/chtRcThermalDiffusivity/chtRcThermalDiffusivityFvPatchScalarField.C
similarity index 75%
rename from applications/solvers/coupled/conjugateHeatTransfer/fvPatchFields/chtRcThermalDiffusivity/chtRcThermalDiffusivityFvPatchScalarField.C
rename to src/conjugateHeatTransfer/fvPatchFields/chtRcThermalDiffusivity/chtRcThermalDiffusivityFvPatchScalarField.C
index fa697a71f..1e65c4727 100644
--- a/applications/solvers/coupled/conjugateHeatTransfer/fvPatchFields/chtRcThermalDiffusivity/chtRcThermalDiffusivityFvPatchScalarField.C
+++ b/src/conjugateHeatTransfer/fvPatchFields/chtRcThermalDiffusivity/chtRcThermalDiffusivityFvPatchScalarField.C
@@ -36,14 +36,10 @@ Author
 #include "radiationConstants.H"
 #include "VectorN.H"
 
-// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
-
-namespace Foam
-{
-
 // * * * * * * * * * * * * * * * * Constructors  * * * * * * * * * * * * * * //
 
-chtRcThermalDiffusivityFvPatchScalarField::chtRcThermalDiffusivityFvPatchScalarField
+Foam::chtRcThermalDiffusivityFvPatchScalarField::
+chtRcThermalDiffusivityFvPatchScalarField
 (
     const fvPatch& p,
     const DimensionedField<scalar, volMesh>& iF
@@ -53,7 +49,8 @@ chtRcThermalDiffusivityFvPatchScalarField::chtRcThermalDiffusivityFvPatchScalarF
 {}
 
 
-chtRcThermalDiffusivityFvPatchScalarField::chtRcThermalDiffusivityFvPatchScalarField
+Foam::chtRcThermalDiffusivityFvPatchScalarField::
+chtRcThermalDiffusivityFvPatchScalarField
 (
     const fvPatch& p,
     const DimensionedField<scalar, volMesh>& iF,
@@ -64,7 +61,8 @@ chtRcThermalDiffusivityFvPatchScalarField::chtRcThermalDiffusivityFvPatchScalarF
 {}
 
 
-chtRcThermalDiffusivityFvPatchScalarField::chtRcThermalDiffusivityFvPatchScalarField
+Foam::chtRcThermalDiffusivityFvPatchScalarField::
+chtRcThermalDiffusivityFvPatchScalarField
 (
     const chtRcThermalDiffusivityFvPatchScalarField& ptf,
     const fvPatch& p,
@@ -76,7 +74,8 @@ chtRcThermalDiffusivityFvPatchScalarField::chtRcThermalDiffusivityFvPatchScalarF
 {}
 
 
-chtRcThermalDiffusivityFvPatchScalarField::chtRcThermalDiffusivityFvPatchScalarField
+Foam::chtRcThermalDiffusivityFvPatchScalarField::
+chtRcThermalDiffusivityFvPatchScalarField
 (
     const chtRcThermalDiffusivityFvPatchScalarField& ptf,
     const DimensionedField<scalar, volMesh>& iF
@@ -88,7 +87,7 @@ chtRcThermalDiffusivityFvPatchScalarField::chtRcThermalDiffusivityFvPatchScalarF
 
 // * * * * * * * * * * * * * * * Member Functions  * * * * * * * * * * * * * //
 
-void chtRcThermalDiffusivityFvPatchScalarField::evaluate
+void Foam::chtRcThermalDiffusivityFvPatchScalarField::evaluate
 (
     const Pstream::commsTypes
 )
@@ -97,7 +96,7 @@ void chtRcThermalDiffusivityFvPatchScalarField::evaluate
 }
 
 
-void chtRcThermalDiffusivityFvPatchScalarField::updateCoeffs()
+void Foam::chtRcThermalDiffusivityFvPatchScalarField::updateCoeffs()
 {
     if (updated())
     {
@@ -109,16 +108,18 @@ void chtRcThermalDiffusivityFvPatchScalarField::updateCoeffs()
 
 
 void
-chtRcThermalDiffusivityFvPatchScalarField::calcThermalDiffusivity
+Foam::chtRcThermalDiffusivityFvPatchScalarField::calcThermalDiffusivity
 (
     chtRegionCoupleBase& owner,
     const chtRegionCoupleBase& neighbour
 ) const
 {
-    if(debug)
+    if (debug)
     {
-        Info << "In chtRcThermalDiffusivityFvPatchScalarField::calcThermalDiffusivity on "
-            << this->dimensionedInternalField().name()
+        InfoIn
+        (
+            "chtRcThermalDiffusivityFvPatchScalarField::calcThermalDiffusivity"
+        )   << "for field " << this->dimensionedInternalField().name()
             << " in " << this->patch().boundaryMesh().mesh().name()
             << endl;
     }
@@ -158,19 +159,19 @@ chtRcThermalDiffusivityFvPatchScalarField::calcThermalDiffusivity
         const scalarField& lfNei = neighbour.originalPatchField();
         scalarField lTcNei = TwOwn.shadowPatchField().patchInternalField();
 
-        forAll(lData, facei)
+        forAll (lData, facei)
         {
             lData[facei][0] = lTcNei[facei];
             lData[facei][1] = lfNei[facei];
         }
 
-        if(TwOwn.shadowPatchField().radiation())
+        if (TwOwn.shadowPatchField().radiation())
         {
             const scalarField& lQrNei =
                 owner.lookupShadowPatchField<volScalarField, scalar>("Qr");
             const scalarField& lTwNei = TwOwn.shadowPatchField();
 
-            forAll(lData, facei)
+            forAll (lData, facei)
             {
                 lData[facei][2] = lTwNei[facei];
                 lData[facei][3] = lQrNei[facei];
@@ -180,15 +181,15 @@ chtRcThermalDiffusivityFvPatchScalarField::calcThermalDiffusivity
         const Field<VectorN<scalar, 4> > iData =
             owner.regionCouplePatch().interpolate(lData);
 
-        forAll(iData, facei)
+        forAll (iData, facei)
         {
             TcNei[facei] = iData[facei][0];
             fNei[facei] = iData[facei][1];
         }
 
-        if(TwOwn.shadowPatchField().radiation())
+        if (TwOwn.shadowPatchField().radiation())
         {
-            forAll(iData, facei)
+            forAll (iData, facei)
             {
                 Qr[facei] += iData[facei][3];
                 fourQro[facei] +=
@@ -205,46 +206,35 @@ chtRcThermalDiffusivityFvPatchScalarField::calcThermalDiffusivity
     const scalarField kOwn = fOwn/(1.0 - p.weights())/mld.magDelta(p.index());
     const scalarField kNei = fNei/p.weights()/mld.magDelta(p.index());
 
-    //Info << "kOwn = " << kOwn << endl;
-    //Info << "kNei = " << kNei << endl;
-    //Info << "TcOwn = " << TcOwn << endl;
-    //Info << "TcNei = " << TcNei << endl;
-    //Info << "DeltaT = " << TcNei - TcOwn << endl;
-
-    //Info << "Qr = " << Qr << endl;
-    //Info << "kOwn + kNei = " << (kOwn + kNei) << endl;
-
-    //Info << "k = " << k << endl;
-
     k = kOwn*(TwOwn*(kNei*(TcNei - TcOwn) + Qr + fourQro) - TcOwn*fourQro);
     k /= stabilise((fourQro + TwOwn*(kOwn + kNei))*(TcNei - TcOwn), SMALL);
     k /= p.deltaCoeffs();
 
     //Info << "k = " << k << endl;
 
-    forAll(k, facei)
+    forAll (k, facei)
     {
         k[facei] = max(min(k[facei], 100*kHarm[facei]), 0.01*kHarm[facei]);
     }
 
-    //Info << "k = " << k << endl;
-
     owner.fvPatchScalarField::updateCoeffs();
 }
 
 
 void
-chtRcThermalDiffusivityFvPatchScalarField::calcTemperature
+Foam::chtRcThermalDiffusivityFvPatchScalarField::calcTemperature
 (
     chtRcTemperatureFvPatchScalarField& TwOwn,
     const chtRcTemperatureFvPatchScalarField& neighbour,
     const chtRegionCoupleBase& ownerK
 ) const
 {
-    if(debug)
+    if (debug)
     {
-        Info << "In chtRcThermalDiffusivityFvPatchScalarField::calcTemperature on "
-            << this->dimensionedInternalField().name()
+        InfoIn
+        (
+            "chtRcThermalDiffusivityFvPatchScalarField::calcTemperature"
+        )   << "for field " << this->dimensionedInternalField().name()
             << " in " << this->patch().boundaryMesh().mesh().name()
             << endl;
     }
@@ -280,19 +270,19 @@ chtRcThermalDiffusivityFvPatchScalarField::calcTemperature
         scalarField lTcNei =
             TwOwn.shadowPatchField().patchInternalField();
 
-        forAll(lData, facei)
+        forAll (lData, facei)
         {
             lData[facei][0] = lTcNei[facei];
             lData[facei][1] = lfNei[facei];
         }
 
-        if(TwOwn.shadowPatchField().radiation())
+        if (TwOwn.shadowPatchField().radiation())
         {
             const scalarField& lTwNei = TwOwn.shadowPatchField();
             const scalarField& lQrNei =
                 TwOwn.lookupShadowPatchField<volScalarField, scalar>("Qr");
 
-            forAll(lData, facei)
+            forAll (lData, facei)
             {
                 lData[facei][2] = lTwNei[facei];
                 lData[facei][3] = lQrNei[facei];
@@ -302,15 +292,15 @@ chtRcThermalDiffusivityFvPatchScalarField::calcTemperature
         const Field<VectorN<scalar, 4> > iData =
             TwOwn.regionCouplePatch().interpolate(lData);
 
-        forAll(iData, facei)
+        forAll (iData, facei)
         {
             TcNei[facei] = iData[facei][0];
             fNei[facei] = iData[facei][1];
         }
 
-        if(TwOwn.shadowPatchField().radiation())
+        if (TwOwn.shadowPatchField().radiation())
         {
-            forAll(iData, facei)
+            forAll (iData, facei)
             {
                 fourQro[facei] +=
                     4.0*radiation::sigmaSB.value()*pow4(iData[facei][2]);
@@ -322,29 +312,15 @@ chtRcThermalDiffusivityFvPatchScalarField::calcTemperature
     const scalarField kOwn = fOwn/(1.0 - p.weights())/mld.magDelta(p.index());
     const scalarField kNei = fNei/p.weights()/mld.magDelta(p.index());
 
-    //Info << "kOwn = " << kOwn << endl;
-    //Info << "kNei = " << kNei << endl;
-    //Info << "TcOwn = " << TcOwn << endl;
-    //Info << "TcNei = " << TcNei << endl;
-    //Info << "Qr = " << Qr << " Sum = " << sum(Qr*p.magSf()) << endl;
-
     TwOwn *=
         (fourQro + Qr + kOwn*TcOwn + kNei*TcNei)
        /(TwOwn*(kOwn + kNei) + fourQro);
 
-    //Info << "TwOwn = " << TwOwn << endl;
-
-    //scalarField q1 = (TwOwn - TcOwn)*kOwn;
-    //Info << "q1 = " << q1 << " Sum = " << sum(q1*p.magSf()) << endl;
-
-    //scalarField q2 = (TcNei - TcOwn)*ownerK*p.deltaCoeffs();
-    //Info << "q2 = " << q2 << " Sum = " << sum(q2*p.magSf()) << endl;
-
     TwOwn.fvPatchScalarField::updateCoeffs();
 }
 
 
-void chtRcThermalDiffusivityFvPatchScalarField::write(Ostream& os) const
+void Foam::chtRcThermalDiffusivityFvPatchScalarField::write(Ostream& os) const
 {
     fvPatchScalarField::write(os);
     os.writeKeyword("remoteField")
@@ -353,19 +329,18 @@ void chtRcThermalDiffusivityFvPatchScalarField::write(Ostream& os) const
 }
 
 
-//- Specify data associated with VectorN<scalar, 4> type is contiguous
-template<>
-inline bool contiguous<VectorN<scalar, 4> >() {return true;}
-
-
 // * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
 
-makePatchTypeField
-(
-    fvPatchScalarField,
-    chtRcThermalDiffusivityFvPatchScalarField
-);
+namespace Foam
+{
+
+    makePatchTypeField
+    (
+        fvPatchScalarField,
+        chtRcThermalDiffusivityFvPatchScalarField
+    );
 
 } // End namespace Foam
 
+
 // ************************************************************************* //
diff --git a/applications/solvers/coupled/conjugateHeatTransfer/fvPatchFields/chtRcThermalDiffusivity/chtRcThermalDiffusivityFvPatchScalarField.H b/src/conjugateHeatTransfer/fvPatchFields/chtRcThermalDiffusivity/chtRcThermalDiffusivityFvPatchScalarField.H
similarity index 93%
rename from applications/solvers/coupled/conjugateHeatTransfer/fvPatchFields/chtRcThermalDiffusivity/chtRcThermalDiffusivityFvPatchScalarField.H
rename to src/conjugateHeatTransfer/fvPatchFields/chtRcThermalDiffusivity/chtRcThermalDiffusivityFvPatchScalarField.H
index eab572340..e8324b1dd 100644
--- a/applications/solvers/coupled/conjugateHeatTransfer/fvPatchFields/chtRcThermalDiffusivity/chtRcThermalDiffusivityFvPatchScalarField.H
+++ b/src/conjugateHeatTransfer/fvPatchFields/chtRcThermalDiffusivity/chtRcThermalDiffusivityFvPatchScalarField.H
@@ -25,10 +25,12 @@ Class
     chtRcThermalDiffusivityFvPatchScalarField
 
 Description
-    Region couple patch field taking radiation into account
+    Diffusivity for the region couple patch field taking radiation into account
+    Master side
 
 Author
     Henrik Rusche, Wikki GmbH.  All rights reserved
+    Cleaned up by Hrvoje Jasak
 
 SourceFiles
     chtRcThermalDiffusivityFvPatchScalarField.C
@@ -46,16 +48,13 @@ namespace Foam
 {
 
 /*---------------------------------------------------------------------------*\
-        Class chtRcThermalDiffusivityFvPatchScalarField Declaration
+          Class chtRcThermalDiffusivityFvPatchScalarField Declaration
 \*---------------------------------------------------------------------------*/
 
 class chtRcThermalDiffusivityFvPatchScalarField
 :
     public chtRegionCoupleBase
 {
-    // Private data
-
-
 public:
 
     //- Runtime type information
@@ -79,7 +78,8 @@ public:
             const dictionary&
         );
 
-        //- Construct by mapping given chtRcThermalDiffusivityFvPatchScalarField onto a new patch
+        //- Construct by mapping given
+        //  chtRcThermalDiffusivityFvPatchScalarField onto a new patch
         chtRcThermalDiffusivityFvPatchScalarField
         (
             const chtRcThermalDiffusivityFvPatchScalarField&,
@@ -119,9 +119,6 @@ public:
 
     // Member functions
 
-        // Access
-
-
         // Evaluation functions
 
             //- Evaluate the patch field
@@ -148,7 +145,6 @@ public:
 
         //- Write
         virtual void write(Ostream&) const;
-
 };
 
 
diff --git a/applications/solvers/coupled/conjugateHeatTransfer/fvPatchFields/chtRcThermalDiffusivityResistance/chtRcThermalDiffusivityResistanceFvPatchScalarField.C b/src/conjugateHeatTransfer/fvPatchFields/chtRcThermalDiffusivityResistance/chtRcThermalDiffusivityResistanceFvPatchScalarField.C
similarity index 100%
rename from applications/solvers/coupled/conjugateHeatTransfer/fvPatchFields/chtRcThermalDiffusivityResistance/chtRcThermalDiffusivityResistanceFvPatchScalarField.C
rename to src/conjugateHeatTransfer/fvPatchFields/chtRcThermalDiffusivityResistance/chtRcThermalDiffusivityResistanceFvPatchScalarField.C
diff --git a/applications/solvers/coupled/conjugateHeatTransfer/fvPatchFields/chtRcThermalDiffusivityResistance/chtRcThermalDiffusivityResistanceFvPatchScalarField.H b/src/conjugateHeatTransfer/fvPatchFields/chtRcThermalDiffusivityResistance/chtRcThermalDiffusivityResistanceFvPatchScalarField.H
similarity index 100%
rename from applications/solvers/coupled/conjugateHeatTransfer/fvPatchFields/chtRcThermalDiffusivityResistance/chtRcThermalDiffusivityResistanceFvPatchScalarField.H
rename to src/conjugateHeatTransfer/fvPatchFields/chtRcThermalDiffusivityResistance/chtRcThermalDiffusivityResistanceFvPatchScalarField.H
diff --git a/applications/solvers/coupled/conjugateHeatTransfer/fvPatchFields/chtRcThermalDiffusivitySlave/chtRcThermalDiffusivitySlaveFvPatchScalarField.C b/src/conjugateHeatTransfer/fvPatchFields/chtRcThermalDiffusivitySlave/chtRcThermalDiffusivitySlaveFvPatchScalarField.C
similarity index 77%
rename from applications/solvers/coupled/conjugateHeatTransfer/fvPatchFields/chtRcThermalDiffusivitySlave/chtRcThermalDiffusivitySlaveFvPatchScalarField.C
rename to src/conjugateHeatTransfer/fvPatchFields/chtRcThermalDiffusivitySlave/chtRcThermalDiffusivitySlaveFvPatchScalarField.C
index 7536afa19..307894477 100644
--- a/applications/solvers/coupled/conjugateHeatTransfer/fvPatchFields/chtRcThermalDiffusivitySlave/chtRcThermalDiffusivitySlaveFvPatchScalarField.C
+++ b/src/conjugateHeatTransfer/fvPatchFields/chtRcThermalDiffusivitySlave/chtRcThermalDiffusivitySlaveFvPatchScalarField.C
@@ -31,14 +31,10 @@ Author
 #include "fvPatchFieldMapper.H"
 #include "volFields.H"
 
-// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
-
-namespace Foam
-{
-
 // * * * * * * * * * * * * * * * * Constructors  * * * * * * * * * * * * * * //
 
-chtRcThermalDiffusivitySlaveFvPatchScalarField::chtRcThermalDiffusivitySlaveFvPatchScalarField
+Foam::chtRcThermalDiffusivitySlaveFvPatchScalarField::
+chtRcThermalDiffusivitySlaveFvPatchScalarField
 (
     const fvPatch& p,
     const DimensionedField<scalar, volMesh>& iF
@@ -48,7 +44,8 @@ chtRcThermalDiffusivitySlaveFvPatchScalarField::chtRcThermalDiffusivitySlaveFvPa
 {}
 
 
-chtRcThermalDiffusivitySlaveFvPatchScalarField::chtRcThermalDiffusivitySlaveFvPatchScalarField
+Foam::chtRcThermalDiffusivitySlaveFvPatchScalarField::
+chtRcThermalDiffusivitySlaveFvPatchScalarField
 (
     const fvPatch& p,
     const DimensionedField<scalar, volMesh>& iF,
@@ -59,7 +56,19 @@ chtRcThermalDiffusivitySlaveFvPatchScalarField::chtRcThermalDiffusivitySlaveFvPa
 {}
 
 
-chtRcThermalDiffusivitySlaveFvPatchScalarField::chtRcThermalDiffusivitySlaveFvPatchScalarField
+Foam::chtRcThermalDiffusivitySlaveFvPatchScalarField::
+chtRcThermalDiffusivitySlaveFvPatchScalarField
+(
+    const chtRcThermalDiffusivitySlaveFvPatchScalarField& ptf,
+    const DimensionedField<scalar, volMesh>& iF
+)
+:
+    chtRegionCoupleBase(ptf, iF)
+{}
+
+
+Foam::chtRcThermalDiffusivitySlaveFvPatchScalarField::
+chtRcThermalDiffusivitySlaveFvPatchScalarField
 (
     const chtRcThermalDiffusivitySlaveFvPatchScalarField& ptf,
     const fvPatch& p,
@@ -71,20 +80,10 @@ chtRcThermalDiffusivitySlaveFvPatchScalarField::chtRcThermalDiffusivitySlaveFvPa
 {}
 
 
-chtRcThermalDiffusivitySlaveFvPatchScalarField::chtRcThermalDiffusivitySlaveFvPatchScalarField
-(
-    const chtRcThermalDiffusivitySlaveFvPatchScalarField& ptf,
-    const DimensionedField<scalar, volMesh>& iF
-)
-:
-    chtRegionCoupleBase(ptf, iF)
-{}
-
-
 // * * * * * * * * * * * * * * * Member Functions  * * * * * * * * * * * * * //
 
-const chtRegionCoupleBase&
-chtRcThermalDiffusivitySlaveFvPatchScalarField::shadowPatchField() const
+const Foam::chtRegionCoupleBase&
+Foam::chtRcThermalDiffusivitySlaveFvPatchScalarField::shadowPatchField() const
 {
     return dynamic_cast<const chtRegionCoupleBase&>
     (
@@ -93,7 +92,7 @@ chtRcThermalDiffusivitySlaveFvPatchScalarField::shadowPatchField() const
 }
 
 
-void chtRcThermalDiffusivitySlaveFvPatchScalarField::evaluate
+void Foam::chtRcThermalDiffusivitySlaveFvPatchScalarField::evaluate
 (
     const Pstream::commsTypes
 )
@@ -102,7 +101,7 @@ void chtRcThermalDiffusivitySlaveFvPatchScalarField::evaluate
 }
 
 
-void chtRcThermalDiffusivitySlaveFvPatchScalarField::updateCoeffs()
+void Foam::chtRcThermalDiffusivitySlaveFvPatchScalarField::updateCoeffs()
 {
     if (updated())
     {
@@ -115,12 +114,16 @@ void chtRcThermalDiffusivitySlaveFvPatchScalarField::updateCoeffs()
 
 // * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
 
-makePatchTypeField
-(
-    fvPatchScalarField,
-    chtRcThermalDiffusivitySlaveFvPatchScalarField
-);
+namespace Foam
+{
+
+    makePatchTypeField
+    (
+        fvPatchScalarField,
+        chtRcThermalDiffusivitySlaveFvPatchScalarField
+    );
 
 } // End namespace Foam
 
+
 // ************************************************************************* //
diff --git a/applications/solvers/coupled/conjugateHeatTransfer/fvPatchFields/chtRcThermalDiffusivitySlave/chtRcThermalDiffusivitySlaveFvPatchScalarField.H b/src/conjugateHeatTransfer/fvPatchFields/chtRcThermalDiffusivitySlave/chtRcThermalDiffusivitySlaveFvPatchScalarField.H
similarity index 93%
rename from applications/solvers/coupled/conjugateHeatTransfer/fvPatchFields/chtRcThermalDiffusivitySlave/chtRcThermalDiffusivitySlaveFvPatchScalarField.H
rename to src/conjugateHeatTransfer/fvPatchFields/chtRcThermalDiffusivitySlave/chtRcThermalDiffusivitySlaveFvPatchScalarField.H
index f30b5ae8a..71e132a7f 100644
--- a/applications/solvers/coupled/conjugateHeatTransfer/fvPatchFields/chtRcThermalDiffusivitySlave/chtRcThermalDiffusivitySlaveFvPatchScalarField.H
+++ b/src/conjugateHeatTransfer/fvPatchFields/chtRcThermalDiffusivitySlave/chtRcThermalDiffusivitySlaveFvPatchScalarField.H
@@ -25,10 +25,12 @@ Class
     chtRcThermalDiffusivitySlaveFvPatchScalarField
 
 Description
-    Region couple patch field taking radiation into account
+    Diffusivity for the region couple patch field taking radiation into account
+    Slave side
 
 Author
     Henrik Rusche, Wikki GmbH.  All rights reserved
+    Cleaned up by Hrvoje Jasak
 
 SourceFiles
     chtRcThermalDiffusivitySlaveFvPatchScalarField.C
@@ -49,15 +51,13 @@ class chtThermalDiffusivityRegionCoupleFvPatchScalarField;
 class chtTemperatureRegionCoupleFvPatchScalarField;
 
 /*---------------------------------------------------------------------------*\
-                  Class chtRcThermalDiffusivitySlaveFvPatchScalarField Declaration
+        Class chtRcThermalDiffusivitySlaveFvPatchScalarField Declaration
 \*---------------------------------------------------------------------------*/
 
 class chtRcThermalDiffusivitySlaveFvPatchScalarField
 :
     public chtRegionCoupleBase
 {
-    // Private data
-
 public:
 
     //- Runtime type information
@@ -81,7 +81,8 @@ public:
             const dictionary&
         );
 
-        //- Construct by mapping given chtRcThermalDiffusivitySlaveFvPatchScalarField onto a new patch
+        //- Construct by mapping given
+        //  chtRcThermalDiffusivitySlaveFvPatchScalarField onto a new patch
         chtRcThermalDiffusivitySlaveFvPatchScalarField
         (
             const chtRcThermalDiffusivitySlaveFvPatchScalarField&,
diff --git a/applications/solvers/coupled/conjugateHeatTransfer/fvPatchFields/chtRegionCoupleBase/chtRegionCoupleBase.C b/src/conjugateHeatTransfer/fvPatchFields/chtRegionCoupleBase/chtRegionCoupleBase.C
similarity index 100%
rename from applications/solvers/coupled/conjugateHeatTransfer/fvPatchFields/chtRegionCoupleBase/chtRegionCoupleBase.C
rename to src/conjugateHeatTransfer/fvPatchFields/chtRegionCoupleBase/chtRegionCoupleBase.C
diff --git a/applications/solvers/coupled/conjugateHeatTransfer/fvPatchFields/chtRegionCoupleBase/chtRegionCoupleBase.H b/src/conjugateHeatTransfer/fvPatchFields/chtRegionCoupleBase/chtRegionCoupleBase.H
similarity index 100%
rename from applications/solvers/coupled/conjugateHeatTransfer/fvPatchFields/chtRegionCoupleBase/chtRegionCoupleBase.H
rename to src/conjugateHeatTransfer/fvPatchFields/chtRegionCoupleBase/chtRegionCoupleBase.H
diff --git a/applications/solvers/coupled/conjugateHeatTransfer/fvPatchFields/controlledParabolicVelocity/controlledParabolicVelocityFvPatchVectorField.C b/src/conjugateHeatTransfer/fvPatchFields/controlledParabolicVelocity/controlledParabolicVelocityFvPatchVectorField.C
similarity index 100%
rename from applications/solvers/coupled/conjugateHeatTransfer/fvPatchFields/controlledParabolicVelocity/controlledParabolicVelocityFvPatchVectorField.C
rename to src/conjugateHeatTransfer/fvPatchFields/controlledParabolicVelocity/controlledParabolicVelocityFvPatchVectorField.C
diff --git a/applications/solvers/coupled/conjugateHeatTransfer/fvPatchFields/controlledParabolicVelocity/controlledParabolicVelocityFvPatchVectorField.H b/src/conjugateHeatTransfer/fvPatchFields/controlledParabolicVelocity/controlledParabolicVelocityFvPatchVectorField.H
similarity index 100%
rename from applications/solvers/coupled/conjugateHeatTransfer/fvPatchFields/controlledParabolicVelocity/controlledParabolicVelocityFvPatchVectorField.H
rename to src/conjugateHeatTransfer/fvPatchFields/controlledParabolicVelocity/controlledParabolicVelocityFvPatchVectorField.H
diff --git a/applications/solvers/coupled/conjugateHeatTransfer/fvPatchFields/extendedWallHeatTransfer/extendedWallHeatTransferFvPatchScalarField.C b/src/conjugateHeatTransfer/fvPatchFields/extendedWallHeatTransfer/extendedWallHeatTransferFvPatchScalarField.C
similarity index 100%
rename from applications/solvers/coupled/conjugateHeatTransfer/fvPatchFields/extendedWallHeatTransfer/extendedWallHeatTransferFvPatchScalarField.C
rename to src/conjugateHeatTransfer/fvPatchFields/extendedWallHeatTransfer/extendedWallHeatTransferFvPatchScalarField.C
diff --git a/applications/solvers/coupled/conjugateHeatTransfer/fvPatchFields/extendedWallHeatTransfer/extendedWallHeatTransferFvPatchScalarField.H b/src/conjugateHeatTransfer/fvPatchFields/extendedWallHeatTransfer/extendedWallHeatTransferFvPatchScalarField.H
similarity index 100%
rename from applications/solvers/coupled/conjugateHeatTransfer/fvPatchFields/extendedWallHeatTransfer/extendedWallHeatTransferFvPatchScalarField.H
rename to src/conjugateHeatTransfer/fvPatchFields/extendedWallHeatTransfer/extendedWallHeatTransferFvPatchScalarField.H
diff --git a/applications/solvers/coupled/conjugateHeatTransfer/fvPatchFields/extendedWallHeatTransfer/viewFactorRadiation.C b/src/conjugateHeatTransfer/fvPatchFields/extendedWallHeatTransfer/viewFactorRadiation.C
similarity index 100%
rename from applications/solvers/coupled/conjugateHeatTransfer/fvPatchFields/extendedWallHeatTransfer/viewFactorRadiation.C
rename to src/conjugateHeatTransfer/fvPatchFields/extendedWallHeatTransfer/viewFactorRadiation.C
diff --git a/applications/solvers/coupled/conjugateHeatTransfer/fvPatchFields/extendedWallHeatTransfer/viewFactorRadiation.H b/src/conjugateHeatTransfer/fvPatchFields/extendedWallHeatTransfer/viewFactorRadiation.H
similarity index 100%
rename from applications/solvers/coupled/conjugateHeatTransfer/fvPatchFields/extendedWallHeatTransfer/viewFactorRadiation.H
rename to src/conjugateHeatTransfer/fvPatchFields/extendedWallHeatTransfer/viewFactorRadiation.H
diff --git a/applications/solvers/coupled/conjugateHeatTransfer/fvPatchFields/externalRadiation/ersConstantFlux/ersConstantFlux.C b/src/conjugateHeatTransfer/fvPatchFields/externalRadiation/ersConstantFlux/ersConstantFlux.C
similarity index 100%
rename from applications/solvers/coupled/conjugateHeatTransfer/fvPatchFields/externalRadiation/ersConstantFlux/ersConstantFlux.C
rename to src/conjugateHeatTransfer/fvPatchFields/externalRadiation/ersConstantFlux/ersConstantFlux.C
diff --git a/applications/solvers/coupled/conjugateHeatTransfer/fvPatchFields/externalRadiation/ersConstantFlux/ersConstantFlux.H b/src/conjugateHeatTransfer/fvPatchFields/externalRadiation/ersConstantFlux/ersConstantFlux.H
similarity index 100%
rename from applications/solvers/coupled/conjugateHeatTransfer/fvPatchFields/externalRadiation/ersConstantFlux/ersConstantFlux.H
rename to src/conjugateHeatTransfer/fvPatchFields/externalRadiation/ersConstantFlux/ersConstantFlux.H
diff --git a/applications/solvers/coupled/conjugateHeatTransfer/fvPatchFields/externalRadiation/ersPlaneToCylinder/ersPlaneToCylinder.C b/src/conjugateHeatTransfer/fvPatchFields/externalRadiation/ersPlaneToCylinder/ersPlaneToCylinder.C
similarity index 100%
rename from applications/solvers/coupled/conjugateHeatTransfer/fvPatchFields/externalRadiation/ersPlaneToCylinder/ersPlaneToCylinder.C
rename to src/conjugateHeatTransfer/fvPatchFields/externalRadiation/ersPlaneToCylinder/ersPlaneToCylinder.C
diff --git a/applications/solvers/coupled/conjugateHeatTransfer/fvPatchFields/externalRadiation/ersPlaneToCylinder/ersPlaneToCylinder.H b/src/conjugateHeatTransfer/fvPatchFields/externalRadiation/ersPlaneToCylinder/ersPlaneToCylinder.H
similarity index 100%
rename from applications/solvers/coupled/conjugateHeatTransfer/fvPatchFields/externalRadiation/ersPlaneToCylinder/ersPlaneToCylinder.H
rename to src/conjugateHeatTransfer/fvPatchFields/externalRadiation/ersPlaneToCylinder/ersPlaneToCylinder.H
diff --git a/applications/solvers/coupled/conjugateHeatTransfer/fvPatchFields/externalRadiation/ersPointSource/ersPointSource.C b/src/conjugateHeatTransfer/fvPatchFields/externalRadiation/ersPointSource/ersPointSource.C
similarity index 100%
rename from applications/solvers/coupled/conjugateHeatTransfer/fvPatchFields/externalRadiation/ersPointSource/ersPointSource.C
rename to src/conjugateHeatTransfer/fvPatchFields/externalRadiation/ersPointSource/ersPointSource.C
diff --git a/applications/solvers/coupled/conjugateHeatTransfer/fvPatchFields/externalRadiation/ersPointSource/ersPointSource.H b/src/conjugateHeatTransfer/fvPatchFields/externalRadiation/ersPointSource/ersPointSource.H
similarity index 100%
rename from applications/solvers/coupled/conjugateHeatTransfer/fvPatchFields/externalRadiation/ersPointSource/ersPointSource.H
rename to src/conjugateHeatTransfer/fvPatchFields/externalRadiation/ersPointSource/ersPointSource.H
diff --git a/applications/solvers/coupled/conjugateHeatTransfer/fvPatchFields/externalRadiation/ersViewFactor/ersViewFactor.C b/src/conjugateHeatTransfer/fvPatchFields/externalRadiation/ersViewFactor/ersViewFactor.C
similarity index 100%
rename from applications/solvers/coupled/conjugateHeatTransfer/fvPatchFields/externalRadiation/ersViewFactor/ersViewFactor.C
rename to src/conjugateHeatTransfer/fvPatchFields/externalRadiation/ersViewFactor/ersViewFactor.C
diff --git a/applications/solvers/coupled/conjugateHeatTransfer/fvPatchFields/externalRadiation/ersViewFactor/ersViewFactor.H b/src/conjugateHeatTransfer/fvPatchFields/externalRadiation/ersViewFactor/ersViewFactor.H
similarity index 100%
rename from applications/solvers/coupled/conjugateHeatTransfer/fvPatchFields/externalRadiation/ersViewFactor/ersViewFactor.H
rename to src/conjugateHeatTransfer/fvPatchFields/externalRadiation/ersViewFactor/ersViewFactor.H
diff --git a/applications/solvers/coupled/conjugateHeatTransfer/fvPatchFields/externalRadiation/externalRadiationSource/externalRadiationSource.C b/src/conjugateHeatTransfer/fvPatchFields/externalRadiation/externalRadiationSource/externalRadiationSource.C
similarity index 100%
rename from applications/solvers/coupled/conjugateHeatTransfer/fvPatchFields/externalRadiation/externalRadiationSource/externalRadiationSource.C
rename to src/conjugateHeatTransfer/fvPatchFields/externalRadiation/externalRadiationSource/externalRadiationSource.C
diff --git a/applications/solvers/coupled/conjugateHeatTransfer/fvPatchFields/externalRadiation/externalRadiationSource/externalRadiationSource.H b/src/conjugateHeatTransfer/fvPatchFields/externalRadiation/externalRadiationSource/externalRadiationSource.H
similarity index 100%
rename from applications/solvers/coupled/conjugateHeatTransfer/fvPatchFields/externalRadiation/externalRadiationSource/externalRadiationSource.H
rename to src/conjugateHeatTransfer/fvPatchFields/externalRadiation/externalRadiationSource/externalRadiationSource.H
diff --git a/applications/solvers/coupled/conjugateHeatTransfer/fvPatchFields/externalRadiation/externalRadiationSource/newExternalRadiationSource.C b/src/conjugateHeatTransfer/fvPatchFields/externalRadiation/externalRadiationSource/newExternalRadiationSource.C
similarity index 100%
rename from applications/solvers/coupled/conjugateHeatTransfer/fvPatchFields/externalRadiation/externalRadiationSource/newExternalRadiationSource.C
rename to src/conjugateHeatTransfer/fvPatchFields/externalRadiation/externalRadiationSource/newExternalRadiationSource.C
diff --git a/applications/solvers/coupled/conjugateHeatTransfer/fvPatchFields/fieldStorage/fieldStorage.C b/src/conjugateHeatTransfer/fvPatchFields/fieldStorage/fieldStorage.C
similarity index 100%
rename from applications/solvers/coupled/conjugateHeatTransfer/fvPatchFields/fieldStorage/fieldStorage.C
rename to src/conjugateHeatTransfer/fvPatchFields/fieldStorage/fieldStorage.C
diff --git a/applications/solvers/coupled/conjugateHeatTransfer/fvPatchFields/fieldStorage/fieldStorage.H b/src/conjugateHeatTransfer/fvPatchFields/fieldStorage/fieldStorage.H
similarity index 100%
rename from applications/solvers/coupled/conjugateHeatTransfer/fvPatchFields/fieldStorage/fieldStorage.H
rename to src/conjugateHeatTransfer/fvPatchFields/fieldStorage/fieldStorage.H
diff --git a/applications/solvers/coupled/conjugateHeatTransfer/fvPatchFields/solidWallHeatFluxTemperature/solidWallHeatFluxTemperatureFvPatchScalarField.C b/src/conjugateHeatTransfer/fvPatchFields/solidWallHeatFluxTemperature/solidWallHeatFluxTemperatureFvPatchScalarField.C
similarity index 100%
rename from applications/solvers/coupled/conjugateHeatTransfer/fvPatchFields/solidWallHeatFluxTemperature/solidWallHeatFluxTemperatureFvPatchScalarField.C
rename to src/conjugateHeatTransfer/fvPatchFields/solidWallHeatFluxTemperature/solidWallHeatFluxTemperatureFvPatchScalarField.C
diff --git a/applications/solvers/coupled/conjugateHeatTransfer/fvPatchFields/solidWallHeatFluxTemperature/solidWallHeatFluxTemperatureFvPatchScalarField.H b/src/conjugateHeatTransfer/fvPatchFields/solidWallHeatFluxTemperature/solidWallHeatFluxTemperatureFvPatchScalarField.H
similarity index 100%
rename from applications/solvers/coupled/conjugateHeatTransfer/fvPatchFields/solidWallHeatFluxTemperature/solidWallHeatFluxTemperatureFvPatchScalarField.H
rename to src/conjugateHeatTransfer/fvPatchFields/solidWallHeatFluxTemperature/solidWallHeatFluxTemperatureFvPatchScalarField.H
diff --git a/applications/solvers/coupled/conjugateHeatTransfer/thermalModel/thermalGaps/constantResistanceThermalGap/constantResistanceThermalGap.C b/src/conjugateHeatTransfer/thermalModel/thermalGaps/constantResistanceThermalGap/constantResistanceThermalGap.C
similarity index 100%
rename from applications/solvers/coupled/conjugateHeatTransfer/thermalModel/thermalGaps/constantResistanceThermalGap/constantResistanceThermalGap.C
rename to src/conjugateHeatTransfer/thermalModel/thermalGaps/constantResistanceThermalGap/constantResistanceThermalGap.C
diff --git a/applications/solvers/coupled/conjugateHeatTransfer/thermalModel/thermalGaps/constantResistanceThermalGap/constantResistanceThermalGap.H b/src/conjugateHeatTransfer/thermalModel/thermalGaps/constantResistanceThermalGap/constantResistanceThermalGap.H
similarity index 100%
rename from applications/solvers/coupled/conjugateHeatTransfer/thermalModel/thermalGaps/constantResistanceThermalGap/constantResistanceThermalGap.H
rename to src/conjugateHeatTransfer/thermalModel/thermalGaps/constantResistanceThermalGap/constantResistanceThermalGap.H
diff --git a/applications/solvers/coupled/conjugateHeatTransfer/thermalModel/thermalGaps/constantThermalGap/constantThermalGap.C b/src/conjugateHeatTransfer/thermalModel/thermalGaps/constantThermalGap/constantThermalGap.C
similarity index 100%
rename from applications/solvers/coupled/conjugateHeatTransfer/thermalModel/thermalGaps/constantThermalGap/constantThermalGap.C
rename to src/conjugateHeatTransfer/thermalModel/thermalGaps/constantThermalGap/constantThermalGap.C
diff --git a/applications/solvers/coupled/conjugateHeatTransfer/thermalModel/thermalGaps/constantThermalGap/constantThermalGap.H b/src/conjugateHeatTransfer/thermalModel/thermalGaps/constantThermalGap/constantThermalGap.H
similarity index 100%
rename from applications/solvers/coupled/conjugateHeatTransfer/thermalModel/thermalGaps/constantThermalGap/constantThermalGap.H
rename to src/conjugateHeatTransfer/thermalModel/thermalGaps/constantThermalGap/constantThermalGap.H
diff --git a/applications/solvers/coupled/conjugateHeatTransfer/thermalModel/thermalGaps/thermalGap/newThermalGap.C b/src/conjugateHeatTransfer/thermalModel/thermalGaps/thermalGap/newThermalGap.C
similarity index 100%
rename from applications/solvers/coupled/conjugateHeatTransfer/thermalModel/thermalGaps/thermalGap/newThermalGap.C
rename to src/conjugateHeatTransfer/thermalModel/thermalGaps/thermalGap/newThermalGap.C
diff --git a/applications/solvers/coupled/conjugateHeatTransfer/thermalModel/thermalGaps/thermalGap/thermalGap.C b/src/conjugateHeatTransfer/thermalModel/thermalGaps/thermalGap/thermalGap.C
similarity index 100%
rename from applications/solvers/coupled/conjugateHeatTransfer/thermalModel/thermalGaps/thermalGap/thermalGap.C
rename to src/conjugateHeatTransfer/thermalModel/thermalGaps/thermalGap/thermalGap.C
diff --git a/applications/solvers/coupled/conjugateHeatTransfer/thermalModel/thermalGaps/thermalGap/thermalGap.H b/src/conjugateHeatTransfer/thermalModel/thermalGaps/thermalGap/thermalGap.H
similarity index 100%
rename from applications/solvers/coupled/conjugateHeatTransfer/thermalModel/thermalGaps/thermalGap/thermalGap.H
rename to src/conjugateHeatTransfer/thermalModel/thermalGaps/thermalGap/thermalGap.H
diff --git a/applications/solvers/coupled/conjugateHeatTransfer/thermalModel/thermalLaws/constantThermal/constantThermal.C b/src/conjugateHeatTransfer/thermalModel/thermalLaws/constantThermal/constantThermal.C
similarity index 100%
rename from applications/solvers/coupled/conjugateHeatTransfer/thermalModel/thermalLaws/constantThermal/constantThermal.C
rename to src/conjugateHeatTransfer/thermalModel/thermalLaws/constantThermal/constantThermal.C
diff --git a/applications/solvers/coupled/conjugateHeatTransfer/thermalModel/thermalLaws/constantThermal/constantThermal.H b/src/conjugateHeatTransfer/thermalModel/thermalLaws/constantThermal/constantThermal.H
similarity index 100%
rename from applications/solvers/coupled/conjugateHeatTransfer/thermalModel/thermalLaws/constantThermal/constantThermal.H
rename to src/conjugateHeatTransfer/thermalModel/thermalLaws/constantThermal/constantThermal.H
diff --git a/applications/solvers/coupled/conjugateHeatTransfer/thermalModel/thermalLaws/multiMaterialThermal/multiMaterialThermal.C b/src/conjugateHeatTransfer/thermalModel/thermalLaws/multiMaterialThermal/multiMaterialThermal.C
similarity index 100%
rename from applications/solvers/coupled/conjugateHeatTransfer/thermalModel/thermalLaws/multiMaterialThermal/multiMaterialThermal.C
rename to src/conjugateHeatTransfer/thermalModel/thermalLaws/multiMaterialThermal/multiMaterialThermal.C
diff --git a/applications/solvers/coupled/conjugateHeatTransfer/thermalModel/thermalLaws/multiMaterialThermal/multiMaterialThermal.H b/src/conjugateHeatTransfer/thermalModel/thermalLaws/multiMaterialThermal/multiMaterialThermal.H
similarity index 100%
rename from applications/solvers/coupled/conjugateHeatTransfer/thermalModel/thermalLaws/multiMaterialThermal/multiMaterialThermal.H
rename to src/conjugateHeatTransfer/thermalModel/thermalLaws/multiMaterialThermal/multiMaterialThermal.H
diff --git a/applications/solvers/coupled/conjugateHeatTransfer/thermalModel/thermalLaws/multiMaterialZonesThermal/multiMaterialZonesThermal.C b/src/conjugateHeatTransfer/thermalModel/thermalLaws/multiMaterialZonesThermal/multiMaterialZonesThermal.C
similarity index 100%
rename from applications/solvers/coupled/conjugateHeatTransfer/thermalModel/thermalLaws/multiMaterialZonesThermal/multiMaterialZonesThermal.C
rename to src/conjugateHeatTransfer/thermalModel/thermalLaws/multiMaterialZonesThermal/multiMaterialZonesThermal.C
diff --git a/applications/solvers/coupled/conjugateHeatTransfer/thermalModel/thermalLaws/multiMaterialZonesThermal/multiMaterialZonesThermal.H b/src/conjugateHeatTransfer/thermalModel/thermalLaws/multiMaterialZonesThermal/multiMaterialZonesThermal.H
similarity index 100%
rename from applications/solvers/coupled/conjugateHeatTransfer/thermalModel/thermalLaws/multiMaterialZonesThermal/multiMaterialZonesThermal.H
rename to src/conjugateHeatTransfer/thermalModel/thermalLaws/multiMaterialZonesThermal/multiMaterialZonesThermal.H
diff --git a/applications/solvers/coupled/conjugateHeatTransfer/thermalModel/thermalLaws/thermalLaw/newThermalLaw.C b/src/conjugateHeatTransfer/thermalModel/thermalLaws/thermalLaw/newThermalLaw.C
similarity index 100%
rename from applications/solvers/coupled/conjugateHeatTransfer/thermalModel/thermalLaws/thermalLaw/newThermalLaw.C
rename to src/conjugateHeatTransfer/thermalModel/thermalLaws/thermalLaw/newThermalLaw.C
diff --git a/applications/solvers/coupled/conjugateHeatTransfer/thermalModel/thermalLaws/thermalLaw/thermalLaw.C b/src/conjugateHeatTransfer/thermalModel/thermalLaws/thermalLaw/thermalLaw.C
similarity index 100%
rename from applications/solvers/coupled/conjugateHeatTransfer/thermalModel/thermalLaws/thermalLaw/thermalLaw.C
rename to src/conjugateHeatTransfer/thermalModel/thermalLaws/thermalLaw/thermalLaw.C
diff --git a/applications/solvers/coupled/conjugateHeatTransfer/thermalModel/thermalLaws/thermalLaw/thermalLaw.H b/src/conjugateHeatTransfer/thermalModel/thermalLaws/thermalLaw/thermalLaw.H
similarity index 100%
rename from applications/solvers/coupled/conjugateHeatTransfer/thermalModel/thermalLaws/thermalLaw/thermalLaw.H
rename to src/conjugateHeatTransfer/thermalModel/thermalLaws/thermalLaw/thermalLaw.H
diff --git a/applications/solvers/coupled/conjugateHeatTransfer/thermalModel/thermalModel.C b/src/conjugateHeatTransfer/thermalModel/thermalModel.C
similarity index 100%
rename from applications/solvers/coupled/conjugateHeatTransfer/thermalModel/thermalModel.C
rename to src/conjugateHeatTransfer/thermalModel/thermalModel.C
diff --git a/applications/solvers/coupled/conjugateHeatTransfer/thermalModel/thermalModel.H b/src/conjugateHeatTransfer/thermalModel/thermalModel.H
similarity index 100%
rename from applications/solvers/coupled/conjugateHeatTransfer/thermalModel/thermalModel.H
rename to src/conjugateHeatTransfer/thermalModel/thermalModel.H
diff --git a/applications/solvers/coupled/conjugateHeatTransfer/thermalModel/thermalSource/constantThermalSource/constantThermalSource.C b/src/conjugateHeatTransfer/thermalModel/thermalSource/constantThermalSource/constantThermalSource.C
similarity index 100%
rename from applications/solvers/coupled/conjugateHeatTransfer/thermalModel/thermalSource/constantThermalSource/constantThermalSource.C
rename to src/conjugateHeatTransfer/thermalModel/thermalSource/constantThermalSource/constantThermalSource.C
diff --git a/applications/solvers/coupled/conjugateHeatTransfer/thermalModel/thermalSource/constantThermalSource/constantThermalSource.H b/src/conjugateHeatTransfer/thermalModel/thermalSource/constantThermalSource/constantThermalSource.H
similarity index 100%
rename from applications/solvers/coupled/conjugateHeatTransfer/thermalModel/thermalSource/constantThermalSource/constantThermalSource.H
rename to src/conjugateHeatTransfer/thermalModel/thermalSource/constantThermalSource/constantThermalSource.H
diff --git a/applications/solvers/coupled/conjugateHeatTransfer/thermalModel/thermalSource/thermalSource/newThermalSource.C b/src/conjugateHeatTransfer/thermalModel/thermalSource/thermalSource/newThermalSource.C
similarity index 100%
rename from applications/solvers/coupled/conjugateHeatTransfer/thermalModel/thermalSource/thermalSource/newThermalSource.C
rename to src/conjugateHeatTransfer/thermalModel/thermalSource/thermalSource/newThermalSource.C
diff --git a/applications/solvers/coupled/conjugateHeatTransfer/thermalModel/thermalSource/thermalSource/thermalSource.C b/src/conjugateHeatTransfer/thermalModel/thermalSource/thermalSource/thermalSource.C
similarity index 100%
rename from applications/solvers/coupled/conjugateHeatTransfer/thermalModel/thermalSource/thermalSource/thermalSource.C
rename to src/conjugateHeatTransfer/thermalModel/thermalSource/thermalSource/thermalSource.C
diff --git a/applications/solvers/coupled/conjugateHeatTransfer/thermalModel/thermalSource/thermalSource/thermalSource.H b/src/conjugateHeatTransfer/thermalModel/thermalSource/thermalSource/thermalSource.H
similarity index 100%
rename from applications/solvers/coupled/conjugateHeatTransfer/thermalModel/thermalSource/thermalSource/thermalSource.H
rename to src/conjugateHeatTransfer/thermalModel/thermalSource/thermalSource/thermalSource.H

From a8599a8f9ae1a23814dd256ea22091a5873f0b84 Mon Sep 17 00:00:00 2001
From: Hrvoje Jasak <h.jasak@wikki.co.uk>
Date: Mon, 6 Feb 2017 11:31:35 +0000
Subject: [PATCH 118/130] Removed local Allwmake - not needed

---
 applications/solvers/coupled/Allwmake | 13 -------------
 1 file changed, 13 deletions(-)
 delete mode 100755 applications/solvers/coupled/Allwmake

diff --git a/applications/solvers/coupled/Allwmake b/applications/solvers/coupled/Allwmake
deleted file mode 100755
index e6320de43..000000000
--- a/applications/solvers/coupled/Allwmake
+++ /dev/null
@@ -1,13 +0,0 @@
-#!/bin/sh
-cd ${0%/*} || exit 1    # run from this directory
-set -x
-
-wmake libso conjugateHeatTransfer
-
-wmake blockCoupledScalarTransportFoam
-wmake conjugateHeatFoam
-wmake conjugateHeatSimpleFoam
-wmake pUCoupledFoam
-wmake MRFPorousFoam
-
-# ----------------------------------------------------------------- end-of-file

From a8ddd0885fe8c5c751fea1c7354c24dae311996f Mon Sep 17 00:00:00 2001
From: Hrvoje Jasak <h.jasak@wikki.co.uk>
Date: Mon, 6 Feb 2017 11:32:06 +0000
Subject: [PATCH 119/130] Added option for zonal under-relaxation

---
 .../fvMatrices/fvMatrix/fvMatrix.C            | 166 ++++++++++++++++--
 .../fvMatrices/fvMatrix/fvMatrix.H            |   8 +
 2 files changed, 158 insertions(+), 16 deletions(-)

diff --git a/src/finiteVolume/fvMatrices/fvMatrix/fvMatrix.C b/src/finiteVolume/fvMatrices/fvMatrix/fvMatrix.C
index 793a55d90..39e7d4447 100644
--- a/src/finiteVolume/fvMatrices/fvMatrix/fvMatrix.C
+++ b/src/finiteVolume/fvMatrices/fvMatrix/fvMatrix.C
@@ -56,7 +56,7 @@ void Foam::fvMatrix<Type>::addToInternalField
             << abort(FatalError);
     }
 
-    forAll(addr, faceI)
+    forAll (addr, faceI)
     {
         intf[addr[faceI]] += pf[faceI];
     }
@@ -96,7 +96,7 @@ void Foam::fvMatrix<Type>::subtractFromInternalField
             << abort(FatalError);
     }
 
-    forAll(addr, faceI)
+    forAll (addr, faceI)
     {
         intf[addr[faceI]] -= pf[faceI];
     }
@@ -124,7 +124,7 @@ void Foam::fvMatrix<Type>::addBoundaryDiag
     const direction solveCmpt
 ) const
 {
-    forAll(internalCoeffs_, patchI)
+    forAll (internalCoeffs_, patchI)
     {
         addToInternalField
         (
@@ -139,7 +139,7 @@ void Foam::fvMatrix<Type>::addBoundaryDiag
 template<class Type>
 void Foam::fvMatrix<Type>::addCmptAvBoundaryDiag(scalarField& diag) const
 {
-    forAll(internalCoeffs_, patchI)
+    forAll (internalCoeffs_, patchI)
     {
         addToInternalField
         (
@@ -158,7 +158,7 @@ void Foam::fvMatrix<Type>::addBoundarySource
     const bool couples
 ) const
 {
-    forAll(psi_.boundaryField(), patchI)
+    forAll (psi_.boundaryField(), patchI)
     {
         const fvPatchField<Type>& ptf = psi_.boundaryField()[patchI];
         const Field<Type>& pbc = boundaryCoeffs_[patchI];
@@ -190,7 +190,7 @@ void Foam::fvMatrix<Type>::correctImplicitBoundarySource
     const direction cmpt
 ) const
 {
-    forAll(psi_.boundaryField(), patchI)
+    forAll (psi_.boundaryField(), patchI)
     {
         const fvPatchField<Type>& ptf = psi_.boundaryField()[patchI];
         const scalarField& pbc = bouCoeffsCmpt[patchI];
@@ -492,7 +492,7 @@ void Foam::fvMatrix<Type>::setValues
             GeometricField<Type, fvPatchField, volMesh>&
         >(psi_).internalField();
 
-    forAll(cellLabels, i)
+    forAll (cellLabels, i)
     {
         label celli = cellLabels[i];
 
@@ -503,7 +503,7 @@ void Foam::fvMatrix<Type>::setValues
         {
             const cell& c = cells[celli];
 
-            forAll(c, j)
+            forAll (c, j)
             {
                 label facei = c[j];
 
@@ -622,7 +622,7 @@ void Foam::fvMatrix<Type>::relax(const scalar alpha)
     sumMagOffDiag(sumOff);
 
     // Handle the boundary contributions to the diagonal
-    forAll(psi_.boundaryField(), patchI)
+    forAll (psi_.boundaryField(), patchI)
     {
         const fvPatchField<Type>& ptf = psi_.boundaryField()[patchI];
 
@@ -637,7 +637,7 @@ void Foam::fvMatrix<Type>::relax(const scalar alpha)
 
                 // For coupled boundaries add the diagonal and
                 // off-diagonal contributions
-                forAll(pa, face)
+                forAll (pa, face)
                 {
                     D[pa[face]] += component(iCoeffs[face], 0);
                     sumOff[pa[face]] += mag(component(pCoeffs[face], 0));
@@ -649,7 +649,7 @@ void Foam::fvMatrix<Type>::relax(const scalar alpha)
                 // contribution off-diagonal sum which avoids having to remove
                 // it from the diagonal later.
                 // Also add the source contribution from the relaxation
-                forAll(pa, face)
+                forAll (pa, face)
                 {
                     Type iCoeff0 = iCoeffs[face];
                     iCoeffs[face] = cmptMag(iCoeffs[face]);
@@ -669,7 +669,7 @@ void Foam::fvMatrix<Type>::relax(const scalar alpha)
     D /= alpha;
 
     // Now remove the diagonal contribution from coupled boundaries
-    forAll(psi_.boundaryField(), patchI)
+    forAll (psi_.boundaryField(), patchI)
     {
         const fvPatchField<Type>& ptf = psi_.boundaryField()[patchI];
 
@@ -680,7 +680,7 @@ void Foam::fvMatrix<Type>::relax(const scalar alpha)
 
             if (ptf.coupled())
             {
-                forAll(pa, face)
+                forAll (pa, face)
                 {
                     D[pa[face]] -= component(iCoeffs[face], 0);
                 }
@@ -715,6 +715,140 @@ void Foam::fvMatrix<Type>::relax()
 }
 
 
+template<class Type>
+void Foam::fvMatrix<Type>::relax
+(
+    const scalar alpha,
+    const word& cellZoneName
+)
+{
+    if (alpha <= 0)
+    {
+        return;
+    }
+
+    // Find cell zone and check it is not empty
+    const label zoneID = psi_.mesh().cellZones().findZoneID(cellZoneName);
+
+    if (zoneID < 0)
+    {
+        WarningIn
+        (
+            "void fvMatrix<Type>::relax\n"
+            "(\n"
+            "    const scalar alpha,\n"
+            "    const word& cellZoneName\n"
+            ")"
+        )   << "Cannot find cell zone " << cellZoneName
+            << ".  No relaxation applied"
+            << endl;
+
+        return;
+    }
+
+    if (psi_.mesh().cellZones()[zoneID].empty())
+    {
+        // Zone empty, skip
+        return;
+    }
+
+    Field<Type>& S = source();
+    scalarField& D = diag();
+
+    // Store the current unrelaxed diagonal for use in updating the source
+    scalarField D0(D);
+
+    // Calculate the sum-mag off-diagonal from the interior faces
+    scalarField sumOff(D.size(), 0.0);
+    sumMagOffDiag(sumOff);
+
+    // Handle the boundary contributions to the diagonal
+    forAll (psi_.boundaryField(), patchI)
+    {
+        const fvPatchField<Type>& ptf = psi_.boundaryField()[patchI];
+
+        if (ptf.size())
+        {
+            const unallocLabelList& pa = lduAddr().patchAddr(patchI);
+            Field<Type>& iCoeffs = internalCoeffs_[patchI];
+
+            if (ptf.coupled())
+            {
+                const Field<Type>& pCoeffs = boundaryCoeffs_[patchI];
+
+                // For coupled boundaries add the diagonal and
+                // off-diagonal contributions
+                forAll (pa, face)
+                {
+                    D[pa[face]] += component(iCoeffs[face], 0);
+                    sumOff[pa[face]] += mag(component(pCoeffs[face], 0));
+                }
+            }
+            else
+            {
+                // For non-coupled boundaries subtract the diagonal
+                // contribution off-diagonal sum which avoids having to remove
+                // it from the diagonal later.
+                // Also add the source contribution from the relaxation
+                forAll (pa, face)
+                {
+                    Type iCoeff0 = iCoeffs[face];
+                    iCoeffs[face] = cmptMag(iCoeffs[face]);
+                    sumOff[pa[face]] -= cmptMin(iCoeffs[face]);
+                    iCoeffs[face] /= alpha;
+                    S[pa[face]] +=
+                        cmptMultiply(iCoeffs[face] - iCoeff0, psi_[pa[face]]);
+                }
+            }
+        }
+    }
+
+    // Under-relax only cells within the zone
+    const labelList& zoneCells = psi_.mesh().cellZones()[zoneID];
+
+    forAll (zoneCells, zcI)
+    {
+        const label cellI = zoneCells[zcI];
+
+        // Ensure the matrix is diagonally dominant...
+        D[cellI] = max(D[cellI], sumOff[cellI]);
+
+        // ... then relax
+        D[cellI] /= alpha;
+    }
+
+    // Now remove the diagonal contribution from coupled boundaries
+    forAll (psi_.boundaryField(), patchI)
+    {
+        const fvPatchField<Type>& ptf = psi_.boundaryField()[patchI];
+
+        if (ptf.size())
+        {
+            const unallocLabelList& pa = lduAddr().patchAddr(patchI);
+            Field<Type>& iCoeffs = internalCoeffs_[patchI];
+
+            if (ptf.coupled())
+            {
+                forAll (pa, face)
+                {
+                    D[pa[face]] -= component(iCoeffs[face], 0);
+                }
+            }
+        }
+    }
+
+    const Field<Type>& psiIn = psi_.internalField();
+
+    forAll (zoneCells, zcI)
+    {
+        const label cellI = zoneCells[zcI];
+
+        // Finally add the relaxation contribution to the source.
+        S[cellI] += (D[cellI] - D0[cellI])*psiIn[cellI];
+    }
+}
+
+
 template<class Type>
 void Foam::fvMatrix<Type>::completeAssembly()
 {
@@ -739,7 +873,7 @@ void Foam::fvMatrix<Type>::completeAssembly()
     typename GeometricField<Type, fvPatchField, volMesh>::
         GeometricBoundaryField& bFields = ncPsi.boundaryField();
 
-    forAll(bFields, patchI)
+    forAll (bFields, patchI)
     {
         bFields[patchI].manipulateMatrix(*this);
     }
@@ -760,7 +894,7 @@ Foam::tmp<Foam::Field<Type> > Foam::fvMatrix<Type>::DD() const
 {
     tmp<Field<Type> > tdiag(pTraits<Type>::one*diag());
 
-    forAll(psi_.boundaryField(), patchI)
+    forAll (psi_.boundaryField(), patchI)
     {
         const fvPatchField<Type>& ptf = psi_.boundaryField()[patchI];
 
@@ -1223,7 +1357,7 @@ void Foam::fvMatrix<Type>::operator*=
     lduMatrix::operator*=(dsf.field());
     source_ *= dsf.field();
 
-    forAll(boundaryCoeffs_, patchI)
+    forAll (boundaryCoeffs_, patchI)
     {
         scalarField psf
         (
diff --git a/src/finiteVolume/fvMatrices/fvMatrix/fvMatrix.H b/src/finiteVolume/fvMatrices/fvMatrix/fvMatrix.H
index 061e8bacf..aad197d93 100644
--- a/src/finiteVolume/fvMatrices/fvMatrix/fvMatrix.H
+++ b/src/finiteVolume/fvMatrices/fvMatrix/fvMatrix.H
@@ -357,6 +357,14 @@ public:
             //  alpha is read from controlDict
             void relax();
 
+            //- Relax matrix only within a cell zone
+            //  Experimental
+            void relax
+            (
+                const scalar alpha,
+                const word& cellZoneName
+            );
+
             //- Complete matrix assembly for solution:
             //  Manipulate based on a boundary field
             void completeAssembly();

From fdffd47248a5de3495a1089b310dd51e94811f67 Mon Sep 17 00:00:00 2001
From: Hrvoje Jasak <h.jasak@wikki.co.uk>
Date: Mon, 6 Feb 2017 11:38:05 +0000
Subject: [PATCH 120/130] Report cluster size in AMG coarsening

---
 .../BlockMatrixAgglomeration.C                     | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/src/foam/matrices/blockLduMatrix/BlockAMG/BlockMatrixCoarsening/BlockMatrixAgglomeration/BlockMatrixAgglomeration.C b/src/foam/matrices/blockLduMatrix/BlockAMG/BlockMatrixCoarsening/BlockMatrixAgglomeration/BlockMatrixAgglomeration.C
index 4a9c13d36..25c22b8fa 100644
--- a/src/foam/matrices/blockLduMatrix/BlockAMG/BlockMatrixCoarsening/BlockMatrixAgglomeration/BlockMatrixAgglomeration.C
+++ b/src/foam/matrices/blockLduMatrix/BlockAMG/BlockMatrixCoarsening/BlockMatrixAgglomeration/BlockMatrixAgglomeration.C
@@ -369,6 +369,20 @@ void Foam::BlockMatrixAgglomeration<Type>::calcAgglomeration()
         {
             Pout << ".  Rejected" << endl;
         }
+
+        // Count cluster size
+        labelList clusterSize(nCoarseEqns_, 0);
+
+        forAll (agglomIndex_, eqnI)
+        {
+            clusterSize[agglomIndex_[eqnI]]++;
+        }
+
+        label minClusterSize = gMin(clusterSize);
+        label maxClusterSize = gMax(clusterSize);
+
+        Info<< "Cluster size: min = " << minClusterSize
+            << " max = " << maxClusterSize << endl;
     }
 }
 

From 13f27d9376c0b1b777614d64a871352c97da36a9 Mon Sep 17 00:00:00 2001
From: Hrvoje Jasak <h.jasak@wikki.co.uk>
Date: Mon, 6 Feb 2017 11:43:00 +0000
Subject: [PATCH 121/130] Added default removte field name for implicitly
 created regionCoupling patch fields

---
 .../constraint/regionCoupling/regionCouplingFvPatchField.C      | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/finiteVolume/fields/fvPatchFields/constraint/regionCoupling/regionCouplingFvPatchField.C b/src/finiteVolume/fields/fvPatchFields/constraint/regionCoupling/regionCouplingFvPatchField.C
index a60d5e22c..4724300b3 100644
--- a/src/finiteVolume/fields/fvPatchFields/constraint/regionCoupling/regionCouplingFvPatchField.C
+++ b/src/finiteVolume/fields/fvPatchFields/constraint/regionCoupling/regionCouplingFvPatchField.C
@@ -84,7 +84,7 @@ regionCouplingFvPatchField<Type>::regionCouplingFvPatchField
 :
     coupledFvPatchField<Type>(p, iF, dict),
     regionCouplePatch_(refCast<const regionCoupleFvPatch>(p)),
-    remoteFieldName_(dict.lookup("remoteField")),
+    remoteFieldName_(dict.lookupOrDefault<word>("remoteField", iF.name())),
     matrixUpdateBuffer_(),
     originalPatchField_(),
     curTimeIndex_(-1)

From e98722db658607a1cb4e2a58377a95617fe7d40d Mon Sep 17 00:00:00 2001
From: Hrvoje Jasak <h.jasak@wikki.co.uk>
Date: Mon, 6 Feb 2017 11:45:17 +0000
Subject: [PATCH 122/130] Bugfix: removed SuSp stabilisation of the momentum
 equation

---
 applications/solvers/coupled/MRFPorousFoam/UEqn.H | 4 +---
 applications/solvers/coupled/pUCoupledFoam/UEqn.H | 4 +---
 2 files changed, 2 insertions(+), 6 deletions(-)

diff --git a/applications/solvers/coupled/MRFPorousFoam/UEqn.H b/applications/solvers/coupled/MRFPorousFoam/UEqn.H
index c40bb8ef2..189324c0b 100644
--- a/applications/solvers/coupled/MRFPorousFoam/UEqn.H
+++ b/applications/solvers/coupled/MRFPorousFoam/UEqn.H
@@ -52,9 +52,7 @@
         trAU().rename("rAU");
     }
 
-    // Insert the additional components. Note this will destroy the H and A
-
-    UEqn += fvm::SuSp(-divPhi, U) + divPhi*U;
+    // Under-relax momentum. Note this will destroy the H and A
     UEqn.relax();
 
     // Insert momentum equation
diff --git a/applications/solvers/coupled/pUCoupledFoam/UEqn.H b/applications/solvers/coupled/pUCoupledFoam/UEqn.H
index 6f4225be4..759e22b83 100644
--- a/applications/solvers/coupled/pUCoupledFoam/UEqn.H
+++ b/applications/solvers/coupled/pUCoupledFoam/UEqn.H
@@ -14,9 +14,7 @@
 
     rAU = 1.0/UEqn.A();
 
-    // Insert the additional components. Note this will destroy the H and A
-
-    UEqn += fvm::SuSp(-divPhi, U) + divPhi*U;
+    // Under-relax momentum. Note this will destroy the H and A
     UEqn.relax();
 
     UpEqn.insertEquation(0, UEqn);

From 6c01d9a6a8d80bebb442f5cf77eb12d5b07039ac Mon Sep 17 00:00:00 2001
From: Hrvoje Jasak <h.jasak@wikki.co.uk>
Date: Mon, 6 Feb 2017 11:47:00 +0000
Subject: [PATCH 123/130] Added rescue procedure on block solver divergence. 
 Experimental

---
 .../coupled/MRFPorousFoam/MRFPorousFoam.C     | 37 +++++++++++++++----
 .../MRFPorousFoam/initConvergenceCheck.H      |  3 +-
 2 files changed, 32 insertions(+), 8 deletions(-)

diff --git a/applications/solvers/coupled/MRFPorousFoam/MRFPorousFoam.C b/applications/solvers/coupled/MRFPorousFoam/MRFPorousFoam.C
index 4545f7093..8e96be5a4 100644
--- a/applications/solvers/coupled/MRFPorousFoam/MRFPorousFoam.C
+++ b/applications/solvers/coupled/MRFPorousFoam/MRFPorousFoam.C
@@ -81,17 +81,40 @@ int main(int argc, char *argv[])
 #       include "couplingTerms.H"
 
         // Solve the block matrix
-        maxResidual = cmptMax(UpEqn.solve().initialResidual());
+        residual = UpEqn.solve();
+        maxResidual = cmptMax(residual.initialResidual());
 
-        // Retrieve solution
-        UpEqn.retrieveSolution(0, U.internalField());
-        UpEqn.retrieveSolution(3, p.internalField());
+        // Check for divergence
+        if (mag(residual.finalResidual() - residual.initialResidual()) > 1)
+        {
+            Info<< "DIVERGENCE: rescue step" << endl;
 
-        U.correctBoundaryConditions();
-        p.correctBoundaryConditions();
+            // Solving potential flow equation and correcting velocities
+            phi = (fvc::interpolate(U) & mesh.Sf());
+            
+            solve
+            (
+                tpEqn
+             ==
+              - fvc::div(U)
+            );
 
-        phi = (fvc::interpolate(U) & mesh.Sf()) + tpEqn().flux() + tpresSource;
+            phi += tpEqn().flux() + tpresSource;
 
+            U = fvc::reconstruct(phi);
+        }
+        else
+        {
+            // Retrieve solution
+            UpEqn.retrieveSolution(0, U.internalField());
+            UpEqn.retrieveSolution(3, p.internalField());
+
+            U.correctBoundaryConditions();
+            p.correctBoundaryConditions();
+
+            phi = (fvc::interpolate(U) & mesh.Sf()) + tpEqn().flux() + tpresSource;
+        }
+        
         // Make flux relative in rotating zones
         mrfZones.relativeFlux(phi);
 
diff --git a/applications/solvers/coupled/MRFPorousFoam/initConvergenceCheck.H b/applications/solvers/coupled/MRFPorousFoam/initConvergenceCheck.H
index 482bc421d..6b4622db6 100644
--- a/applications/solvers/coupled/MRFPorousFoam/initConvergenceCheck.H
+++ b/applications/solvers/coupled/MRFPorousFoam/initConvergenceCheck.H
@@ -1,4 +1,5 @@
-// initialize values for convergence checks
+    // initialize values for convergence checks
+    BlockSolverPerformance<vector4> residual;
 
     scalar maxResidual = 0;
     scalar convergenceCriterion = 0;

From a2919d36e576b366b3d2751914295740b8e4f6ab Mon Sep 17 00:00:00 2001
From: Hrvoje Jasak <h.jasak@wikki.co.uk>
Date: Thu, 9 Feb 2017 13:40:12 +0000
Subject: [PATCH 124/130] Bugfix: updated list of contributors

---
 ListOfContributors | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/ListOfContributors b/ListOfContributors
index b0927951e..f8344c056 100644
--- a/ListOfContributors
+++ b/ListOfContributors
@@ -1,8 +1,8 @@
 #                            -*- mode: org; -*-
 #
-#+TITLE:          *Cumulative List of Contributors for foam-extend-3.0*
+#+TITLE:          *Cumulative List of Contributors for foam-extend
 #+AUTHOR:                      Hrvoje Jasak.
-#+DATE:                       10 January 2014
+#+DATE:                       9 February 2017
 #+LINK:                  http://foam-extend.org
 #+OPTIONS: author:nil
 
@@ -91,4 +91,7 @@ Contents:
     Alexander Vakhrushev
     Vuko Vukcevic
     Niklas Wikstrom
-
+    Celio Fernandes
+    Luis Ferras
+    Manoel de Araujo
+    J.Miguel Nobrega

From 6653e9a5d47abacf0d27dc8c4a48a92a3a11634c Mon Sep 17 00:00:00 2001
From: Hrvoje Jasak <h.jasak@wikki.co.uk>
Date: Fri, 10 Feb 2017 09:59:29 +0000
Subject: [PATCH 125/130] Bugfix: Aitken relaxation

---
 src/ODE/sixDOF/sixDOFqODE/sixDOFqODE.C | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/ODE/sixDOF/sixDOFqODE/sixDOFqODE.C b/src/ODE/sixDOF/sixDOFqODE/sixDOFqODE.C
index 0a40811ab..e6b54b275 100644
--- a/src/ODE/sixDOF/sixDOFqODE/sixDOFqODE.C
+++ b/src/ODE/sixDOF/sixDOFqODE/sixDOFqODE.C
@@ -598,7 +598,7 @@ void Foam::sixDOFqODE::relaxAcceleration
     const scalar maxRelFactor
 )
 {
-    if (minRelFactor - maxRelFactor < SMALL)
+    if (mag(minRelFactor - maxRelFactor) < SMALL)
     {
         // Fixed relaxation
        relaxFactorT_ = minRelFactor;

From c373b855ba586cc4529f7b4a4ec502bb3bc0040e Mon Sep 17 00:00:00 2001
From: Hrvoje Jasak <h.jasak@wikki.co.uk>
Date: Fri, 10 Feb 2017 10:00:24 +0000
Subject: [PATCH 126/130] Bugfix: do not read fluxRequired from dictionary

---
 .../finiteVolume/fvSchemes/fvSchemes.C        | 33 ++++++++-----------
 1 file changed, 13 insertions(+), 20 deletions(-)

diff --git a/src/finiteVolume/finiteVolume/fvSchemes/fvSchemes.C b/src/finiteVolume/finiteVolume/fvSchemes/fvSchemes.C
index 60e5b94cb..a909634c0 100644
--- a/src/finiteVolume/finiteVolume/fvSchemes/fvSchemes.C
+++ b/src/finiteVolume/finiteVolume/fvSchemes/fvSchemes.C
@@ -166,14 +166,7 @@ Foam::fvSchemes::fvSchemes(const objectRegistry& obr)
         laplacianSchemes_.name() + "::default",
         tokenList()
     ),
-    fluxRequired_
-    (
-        ITstream
-        (
-            objectPath() + "::fluxRequired",
-            tokenList()
-        )()
-    ),
+    fluxRequired_(),  // Do not read flux required option
     defaultFluxRequired_(false)
 {
     if (!headerOk())
@@ -386,19 +379,19 @@ bool Foam::fvSchemes::read()
         }
 
 
-        if (dict.found("fluxRequired"))
-        {
-            fluxRequired_ = dict.subDict("fluxRequired");
+        // if (dict.found("fluxRequired"))
+        // {
+        //     fluxRequired_ = dict.subDict("fluxRequired");
 
-            if
-            (
-                fluxRequired_.found("default")
-             && word(fluxRequired_.lookup("default")) != "none"
-            )
-            {
-                defaultFluxRequired_ = Switch(fluxRequired_.lookup("default"));
-            }
-        }
+        //     if
+        //     (
+        //         fluxRequired_.found("default")
+        //      && word(fluxRequired_.lookup("default")) != "none"
+        //     )
+        //     {
+        //         defaultFluxRequired_ = Switch(fluxRequired_.lookup("default"));
+        //     }
+        // }
 
         return true;
     }

From f9540ece34b2c668dcae986d0b8e485a7eb887c0 Mon Sep 17 00:00:00 2001
From: Hrvoje Jasak <h.jasak@wikki.co.uk>
Date: Fri, 10 Feb 2017 10:01:22 +0000
Subject: [PATCH 127/130] Bugfix: Added handling for matrix constraints

---
 .../lduInterfaceField/lduInterfaceField.H     |  9 +++++++++
 .../matrices/lduMatrix/lduMatrix/lduMatrix.H  |  9 +++++++++
 .../lduMatrixUpdateMatrixInterfaces.C         | 20 +++++++++++++++++++
 3 files changed, 38 insertions(+)

diff --git a/src/foam/matrices/lduMatrix/lduAddressing/lduInterfaceFields/lduInterfaceField/lduInterfaceField.H b/src/foam/matrices/lduMatrix/lduAddressing/lduInterfaceFields/lduInterfaceField/lduInterfaceField.H
index facf60c09..e9a0273ab 100644
--- a/src/foam/matrices/lduMatrix/lduAddressing/lduInterfaceFields/lduInterfaceField/lduInterfaceField.H
+++ b/src/foam/matrices/lduMatrix/lduAddressing/lduInterfaceFields/lduInterfaceField/lduInterfaceField.H
@@ -161,6 +161,15 @@ public:
                 const Pstream::commsTypes commsType,
                 const bool switchToLhs
             ) const = 0;
+
+            //- Update the constraints
+            virtual void updateConstraints
+            (
+                const scalarField&,
+                scalarField&,
+                const label
+            ) const
+            {}
 };
 
 
diff --git a/src/foam/matrices/lduMatrix/lduMatrix/lduMatrix.H b/src/foam/matrices/lduMatrix/lduMatrix/lduMatrix.H
index 09c9805c7..8bcf13a0e 100644
--- a/src/foam/matrices/lduMatrix/lduMatrix/lduMatrix.H
+++ b/src/foam/matrices/lduMatrix/lduMatrix/lduMatrix.H
@@ -838,6 +838,15 @@ public:
                 const bool switchToLhs = false
             ) const;
 
+            //- Update the constraints operating on the solution and the source
+            //  of the system
+            void updateInterfaceConstraints
+            (
+                const scalarField& xif,
+                scalarField& residual,
+                const lduInterfaceFieldPtrsList& interfaces
+            ) const;
+
 
             template<class Type>
             tmp<Field<Type> > H(const Field<Type>&) const;
diff --git a/src/foam/matrices/lduMatrix/lduMatrix/lduMatrixUpdateMatrixInterfaces.C b/src/foam/matrices/lduMatrix/lduMatrix/lduMatrixUpdateMatrixInterfaces.C
index 39995364b..557663e49 100644
--- a/src/foam/matrices/lduMatrix/lduMatrix/lduMatrixUpdateMatrixInterfaces.C
+++ b/src/foam/matrices/lduMatrix/lduMatrix/lduMatrixUpdateMatrixInterfaces.C
@@ -285,4 +285,24 @@ void Foam::lduMatrix::updateMatrixInterfaces
 }
 
 
+void Foam::lduMatrix::updateInterfaceConstraints
+(
+    const scalarField& xif,
+    scalarField& residual,
+    const lduInterfaceFieldPtrsList& interfaces
+) const
+{
+    label counter = 0;
+
+    forAll (interfaces, interfaceI)
+    {
+        if (interfaces.set(interfaceI))
+        {
+            interfaces[interfaceI].updateConstraints(xif, residual, counter);
+            ++counter;
+        }
+    }
+}
+
+
 // ************************************************************************* //

From c2e734d4be466f51bab24c926361a28547a94298 Mon Sep 17 00:00:00 2001
From: Hrvoje Jasak <h.jasak@wikki.co.uk>
Date: Fri, 10 Feb 2017 10:01:49 +0000
Subject: [PATCH 128/130] Formatting

---
 applications/utilities/preProcessing/convertPhi/convertPhi.C | 1 -
 1 file changed, 1 deletion(-)

diff --git a/applications/utilities/preProcessing/convertPhi/convertPhi.C b/applications/utilities/preProcessing/convertPhi/convertPhi.C
index ccf61d047..55b4b4937 100644
--- a/applications/utilities/preProcessing/convertPhi/convertPhi.C
+++ b/applications/utilities/preProcessing/convertPhi/convertPhi.C
@@ -46,7 +46,6 @@ int main(int argc, char *argv[])
         FatalError.exit();
     }
 
-
     dimensionedScalar rhoRef
     (
         "rhoRef",

From fc2276c2a9d19c2c26235b27c82579fa13179ffe Mon Sep 17 00:00:00 2001
From: Hrvoje Jasak <h.jasak@wikki.co.uk>
Date: Fri, 10 Feb 2017 10:02:07 +0000
Subject: [PATCH 129/130] Bugfix: order of operators for dot-product

---
 applications/solvers/compressible/steadyCompressibleFoam/pEqn.H | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/applications/solvers/compressible/steadyCompressibleFoam/pEqn.H b/applications/solvers/compressible/steadyCompressibleFoam/pEqn.H
index 2a46da754..e954c7b13 100644
--- a/applications/solvers/compressible/steadyCompressibleFoam/pEqn.H
+++ b/applications/solvers/compressible/steadyCompressibleFoam/pEqn.H
@@ -13,7 +13,7 @@
         U = rUA*UEqn.H();
 
         // Calculate phi for boundary conditions
-        phi = rhof*fvc::interpolate(U) & mesh.Sf();
+        phi = rhof*(fvc::interpolate(U) & mesh.Sf());
 
         surfaceScalarField phid2 = rhoReff/rhof*phi;
 

From 6df1d9861d1a2cd819cf3cedffc6dcba7b928b9e Mon Sep 17 00:00:00 2001
From: Hrvoje Jasak <h.jasak@wikki.co.uk>
Date: Fri, 10 Feb 2017 10:03:04 +0000
Subject: [PATCH 130/130] Added option to relax only a part of the matrix

---
 .../fvMatrices/fvMatrix/fvMatrix.C            | 24 ++-----------------
 .../fvMatrices/fvMatrix/fvMatrix.H            |  2 +-
 2 files changed, 3 insertions(+), 23 deletions(-)

diff --git a/src/finiteVolume/fvMatrices/fvMatrix/fvMatrix.C b/src/finiteVolume/fvMatrices/fvMatrix/fvMatrix.C
index 39e7d4447..2909f3fea 100644
--- a/src/finiteVolume/fvMatrices/fvMatrix/fvMatrix.C
+++ b/src/finiteVolume/fvMatrices/fvMatrix/fvMatrix.C
@@ -719,7 +719,7 @@ template<class Type>
 void Foam::fvMatrix<Type>::relax
 (
     const scalar alpha,
-    const word& cellZoneName
+    const labelList& zoneCells
 )
 {
     if (alpha <= 0)
@@ -727,26 +727,7 @@ void Foam::fvMatrix<Type>::relax
         return;
     }
 
-    // Find cell zone and check it is not empty
-    const label zoneID = psi_.mesh().cellZones().findZoneID(cellZoneName);
-
-    if (zoneID < 0)
-    {
-        WarningIn
-        (
-            "void fvMatrix<Type>::relax\n"
-            "(\n"
-            "    const scalar alpha,\n"
-            "    const word& cellZoneName\n"
-            ")"
-        )   << "Cannot find cell zone " << cellZoneName
-            << ".  No relaxation applied"
-            << endl;
-
-        return;
-    }
-
-    if (psi_.mesh().cellZones()[zoneID].empty())
+    if (zoneCells.empty())
     {
         // Zone empty, skip
         return;
@@ -804,7 +785,6 @@ void Foam::fvMatrix<Type>::relax
     }
 
     // Under-relax only cells within the zone
-    const labelList& zoneCells = psi_.mesh().cellZones()[zoneID];
 
     forAll (zoneCells, zcI)
     {
diff --git a/src/finiteVolume/fvMatrices/fvMatrix/fvMatrix.H b/src/finiteVolume/fvMatrices/fvMatrix/fvMatrix.H
index aad197d93..c86d2e10c 100644
--- a/src/finiteVolume/fvMatrices/fvMatrix/fvMatrix.H
+++ b/src/finiteVolume/fvMatrices/fvMatrix/fvMatrix.H
@@ -362,7 +362,7 @@ public:
             void relax
             (
                 const scalar alpha,
-                const word& cellZoneName
+                const labelList& zoneCells
             );
 
             //- Complete matrix assembly for solution: