Load balancing development 1

src/decompositionMethods/Allwmake

@@ -2,6 +2,8 @@
 cd ${0%/*} || exit 1    # run from this directory
 set -x
 
+wmake libso decomposeReconstruct
+
 wmakeLnInclude decompositionMethods
 
 wmake libso decompositionMethods

src/decompositionMethods/AllwmakeLnInclude

@@ -2,6 +2,8 @@
 cd ${0%/*} || exit 1    # run from this directory
 set -x
 
+wmakeLnInclude decomposeReconstruct
+
 wmakeLnInclude decompositionMethods
 wmakeLnInclude metisDecomp
 wmakeLnInclude parMetisDecomp

src/decompositionMethods/decomposeReconstruct/Make/files

@@ -8,6 +8,7 @@ decomposeTools/point/pointFieldDecomposer.C
 decomposeTools/tetFiniteElement/tetPointFieldDecomposer.C
 decomposeTools/lagrangian/lagrangianFieldDecomposer.C
 
+reconstructTools/finiteVolume/sharedPoints.C
 reconstructTools/finiteVolume/processorMeshesReconstructor.C
 reconstructTools/finiteVolume/processorMeshesRebuild.C
 reconstructTools/finiteVolume/fvFieldReconstructor.C

src/decompositionMethods/decomposeReconstruct/Make/options

@@ -1,4 +1,4 @@
-EXE_INC = \
+EXE_INC = -std=c++11 \
     -I$(LIB_SRC)/decompositionMethods/decompositionMethods/lnInclude \
     -I$(LIB_SRC)/finiteVolume/lnInclude \
     -I$(LIB_SRC)/meshTools/lnInclude \

src/decompositionMethods/decomposeReconstruct/decomposeTools/finiteVolume/distributeCells.C

@@ -107,8 +107,6 @@ void Foam::domainDecomposition::distributeCells()
 
     // Construct decomposition method and either do decomposition on
     // cell centres or on agglomeration
-
-
     autoPtr<decompositionMethod> decomposePtr = decompositionMethod::New
     (
         decompositionDict_,

src/decompositionMethods/decomposeReconstruct/decomposeTools/finiteVolume/domainDecomposition.C

@@ -83,7 +83,8 @@ Foam::domainDecomposition::~domainDecomposition()
 Foam::autoPtr<Foam::fvMesh> Foam::domainDecomposition::processorMesh
 (
     const label procI,
-    const Time& processorDb
+    const Time& processorDb,
+    const word& regionName
 ) const
 {
     // Create processor points
@@ -210,7 +211,7 @@ Foam::autoPtr<Foam::fvMesh> Foam::domainDecomposition::processorMesh
         (
             IOobject
             (
-                mesh_.polyMesh::name(),  // region name of undecomposed mesh
+                regionName,
                 mesh_.pointsInstance(),
                 processorDb
             ),
@@ -283,8 +284,8 @@ Foam::autoPtr<Foam::fvMesh> Foam::domainDecomposition::processorMesh
         nPatches++;
     }
 
-    // Add boundary patches
+    // Add boundary patches to polyMesh and fvMesh
-    procMesh.addPatches(procPatches);
+    procMesh.addFvPatches(procPatches);
 
     // Create and add zones
 
@@ -531,7 +532,12 @@ bool Foam::domainDecomposition::writeDecomposition()
         // Set the precision of the points data to 10
         IOstream::defaultPrecision(10);
 
-        autoPtr<fvMesh> procMeshPtr = processorMesh(procI, processorDb);
+        autoPtr<fvMesh> procMeshPtr = processorMesh
+        (
+            procI,
+            processorDb,
+            mesh_.polyMesh::name()     // region name of undecomposed mesh
+        );
         fvMesh& procMesh = procMeshPtr();
 
         procMesh.write();

src/decompositionMethods/decomposeReconstruct/decomposeTools/finiteVolume/domainDecomposition.H

@@ -177,11 +177,12 @@ public:
 
         // Decomposed mesh and addressing
 
-            //- Decompose mesh for a given processor
+            //- Create a decomposed mesh for a given processor index
             autoPtr<fvMesh> processorMesh
             (
                 const label procI,
-                const Time& procDb
+                const Time& procDb,
+                const word& regionName
             ) const;
 
             //- Return processor point addressing

src/decompositionMethods/decomposeReconstruct/decomposeTools/finiteVolume/processorMeshData.H

@@ -0,0 +1,184 @@
+/*---------------------------------------------------------------------------*\
+  =========                 |
+  \\      /  F ield         | foam-extend: Open Source CFD
+   \\    /   O peration     | Version:     4.0
+    \\  /    A nd           | Web:         http://www.foam-extend.org
+     \\/     M anipulation  | For copyright notice see file Copyright
+-------------------------------------------------------------------------------
+License
+    This file is part of foam-extend.
+
+    foam-extend is free software: you can redistribute it and/or modify it
+    under the terms of the GNU General Public License as published by the
+    Free Software Foundation, either version 3 of the License, or (at your
+    option) any later version.
+
+    foam-extend is distributed in the hope that it will be useful, but
+    WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+    General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with foam-extend.  If not, see <http://www.gnu.org/licenses/>.
+
+Class
+    Foam::processorMeshData
+
+Description
+    The class holds mesh data required to assemble a single processor mesh
+    from the decomposition.  The data is raw but can be assembled into
+    a complete mesh
+
+Author
+    Hrvoje Jasak, Wikki Ltd.
+
+SourceFiles
+    processorMeshData.C
+
+\*---------------------------------------------------------------------------*/
+
+#ifndef processorMeshData_H
+#define processorMeshData_H
+
+#include ".H"
+
+// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
+
+namespace Foam
+{
+
+/*---------------------------------------------------------------------------*\
+                     Class processorMeshData Declaration
+\*---------------------------------------------------------------------------*/
+
+class processorMeshData
+{
+    // Private data
+
+        // Internal mesh data
+    
+            //- Points
+            pointField procPoints_;
+
+            //- Faces
+            faceList procFaces_;
+
+            //- Face owner
+            labelList procOwner_;
+
+            //- Face neighbour
+            labelList procNeighbour_;
+
+
+        // Boundary data, separated into existing and new patches
+    
+            //- Boundary addressing
+            labelList boundaryAddressing_;
+
+            //- Regular patch sizes
+            labelList patchSizes_;
+
+            //- Regular patch starts
+            labelList patchStarts_;
+    
+            //- Processor patch sizes
+            labelList procPatchSizes_;
+    
+            //- Processor patch starts
+            labelList procPatchSizes_;
+
+    
+        // Zones
+
+            //- Point zones
+            labelListList pointZones_;
+    
+            //- Face zones
+            labelListList faceZones_;
+
+            //- Face zone flips
+            labelListList faceZoneFlips_;
+
+            //- Cell zones
+            labelListList cellZones_;
+
+    
+    // Private Member Functions
+
+        //- Disallow default bitwise copy construct
+        processorMeshData(const processorMeshData&);
+
+        //- Disallow default bitwise assignment
+        void operator=(const processorMeshData&);
+
+
+public:
+
+    // Static data members
+
+        //- Static data staticData
+        static const dataType staticData;
+
+
+    // Constructors
+
+        //- Construct from components
+        processorMeshData(const dataType& data);
+
+        //- Construct from Istream
+        processorMeshData(Istream&);
+
+        //- Construct as copy
+        processorMeshData(const processorMeshData&);
+
+
+    // Selectors
+
+        //- Select null constructed
+        static autoPtr<processorMeshData> New();
+
+
+    //- Destructor
+    ~processorMeshData();
+
+
+    // Member Functions
+
+        // Access
+
+        // Check
+
+        // Edit
+
+        // Write
+
+
+    // Member Operators
+
+        void operator=(const processorMeshData&);
+
+
+    // Friend Functions
+
+    // Friend Operators
+
+    // IOstream Operators
+
+        friend Istream& operator>>(Istream&, processorMeshData&);
+        friend Ostream& operator<<(Ostream&, const processorMeshData&);
+};
+
+
+// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
+
+} // End namespace Foam
+
+// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
+
+#include "processorMeshDataI.H"
+
+// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
+
+#endif
+
+// ************************************************************************* //

src/decompositionMethods/decomposeReconstruct/reconstructTools/finiteVolume/processorMeshesRebuild.C

@@ -25,7 +25,7 @@ License
 
 #include "processorMeshesReconstructor.H"
 #include "processorPolyPatch.H"
-#include "globalMeshData.H"
+#include "sharedPoints.H"
 
 // * * * * * * * * * * * * * Private Member Functions  * * * * * * * * * * * //
 
@@ -494,8 +494,9 @@ Foam::processorMeshesReconstructor::reconstructMesh(const Time& db)
         << "    nPatches = " << nReconPatches << nl
         << "    nPatchFaces = " << reconPatchSizes << endl;
 
-    // Note: for easier debugging, set owner and neighbour to -1
+    // Note: for easier debugging, set mapping, owner and neighbour to -1
     pointField reconPoints(nReconPoints);
+    labelList globalPointMapping(nReconPoints, -1);
     faceList reconFaces(nReconFaces);
     labelList cellOffset(meshes_.size(), 0);
     labelList reconOwner(nReconFaces, -1);
@@ -604,8 +605,10 @@ Foam::processorMeshesReconstructor::reconstructMesh(const Time& db)
 
     reconPatchSizes = 0;
 
-    // Prepare handling for globally shared points
+    // Prepare handling for globally shared points.  This is equivalent
-    labelList globalPointMapping;
+    // to parallel processor points, but working on a PtrList of meshes
+    // on the same processor
+    sharedPoints sharedData(meshes_);
 
     // Dump first valid mesh without checking
     {
@@ -619,10 +622,6 @@ Foam::processorMeshesReconstructor::reconstructMesh(const Time& db)
         labelList& cpAddr = cellProcAddressing_[fvmId];
         labelList& bpAddr = boundaryProcAddressing_[fvmId];
 
-        // Prepare handling for global mesh data: set to -1
-        globalPointMapping.setSize(curMesh.globalData().nGlobalPoints());
-        globalPointMapping = -1;
-
         // Dump all points into the global point list
         // Reconstruct only live points.  HJ, 7/Mar/2011
         const pointField& curPoints = curMesh.points();
@@ -636,8 +635,7 @@ Foam::processorMeshesReconstructor::reconstructMesh(const Time& db)
         }
 
         // Collect globally shared point labels
-        // CHANGE HERE!!!
+        const labelList& curSpl = sharedData.sharedPointLabels()[fvmId];
-        const labelList& curSpl = curMesh.globalData().sharedPointLabels();
 
         forAll (curSpl, splI)
         {
@@ -816,7 +814,8 @@ Foam::processorMeshesReconstructor::reconstructMesh(const Time& db)
 
                         forAll (reversedFaces, faceI)
                         {
-                            reversedFaces[faceI] = procPatch[faceI].reverseFace();
+                            reversedFaces[faceI] =
+                                procPatch[faceI].reverseFace();
                         }
 
                         primitiveFacePatch reversedPatch
@@ -828,8 +827,9 @@ Foam::processorMeshesReconstructor::reconstructMesh(const Time& db)
                         // Insert addressing from master side into
                         // local point addressing.  Each face of reversed patch
                         // now matches the master face.
-                        // Note: this is done by visiting faces, since meshPoints
+                        // Note: this is done by visiting faces, since
-                        // are ordered in increasing order.  HJ, 10/Mar/2011
+                        // meshPoints are ordered in increasing order.
+                        // HJ, 10/Mar/2011
 
                         forAll (reversedFaces, faceI)
                         {
@@ -879,7 +879,7 @@ Foam::processorMeshesReconstructor::reconstructMesh(const Time& db)
             fpAddr.setSize(curFaces.size());
 
             // Collect globally shared point labels
-            const labelList& curSpl = curMesh.globalData().sharedPointLabels();
+            const labelList& curSpl = sharedData.sharedPointLabels()[procI];
 
             forAll (curSpl, splI)
             {
@@ -893,7 +893,11 @@ Foam::processorMeshesReconstructor::reconstructMesh(const Time& db)
                 else
                 {
                     // Compare.  Is this needed - should always be OK.
-                    if (globalPointMapping[curSpl[splI]] != ppAddr[curSpl[splI]])
+                    if
+                    (
+                        globalPointMapping[curSpl[splI]]
+                     != ppAddr[curSpl[splI]]
+                    )
                     {
                         WarningIn
                         (
@@ -963,7 +967,8 @@ Foam::processorMeshesReconstructor::reconstructMesh(const Time& db)
 
                         const label masterProcID = procPatch.neighbProcNo();
 
-                        // Get local face-cell addressing: it will become neighbour
+                        // Get local face-cell addressing: it will become
+                        // a neighbour
                         // addressing for the already inserted faces
                         const labelList procFaceCells = procPatch.faceCells();
 
@@ -1039,7 +1044,8 @@ Foam::processorMeshesReconstructor::reconstructMesh(const Time& db)
                 cpAddr[cellI] = cellI + cellOffset[procI];
             }
 
-            // Sort out boundary addressing: i for live patches, -1 for processor
+            // Sort out boundary addressing: i for live patches,
+            // -1 for processor
             bpAddr = -1;
 
             // Note: loop over mapped patches

src/decompositionMethods/decomposeReconstruct/reconstructTools/finiteVolume/processorMeshesReconstructor.C

@@ -27,9 +27,9 @@ License
 
 // * * * * * * * * * * * * * Private Member Functions  * * * * * * * * * * * //
 
-void Foam::processorMeshesReconstructor::readMeshes()
+void Foam::processorMeshesReconstructor::readMeshes(PtrList<Time>& databases)
 {
-    forAll (databases_, procI)
+    forAll (databases, procI)
     {
         Info<< "Reading mesh for processor " << procI << endl;
         meshes_.set
@@ -40,8 +40,8 @@ void Foam::processorMeshesReconstructor::readMeshes()
                 IOobject
                 (
                     meshName_,
-                    databases_[procI].timeName(),
+                    databases[procI].timeName(),
-                    databases_[procI]
+                    databases[procI]
                 )
             )
         );
@@ -66,12 +66,24 @@ void Foam::processorMeshesReconstructor::clearMaps()
 
 Foam::processorMeshesReconstructor::processorMeshesReconstructor
 (
-    PtrList<Time>& databases,
+    const word& meshName
-    const word& meshName,
+)
-    const bool read
+:
+    meshName_(meshName),
+    meshes_(),
+    pointProcAddressing_(),
+    faceProcAddressing_(),
+    cellProcAddressing_(),
+    boundaryProcAddressing_()
+{}
+
+
+Foam::processorMeshesReconstructor::processorMeshesReconstructor
+(
+    PtrList<Time>& databases,
+    const word& meshName
 )
 :
-    databases_(databases),
     meshName_(meshName),
     meshes_(databases.size()),
     pointProcAddressing_(),
@@ -79,10 +91,7 @@ Foam::processorMeshesReconstructor::processorMeshesReconstructor
     cellProcAddressing_(),
     boundaryProcAddressing_()
 {
-    if (read)
+    readMeshes(databases);
-    {
-        readMeshes();
-    }
 }
 
 
@@ -93,10 +102,10 @@ Foam::processorMeshesReconstructor::readUpdate()
 {
     polyMesh::readUpdateState stat = polyMesh::UNCHANGED;
 
-    forAll (databases_, procI)
+    forAll (meshes_, procI)
     {
         // Only do action if database has been set
-        if (databases_.set(procI))
+        if (meshes_.set(procI))
         {
             // Check if any new meshes need to be read.
             polyMesh::readUpdateState procStat = meshes_[procI].readUpdate();
@@ -113,9 +122,10 @@ Foam::processorMeshesReconstructor::readUpdate()
                     FatalErrorIn("processorMeshesReconstructor::readUpdate()")
                         << "Processor " << procI
                         << " has a different polyMesh at time "
-                        << databases_[procI].timeName()
+                        << meshes_[procI].time().timeName()
                         << " compared to any previous processors." << nl
-                        << "Please check time " << databases_[procI].timeName()
+                        << "Please check time "
+                        << meshes_[procI].time().timeName()
                         << " directories on all processors for consistent"
                         << " mesh files."
                         << exit(FatalError);

src/decompositionMethods/decomposeReconstruct/reconstructTools/finiteVolume/processorMeshesReconstructor.H

@@ -58,9 +58,6 @@ class processorMeshesReconstructor
 {
     // Private data
 
-        //- Processor databases
-        PtrList<Time>& databases_;
-
         //- Name of mesh region to reconstruct
         const word meshName_;
 
@@ -90,7 +87,7 @@ class processorMeshesReconstructor
 
 
         //- Read all meshes
-        void readMeshes();
+        void readMeshes(PtrList<Time>& databases);
 
         //- Attempt to read mapping.  If not available, return false
         bool readMapping();
@@ -118,12 +115,14 @@ public:
 
     // Constructors
 
-        //- Construct reading mesh names
+        //- Construct given name.  Set meshes later
+        processorMeshesReconstructor(const word& meshName);
+
+        //- Construct reading meshes from databases
         processorMeshesReconstructor
         (
             PtrList<Time>& databases,
-            const word& meshName,
+            const word& meshName
-            const bool read
         );
 
 

src/decompositionMethods/decomposeReconstruct/reconstructTools/finiteVolume/sharedPoints.C

@@ -0,0 +1,439 @@
+/*---------------------------------------------------------------------------*\
+  =========                 |
+  \\      /  F ield         | foam-extend: Open Source CFD
+   \\    /   O peration     | Version:     4.0
+    \\  /    A nd           | Web:         http://www.foam-extend.org
+     \\/     M anipulation  | For copyright notice see file Copyright
+-------------------------------------------------------------------------------
+License
+    This file is part of foam-extend.
+
+    foam-extend is free software: you can redistribute it and/or modify it
+    under the terms of the GNU General Public License as published by the
+    Free Software Foundation, either version 3 of the License, or (at your
+    option) any later version.
+
+    foam-extend is distributed in the hope that it will be useful, but
+    WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+    General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with foam-extend.  If not, see <http://www.gnu.org/licenses/>.
+
+\*---------------------------------------------------------------------------*/
+
+#include "sharedPoints.H"
+#include "processorPolyPatch.H"
+
+// * * * * * * * * * * * * * * Static Data Members * * * * * * * * * * * * * //
+
+
+// * * * * * * * * * * * * * Static Member Functions * * * * * * * * * * * * //
+
+
+// * * * * * * * * * * * * * Private Member Functions  * * * * * * * * * * * //
+
+Foam::labelListList Foam::sharedPoints::procPatchPairs() const
+{
+    labelListList patchPairs(meshes_.size());
+
+    // Initialise patch pair indices to -1
+    forAll (meshes_, meshI)
+    {
+        if (meshes_.set(meshI))
+        {
+            patchPairs[meshI].setSize(meshes_[meshI].boundaryMesh().size(), -1);
+        }
+    }
+
+    forAll (meshes_, meshI)
+    {
+        if (meshes_.set(meshI))
+        {
+            const polyMesh& curMesh = meshes_[meshI];
+            const polyBoundaryMesh& curPatches = curMesh.boundaryMesh();
+
+            forAll (curPatches, patchI)
+            {
+                if (isA<processorPolyPatch>(curPatches[patchI]))
+                {
+                    // Found processor patch
+                    if (patchPairs[meshI][patchI] == -1)
+                    {
+                        // Neighbour not found.  Find one
+                        const processorPolyPatch& myProcPatch =
+                            refCast<const processorPolyPatch>
+                            (
+                                curPatches[patchI]
+                            );
+
+                        const int myProcID = meshI;
+                        const int nbrProcID = myProcPatch.neighbProcNo();
+
+                        // Get the other mesh
+                        if (!meshes_.set(nbrProcID))
+                        {
+                            FatalErrorIn
+                            (
+                                "labelListList sharedPoints::procPatchPairs()"
+                            )   << "Neighbour mesh does not exist for proc "
+                                << meshI << " patch " << patchI
+                                << " and neighbour " << nbrProcID
+                                << abort(FatalError);
+                        }
+
+                        const polyMesh& nbrMesh = meshes_[nbrProcID];
+                        const polyBoundaryMesh& nbrPatches =
+                            nbrMesh.boundaryMesh();
+
+                        // Check all neighbour processor patches until a match
+                        // is found
+                        bool found = false;
+
+                        forAll (nbrPatches, nbrPatchI)
+                        {
+                            if (isA<processorPolyPatch>(nbrPatches[nbrPatchI]))
+                            {
+                                const processorPolyPatch& nbrProcPatch =
+                                    refCast<const processorPolyPatch>
+                                    (
+                                        nbrPatches[nbrPatchI]
+                                    );
+
+                                if (nbrProcPatch.neighbProcNo() == myProcID)
+                                {
+                                    // Pair found.  Record it twice
+                                    patchPairs[myProcID][patchI] = nbrPatchI;
+
+                                    patchPairs[nbrProcID][nbrPatchI] = patchI;
+
+                                    found = true;
+                                }
+                            }
+
+                            if (found)
+                            {
+                                break;
+                            }
+                        }
+
+                        if (!found)
+                        {
+                            FatalErrorIn
+                            (
+                                "labelListList sharedPoints::procPatchPairs()"
+                            )   << "Neighbour patch does not exist for proc "
+                                << meshI << " patch " << patchI
+                                << " and neighbour " << nbrProcID
+                                << abort(FatalError);
+                        }
+                    }
+                }
+            }
+        }
+    }
+    Info<< "patch pairs: " << patchPairs << endl;
+    return patchPairs;
+}
+
+
+void Foam::sharedPoints::calcSharedPoints()
+{
+    // Algorithm
+    // Go through all processor patches and mark local points that are used
+    // by more than one processor patch and mark them as globally shared
+    // Pass the data to other processors.  Mark the locally multiply shared
+    // points and pass on the data
+    // Once all the data is passed forwards and back, check all points on
+    // all processors.  Record globally shared point, its local label and its
+    // slot in the globally shared point list
+
+    // Mark-up:
+    // 0 = point does not touch a processor boundary
+    // 1 = point on only one processor boundary: not locally shared
+    // 2 = locally detected global point
+
+    // Mark-up array: procI, procJ,
+    labelListList markedPoints(meshes_.size());
+
+    forAll (meshes_, meshI)
+    {
+        if (meshes_.set(meshI))
+        {
+            markedPoints[meshI].setSize(meshes_[meshI].nPoints(), 0);
+        }
+    }
+
+    // Mark up points for the first time
+    forAll (meshes_, meshI)
+    {
+        if (meshes_.set(meshI))
+        {
+            const polyMesh& curMesh = meshes_[meshI];
+            const polyBoundaryMesh& patches = curMesh.boundaryMesh();
+
+            // Mark points belonging to processor patches.  If the point
+            // is marked more than once, it may be a globally shared point
+            labelList& curMarkedPoints = markedPoints[meshI];
+
+            forAll (patches, patchI)
+            {
+                const polyPatch& pp = patches[patchI];
+
+                if (isA<processorPolyPatch>(pp))
+                {
+                    // Found processor patch
+                    const labelList& patchMeshPoints = pp.meshPoints();
+
+                    forAll (patchMeshPoints, mpI)
+                    {
+                        // Mark the point
+                        curMarkedPoints[patchMeshPoints[mpI]]++;
+                    }
+                }
+            }
+        }
+    }
+
+    // Get processor patch to neighbour processor patch addressing
+    labelListList patchPairs = procPatchPairs();
+
+    // Communicate and count global points
+    labelList nGlobalPointsPerProc(meshes_.size(), 0);
+
+    // Identify, count and communicate points across processor boundaries
+    // Repeat until the number of points per processor stabilises,
+    // ie. no further points are found through communication
+    label oldNTotalPoints, newNTotalPoints;
+    do
+    {
+        oldNTotalPoints = sum(nGlobalPointsPerProc);
+
+        // Reset the list
+        nGlobalPointsPerProc = 0;
+
+        forAll (meshes_, meshI)
+        {
+            if (meshes_.set(meshI))
+            {
+                const polyMesh& curMesh = meshes_[meshI];
+                const polyBoundaryMesh& patches = curMesh.boundaryMesh();
+
+                labelList& curMarkedPoints = markedPoints[meshI];
+
+                // Collect the points that have been addressed multiple times
+                forAll (patches, patchI)
+                {
+                    const polyPatch& pp = patches[patchI];
+
+                    if (isA<processorPolyPatch>(pp))
+                    {
+                        // Found processor patch
+                        const labelList& patchMeshPoints = pp.meshPoints();
+
+                        // My processor patch
+                        const processorPolyPatch& myProcPatch =
+                            refCast<const processorPolyPatch>(pp);
+
+                        // Get neighbour processor ID
+                        const int nbrProcID = myProcPatch.neighbProcNo();
+
+                        // Neighbour patch
+                        const polyPatch& nbrPatch =
+                            meshes_[nbrProcID].boundaryMesh()
+                            [patchPairs[meshI][patchI]];
+
+                        const labelList& nbrMeshPoints = nbrPatch.meshPoints();
+
+                        forAll (patchMeshPoints, mpI)
+                        {
+                            if (curMarkedPoints[patchMeshPoints[mpI]] > 1)
+                            {
+                                // Mark the point on the other processor/side
+                                markedPoints[nbrProcID][nbrMeshPoints[mpI]] =
+                                    curMarkedPoints[patchMeshPoints[mpI]];
+                            }
+                        }
+                    }
+                }
+
+                // Count number of shared points per processor
+                forAll (curMarkedPoints, cpI)
+                {
+                    if (curMarkedPoints[cpI] > 1)
+                    {
+                        nGlobalPointsPerProc[meshI]++;
+                    }
+                }
+            }
+        }
+
+        newNTotalPoints = sum(nGlobalPointsPerProc);
+
+        Info<< "Proc merge pass: " << oldNTotalPoints << " "
+            << newNTotalPoints << endl;
+    } while (oldNTotalPoints != newNTotalPoints);
+
+    Info<< "Number of shared points per processor: " << nGlobalPointsPerProc
+        << endl;
+
+    // Collect points for every processor, in order to re-use the markedPoints
+    // list.  Note: the list of global labels of shared points
+    // will be collected later
+    forAll (meshes_, meshI)
+    {
+        if (meshes_.set(meshI))
+        {
+            labelList& curSharedPoints = sharedPointLabels_[meshI];
+
+            curSharedPoints.setSize(nGlobalPointsPerProc[meshI]);
+
+            // Count inserted points
+            label nShared = 0;
+
+            // Get point marking
+            const labelList& curMarkedPoints = markedPoints[meshI];
+
+            forAll (curMarkedPoints, pointI)
+            {
+                if (curMarkedPoints[pointI] > 1)
+                {
+                    curSharedPoints[nShared] = pointI;
+                    nShared++;
+                }
+            }
+        }
+    }
+
+    // Clear markup list.  It will be used for the global processor point
+    forAll (markedPoints, meshI)
+    {
+        markedPoints[meshI] = -1;
+    }
+
+    // Provide global mark for all processors and communicate it across
+    // processor boundaries
+    forAll (meshes_, meshI)
+    {
+        if (meshes_.set(meshI))
+        {
+            const polyMesh& curMesh = meshes_[meshI];
+            const polyBoundaryMesh& patches = curMesh.boundaryMesh();
+
+            // Get shared points and assign global shared point index
+            const labelList& curSharedPoints = sharedPointLabels_[meshI];
+
+            // Prepare addressing into the global shared point list
+            labelList& curSharedAddr = sharedPointAddr_[meshI];
+            curSharedAddr.setSize(curSharedPoints.size());
+
+            labelList& curMarkedPoints = markedPoints[meshI];
+
+            forAll (curSharedPoints, spI)
+            {
+                if (curMarkedPoints[curSharedPoints[spI]] == -1)
+                {
+                    // Found new point.  Mark it and collect addressing
+                    curMarkedPoints[curSharedPoints[spI]] = nGlobalPoints_;
+
+                    // Collect addressing
+                    curSharedAddr[spI] = nGlobalPoints_;
+
+                    nGlobalPoints_++;
+                }
+                else
+                {
+                    // Point already marked.  Collect addressing
+                    curSharedAddr[spI] = curMarkedPoints[curSharedPoints[spI]];
+                }
+            }
+
+            // Communicate labels accross the boundary using processor patches
+            forAll (patches, patchI)
+            {
+                const polyMesh& curMesh = meshes_[meshI];
+                const polyBoundaryMesh& patches = curMesh.boundaryMesh();
+
+                // Get point marking
+                const labelList& curMarkedPoints = markedPoints[meshI];
+
+                const polyPatch& pp = patches[patchI];
+
+                if (isA<processorPolyPatch>(pp))
+                {
+                    // Found processor patch
+
+                    // My processor patch
+                    const processorPolyPatch& myProcPatch =
+                        refCast<const processorPolyPatch>(pp);
+
+                    const labelList& patchMeshPoints = pp.meshPoints();
+
+                    // Get neighbour processor ID
+                    const int nbrProcID = myProcPatch.neighbProcNo();
+
+                    // Neighbour patch
+                    const polyPatch& nbrPatch =
+                        meshes_[nbrProcID].boundaryMesh()
+                        [patchPairs[meshI][patchI]];
+
+                    const labelList& nbrMeshPoints = nbrPatch.meshPoints();
+
+                    forAll (patchMeshPoints, mpI)
+                    {
+                        if (curMarkedPoints[patchMeshPoints[mpI]] > -1)
+                        {
+                            // Mark opposite side
+                            markedPoints[nbrProcID][nbrMeshPoints[mpI]] =
+                                curMarkedPoints[patchMeshPoints[mpI]];
+                        }
+                    }
+                }
+            }
+        }
+    }
+
+    Info<< "nGlobalPoints_: " << nGlobalPoints_ << nl
+        << "sharedPointLabels_: " << sharedPointLabels_ << nl
+        << "sharedPointAddr_: " << sharedPointAddr_ << endl;
+}
+
+
+// * * * * * * * * * * * * Protected Member Functions  * * * * * * * * * * * //
+
+
+// * * * * * * * * * * * * * * * * Constructors  * * * * * * * * * * * * * * //
+
+Foam::sharedPoints::sharedPoints(const PtrList<fvMesh>& meshes)
+:
+    meshes_(meshes),
+    sharedPointAddr_(meshes_.size()),
+    sharedPointLabels_(meshes_.size()),
+    nGlobalPoints_(0)
+{
+    calcSharedPoints();
+}
+
+
+// * * * * * * * * * * * * * * * * Destructor  * * * * * * * * * * * * * * * //
+
+// Foam::sharedPoints::~sharedPoints()
+// {}
+
+
+// * * * * * * * * * * * * * * Member Functions  * * * * * * * * * * * * * * //
+
+// void Foam::sharedPoints::operator=(const sharedPoints& rhs)
+// {
+//     // Check for assignment to self
+//     if (this == &rhs)
+//     {
+//         FatalErrorIn("Foam::sharedPoints::operator=(const Foam::sharedPoints&)")
+//             << "Attempted assignment to self"
+//             << abort(FatalError);
+//     }
+// }
+
+
+// ************************************************************************* //

src/decompositionMethods/decomposeReconstruct/reconstructTools/finiteVolume/sharedPoints.H

@@ -0,0 +1,166 @@
+/*---------------------------------------------------------------------------*\
+  =========                 |
+  \\      /  F ield         | foam-extend: Open Source CFD
+   \\    /   O peration     | Version:     4.0
+    \\  /    A nd           | Web:         http://www.foam-extend.org
+     \\/     M anipulation  | For copyright notice see file Copyright
+-------------------------------------------------------------------------------
+License
+    This file is part of foam-extend.
+
+    foam-extend is free software: you can redistribute it and/or modify it
+    under the terms of the GNU General Public License as published by the
+    Free Software Foundation, either version 3 of the License, or (at your
+    option) any later version.
+
+    foam-extend is distributed in the hope that it will be useful, but
+    WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+    General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with foam-extend.  If not, see <http://www.gnu.org/licenses/>.
+
+Class
+    Foam::sharedPoints
+
+Description
+    Calculates points shared by more than two processor patches from a list
+    of meshes.
+
+    Works purely on topology.
+    Needs:
+      - domain to be one single domain (i.e. all faces can be reached through
+        face-cell walk).
+      - patch face ordering to be ok
+      - f[0] ordering on patch faces to be ok.
+
+    Works by constructing equivalence lists for all the points on processor
+    patches. These list are procPointList and give processor and meshPoint
+    label on that processor.
+    E.g.
+    @verbatim
+          ((7 93)(4 731)(3 114))
+    @endverbatim
+
+    means point 93 on proc7 is connected to point 731 on proc4 and 114 on proc3.
+    It then gets the lowest numbered processor (the 'master') to request a
+    sharedPoint label from processor0 and it redistributes this label back to
+    the other processors in the equivalence list.
+
+Note
+    Currently operating with a PtrList<fvMesh>, whereas the operation actually
+    works on PtrList<polyMesh> as well, due to the calling side
+
+Author
+    Hrvoje Jasak, Wikki Ltd.  All rights reserved
+
+SourceFiles
+    sharedPoints.C
+
+\*---------------------------------------------------------------------------*/
+
+#ifndef sharedPoints_H
+#define sharedPoints_H
+
+#include "PtrList.H"
+#include "fvMesh.H"
+
+// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
+
+namespace Foam
+{
+
+/*---------------------------------------------------------------------------*\
+                          Class sharedPoints Declaration
+\*---------------------------------------------------------------------------*/
+
+class sharedPoints
+{
+    // Private typedefs
+
+        //- Define procPointList as holding a list of meshPoint/processor labels
+        typedef FixedList<label, 2> procPoint;
+        typedef List<procPoint> procPointList;
+
+
+    // Private data
+
+        //- List of meshes
+        const PtrList<fvMesh>& meshes_;
+
+        //- Shared points used by this processor (= global point number)
+        labelListList sharedPointAddr_;
+
+        //- My mesh points corresponding to the shared points
+        labelListList sharedPointLabels_;
+
+        //- Total number of shared points.
+        label nGlobalPoints_;
+
+
+    // Private Member Functions
+
+        //- Disallow default bitwise copy construct
+        sharedPoints(const sharedPoints&) = delete;
+
+        //- Disallow default bitwise assignment
+        void operator=(const sharedPoints&) = delete;
+
+
+        //- Calculate processor patch pairs
+        //  For each processor patch, find equivalent patch on other mesh
+        labelListList procPatchPairs() const;
+
+        //- Calculate shared points
+        void calcSharedPoints();
+
+
+public:
+
+    // Static data members
+
+
+    // Constructors
+
+        //- Construct from the list of meshes
+        sharedPoints(const PtrList<fvMesh>& meshes);
+
+
+    //- Destructor
+    ~sharedPoints() = default;
+
+
+    // Member Functions
+
+        // Access
+
+            //- Shared points used by this processor (= global point number)
+            inline const labelListList& sharedPointAddr() const
+            {
+                return sharedPointAddr_;
+            }
+
+            //- Local mesh points corresponding to the shared points
+            inline const labelListList& sharedPointLabels() const
+            {
+                return sharedPointLabels_;
+            }
+
+            //- Number of globally shared points
+            inline label nGlobalPoints() const
+            {
+                return nGlobalPoints_;
+            }
+};
+
+
+// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
+
+} // End namespace Foam
+
+// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
+
+#endif
+
+// ************************************************************************* //