From 7f08269ab267b8819b4f22a11d0e541aaa7ce744 Mon Sep 17 00:00:00 2001
From: Vuko Vukcevic <v.vukcevic@wikki.co.uk>
Date: Tue, 19 Feb 2019 22:15:16 +0100
Subject: [PATCH] Two bugfixes:

1. polyTopoChanger::changeMesh -> call to polyTopoChanger::update needs to
   happen after polyMesh::updateMesh (more specifically,
   polyBoundaryMesh::updateMesh) in order to correctly build the processor data,
2. refinement::updateMesh -> now that number 1 is fixed, point sync happens in
   updateMesh instead of setRefinement.

These two combined were responsible for hard-to-track errors in AMR + DLB
---
 .../polyhedralRefinement.C                    |  6 ++-
 .../refinement/refinement/refinement.C        | 37 ++++++++++---------
 .../refinement/refinement/refinement.H        |  4 +-
 .../polyTopoChangerChangeMesh.C               |  8 +++-
 .../topoChangerFvMeshLoadBalanceTemplates.C   |  1 -
 5 files changed, 31 insertions(+), 25 deletions(-)

diff --git a/src/dynamicMesh/dynamicMesh/polyMeshModifiers/refinement/polyhedralRefinement/polyhedralRefinement.C b/src/dynamicMesh/dynamicMesh/polyMeshModifiers/refinement/polyhedralRefinement/polyhedralRefinement.C
index d52bb93af..777b60f4f 100644
--- a/src/dynamicMesh/dynamicMesh/polyMeshModifiers/refinement/polyhedralRefinement/polyhedralRefinement.C
+++ b/src/dynamicMesh/dynamicMesh/polyMeshModifiers/refinement/polyhedralRefinement/polyhedralRefinement.C
@@ -2088,7 +2088,8 @@ void Foam::polyhedralRefinement::setCellsToRefine
 
 	if (debug)
         {
-            Info<< "Added " << nAddCells << " in iteration " << nIters << nl;
+            Info<< "Added " << nAddCells << " cells in iteration "
+                << nIters << nl;
         }
 
     } while (nAddCells > 0);
@@ -2375,7 +2376,8 @@ void Foam::polyhedralRefinement::setSplitPointsToUnrefine
 
         if (debug)
         {
-            Info<< "Removed " << nRemCells << " in iteration " << nIters << nl;
+            Info<< "Removed " << nRemCells << " cells in iteration "
+                << nIters << nl;
         }
 
     } while (nRemCells > 0);
diff --git a/src/dynamicMesh/dynamicMesh/polyMeshModifiers/refinement/refinement/refinement.C b/src/dynamicMesh/dynamicMesh/polyMeshModifiers/refinement/refinement/refinement.C
index b2d310f7d..740aaced0 100644
--- a/src/dynamicMesh/dynamicMesh/polyMeshModifiers/refinement/refinement/refinement.C
+++ b/src/dynamicMesh/dynamicMesh/polyMeshModifiers/refinement/refinement/refinement.C
@@ -1057,6 +1057,10 @@ Foam::label Foam::refinement::faceConsistentUnrefinement
                         << "This is probably because the refinement and "
                         << "unrefinement regions are very close." << nl
                         << "Try increasing nUnrefinementBufferLayers. "
+                        << nl
+                        << "Another possibility is that you are running "
+                        << "with Dynamic Load Balancing, in which case "
+                        << "this should be fine."
                         << endl;
                 }
             }
@@ -1443,20 +1447,6 @@ bool Foam::refinement::changeTopology() const
 
 void Foam::refinement::setRefinement(polyTopoChange& ref) const
 {
-    // Make sure that the point levels are updated across coupled patches before
-    // setting refinement and unrefinement. Note: not sure why the sync is not
-    // performed correctly if I do it in updateMesh. This is a temporary
-    // solution, need to investigate in detail, but I assume something is not
-    // updated yet in that case. VV, 31/Jan/2018.
-    syncTools::syncPointList
-    (
-        mesh_,
-        pointLevel_,
-        maxEqOp<label>(),
-        0,   // Null value
-        true // Apply separation for parallel cyclics
-    );
-
     // Set refinement and unrefinement
     this->setRefinementInstruction(ref);
     this->setUnrefinementInstruction(ref);
@@ -1610,10 +1600,21 @@ void Foam::refinement::updateMesh(const mapPolyMesh& map)
         }
     }
 
-    // Note: new point level is going to be synced at processor boundaries just
-    // before the next step in setRefinement. Need to investigate why the sync
-    // is not done properly if I put it here. Something is not updated yet.
-    // VV, 31/Jan/2018.
+    // Sync the new point level. Note: here, we assume that the call to
+    // updateMesh happened after polyBoundaryMesh::updateMesh where the
+    // processor data is fully rebuilt. If this is not the case, the point
+    // level will remain unsynced and will cause all kinds of trouble that
+    // are extremely difficult to spot. See the change in
+    // polyTopoChanger::changeMesh order of calling polyMesh::updateMesh and
+    // polyTopoChanger::update. VV, 19/Feb/2019.
+    syncTools::syncPointList
+    (
+        mesh_,
+        newPointLevel,
+        maxEqOp<label>(),
+        0,   // Null value
+        true // Apply separation for parallel cyclics
+    );
 
     // Transfer the new point level into the data member
     pointLevel_.transfer(newPointLevel);
diff --git a/src/dynamicMesh/dynamicMesh/polyMeshModifiers/refinement/refinement/refinement.H b/src/dynamicMesh/dynamicMesh/polyMeshModifiers/refinement/refinement/refinement.H
index 74b79cf92..9cd293a39 100644
--- a/src/dynamicMesh/dynamicMesh/polyMeshModifiers/refinement/refinement/refinement.H
+++ b/src/dynamicMesh/dynamicMesh/polyMeshModifiers/refinement/refinement/refinement.H
@@ -189,8 +189,8 @@ protected:
                 const label nei
             ) const;
 
-            //- Modifies existing faceI for either new owner/neighbour or new face
-            //  points. Reverses if nessecary
+            //- Modifies existing faceI for either new owner/neighbour or new
+            //  face points. Reverses if nessecary
             void modifyFace
             (
                 polyTopoChange& ref,
diff --git a/src/dynamicMesh/dynamicMesh/polyTopoChange/polyTopoChanger/polyTopoChangerChangeMesh.C b/src/dynamicMesh/dynamicMesh/polyTopoChange/polyTopoChanger/polyTopoChangerChangeMesh.C
index 8b89f320b..e909a6369 100644
--- a/src/dynamicMesh/dynamicMesh/polyTopoChange/polyTopoChanger/polyTopoChangerChangeMesh.C
+++ b/src/dynamicMesh/dynamicMesh/polyTopoChange/polyTopoChanger/polyTopoChangerChangeMesh.C
@@ -2490,10 +2490,14 @@ Foam::autoPtr<Foam::mapPolyMesh> Foam::polyTopoChanger::changeMesh()
             topoChangeRequest()()
         );
 
-        update(topoChangeMap());
-
         mesh_.updateMesh(topoChangeMap());
 
+        // Bugfix: call to polyTopoChanger::update must happen after
+        // polyMesh::updateMesh where all the relevant mesh bits for parallel
+        // comms are updated. First noticed when the syncying of pointLevel in
+        // refinement::updateMesh was syncying properly. VV, 19/Feb/2019
+        update(topoChangeMap());
+
         // Increment the morph index
         morphIndex_++;
 
diff --git a/src/dynamicMesh/topoChangerFvMesh/topoChangerFvMesh/topoChangerFvMeshLoadBalanceTemplates.C b/src/dynamicMesh/topoChangerFvMesh/topoChangerFvMesh/topoChangerFvMeshLoadBalanceTemplates.C
index ee2db7c26..9deb284c4 100644
--- a/src/dynamicMesh/topoChangerFvMesh/topoChangerFvMesh/topoChangerFvMeshLoadBalanceTemplates.C
+++ b/src/dynamicMesh/topoChangerFvMesh/topoChangerFvMesh/topoChangerFvMeshLoadBalanceTemplates.C
@@ -89,7 +89,6 @@ void Foam::topoChangerFvMesh::insertFields
         iter
     )
     {
-
         localFields[fI].set
         (
             Pstream::myProcNo(),