From 65e6031bcbf77772ae7becaacc0c5763f551ed81 Mon Sep 17 00:00:00 2001 From: Hrvoje Jasak Date: Thu, 11 Aug 2016 15:06:26 +0100 Subject: [PATCH] Feature: comms update for immersed boundary solver --- .../immersedBoundaryFvPatch.C | 270 +++--------------- .../immersedBoundaryFvPatch.H | 12 +- .../immersedBoundaryFvPatchLeastSquaresFit.C | 4 +- .../immersedBoundaryFvPatchSamplingWeights.C | 4 +- .../immersedBoundaryFvPatchTemplates.C | 3 + .../immersedBoundaryFvPatchTriAddressing.C | 56 ++-- .../sphereInChannel/system/controlDict | 1 - 7 files changed, 85 insertions(+), 265 deletions(-) diff --git a/src/immersedBoundary/immersedBoundary/immersedBoundaryFvPatch/immersedBoundaryFvPatch.C b/src/immersedBoundary/immersedBoundary/immersedBoundaryFvPatch/immersedBoundaryFvPatch.C index 7977da047..b57945380 100644 --- a/src/immersedBoundary/immersedBoundary/immersedBoundaryFvPatch/immersedBoundaryFvPatch.C +++ b/src/immersedBoundary/immersedBoundary/immersedBoundaryFvPatch/immersedBoundaryFvPatch.C @@ -1095,21 +1095,11 @@ void Foam::immersedBoundaryFvPatch::makeIbCellCells() const ibCellCellsPtr_ = new labelListList(ibc.size()); labelListList& cellCells = *ibCellCellsPtr_; - ibProcCentresPtr_ = new FieldField(Pstream::nProcs()); - FieldField& procCentres = *ibProcCentresPtr_; + ibProcCentresPtr_ = new vectorListList(Pstream::nProcs()); + vectorListList& procCentres = *ibProcCentresPtr_; - forAll (procCentres, procI) - { - procCentres.set(procI, new vectorField(0)); - } - - ibProcGammaPtr_ = new FieldField(Pstream::nProcs()); - FieldField& procGamma = *ibProcGammaPtr_; - - forAll (procGamma, procI) - { - procGamma.set(procI, new scalarField(0)); - } + ibProcGammaPtr_ = new scalarListList(Pstream::nProcs()); + scalarListList& procGamma = *ibProcGammaPtr_; ibCellProcCellsPtr_ = new List >(ibc.size()); @@ -1250,113 +1240,29 @@ void Foam::immersedBoundaryFvPatch::makeIbCellCells() const labelList procIbCells = procIbCellsSet.toc(); sort(procIbCells); - // Note: consider more sophisticated gather-scatter - // HJ, 18/Jun/2015 - - // Send and receive number of immersed boundary cells - // next to processor boundaries - for (label procI = 0; procI < Pstream::nProcs(); procI++) - { - if (procI != Pstream::myProcNo()) - { - // Parallel data exchange - { - OPstream toProc - ( - Pstream::blocking, - procI, - sizeof(label) - ); - - toProc << procIbCells.size(); - } - } - } - - labelList sizes(Pstream::nProcs(), 0); - for (label procI = 0; procI < Pstream::nProcs(); procI++) - { - if (procI != Pstream::myProcNo()) - { - // Parallel data exchange - { - IPstream fromProc - ( - Pstream::blocking, - procI, - sizeof(label) - ); - - fromProc >> sizes[procI]; - } - } - } + // Note: new gather-scatter operations + // HJ, 11/Aug/2016 // Send and receive ibc centres and radii - vectorField centres(procIbCells.size(), vector::zero); + vectorListList ctrs(Pstream::nProcs()); + + ctrs[Pstream::myProcNo()].setSize(procIbCells.size()); + vectorList& centres = ctrs[Pstream::myProcNo()]; + forAll (centres, cellI) { centres[cellI] = C[ibc[procIbCells[cellI]]]; } - scalarField procRMax(rM, procIbCells); - for (label procI = 0; procI < Pstream::nProcs(); procI++) - { - if (procI != Pstream::myProcNo()) - { - // Parallel data exchange - { - OPstream toProc - ( - Pstream::blocking, - procI, - centres.size()*sizeof(vector) - + procRMax.size()*sizeof(scalar) - ); + Pstream::gatherList(ctrs); + Pstream::scatterList(ctrs); - toProc << centres << procRMax; - } - } - } + scalarListList rMax(Pstream::nProcs()); - FieldField ctrs(Pstream::nProcs()); - FieldField rMax(Pstream::nProcs()); + rMax[Pstream::myProcNo()] = scalarField(rM, procIbCells); - for (label procI = 0; procI < Pstream::nProcs(); procI++) - { - if (procI != Pstream::myProcNo()) - { - ctrs.set - ( - procI, - new vectorField(sizes[procI], vector::zero) - ); - - rMax.set - ( - procI, - new scalarField(sizes[procI], 0) - ); - - // Parallel data exchange - { - IPstream fromProc - ( - Pstream::blocking, - procI, - sizes[procI]*sizeof(vector) - + sizes[procI]*sizeof(scalar) - ); - - fromProc >> ctrs[procI] >> rMax[procI]; - } - } - else - { - ctrs.set(procI, new vectorField(0)); - rMax.set(procI, new scalarField(0)); - } - } + Pstream::gatherList(rMax); + Pstream::scatterList(rMax); // Find cells needed by other processors if (ibProcCellsPtr_) @@ -1431,131 +1337,27 @@ void Foam::immersedBoundaryFvPatch::makeIbCellCells() const procCells[procI] = procCellSet.toc(); } + Pstream::gatherList(procCells); + Pstream::scatterList(procCells); - // Send and receive sizes - for (label procI = 0; procI < Pstream::nProcs(); procI++) - { - if (procI != Pstream::myProcNo()) - { - // Parallel data exchange - { - OPstream toProc - ( - Pstream::blocking, - procI, - sizeof(label) - ); + procCentres[Pstream::myProcNo()] = + vectorField + ( + C, + procCells[Pstream::myProcNo()] + ); - toProc << procCells[procI].size(); - } - } - } - - labelList procSizes(Pstream::nProcs(), 0); - for (label procI = 0; procI < Pstream::nProcs(); procI++) - { - if (procI != Pstream::myProcNo()) - { - // Parallel data exchange - { - IPstream fromProc - ( - Pstream::blocking, - procI, - sizeof(label) - ); - - fromProc >> procSizes[procI]; - } - } - } - - // Send cell centres - for (label procI = 0; procI < Pstream::nProcs(); procI++) - { - if (procI != Pstream::myProcNo()) - { - vectorField centres(C, procCells[procI]); - - // Parallel data exchange - { - OPstream toProc - ( - Pstream::blocking, - procI, - centres.size()*sizeof(vector) - ); - - toProc << centres; - } - } - } - - // Receive cell centres - for (label procI = 0; procI < Pstream::nProcs(); procI++) - { - if (procI != Pstream::myProcNo()) - { - procCentres[procI].setSize(procSizes[procI]); - - // Parallel data exchange - { - IPstream fromProc - ( - Pstream::blocking, - procI, - procSizes[procI]*sizeof(vector) - ); - - fromProc >> procCentres[procI]; - } - } - // else: already set to zero-size field - } + Pstream::gatherList(procCentres); + Pstream::scatterList(procCentres); // Send cell gamma - const scalarField& gammaI = gamma().internalField(); - for (label procI = 0; procI < Pstream::nProcs(); procI++) - { - if (procI != Pstream::myProcNo()) - { - scalarField gamma(gammaI, procCells[procI]); - - // Parallel data exchange - { - OPstream toProc - ( - Pstream::blocking, - procI, - gamma.size()*sizeof(scalar) - ); - - toProc << gamma; - } - } - } - - // Receive cell gamma - for (label procI = 0; procI < Pstream::nProcs(); procI++) - { - if (procI != Pstream::myProcNo()) - { - procGamma[procI].setSize(procSizes[procI]); - - // Parallel data exchange - { - IPstream fromProc - ( - Pstream::blocking, - procI, - procSizes[procI]*sizeof(scalar) - ); - - fromProc >> procGamma[procI]; - } - } - // else: already set to zero-size field - } + procGamma[Pstream::myProcNo()] = + scalarField + ( + gamma().internalField(), + procCells[Pstream::myProcNo()] + ); + // Reset own size to zero? HJ, 11/Aug/2016 // Cell-procCells addressing forAll (cellProcCells, cellI) @@ -2410,7 +2212,7 @@ const Foam::labelListList& Foam::immersedBoundaryFvPatch::ibCellCells() const } -const Foam::FieldField& +const Foam::vectorListList& Foam::immersedBoundaryFvPatch::ibProcCentres() const { if (!ibProcCentresPtr_) @@ -2422,7 +2224,7 @@ Foam::immersedBoundaryFvPatch::ibProcCentres() const } -const Foam::FieldField& +const Foam::scalarListList& Foam::immersedBoundaryFvPatch::ibProcGamma() const { if (!ibProcGammaPtr_) diff --git a/src/immersedBoundary/immersedBoundary/immersedBoundaryFvPatch/immersedBoundaryFvPatch.H b/src/immersedBoundary/immersedBoundary/immersedBoundaryFvPatch/immersedBoundaryFvPatch.H index af5ef486a..d6dbbc41b 100644 --- a/src/immersedBoundary/immersedBoundary/immersedBoundaryFvPatch/immersedBoundaryFvPatch.H +++ b/src/immersedBoundary/immersedBoundary/immersedBoundaryFvPatch/immersedBoundaryFvPatch.H @@ -49,6 +49,8 @@ SourceFiles #include "surfaceFieldsFwd.H" #include "dynamicLabelList.H" #include "labelPair.H" +#include "scalarList.H" +#include "vectorList.H" #include "FieldFields.H" #include "scalarMatrices.H" @@ -70,7 +72,7 @@ class immersedBoundaryFvPatch { // Private data - //- Reference to processor patch + //- Reference to immersed boundary patch const immersedBoundaryPolyPatch& ibPolyPatch_; //- Finite volume mesh reference @@ -168,10 +170,10 @@ class immersedBoundaryFvPatch mutable labelListList* ibProcCellsPtr_; //- Centres of cells from neighbour processors - mutable FieldField* ibProcCentresPtr_; + mutable vectorListList* ibProcCentresPtr_; //- Gamma of cells from neighbour processors - mutable FieldField* ibProcGammaPtr_; + mutable scalarListList* ibProcGammaPtr_; //- Cell-proc-cell addressing mutable List >* ibCellProcCellsPtr_; @@ -447,12 +449,12 @@ public: //- Return neighbour proc centres // These are centres from neighbouring processors the // local processor needs to receive - const FieldField& ibProcCentres() const; + const vectorListList& ibProcCentres() const; //- Return neighbour proc gamma // These are gamma values from neighbouring processors the // local processor needs to receive - const FieldField& ibProcGamma() const; + const scalarListList& ibProcGamma() const; //- Return neighbour cell addressing const List >& ibCellProcCells() const; diff --git a/src/immersedBoundary/immersedBoundary/immersedBoundaryFvPatch/immersedBoundaryFvPatchLeastSquaresFit.C b/src/immersedBoundary/immersedBoundary/immersedBoundaryFvPatch/immersedBoundaryFvPatchLeastSquaresFit.C index fdd78c2f5..feabbf96c 100644 --- a/src/immersedBoundary/immersedBoundary/immersedBoundaryFvPatch/immersedBoundaryFvPatchLeastSquaresFit.C +++ b/src/immersedBoundary/immersedBoundary/immersedBoundaryFvPatch/immersedBoundaryFvPatchLeastSquaresFit.C @@ -66,7 +66,7 @@ void Foam::immersedBoundaryFvPatch::makeInvDirichletMatrices() const // Initialize maxRowSum for debug scalar maxRowSum = 0.0; - const FieldField& procC = ibProcCentres(); + const vectorListList& procC = ibProcCentres(); label nCoeffs = 5; @@ -305,7 +305,7 @@ void Foam::immersedBoundaryFvPatch::makeInvNeumannMatrices() const // Initialize maxRowSum for debug scalar maxRowSum = 0.0; - const FieldField& procC = ibProcCentres(); + const vectorListList& procC = ibProcCentres(); label nCoeffs = 6; diff --git a/src/immersedBoundary/immersedBoundary/immersedBoundaryFvPatch/immersedBoundaryFvPatchSamplingWeights.C b/src/immersedBoundary/immersedBoundary/immersedBoundaryFvPatch/immersedBoundaryFvPatchSamplingWeights.C index 12070fb4d..f70cecc02 100644 --- a/src/immersedBoundary/immersedBoundary/immersedBoundaryFvPatch/immersedBoundaryFvPatchSamplingWeights.C +++ b/src/immersedBoundary/immersedBoundary/immersedBoundaryFvPatch/immersedBoundaryFvPatchSamplingWeights.C @@ -74,8 +74,8 @@ void Foam::immersedBoundaryFvPatch::makeIbSamplingWeights() const const scalarField& gammaIn = gamma().internalField(); const vectorField& CIn = mesh_.C().internalField(); - const FieldField& gammaProc = ibProcGamma(); - const FieldField& CProc = ibProcCentres(); + const scalarListList& gammaProc = ibProcGamma(); + const vectorListList& CProc = ibProcCentres(); // Go through all cellCells and calculate inverse distance for // all live points diff --git a/src/immersedBoundary/immersedBoundary/immersedBoundaryFvPatch/immersedBoundaryFvPatchTemplates.C b/src/immersedBoundary/immersedBoundary/immersedBoundaryFvPatch/immersedBoundaryFvPatchTemplates.C index c7745b9a8..4e3f612af 100644 --- a/src/immersedBoundary/immersedBoundary/immersedBoundaryFvPatch/immersedBoundaryFvPatchTemplates.C +++ b/src/immersedBoundary/immersedBoundary/immersedBoundaryFvPatch/immersedBoundaryFvPatchTemplates.C @@ -41,6 +41,9 @@ Foam::immersedBoundaryFvPatch::sendAndReceive ); FieldField& procPsi = tprocPsi(); + // This requires a rewrite useng mapDistribute + // HJ, 11/Aug/2016 + forAll (procPsi, procI) { procPsi.set diff --git a/src/immersedBoundary/immersedBoundary/immersedBoundaryFvPatch/immersedBoundaryFvPatchTriAddressing.C b/src/immersedBoundary/immersedBoundary/immersedBoundaryFvPatch/immersedBoundaryFvPatchTriAddressing.C index d1b76194a..c7f669bd2 100644 --- a/src/immersedBoundary/immersedBoundary/immersedBoundaryFvPatch/immersedBoundaryFvPatchTriAddressing.C +++ b/src/immersedBoundary/immersedBoundary/immersedBoundaryFvPatch/immersedBoundaryFvPatchTriAddressing.C @@ -61,7 +61,7 @@ void Foam::immersedBoundaryFvPatch::makeTriAddressing() const { hitTris[hf[hfI]] = hfI; } - + Info<< "triangles: " << triPatch.size() << " hit: " << hf.size() << endl; // Allocate storage cellsToTriAddrPtr_ = new labelListList(triPatch.size()); labelListList& addr = *cellsToTriAddrPtr_; @@ -84,8 +84,16 @@ void Foam::immersedBoundaryFvPatch::makeTriAddressing() const label faceIndex = 0; label counter = 0; - forAll (triPatch, triI) + boolList visited(triPatch.size(), false); + + register label curTri; + + // Only search for tri faces in the mesh + forAll (triFacesInMesh, tfimI) { + + const label triI = triFacesInMesh[tfimI]; + if (hitTris[triI] > -1) { // Triangle contains IB point @@ -99,8 +107,8 @@ void Foam::immersedBoundaryFvPatch::makeTriAddressing() const { // No direct hit. Start a neighbourhood search - // Record already visited faces - labelHashSet visited; + // Reset visited faces + visited = false; // Collect new faces to visit SLList