From 927c94c6d858cea9caa81c0c13ab20c30f6adae9 Mon Sep 17 00:00:00 2001 From: Hrvoje Jasak Date: Fri, 14 Oct 2011 13:00:53 +0100 Subject: [PATCH] Patch constrained decomposition --- .../decompositionMethods/Make/files | 3 +- .../decompositionMethod/decompositionMethod.C | 437 ++++++++++++++---- .../decompositionMethod/decompositionMethod.H | 90 ++-- .../geomDecomp/geomDecomp.C | 3 +- .../geomDecomp/geomDecomp.H | 19 +- .../hierarchGeomDecomp/hierarchGeomDecomp.C | 21 +- .../hierarchGeomDecomp/hierarchGeomDecomp.H | 43 +- .../manualDecomp/manualDecomp.C | 32 +- .../manualDecomp/manualDecomp.H | 21 +- .../patchConstrainedDecomp.C | 120 +++++ .../patchConstrainedDecomp.H | 142 ++++++ .../simpleGeomDecomp/simpleGeomDecomp.C | 8 +- .../simpleGeomDecomp/simpleGeomDecomp.H | 10 +- 13 files changed, 753 insertions(+), 196 deletions(-) create mode 100644 src/decompositionMethods/decompositionMethods/patchConstrainedDecomp/patchConstrainedDecomp.C create mode 100644 src/decompositionMethods/decompositionMethods/patchConstrainedDecomp/patchConstrainedDecomp.H diff --git a/src/decompositionMethods/decompositionMethods/Make/files b/src/decompositionMethods/decompositionMethods/Make/files index 43bb9a25c..28ba31a12 100644 --- a/src/decompositionMethods/decompositionMethods/Make/files +++ b/src/decompositionMethods/decompositionMethods/Make/files @@ -1,7 +1,8 @@ decompositionMethod/decompositionMethod.C +manualDecomp/manualDecomp.C geomDecomp/geomDecomp.C simpleGeomDecomp/simpleGeomDecomp.C hierarchGeomDecomp/hierarchGeomDecomp.C -manualDecomp/manualDecomp.C +patchConstrainedDecomp/patchConstrainedDecomp.C LIB = $(FOAM_LIBBIN)/libdecompositionMethods diff --git a/src/decompositionMethods/decompositionMethods/decompositionMethod/decompositionMethod.C b/src/decompositionMethods/decompositionMethods/decompositionMethod/decompositionMethod.C index 05a206020..5adf0f20c 100644 --- a/src/decompositionMethods/decompositionMethods/decompositionMethod/decompositionMethod.C +++ b/src/decompositionMethods/decompositionMethods/decompositionMethod/decompositionMethod.C @@ -28,6 +28,9 @@ InClass \*---------------------------------------------------------------------------*/ #include "decompositionMethod.H" +#include "cyclicPolyPatch.H" +#include "syncTools.H" +#include "globalIndex.H" // * * * * * * * * * * * * * * Static Data Members * * * * * * * * * * * * * // @@ -38,7 +41,336 @@ namespace Foam defineRunTimeSelectionTable(decompositionMethod, dictionaryMesh); } -// * * * * * * * * * * * * * * * Member Functions * * * * * * * * * * * * * // + +// * * * * * * * * * * * * Protected Member Functions * * * * * * * * * * * // + +void Foam::decompositionMethod::calcCSR +( + const labelListList& cellCells, + List& adjncy, + List& xadj +) +{ + // Count number of internal faces + label nConnections = 0; + + forAll(cellCells, coarseI) + { + nConnections += cellCells[coarseI].size(); + } + + // Create the adjncy array as twice the size of the total number of + // internal faces + adjncy.setSize(nConnections); + + xadj.setSize(cellCells.size() + 1); + + + // Fill in xadj + // ~~~~~~~~~~~~ + label freeAdj = 0; + + forAll(cellCells, coarseI) + { + xadj[coarseI] = freeAdj; + + const labelList& cCells = cellCells[coarseI]; + + forAll(cCells, i) + { + adjncy[freeAdj++] = cCells[i]; + } + } + xadj[cellCells.size()] = freeAdj; +} + + + +void Foam::decompositionMethod::calcCSR +( + const polyMesh& mesh, + List& adjncy, + List& xadj +) +{ + // Make Metis CSR (Compressed Storage Format) storage + // adjncy : contains neighbours (= edges in graph) + // xadj(celli) : start of information in adjncy for celli + + xadj.setSize(mesh.nCells() + 1); + + // Initialise the number of internal faces of the cells to twice the + // number of internal faces + label nInternalFaces = 2*mesh.nInternalFaces(); + + // Check the boundary for coupled patches and add to the number of + // internal faces + const polyBoundaryMesh& pbm = mesh.boundaryMesh(); + + forAll(pbm, patchi) + { + if (isA(pbm[patchi])) + { + nInternalFaces += pbm[patchi].size(); + } + } + + // Create the adjncy array the size of the total number of internal and + // coupled faces + adjncy.setSize(nInternalFaces); + + // Fill in xadj + // ~~~~~~~~~~~~ + label freeAdj = 0; + + for (label cellI = 0; cellI < mesh.nCells(); cellI++) + { + xadj[cellI] = freeAdj; + + const labelList& cFaces = mesh.cells()[cellI]; + + forAll(cFaces, i) + { + label faceI = cFaces[i]; + + if + ( + mesh.isInternalFace(faceI) + || isA(pbm[pbm.whichPatch(faceI)]) + ) + { + freeAdj++; + } + } + } + xadj[mesh.nCells()] = freeAdj; + + + // Fill in adjncy + // ~~~~~~~~~~~~~~ + + labelList nFacesPerCell(mesh.nCells(), 0); + + // Internal faces + for (label faceI = 0; faceI < mesh.nInternalFaces(); faceI++) + { + label own = mesh.faceOwner()[faceI]; + label nei = mesh.faceNeighbour()[faceI]; + + adjncy[xadj[own] + nFacesPerCell[own]++] = nei; + adjncy[xadj[nei] + nFacesPerCell[nei]++] = own; + } + + // Coupled faces. Only cyclics done. + forAll(pbm, patchi) + { + if (isA(pbm[patchi])) + { + const unallocLabelList& faceCells = pbm[patchi].faceCells(); + + label sizeby2 = faceCells.size()/2; + + for (label facei=0; facei& adjncy, + List& xadj +) +{ + // Create global cell numbers + // ~~~~~~~~~~~~~~~~~~~~~~~~~~ + + globalIndex globalCells(mesh.nCells()); + + + // + // Make Metis Distributed CSR (Compressed Storage Format) storage + // adjncy : contains cellCells (= edges in graph) + // xadj(celli) : start of information in adjncy for celli + // + + + const labelList& faceOwner = mesh.faceOwner(); + const labelList& faceNeighbour = mesh.faceNeighbour(); + const polyBoundaryMesh& patches = mesh.boundaryMesh(); + + + // Get renumbered owner on other side of coupled faces + // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + List globalNeighbour(mesh.nFaces()-mesh.nInternalFaces()); + + forAll(patches, patchI) + { + const polyPatch& pp = patches[patchI]; + + if (pp.coupled()) + { + label faceI = pp.start(); + label bFaceI = pp.start() - mesh.nInternalFaces(); + + forAll(pp, i) + { + globalNeighbour[bFaceI++] = globalCells.toGlobal + ( + faceOwner[faceI++] + ); + } + } + } + + // Get the cell on the other side of coupled patches + syncTools::swapBoundaryFaceList(mesh, globalNeighbour, false); + + + // Count number of faces (internal + coupled) + // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + // Number of faces per cell + List nFacesPerCell(mesh.nCells(), 0); + + // Number of coupled faces + label nCoupledFaces = 0; + + for (label faceI = 0; faceI < mesh.nInternalFaces(); faceI++) + { + nFacesPerCell[faceOwner[faceI]]++; + nFacesPerCell[faceNeighbour[faceI]]++; + } + // Handle coupled faces + forAll(patches, patchI) + { + const polyPatch& pp = patches[patchI]; + + if (pp.coupled()) + { + label faceI = pp.start(); + + forAll(pp, i) + { + nCoupledFaces++; + nFacesPerCell[faceOwner[faceI++]]++; + } + } + } + + + // Fill in xadj + // ~~~~~~~~~~~~ + + xadj.setSize(mesh.nCells() + 1); + + int freeAdj = 0; + + for (label cellI = 0; cellI < mesh.nCells(); cellI++) + { + xadj[cellI] = freeAdj; + + freeAdj += nFacesPerCell[cellI]; + } + xadj[mesh.nCells()] = freeAdj; + + + + // Fill in adjncy + // ~~~~~~~~~~~~~~ + + adjncy.setSize(2*mesh.nInternalFaces() + nCoupledFaces); + + nFacesPerCell = 0; + + // For internal faces is just offsetted owner and neighbour + for (label faceI = 0; faceI < mesh.nInternalFaces(); faceI++) + { + label own = faceOwner[faceI]; + label nei = faceNeighbour[faceI]; + + adjncy[xadj[own] + nFacesPerCell[own]++] = globalCells.toGlobal(nei); + adjncy[xadj[nei] + nFacesPerCell[nei]++] = globalCells.toGlobal(own); + } + // For boundary faces is offsetted coupled neighbour + forAll(patches, patchI) + { + const polyPatch& pp = patches[patchI]; + + if (pp.coupled()) + { + label faceI = pp.start(); + label bFaceI = pp.start()-mesh.nInternalFaces(); + + forAll(pp, i) + { + label own = faceOwner[faceI]; + adjncy[xadj[own] + nFacesPerCell[own]++] = + globalNeighbour[bFaceI]; + + faceI++; + bFaceI++; + } + } + } +} + + +void Foam::decompositionMethod::calcCellCells +( + const polyMesh& mesh, + const labelList& fineToCoarse, + const label nCoarse, + labelListList& cellCells +) +{ + if (fineToCoarse.size() != mesh.nCells()) + { + FatalErrorIn + ( + "decompositionMethod::calcCellCells" + "(const labelList&, labelListList&) const" + ) << "Only valid for mesh agglomeration." << exit(FatalError); + } + + List > dynCellCells(nCoarse); + + forAll(mesh.faceNeighbour(), faceI) + { + label own = fineToCoarse[mesh.faceOwner()[faceI]]; + label nei = fineToCoarse[mesh.faceNeighbour()[faceI]]; + + if (own != nei) + { + if (findIndex(dynCellCells[own], nei) == -1) + { + dynCellCells[own].append(nei); + } + if (findIndex(dynCellCells[nei], own) == -1) + { + dynCellCells[nei].append(own); + } + } + } + + cellCells.setSize(dynCellCells.size()); + forAll(dynCellCells, coarseI) + { + cellCells[coarseI].transfer(dynCellCells[coarseI]); + } +} + + +// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * // Foam::autoPtr Foam::decompositionMethod::New ( @@ -102,17 +434,40 @@ Foam::autoPtr Foam::decompositionMethod::New } +// * * * * * * * * * * * * * * * Member Functions * * * * * * * * * * * * * // + Foam::labelList Foam::decompositionMethod::decompose ( const pointField& points ) { - scalarField weights(0); + scalarField weights(points.size(), 1); return decompose(points, weights); } +Foam::labelList Foam::decompositionMethod::decompose +( + const labelList& fineToCoarse, + const pointField& coarsePoints +) +{ + // Decompose based on agglomerated points + labelList coarseDistribution(decompose(coarsePoints)); + + // Rework back into decomposition for original mesh_ + labelList fineDistribution(fineToCoarse.size()); + + forAll(fineDistribution, i) + { + fineDistribution[i] = coarseDistribution[fineToCoarse[i]]; + } + + return fineDistribution; +} + + Foam::labelList Foam::decompositionMethod::decompose ( const labelList& fineToCoarse, @@ -135,82 +490,4 @@ Foam::labelList Foam::decompositionMethod::decompose } -Foam::labelList Foam::decompositionMethod::decompose -( - const labelList& fineToCoarse, - const pointField& coarsePoints -) -{ - // Decompose based on agglomerated points - labelList coarseDistribution(decompose(coarsePoints)); - - // Rework back into decomposition for original mesh_ - labelList fineDistribution(fineToCoarse.size()); - - forAll(fineDistribution, i) - { - fineDistribution[i] = coarseDistribution[fineToCoarse[i]]; - } - - return fineDistribution; -} - - -void Foam::decompositionMethod::calcCellCells -( - const polyMesh& mesh, - const labelList& fineToCoarse, - const label nCoarse, - labelListList& cellCells -) -{ - if (fineToCoarse.size() != mesh.nCells()) - { - FatalErrorIn - ( - "decompositionMethod::calcCellCells" - "(const labelList&, labelListList&) const" - ) << "Only valid for mesh agglomeration." << exit(FatalError); - } - - List > dynCellCells(nCoarse); - - forAll(mesh.faceNeighbour(), faceI) - { - label own = fineToCoarse[mesh.faceOwner()[faceI]]; - label nei = fineToCoarse[mesh.faceNeighbour()[faceI]]; - - if (own != nei) - { - if (findIndex(dynCellCells[own], nei) == -1) - { - dynCellCells[own].append(nei); - } - if (findIndex(dynCellCells[nei], own) == -1) - { - dynCellCells[nei].append(own); - } - } - } - - cellCells.setSize(dynCellCells.size()); - forAll(dynCellCells, coarseI) - { - cellCells[coarseI].transfer(dynCellCells[coarseI]); - } -} - - -Foam::labelList Foam::decompositionMethod::decompose -( - const labelListList& globalCellCells, - const pointField& cc -) -{ - scalarField cWeights(0); - - return decompose(globalCellCells, cc, cWeights); -} - - // ************************************************************************* // diff --git a/src/decompositionMethods/decompositionMethods/decompositionMethod/decompositionMethod.H b/src/decompositionMethods/decompositionMethods/decompositionMethod/decompositionMethod.H index e241ffd63..7957a96a6 100644 --- a/src/decompositionMethods/decompositionMethods/decompositionMethod/decompositionMethod.H +++ b/src/decompositionMethods/decompositionMethods/decompositionMethod/decompositionMethod.H @@ -48,30 +48,64 @@ namespace Foam class decompositionMethod { - protected: // Protected data + //- Decomposition dictionary const dictionary& decompositionDict_; + + //- Number of processors in decomposition label nProcessors_; - //- Helper: determine (non-parallel) cellCells from mesh agglomeration. + //- Helper to convert connectivity supplied as cellCells into + // simple CSR (Metis, scotch) storage + static void calcCSR + ( + const labelListList& globalCellCells, + List& adjncy, + List& xadj + ); + + //- Helper: convert local connectivity from the mesh + // into CSR (Metis, scotch) storage + // Treats cyclics as coupled, but not processor patches + static void calcCSR + ( + const polyMesh& mesh, + List& adjncy, + List& xadj + ); + + //- Helper: convert mesh connectivity into distributed CSR + // Very dubious coding. HJ, 1/Mar/2011 + static void calcDistributedCSR + ( + const polyMesh&, + List& adjncy, + List& xadj + ); + + //- Helper: determine (non-parallel) cellCells from mesh + // with agglomeration static void calcCellCells ( const polyMesh& mesh, - const labelList& agglom, + const labelList& fineToCoarse, const label nCoarse, labelListList& cellCells ); + private: // Private Member Functions - //- Disallow default bitwise copy construct and assignment + //- Disallow default bitwise copy construct decompositionMethod(const decompositionMethod&); + + //- Disallow default bitwise assignment void operator=(const decompositionMethod&); @@ -109,7 +143,7 @@ public: // Selectors - //- Return a reference to the selected decomposition method + //- Return a pointer to the selected decomposition method static autoPtr New ( const dictionary& decompositionDict @@ -148,46 +182,37 @@ public: // proc boundaries) virtual bool parallelAware() const = 0; - //- Return for every coordinate the wanted processor number. Use the - // mesh connectivity (if needed) + //- Decompose cells + // If needed, use connectivity directly from the mesh + // Calls decompose (below) with uniform weights + virtual labelList decompose(const pointField&); + + //- Decompose cells with weights virtual labelList decompose ( const pointField& points, const scalarField& pointWeights ) = 0; - //- Like decompose but with uniform weights on the points - virtual labelList decompose(const pointField&); - - //- Return for every coordinate the wanted processor number. Gets - // passed agglomeration map (from fine to coarse cells) and coarse cell - // location. Can be overridden by decomposers that provide this - // functionality natively. Coarse cells are local to the processor - // (if in parallel). If you want to have coarse cells spanning - // processors use the globalCellCells instead. + //- Decompose cell clusters + // Calls decompose (below) with uniform weights virtual labelList decompose ( - const labelList& cellToRegion, - const pointField& regionPoints, - const scalarField& regionWeights + const labelList& fineToCoarse, + const pointField& coarsePoints ); - //- Like decompose but with uniform weights on the regions + //- Decompose cell clusters with weights on clusters virtual labelList decompose ( - const labelList& cellToRegion, - const pointField& regionPoints + const labelList& fineToCoarse, + const pointField& coarsePoints, + const scalarField& coarseWeights ); - //- Return for every coordinate the wanted processor number. Explicitly - // provided connectivity - does not use mesh_. - // The connectivity is equal to mesh.cellCells() except for - // - in parallel the cell numbers are global cell numbers (starting - // from 0 at processor0 and then incrementing all through the - // processors) - // - the connections are across coupled patches + //- Decompose cells with weights with explicitly provided connectivity virtual labelList decompose ( const labelListList& globalCellCells, @@ -195,13 +220,6 @@ public: const scalarField& cWeights ) = 0; - //- Like decompose but with uniform weights on the cells - virtual labelList decompose - ( - const labelListList& globalCellCells, - const pointField& cc - ); - }; diff --git a/src/decompositionMethods/decompositionMethods/geomDecomp/geomDecomp.C b/src/decompositionMethods/decompositionMethods/geomDecomp/geomDecomp.C index 4202c9b2a..bc6697a55 100644 --- a/src/decompositionMethods/decompositionMethods/geomDecomp/geomDecomp.C +++ b/src/decompositionMethods/decompositionMethods/geomDecomp/geomDecomp.C @@ -40,8 +40,7 @@ Foam::geomDecomp::geomDecomp delta_(readScalar(geomDecomDict_.lookup("delta"))), rotDelta_(I) { - // check that the case makes sense : - + // Check that the case makes sense if (nProcessors_ != n_.x()*n_.y()*n_.z()) { FatalErrorIn diff --git a/src/decompositionMethods/decompositionMethods/geomDecomp/geomDecomp.H b/src/decompositionMethods/decompositionMethods/geomDecomp/geomDecomp.H index dee0ea40f..060898e94 100644 --- a/src/decompositionMethods/decompositionMethods/geomDecomp/geomDecomp.H +++ b/src/decompositionMethods/decompositionMethods/geomDecomp/geomDecomp.H @@ -37,7 +37,7 @@ SourceFiles #define geomDecomp_H #include "decompositionMethod.H" -#include "Vector.H" +#include "labelVector.H" namespace Foam { @@ -50,17 +50,32 @@ class geomDecomp : public decompositionMethod { + // Private Member Functions + + //- Disallow default bitwise copy construct + geomDecomp(const geomDecomp&); + + //- Disallow default bitwise assignment + void operator=(const geomDecomp&); + protected: // Protected data + //- Geometric decomposition dictionary const dictionary& geomDecomDict_; - Vector