diff --git a/src/finiteVolume/fvMesh/fvPatches/constraint/ggi/ggiFvPatch.C b/src/finiteVolume/fvMesh/fvPatches/constraint/ggi/ggiFvPatch.C index 05ad430ce..09501ab16 100644 --- a/src/finiteVolume/fvMesh/fvPatches/constraint/ggi/ggiFvPatch.C +++ b/src/finiteVolume/fvMesh/fvPatches/constraint/ggi/ggiFvPatch.C @@ -271,6 +271,12 @@ const Foam::scalarListList& Foam::ggiFvPatch::weights() const } +void Foam::ggiFvPatch::expandAddrToZone(labelField& lf) const +{ + lf = ggiPolyPatch_.fastExpand(lf); +} + + Foam::tmp Foam::ggiFvPatch::interfaceInternalField ( const unallocLabelList& internalData @@ -306,6 +312,7 @@ void Foam::ggiFvPatch::initInternalFieldTransfer const unallocLabelList& iF ) const { + // Label transfer is local without global reduction labelTransferBuffer_ = patchInternalField(iF); } diff --git a/src/finiteVolume/fvMesh/fvPatches/constraint/ggi/ggiFvPatch.H b/src/finiteVolume/fvMesh/fvPatches/constraint/ggi/ggiFvPatch.H index 6aa101dd3..740f0f208 100644 --- a/src/finiteVolume/fvMesh/fvPatches/constraint/ggi/ggiFvPatch.H +++ b/src/finiteVolume/fvMesh/fvPatches/constraint/ggi/ggiFvPatch.H @@ -187,6 +187,11 @@ public: return coupledFvPatch::reverseT(); } + //- Expand addressing to zone + // Used in optimised AMG coarsening + virtual void expandAddrToZone(labelField&) const; + + //- Return the values of the given internal data adjacent to // the interface as a field virtual tmp interfaceInternalField diff --git a/src/finiteVolume/fvMesh/fvPatches/constraint/regionCouple/regionCoupleFvPatch.C b/src/finiteVolume/fvMesh/fvPatches/constraint/regionCouple/regionCoupleFvPatch.C index 41e8a58c1..38f5ba27d 100644 --- a/src/finiteVolume/fvMesh/fvPatches/constraint/regionCouple/regionCoupleFvPatch.C +++ b/src/finiteVolume/fvMesh/fvPatches/constraint/regionCouple/regionCoupleFvPatch.C @@ -315,6 +315,16 @@ const Foam::scalarListList& Foam::regionCoupleFvPatch::weights() const } +void Foam::regionCoupleFvPatch::expandAddrToZone(labelField& lf) const +{ + // Missing code. Activate for AMG solvers across regionCoupleFvPatch + notImplemented + ( + "void regionCoupleFvPatch::expandAddrToZone(labelField& lf) const" + ); +} + + Foam::tmp Foam::regionCoupleFvPatch::interfaceInternalField ( const unallocLabelList& internalData diff --git a/src/finiteVolume/fvMesh/fvPatches/constraint/regionCouple/regionCoupleFvPatch.H b/src/finiteVolume/fvMesh/fvPatches/constraint/regionCouple/regionCoupleFvPatch.H index f09721e88..85a0fba14 100644 --- a/src/finiteVolume/fvMesh/fvPatches/constraint/regionCouple/regionCoupleFvPatch.H +++ b/src/finiteVolume/fvMesh/fvPatches/constraint/regionCouple/regionCoupleFvPatch.H @@ -196,6 +196,11 @@ public: return coupledFvPatch::reverseT(); } + //- Expand addressing to zone + // Used in optimised AMG coarsening + virtual void expandAddrToZone(labelField&) const; + + //- Return the values of the given internal data adjacent to // the interface as a field virtual tmp interfaceInternalField diff --git a/src/foam/matrices/lduMatrix/lduAddressing/lduInterfaces/ggiLduInterface/ggiLduInterface.H b/src/foam/matrices/lduMatrix/lduAddressing/lduInterfaces/ggiLduInterface/ggiLduInterface.H index 4ad62d264..f13268ce0 100644 --- a/src/foam/matrices/lduMatrix/lduAddressing/lduInterfaces/ggiLduInterface/ggiLduInterface.H +++ b/src/foam/matrices/lduMatrix/lduAddressing/lduInterfaces/ggiLduInterface/ggiLduInterface.H @@ -121,6 +121,10 @@ public: //- Return face reverse transformation tensor virtual const tensorField& reverseT() const = 0; + //- Expand addressing to zone + // Used in optimised AMG coarsening + virtual void expandAddrToZone(labelField&) const = 0; + // Transfer buffer access diff --git a/src/foam/matrices/lduMatrix/lduAddressing/lduInterfaces/overlapGGILduInterface/overlapGGILduInterface.H b/src/foam/matrices/lduMatrix/lduAddressing/lduInterfaces/overlapGGILduInterface/overlapGGILduInterface.H index c7a4b181b..ddaa2890c 100644 --- a/src/foam/matrices/lduMatrix/lduAddressing/lduInterfaces/overlapGGILduInterface/overlapGGILduInterface.H +++ b/src/foam/matrices/lduMatrix/lduAddressing/lduInterfaces/overlapGGILduInterface/overlapGGILduInterface.H @@ -47,7 +47,7 @@ namespace Foam { /*---------------------------------------------------------------------------*\ - Class overlapGGILduInterface Declaration + Class overlapGGILduInterface Declaration \*---------------------------------------------------------------------------*/ class overlapGGILduInterface diff --git a/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/AMGInterface/AMGInterface.H b/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/AMGInterface/AMGInterface.H index f5b2557a9..3ef010832 100644 --- a/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/AMGInterface/AMGInterface.H +++ b/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/AMGInterface/AMGInterface.H @@ -82,9 +82,21 @@ protected: labelField faceCells_; //- Fine addressing + // On GGI interfaces, a single fine coefficient may contribute to + // multiple coarse coefficients using different weights. + // To hanld this, a fine coefficient may be visited multiple times + // which is recorded in fineAddressing. + // For simple (matching) interfaces, fineAddressing_[i] = i + // HJ, 21/Jun/2016 labelField fineAddressing_; //- Restrict addressing + // For each fine coefficient, list coarse cluster index it will be + // agglomerated into + // For cases where the fineAddressing is used, restrict addressing + // and weights are expanded to match multiple hits for a single + // fine coefficient, as dictated by fineAddressing + // HJ, 21/Jun/2016 labelField restrictAddressing_; //- Fine level agglomeration weights diff --git a/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/ggiAMGInterface/ggiAMGInterface.C b/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/ggiAMGInterface/ggiAMGInterface.C index 1f10a8724..82527393c 100644 --- a/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/ggiAMGInterface/ggiAMGInterface.C +++ b/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/ggiAMGInterface/ggiAMGInterface.C @@ -203,11 +203,6 @@ Foam::ggiAMGInterface::ggiAMGInterface zoneAddressing_(), mapPtr_(NULL) { - // Note. - // All processors will do the same coarsening and then filter - // the addressing to the local processor - // HJ, 1/Apr/2009 - // Note. // Signalling in global clustering requires me to recognise clustering // from separate processors as separate. In the first phase, this will be @@ -222,7 +217,15 @@ Foam::ggiAMGInterface::ggiAMGInterface // larger max int, which can be changed on request // HJ, 1/Apr/2009 - // Expand the local and neighbour addressing to full zone size + // New algorithm will assemble local clusters and then create a global + // zone ordering by collecting all faces (coarse pairs) from proc0, + // followed by proc 1 etc. This avoids global communication and allows + // each processor only to perform the analysis on locally created coarse + // faces + // HJ, 13/Jun/2016 + + // To help with analysis, expand the local and neighbour addressing + // to full zone size labelField localExpandAddressing(fineGgiInterface_.zoneSize(), 0); // Memory management, local @@ -235,60 +238,70 @@ Foam::ggiAMGInterface::ggiAMGInterface localRestrictAddressing[i] + procOffset*Pstream::myProcNo(); } - if (!localParallel()) - { - // Optimisation of this comms call is needed - // HJ, 9/Jun/2016 - reduce(localExpandAddressing, sumOp()); - } + // Removed global reduce. Only local faces will be analysed. + // HJ, 13/Jun/2016 } + // Create addressing for neighbour faces. Note: expandAddrToZone will + // expand the addressing to zone size. HJ, 13/Jun/2016 labelField neighbourExpandAddressing ( - fineGgiInterface_.shadowInterface().zoneSize(), - 0 + fineGgiInterface_.shadowInterface().interfaceSize() ); + // Fill local cluster ID with a combination of a local ID and processor + // offset // Memory management, neighbour { - const labelList& addr = - fineGgiInterface_.shadowInterface().zoneAddressing(); - - forAll (addr, i) + forAll (neighbourExpandAddressing, i) { - neighbourExpandAddressing[addr[i]] = + neighbourExpandAddressing[i] = neighbourRestrictAddressing[i] + procOffset*Pstream::myProcNo(); } - if (!localParallel()) - { - // Optimisation of this comms call is needed - // HJ, 9/Jun/2016 - reduce(neighbourExpandAddressing, sumOp()); - } + // Expand neighbour side to get all the data required from other + // processors. Note: neigbour is now the size of remote zone + fineGgiInterface_.shadowInterface().expandAddrToZone + ( + neighbourExpandAddressing + ); } + // DEBUG: Check that all sizes are at zone size. + Info<< "Sizes check: local zone size " + << fineGgiInterface_.zoneSize() + << " " << localExpandAddressing << nl + << "shadow zone size " + << fineGgiInterface_.shadowInterface().zoneSize() + << " " << neighbourExpandAddressing + << endl; + + + // Note: neighbourExpandAddressing will be filled with NaNs for faces which + // not local + + Info<< "End of reduce" << endl; // Make a lookup table of entries for owner/neighbour. // All sizes are guessed at the size of fine interface // HJ, 19/Feb/2009 HashTable, label, Hash