diff --git a/src/foam/matrices/blockLduMatrix/BlockAMG/BlockMatrixCoarsening/BlockMatrixAgglomeration/BlockMatrixAgglomeration.C b/src/foam/matrices/blockLduMatrix/BlockAMG/BlockMatrixCoarsening/BlockMatrixAgglomeration/BlockMatrixAgglomeration.C index 9278ad63b..792a39c0f 100644 --- a/src/foam/matrices/blockLduMatrix/BlockAMG/BlockMatrixCoarsening/BlockMatrixAgglomeration/BlockMatrixAgglomeration.C +++ b/src/foam/matrices/blockLduMatrix/BlockAMG/BlockMatrixCoarsening/BlockMatrixAgglomeration/BlockMatrixAgglomeration.C @@ -950,6 +950,7 @@ Foam::BlockMatrixAgglomeration::restrictMatrix() const AMGInterface::New ( coarseAddrPtr(), + coarseInterfaces, fineInterface, fineInterface.interfaceInternalField(agglomIndex_), fineInterfaceAddr[intI] diff --git a/src/foam/matrices/lduMatrix/solvers/AMG/AMGAgglomerations/GAMGAgglomeration/GAMGAgglomerateLduAddressing.C b/src/foam/matrices/lduMatrix/solvers/AMG/AMGAgglomerations/GAMGAgglomeration/GAMGAgglomerateLduAddressing.C index e93172b11..9ba20ce1f 100644 --- a/src/foam/matrices/lduMatrix/solvers/AMG/AMGAgglomerations/GAMGAgglomeration/GAMGAgglomerateLduAddressing.C +++ b/src/foam/matrices/lduMatrix/solvers/AMG/AMGAgglomerations/GAMGAgglomeration/GAMGAgglomerateLduAddressing.C @@ -292,6 +292,7 @@ void Foam::GAMGAgglomeration::agglomerateLduAddressing AMGInterface::New ( meshLevels_[fineLevelIndex], + coarseInterfaces, fineInterfaces[inti], fineInterfaces[inti].interfaceInternalField ( diff --git a/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/AMGInterface/AMGInterface.H b/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/AMGInterface/AMGInterface.H index 3ef010832..639a04314 100644 --- a/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/AMGInterface/AMGInterface.H +++ b/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/AMGInterface/AMGInterface.H @@ -129,12 +129,14 @@ public: lduInterface, ( const lduPrimitiveMesh& lduMesh, + const lduInterfacePtrsList& coarseInterfaces, const lduInterface& fineInterface, const labelField& localRestrictAddressing, const labelField& neighbourRestrictAddressing ), ( lduMesh, + coarseInterfaces, fineInterface, localRestrictAddressing, neighbourRestrictAddressing @@ -149,6 +151,7 @@ public: static autoPtr New ( const lduPrimitiveMesh& lduMesh, + const lduInterfacePtrsList& coarseInterfaces, const lduInterface& fineInterface, const labelField& localRestrictAddressing, const labelField& neighbourRestrictAddressing @@ -165,6 +168,11 @@ public: {} + //- Destructor + virtual ~AMGInterface() + {} + + // Member Functions // Access diff --git a/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/AMGInterface/newAMGInterface.C b/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/AMGInterface/newAMGInterface.C index 4a8f5236f..2f6dc2548 100644 --- a/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/AMGInterface/newAMGInterface.C +++ b/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/AMGInterface/newAMGInterface.C @@ -32,6 +32,7 @@ License Foam::autoPtr Foam::AMGInterface::New ( const lduPrimitiveMesh& lduMesh, + const lduInterfacePtrsList& coarseInterfaces, const lduInterface& fineInterface, const labelField& localRestrictAddressing, const labelField& neighbourRestrictAddressing @@ -46,11 +47,14 @@ Foam::autoPtr Foam::AMGInterface::New { FatalErrorIn ( - "AMGInterface::New" - "(const lduPrimitiveMesh& lduMesh," - "const lduInterface& fineInterface," - "const labelField& localRestrictAddressing," - "const labelField& neighbourRestrictAddressing)" + "AMGInterface::New\n" + "(\n" + " const lduPrimitiveMesh& lduMesh,\n" + " const lduInterfacePtrsList& coarseInterfaces,\n" + " const lduInterface& fineInterface,\n" + " const labelField& localRestrictAddressing,\n" + " const labelField& neighbourRestrictAddressing\n" + ")" ) << "Unknown AMGInterface type " << coupleType << ".\n" << "Valid AMGInterface types are :" << lduInterfaceConstructorTablePtr_->sortedToc() @@ -62,6 +66,7 @@ Foam::autoPtr Foam::AMGInterface::New cstrIter() ( lduMesh, + coarseInterfaces, fineInterface, localRestrictAddressing, neighbourRestrictAddressing diff --git a/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/cyclicAMGInterface/cyclicAMGInterface.C b/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/cyclicAMGInterface/cyclicAMGInterface.C index 2ae72dbaa..72fe4cf0b 100644 --- a/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/cyclicAMGInterface/cyclicAMGInterface.C +++ b/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/cyclicAMGInterface/cyclicAMGInterface.C @@ -45,6 +45,7 @@ namespace Foam Foam::cyclicAMGInterface::cyclicAMGInterface ( const lduPrimitiveMesh& lduMesh, + const lduInterfacePtrsList& coarseInterfaces, const lduInterface& fineInterface, const labelField& localRestrictAddressing, const labelField& neighbourRestrictAddressing diff --git a/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/cyclicAMGInterface/cyclicAMGInterface.H b/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/cyclicAMGInterface/cyclicAMGInterface.H index 3e983f19b..0a3978e47 100644 --- a/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/cyclicAMGInterface/cyclicAMGInterface.H +++ b/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/cyclicAMGInterface/cyclicAMGInterface.H @@ -81,6 +81,7 @@ public: cyclicAMGInterface ( const lduPrimitiveMesh& lduMesh, + const lduInterfacePtrsList& coarseInterfaces, const lduInterface& fineInterface, const labelField& localRestrictAddressing, const labelField& neighbourRestrictAddressing diff --git a/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/cyclicGGIAMGInterface/cyclicGGIAMGInterface.C b/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/cyclicGGIAMGInterface/cyclicGGIAMGInterface.C index 6f1df8b3d..c64227099 100644 --- a/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/cyclicGGIAMGInterface/cyclicGGIAMGInterface.C +++ b/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/cyclicGGIAMGInterface/cyclicGGIAMGInterface.C @@ -45,6 +45,7 @@ namespace Foam Foam::cyclicGGIAMGInterface::cyclicGGIAMGInterface ( const lduPrimitiveMesh& lduMesh, + const lduInterfacePtrsList& coarseInterfaces, const lduInterface& fineInterface, const labelField& localRestrictAddressing, const labelField& neighbourRestrictAddressing @@ -53,6 +54,7 @@ Foam::cyclicGGIAMGInterface::cyclicGGIAMGInterface ggiAMGInterface ( lduMesh, + coarseInterfaces, fineInterface, localRestrictAddressing, neighbourRestrictAddressing diff --git a/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/cyclicGGIAMGInterface/cyclicGGIAMGInterface.H b/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/cyclicGGIAMGInterface/cyclicGGIAMGInterface.H index 19f2b92d7..2498c2282 100644 --- a/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/cyclicGGIAMGInterface/cyclicGGIAMGInterface.H +++ b/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/cyclicGGIAMGInterface/cyclicGGIAMGInterface.H @@ -67,6 +67,7 @@ public: cyclicGGIAMGInterface ( const lduPrimitiveMesh& lduMesh, + const lduInterfacePtrsList& coarseInterfaces, const lduInterface& fineInterface, const labelField& localRestrictAddressing, const labelField& neighbourRestrictAddressing diff --git a/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/ggiAMGInterface/ggiAMGInterface.C b/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/ggiAMGInterface/ggiAMGInterface.C index 82527393c..cc77d00b0 100644 --- a/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/ggiAMGInterface/ggiAMGInterface.C +++ b/src/foam/matrices/lduMatrix/solvers/AMG/interfaces/ggiAMGInterface/ggiAMGInterface.C @@ -192,6 +192,7 @@ void Foam::ggiAMGInterface::initFastReduce() const Foam::ggiAMGInterface::ggiAMGInterface ( const lduPrimitiveMesh& lduMesh, + const lduInterfacePtrsList& coarseInterfaces, const lduInterface& fineInterface, const labelField& localRestrictAddressing, const labelField& neighbourRestrictAddressing @@ -201,107 +202,87 @@ Foam::ggiAMGInterface::ggiAMGInterface fineGgiInterface_(refCast(fineInterface)), zoneSize_(0), zoneAddressing_(), + procMasterFaces_(), mapPtr_(NULL) { - // Note. - // Signalling in global clustering requires me to recognise clustering - // from separate processors as separate. In the first phase, this will be - // used to recognise cluster from each processor as separate and in the - // second phase it will be used to filter local processor faces from - // the global patch. - // Currently, I am calculating unique cluster index as: - // - // id = cluster + procOffset*myProcID - // With procOffset = 1 million, this should be sufficient for 2000 CPUs - // with 2 million coarse cells each. For larger numbers, I need a - // larger max int, which can be changed on request - // HJ, 1/Apr/2009 - - // New algorithm will assemble local clusters and then create a global - // zone ordering by collecting all faces (coarse pairs) from proc0, + // New algorithm will assemble local clusters on the master side and + // create zone ordering by collecting all faces (coarse pairs) from proc0, // followed by proc 1 etc. This avoids global communication and allows // each processor only to perform the analysis on locally created coarse // faces // HJ, 13/Jun/2016 - // To help with analysis, expand the local and neighbour addressing - // to full zone size - labelField localExpandAddressing(fineGgiInterface_.zoneSize(), 0); - - // Memory management, local - { - const labelList& addr = fineGgiInterface_.zoneAddressing(); - - forAll (addr, i) - { - localExpandAddressing[addr[i]] = - localRestrictAddressing[i] + procOffset*Pstream::myProcNo(); - } - - // Removed global reduce. Only local faces will be analysed. - // HJ, 13/Jun/2016 - } + // Note: local addressing contains only local faces + const labelList& fineZa = fineGgiInterface_.zoneAddressing(); // Create addressing for neighbour faces. Note: expandAddrToZone will - // expand the addressing to zone size. HJ, 13/Jun/2016 - labelField neighbourExpandAddressing + // expand the addressing to zone size, including communications. + // Faces which are not used locally will be marked by NaNs + // HJ, 13/Jun/2016 + labelField neighbourExpandAddressing = neighbourRestrictAddressing; + + // Expand neighbour side to get all the data required from other + // processors. + // Note: neigbour is now the size of remote zone + fineGgiInterface_.shadowInterface().expandAddrToZone ( - fineGgiInterface_.shadowInterface().interfaceSize() + neighbourExpandAddressing ); - // Fill local cluster ID with a combination of a local ID and processor - // offset - // Memory management, neighbour - { - forAll (neighbourExpandAddressing, i) - { - neighbourExpandAddressing[i] = - neighbourRestrictAddressing[i] - + procOffset*Pstream::myProcNo(); - } + // Create addressing for neighbour processors. Note: expandAddrToZone will + // expand the addressing to zone size. HJ, 13/Jun/2016 + labelField neighbourExpandProc + ( + fineGgiInterface_.shadowInterface().interfaceSize(), + Pstream::myProcNo() + ); - // Expand neighbour side to get all the data required from other - // processors. Note: neigbour is now the size of remote zone - fineGgiInterface_.shadowInterface().expandAddrToZone - ( - neighbourExpandAddressing - ); - } + // Expand neighbour side to get all the data required from other + // processors. + // Note: neigbour is now the size of remote zone + fineGgiInterface_.shadowInterface().expandAddrToZone + ( + neighbourExpandProc + ); - // DEBUG: Check that all sizes are at zone size. - Info<< "Sizes check: local zone size " - << fineGgiInterface_.zoneSize() - << " " << localExpandAddressing << nl - << "shadow zone size " - << fineGgiInterface_.shadowInterface().zoneSize() - << " " << neighbourExpandAddressing - << endl; + // Note: neighbourExpandAddressing and neighbourExpandProc + // will be filled with NaNs for faces which are not local - - // Note: neighbourExpandAddressing will be filled with NaNs for faces which - // not local - - Info<< "End of reduce" << endl; // Make a lookup table of entries for owner/neighbour. // All sizes are guessed at the size of fine interface // HJ, 19/Feb/2009 - HashTable, label, Hash