/*---------------------------------------------------------------------------*\ ========= | \\ / F ield | OpenFOAM: The Open Source CFD Toolbox \\ / O peration | \\ / A nd | Copyright held by original author \\/ M anipulation | ------------------------------------------------------------------------------- License This file is part of OpenFOAM. OpenFOAM is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; either version 2 of the License, or (at your option) any later version. OpenFOAM is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with OpenFOAM; if not, write to the Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA Application decomposePar Description Automatically decomposes a mesh and fields of a case for parallel execution of OpenFOAM. Usage - decomposePar [OPTION] @param -region \n Region to decompose. @param -cellDist \n Write the cell distribution as a labelList for use with 'manual' decomposition method and as a volScalarField for post-processing. @param -copyUniform \n Copy any @a uniform directories too. @param -fields \n Use existing geometry decomposition and convert fields only. @param -filterPatches \n Remove empty patches when decomposing the geometry. @param -force \n Remove any existing @a processor subdirectories before decomposing the geometry. @param -ifRequired \n Only decompose the geometry if the number of domains has changed from a previous decomposition. No @a processor subdirectories will be removed unless the @a -force option is also specified. This option can be used to avoid redundant geometry decomposition (eg, in scripts), but should be used with caution when the underlying (serial) geometry or the decomposition method etc. have been changed between decompositions. \*---------------------------------------------------------------------------*/ #include "OSspecific.H" #include "fvCFD.H" #include "IOobjectList.H" #include "processorFvPatchFields.H" #include "domainDecomposition.H" #include "labelIOField.H" #include "scalarIOField.H" #include "vectorIOField.H" #include "sphericalTensorIOField.H" #include "symmTensorIOField.H" #include "tensorIOField.H" #include "tetPointFields.H" #include "elementFields.H" #include "tetFemMatrices.H" #include "tetPointFieldDecomposer.H" #include "pointFields.H" #include "readFields.H" #include "fvFieldDecomposer.H" #include "pointFieldDecomposer.H" #include "lagrangianFieldDecomposer.H" #include "faCFD.H" #include "emptyFaPatch.H" #include "faMeshDecomposition.H" #include "faFieldDecomposer.H" // * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * // int main(int argc, char *argv[]) { argList::noParallel(); argList::validOptions.insert("region", "name"); argList::validOptions.insert("cellDist", ""); argList::validOptions.insert("copyUniform", ""); argList::validOptions.insert("fields", ""); argList::validOptions.insert("filterPatches", ""); argList::validOptions.insert("force", ""); argList::validOptions.insert("ifRequired", ""); # include "setRootCase.H" bool writeCellDist(args.options().found("cellDist")); bool copyUniform(args.options().found("copyUniform")); bool decomposeFieldsOnly(args.options().found("fields")); bool filterPatches(args.options().found("filterPatches")); bool forceOverwrite(args.options().found("force")); bool ifRequiredDecomposition(args.options().found("ifRequired")); // Get region name word regionName = fvMesh::defaultRegion; if (args.options().found("region")) { regionName = args.options()["region"]; } # include "createTime.H" Info<< "Time = " << runTime.timeName() << endl; // determine the existing processor count directly label nProcs = 0; while (dir(runTime.path()/(word("processor") + name(nProcs)))) { ++nProcs; } // get requested numberOfSubdomains label nDomains = 0; { IOdictionary decompDict ( IOobject ( "decomposeParDict", runTime.time().system(), runTime, IOobject::MUST_READ, IOobject::NO_WRITE, false ) ); decompDict.lookup("numberOfSubdomains") >> nDomains; } if (decomposeFieldsOnly) { // Sanity check on previously decomposed case if (nProcs != nDomains) { FatalErrorIn(args.executable()) << "Specified -fields, but the case was decomposed with " << nProcs << " domains" << nl << "instead of " << nDomains << " domains as specified in decomposeParDict" << nl << exit(FatalError); } } else if (nProcs) { bool procDirsProblem = true; if (regionName != fvMesh::defaultRegion) { decomposeFieldsOnly = false; procDirsProblem = false; } if (ifRequiredDecomposition && nProcs == nDomains) { // we can reuse the decomposition decomposeFieldsOnly = true; procDirsProblem = false; forceOverwrite = false; Info<< "Using existing processor directories" << nl; } // if (forceOverwrite) // { // Info<< "Removing " << nProcs // << " existing processor directories" << endl; // // remove existing processor dirs // // reverse order to avoid gaps if someone interrupts the process // for (label procI = nProcs-1; procI >= 0; --procI) // { // fileName procDir // ( // runTime.path()/(word("processor") + name(procI)) // ); // rmDir(procDir); // } // procDirsProblem = false; // } // if (procDirsProblem) // { // FatalErrorIn(args.executable()) // << "Case is already decomposed with " << nProcs // << " domains, use the -force option or manually" << nl // << "remove processor directories before decomposing. e.g.," // << nl // << " rm -rf " << runTime.path().c_str() << "/processor*" // << nl // << exit(FatalError); // } } Info<< "Create mesh for region " << regionName << endl; domainDecomposition mesh ( IOobject ( regionName, runTime.timeName(), runTime ) ); // Decompose the mesh if (!decomposeFieldsOnly) { mesh.decomposeMesh(filterPatches); mesh.writeDecomposition(); if (writeCellDist) { // Write the decomposition as labelList for use with 'manual' // decomposition method. // FIXME: may attempt to write to a non-existent "region0/" OFstream os ( runTime.path() / mesh.facesInstance() / regionName / "cellDecomposition" ); os << mesh.cellToProc(); Info<< nl << "Wrote decomposition to " << os.name() << " for use in manual decomposition." << endl; // Write as volScalarField for postprocessing. volScalarField cellDist ( IOobject ( "cellDist", runTime.timeName(), mesh, IOobject::NO_READ, IOobject::AUTO_WRITE ), mesh, dimensionedScalar("cellDist", dimless, 0), zeroGradientFvPatchScalarField::typeName ); const labelList& procIds = mesh.cellToProc(); forAll(procIds, celli) { cellDist[celli] = procIds[celli]; } cellDist.write(); Info<< nl << "Wrote decomposition as volScalarField to " << cellDist.name() << " for use in postprocessing." << endl; } } // Search for list of objects for this time IOobjectList objects(mesh, runTime.timeName()); // Construct the vol fields // ~~~~~~~~~~~~~~~~~~~~~~~~ PtrList volScalarFields; readFields(mesh, objects, volScalarFields); PtrList volVectorFields; readFields(mesh, objects, volVectorFields); PtrList volSphericalTensorFields; readFields(mesh, objects, volSphericalTensorFields); PtrList volSymmTensorFields; readFields(mesh, objects, volSymmTensorFields); PtrList volTensorFields; readFields(mesh, objects, volTensorFields); // Construct the surface fields // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ PtrList surfaceScalarFields; readFields(mesh, objects, surfaceScalarFields); // Construct the point fields // ~~~~~~~~~~~~~~~~~~~~~~~~~~ pointMesh pMesh(mesh); PtrList pointScalarFields; readFields(pMesh, objects, pointScalarFields); PtrList pointVectorFields; readFields(pMesh, objects, pointVectorFields); PtrList pointSphericalTensorFields; readFields(pMesh, objects, pointSphericalTensorFields); PtrList pointSymmTensorFields; readFields(pMesh, objects, pointSymmTensorFields); PtrList pointTensorFields; readFields(pMesh, objects, pointTensorFields); // Construct the tetPoint fields // ~~~~~~~~~~~~~~~~~~~~~~~~~~ tetPolyMesh* tetMeshPtr = NULL; PtrList tetPointScalarFields; PtrList tetPointVectorFields; PtrList tetPointSphericalTensorFields; PtrList tetPointSymmTensorFields; PtrList tetPointTensorFields; PtrList elementScalarFields; PtrList elementVectorFields; if ( objects.lookupClass("tetPointScalarField").size() > 0 || objects.lookupClass("tetPointVectorField").size() > 0 || objects.lookupClass("tetPointSphericalTensorField").size() > 0 || objects.lookupClass("tetPointSymmTensorField").size() > 0 || objects.lookupClass("tetPointTensorField").size() > 0 || objects.lookupClass("elementScalarField").size() > 0 || objects.lookupClass("elementVectorField").size() > 0 ) { tetMeshPtr = new tetPolyMesh(mesh); tetPolyMesh& tetMesh = *tetMeshPtr; readFields(tetMesh, objects, tetPointScalarFields); readFields(tetMesh, objects, tetPointVectorFields); readFields(tetMesh, objects, tetPointSphericalTensorFields); readFields(tetMesh, objects, tetPointSymmTensorFields); readFields(tetMesh, objects, tetPointTensorFields); readFields(tetMesh, objects, elementScalarFields); readFields(tetMesh, objects, elementVectorFields); } // Construct the Lagrangian fields // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ fileNameList cloudDirs ( readDir(runTime.timePath()/"lagrangian", fileName::DIRECTORY) ); // Particles PtrList > lagrangianPositions(cloudDirs.size()); // Particles per cell PtrList< List*> > cellParticles(cloudDirs.size()); PtrList > lagrangianLabelFields(cloudDirs.size()); PtrList > lagrangianScalarFields(cloudDirs.size()); PtrList > lagrangianVectorFields(cloudDirs.size()); PtrList > lagrangianSphericalTensorFields ( cloudDirs.size() ); PtrList > lagrangianSymmTensorFields ( cloudDirs.size() ); PtrList > lagrangianTensorFields ( cloudDirs.size() ); label cloudI = 0; forAll(cloudDirs, i) { IOobjectList sprayObjs ( mesh, runTime.timeName(), "lagrangian"/cloudDirs[i] ); IOobject* positionsPtr = sprayObjs.lookup("positions"); if (positionsPtr) { // Read lagrangian particles // ~~~~~~~~~~~~~~~~~~~~~~~~~ Info<< "Identified lagrangian data set: " << cloudDirs[i] << endl; lagrangianPositions.set ( cloudI, new Cloud ( mesh, cloudDirs[i], false ) ); // Sort particles per cell // ~~~~~~~~~~~~~~~~~~~~~~~ cellParticles.set ( cloudI, new List*> ( mesh.nCells(), static_cast*>(NULL) ) ); label i = 0; forAllIter(Cloud, lagrangianPositions[cloudI], iter) { iter().index() = i++; label celli = iter().cell(); // Check if (celli < 0 || celli >= mesh.nCells()) { FatalErrorIn(args.executable()) << "Illegal cell number " << celli << " for particle with index " << iter().index() << " at position " << iter().position() << nl << "Cell number should be between 0 and " << mesh.nCells()-1 << nl << "On this mesh the particle should be in cell " << mesh.findCell(iter().position()) << exit(FatalError); } if (!cellParticles[cloudI][celli]) { cellParticles[cloudI][celli] = new SLList(); } cellParticles[cloudI][celli]->append(&iter()); } // Read fields // ~~~~~~~~~~~ IOobjectList lagrangianObjects ( mesh, runTime.timeName(), "lagrangian"/cloudDirs[cloudI] ); lagrangianFieldDecomposer::readFields ( cloudI, lagrangianObjects, lagrangianLabelFields ); lagrangianFieldDecomposer::readFields ( cloudI, lagrangianObjects, lagrangianScalarFields ); lagrangianFieldDecomposer::readFields ( cloudI, lagrangianObjects, lagrangianVectorFields ); lagrangianFieldDecomposer::readFields ( cloudI, lagrangianObjects, lagrangianSphericalTensorFields ); lagrangianFieldDecomposer::readFields ( cloudI, lagrangianObjects, lagrangianSymmTensorFields ); lagrangianFieldDecomposer::readFields ( cloudI, lagrangianObjects, lagrangianTensorFields ); cloudI++; } } lagrangianPositions.setSize(cloudI); cellParticles.setSize(cloudI); lagrangianLabelFields.setSize(cloudI); lagrangianScalarFields.setSize(cloudI); lagrangianVectorFields.setSize(cloudI); lagrangianSphericalTensorFields.setSize(cloudI); lagrangianSymmTensorFields.setSize(cloudI); lagrangianTensorFields.setSize(cloudI); // Any uniform data to copy/link? fileName uniformDir("uniform"); if (dir(runTime.timePath()/uniformDir)) { Info<< "Detected additional non-decomposed files in " << runTime.timePath()/uniformDir << endl; } else { uniformDir.clear(); } Info<< endl; // split the fields over processors for (label procI = 0; procI < mesh.nProcs(); procI++) { Info<< "Processor " << procI << ": field transfer" << endl; // open the database Time processorDb ( Time::controlDictName, args.rootPath(), args.caseName()/fileName(word("processor") + name(procI)) ); processorDb.setTime(runTime); // read the mesh fvMesh procMesh ( IOobject ( regionName, processorDb.timeName(), processorDb ) ); labelIOList cellProcAddressing ( IOobject ( "cellProcAddressing", "constant", procMesh.meshSubDir, procMesh, IOobject::MUST_READ, IOobject::NO_WRITE ) ); labelIOList boundaryProcAddressing ( IOobject ( "boundaryProcAddressing", "constant", procMesh.meshSubDir, procMesh, IOobject::MUST_READ, IOobject::NO_WRITE ) ); // FV fields if ( volScalarFields.size() || volVectorFields.size() || volSphericalTensorFields.size() || volSymmTensorFields.size() || volTensorFields.size() || surfaceScalarFields.size() ) { labelIOList faceProcAddressing ( IOobject ( "faceProcAddressing", "constant", procMesh.meshSubDir, procMesh, IOobject::MUST_READ, IOobject::NO_WRITE ) ); fvFieldDecomposer fieldDecomposer ( mesh, procMesh, faceProcAddressing, cellProcAddressing, boundaryProcAddressing ); fieldDecomposer.decomposeFields(volScalarFields); fieldDecomposer.decomposeFields(volVectorFields); fieldDecomposer.decomposeFields(volSphericalTensorFields); fieldDecomposer.decomposeFields(volSymmTensorFields); fieldDecomposer.decomposeFields(volTensorFields); fieldDecomposer.decomposeFields(surfaceScalarFields); } // Point fields if ( pointScalarFields.size() || pointVectorFields.size() || pointSphericalTensorFields.size() || pointSymmTensorFields.size() || pointTensorFields.size() ) { labelIOList pointProcAddressing ( IOobject ( "pointProcAddressing", "constant", procMesh.meshSubDir, procMesh, IOobject::MUST_READ, IOobject::NO_WRITE ) ); pointMesh procPMesh(procMesh, true); pointFieldDecomposer fieldDecomposer ( pMesh, procPMesh, pointProcAddressing, boundaryProcAddressing ); fieldDecomposer.decomposeFields(pointScalarFields); fieldDecomposer.decomposeFields(pointVectorFields); fieldDecomposer.decomposeFields(pointSphericalTensorFields); fieldDecomposer.decomposeFields(pointSymmTensorFields); fieldDecomposer.decomposeFields(pointTensorFields); } // tetPoint fields if (tetMeshPtr) { const tetPolyMesh& tetMesh = *tetMeshPtr; tetPolyMesh procTetMesh(procMesh); // Read the point addressing information labelIOList pointProcAddressing ( IOobject ( "pointProcAddressing", "constant", procMesh.meshSubDir, procMesh, IOobject::MUST_READ, IOobject::NO_WRITE ) ); // Read the point addressing information labelIOList faceProcAddressing ( IOobject ( "faceProcAddressing", "constant", procMesh.meshSubDir, procMesh, IOobject::MUST_READ, IOobject::NO_WRITE ) ); tetPointFieldDecomposer fieldDecomposer ( tetMesh, procTetMesh, pointProcAddressing, faceProcAddressing, cellProcAddressing, boundaryProcAddressing ); fieldDecomposer.decomposeFields(tetPointScalarFields); fieldDecomposer.decomposeFields(tetPointVectorFields); fieldDecomposer.decomposeFields(tetPointSphericalTensorFields); fieldDecomposer.decomposeFields(tetPointSymmTensorFields); fieldDecomposer.decomposeFields(tetPointTensorFields); fieldDecomposer.decomposeFields(elementScalarFields); fieldDecomposer.decomposeFields(elementVectorFields); } // If there is lagrangian data write it out forAll(lagrangianPositions, cloudI) { if (lagrangianPositions[cloudI].size()) { lagrangianFieldDecomposer fieldDecomposer ( mesh, procMesh, cellProcAddressing, cloudDirs[cloudI], lagrangianPositions[cloudI], cellParticles[cloudI] ); // Lagrangian fields if ( lagrangianLabelFields[cloudI].size() || lagrangianScalarFields[cloudI].size() || lagrangianVectorFields[cloudI].size() || lagrangianSphericalTensorFields[cloudI].size() || lagrangianSymmTensorFields[cloudI].size() || lagrangianTensorFields[cloudI].size() ) { fieldDecomposer.decomposeFields ( cloudDirs[cloudI], lagrangianLabelFields[cloudI] ); fieldDecomposer.decomposeFields ( cloudDirs[cloudI], lagrangianScalarFields[cloudI] ); fieldDecomposer.decomposeFields ( cloudDirs[cloudI], lagrangianVectorFields[cloudI] ); fieldDecomposer.decomposeFields ( cloudDirs[cloudI], lagrangianSphericalTensorFields[cloudI] ); fieldDecomposer.decomposeFields ( cloudDirs[cloudI], lagrangianSymmTensorFields[cloudI] ); fieldDecomposer.decomposeFields ( cloudDirs[cloudI], lagrangianTensorFields[cloudI] ); } } } // Any non-decomposed data to copy? if (uniformDir.size()) { const fileName timePath = processorDb.timePath(); if (copyUniform || mesh.distributed()) { cp ( runTime.timePath()/uniformDir, timePath/uniformDir ); } else { // link with relative paths const string parentPath = string("..")/".."; fileName currentDir(cwd()); chDir(timePath); ln ( parentPath/runTime.timeName()/uniformDir, uniformDir ); chDir(currentDir); } } } if (tetMeshPtr) { delete tetMeshPtr; tetMeshPtr = NULL; } // Finite area mesh and field decomposition IOobject faMeshBoundaryIOobj ( "boundary", mesh.time().findInstance(mesh.dbDir()/fvMesh::meshSubDir, "boundary"), faMesh::meshSubDir, mesh, IOobject::MUST_READ, IOobject::NO_WRITE ); if(faMeshBoundaryIOobj.headerOk()) { Info << "\nFinite area mesh decomposition" << endl; faMeshDecomposition aMesh(mesh); aMesh.decomposeMesh(filterPatches); aMesh.writeDecomposition(); // Construct the area fields // ~~~~~~~~~~~~~~~~~~~~~~~~ PtrList areaScalarFields; readFields(aMesh, objects, areaScalarFields); PtrList areaVectorFields; readFields(aMesh, objects, areaVectorFields); PtrList areaSphericalTensorFields; readFields(aMesh, objects, areaSphericalTensorFields); PtrList areaSymmTensorFields; readFields(aMesh, objects, areaSymmTensorFields); PtrList areaTensorFields; readFields(aMesh, objects, areaTensorFields); // Construct the edge fields // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ PtrList edgeScalarFields; readFields(aMesh, objects, edgeScalarFields); Info << endl; // Split the fields over processors for (label procI = 0; procI < mesh.nProcs(); procI++) { Info<< "Processor " << procI << ": finite area field transfer" << endl; // open the database Time processorDb ( Time::controlDictName, args.rootPath(), args.caseName()/fileName(word("processor") + name(procI)) ); processorDb.setTime(runTime); // Read the mesh fvMesh procFvMesh ( IOobject ( regionName, processorDb.timeName(), processorDb ) ); faMesh procMesh(procFvMesh); labelIOList faceProcAddressing ( IOobject ( "faceProcAddressing", "constant", procMesh.meshSubDir, procFvMesh, IOobject::MUST_READ, IOobject::NO_WRITE ) ); labelIOList boundaryProcAddressing ( IOobject ( "boundaryProcAddressing", "constant", procMesh.meshSubDir, procFvMesh, IOobject::MUST_READ, IOobject::NO_WRITE ) ); // FA fields if ( areaScalarFields.size() || areaVectorFields.size() || areaSphericalTensorFields.size() || areaSymmTensorFields.size() || areaTensorFields.size() || edgeScalarFields.size() ) { labelIOList edgeProcAddressing ( IOobject ( "edgeProcAddressing", "constant", procMesh.meshSubDir, procFvMesh, IOobject::MUST_READ, IOobject::NO_WRITE ) ); faFieldDecomposer fieldDecomposer ( aMesh, procMesh, edgeProcAddressing, faceProcAddressing, boundaryProcAddressing ); fieldDecomposer.decomposeFields(areaScalarFields); fieldDecomposer.decomposeFields(areaVectorFields); fieldDecomposer.decomposeFields(areaSphericalTensorFields); fieldDecomposer.decomposeFields(areaSymmTensorFields); fieldDecomposer.decomposeFields(areaTensorFields); fieldDecomposer.decomposeFields(edgeScalarFields); } } } Info<< "\nEnd.\n" << endl; return(0); } // ************************************************************************* //