From d001a8945a9bfa6682e607cca0607565b99dc3ae Mon Sep 17 00:00:00 2001 From: eugeneswalker <38933153+eugeneswalker@users.noreply.github.com> Date: Tue, 9 Mar 2021 12:19:59 -0800 Subject: [PATCH] nalu-wind: add cuda variant (#22184) --- .../builtin/packages/nalu-wind/package.py | 150 +++++++++--------- 1 file changed, 71 insertions(+), 79 deletions(-) diff --git a/var/spack/repos/builtin/packages/nalu-wind/package.py b/var/spack/repos/builtin/packages/nalu-wind/package.py index 2d0347e603..93e155412d 100644 --- a/var/spack/repos/builtin/packages/nalu-wind/package.py +++ b/var/spack/repos/builtin/packages/nalu-wind/package.py @@ -7,13 +7,20 @@ import sys -class NaluWind(CMakePackage): +def _parse_float(val): + try: + return float(val) > 0.0 + except ValueError: + return False + + +class NaluWind(CMakePackage, CudaPackage): """Nalu-Wind: Wind energy focused variant of Nalu.""" - homepage = "https://github.com/exawind/nalu-wind" + homepage = "https://nalu-wind.readthedocs.io" git = "https://github.com/exawind/nalu-wind.git" - maintainers = ['jrood-nrel'] + maintainers = ['jrood-nrel', 'sayerhs'] tags = ['ecp', 'ecp-apps'] @@ -24,8 +31,12 @@ class NaluWind(CMakePackage): description='Build dependencies as shared libraries') variant('pic', default=True, description='Position independent code') - variant('test_tol', default='default', - description='Tolerance for regression tests') + variant('abs_tol', default=1.0e-15, + values=_parse_float, + description='Absolute tolerance for regression tests') + variant('rel_tol', default=1.0e-12, + values=_parse_float, + description='Relative tolerance for regression tests') # Third party libraries variant('openfast', default=False, description='Compile with OpenFAST support') @@ -37,6 +48,10 @@ class NaluWind(CMakePackage): description='Compile with Catalyst support') variant('fftw', default=False, description='Compile with FFTW support') + variant('openmp', default=False, + description='Compile with OpenMP support') + variant('boost', default=False, + description='Enable Boost integration') variant('wind-utils', default=False, description='Build wind-utils') @@ -48,102 +63,79 @@ class NaluWind(CMakePackage): # Cannot build Trilinos as a shared library with STK on Darwin # which is why we have a 'shared' variant for Nalu-Wind # https://github.com/trilinos/Trilinos/issues/2994 - depends_on('trilinos+exodus+tpetra+muelu+belos+ifpack2+amesos2+zoltan+stk+boost~superlu-dist+superlu+hdf5+zlib+pnetcdf+shards~hypre@master,develop', when='+shared') - depends_on('trilinos~shared+exodus+tpetra+muelu+belos+ifpack2+amesos2+zoltan+stk+boost~superlu-dist+superlu+hdf5+zlib+pnetcdf+shards~hypre@master,develop', when='~shared') + depends_on('trilinos@master,develop ~cuda~wrapper+shared+exodus+tpetra+muelu+belos+ifpack2+amesos2+zoltan+stk+boost~superlu-dist+superlu+hdf5+zlib+pnetcdf+shards~hypre', when='~cuda+shared') + depends_on('trilinos@master,develop ~cuda~wrapper~shared+exodus+tpetra+muelu+belos+ifpack2+amesos2+zoltan+stk+boost~superlu-dist+superlu+hdf5+zlib+pnetcdf+shards~hypre', when='~cuda~shared') # Optional dependencies - depends_on('openfast+cxx', when='+openfast+shared') - depends_on('openfast+cxx~shared', when='+openfast~shared') - depends_on('tioga', when='+tioga+shared') - depends_on('tioga~shared', when='+tioga~shared') - depends_on('hypre+mpi+int64~superlu-dist', when='+hypre+shared') - depends_on('hypre+mpi+int64~superlu-dist~shared', when='+hypre~shared') + depends_on('openfast@develop +cxx+shared', when='+openfast+shared') + depends_on('openfast@develop +cxx~shared', when='+openfast~shared') + depends_on('tioga@master +shared', when='+tioga+shared~cuda') + depends_on('tioga@master ~shared', when='+tioga~shared~cuda') + depends_on('tioga@develop ~shared', when='+tioga~shared+cuda') + depends_on('hypre@2.18.2: +mpi~cuda+int64~superlu-dist+shared', when='+hypre~cuda+shared') + depends_on('hypre@2.18.2: +mpi~cuda+int64~superlu-dist~shared', when='+hypre~cuda~shared') + depends_on('kokkos-nvcc-wrapper', type='build', when='+cuda') + for _arch in CudaPackage.cuda_arch_values: + depends_on('trilinos@master,develop ~shared+exodus+tpetra+muelu+belos+ifpack2+amesos2+zoltan+stk+boost~superlu-dist+superlu+hdf5+zlib+pnetcdf+shards~hypre+cuda+cuda_rdc+wrapper cuda_arch={0}'.format(_arch), + when='+cuda cuda_arch={0}'.format(_arch)) + depends_on('hypre@develop +mpi+cuda~int64~superlu-dist cuda_arch={0}'.format(_arch), + when='+hypre+cuda cuda_arch={0}'.format(_arch)) depends_on('trilinos-catalyst-ioss-adapter', when='+catalyst') # FFTW doesn't have a 'shared' variant at this moment depends_on('fftw+mpi', when='+fftw') + depends_on('boost cxxstd=14', when='+boost') + + conflicts('+cuda', when='+shared') + def cmake_args(self): spec = self.spec - options = [] - options.extend([ - '-DTrilinos_DIR:PATH=%s' % spec['trilinos'].prefix, - '-DYAML_DIR:PATH=%s' % spec['yaml-cpp'].prefix, - '-DCMAKE_C_COMPILER=%s' % spec['mpi'].mpicc, - '-DCMAKE_CXX_COMPILER=%s' % spec['mpi'].mpicxx, - '-DCMAKE_Fortran_COMPILER=%s' % spec['mpi'].mpifc, - '-DMPI_C_COMPILER=%s' % spec['mpi'].mpicc, - '-DMPI_CXX_COMPILER=%s' % spec['mpi'].mpicxx, - '-DMPI_Fortran_COMPILER=%s' % spec['mpi'].mpifc, - '-DCMAKE_POSITION_INDEPENDENT_CODE:BOOL=%s' % ( - 'ON' if '+pic' in spec else 'OFF'), - ]) + args = [ + self.define_from_variant('BUILD_SHARED_LIBS', 'shared'), + self.define_from_variant('CMAKE_POSITION_INDEPENDENT_CODE', 'pic'), + self.define('CMAKE_CXX_COMPILER', spec['mpi'].mpicxx), + self.define('CMAKE_C_COMPILER', spec['mpi'].mpicc), + self.define('CMAKE_Fortran_COMPILER', spec['mpi'].mpifc), + self.define('Trilinos_DIR', spec['trilinos'].prefix), + self.define('YAML_DIR', spec['yaml-cpp'].prefix), + self.define_from_variant('ENABLE_CUDA', 'cuda'), + self.define_from_variant('ENABLE_WIND_UTILS', 'wind-utils'), + self.define_from_variant('ENABLE_BOOST', 'boost'), + self.define_from_variant('ENABLE_OPENMP', 'openmp'), + ] + args.append(self.define_from_variant('ENABLE_OPENFAST', 'openfast')) if '+openfast' in spec: - options.extend([ - '-DENABLE_OPENFAST:BOOL=ON', - '-DOpenFAST_DIR:PATH=%s' % spec['openfast'].prefix - ]) - else: - options.append('-DENABLE_OPENFAST:BOOL=OFF') + args.append(self.define('OpenFAST_DIR', spec['openfast'].prefix)) + args.append(self.define_from_variant('ENABLE_TIOGA', 'tioga')) if '+tioga' in spec: - options.extend([ - '-DENABLE_TIOGA:BOOL=ON', - '-DTIOGA_DIR:PATH=%s' % spec['tioga'].prefix - ]) - else: - options.append('-DENABLE_TIOGA:BOOL=OFF') + args.append(self.define('TIOGA_DIR', spec['tioga'].prefix)) + args.append(self.define_from_variant('ENABLE_HYPRE', 'hypre')) if '+hypre' in spec: - options.extend([ - '-DENABLE_HYPRE:BOOL=ON', - '-DHYPRE_DIR:PATH=%s' % spec['hypre'].prefix - ]) - else: - options.append('-DENABLE_HYPRE:BOOL=OFF') + args.append(self.define('HYPRE_DIR', spec['hypre'].prefix)) + args.append(self.define_from_variant('ENABLE_PARAVIEW_CATALYST', 'catalyst')) if '+catalyst' in spec: - options.extend([ - '-DENABLE_PARAVIEW_CATALYST:BOOL=ON', - '-DPARAVIEW_CATALYST_INSTALL_PATH:PATH=%s' % - spec['trilinos-catalyst-ioss-adapter'].prefix - ]) - else: - options.append('-DENABLE_PARAVIEW_CATALYST:BOOL=OFF') + args.append(self.define('PARAVIEW_CATALYST_INSTALL_PATH', + spec['trilinos-catalyst-ioss-adapter'].prefix)) + args.append(self.define_from_variant('ENABLE_FFTW', 'fftw')) if '+fftw' in spec: - options.extend([ - '-DENABLE_FFTW:BOOL=ON', - '-DFFTW_DIR:PATH=%s' % spec['fftw'].prefix - ]) - else: - options.append('-DENABLE_FFTW:BOOL=OFF') + args.append(self.define('FFTW_DIR', spec['fftw'].prefix)) - if '+wind-utils' in spec: - options.append('-DENABLE_WIND_UTILS=ON') - else: - options.append('-DENABLE_WIND_UTILS=OFF') + args.append(self.define('ENABLE_TESTS', self.run_tests)) + if self.run_tests: + args.extend([ + self.define('TEST_TOLERANCE', spec.variants['abs_tol'].value), + self.define('TEST_REL_TOL', spec.variants['rel_tol'].value), + ]) if 'darwin' in spec.architecture: - options.append('-DCMAKE_MACOSX_RPATH:BOOL=ON') + args.append(self.define('CMAKE_MACOSX_RPATH', 'ON')) - if self.run_tests: - options.append('-DENABLE_TESTS:BOOL=ON') - else: - options.append('-DENABLE_TESTS:BOOL=OFF') - - if self.spec.variants['test_tol'].value != 'default': - try: - test_tol = float(self.spec.variants['test_tol'].value) - if test_tol <= 0.0: - raise ValueError - options.append('-DTEST_TOLERANCE:STRING={tol}'.format( - tol=test_tol)) - except ValueError: - print("Specified test_tol must be a positive float. " - "Using the default.") - - return options + return args @run_before('cmake') def add_submodules(self):