From e22e037e304015243bf5f33197fa3780902430ad Mon Sep 17 00:00:00 2001 From: Danny Taller <66029857+dtaller@users.noreply.github.com> Date: Wed, 2 Dec 2020 17:07:56 -0800 Subject: [PATCH] Add CARE package, fixes for ROCmPackage and subclasses (#20070) * use develop version of blt with fixes for rocm * package updates for care+rocm * fixes for plain cpu build * add camp dependency on raja --- lib/spack/spack/build_systems/rocm.py | 2 + .../repos/builtin/packages/camp/package.py | 10 +- .../repos/builtin/packages/care/package.py | 136 ++++++++++++++++++ .../repos/builtin/packages/chai/package.py | 23 +-- .../repos/builtin/packages/raja/package.py | 24 +++- .../repos/builtin/packages/umpire/package.py | 1 + 6 files changed, 180 insertions(+), 16 deletions(-) create mode 100644 var/spack/repos/builtin/packages/care/package.py diff --git a/lib/spack/spack/build_systems/rocm.py b/lib/spack/spack/build_systems/rocm.py index 0107c6376b..ae2e7589dc 100644 --- a/lib/spack/spack/build_systems/rocm.py +++ b/lib/spack/spack/build_systems/rocm.py @@ -102,6 +102,8 @@ class ROCmPackage(PackageBase): depends_on('hsa-rocr-dev', when='+rocm') depends_on('hip', when='+rocm') + conflicts('^blt@:0.3.6', when='+rocm') + # need amd gpu type for rocm builds conflicts('amdgpu_target=none', when='+rocm') diff --git a/var/spack/repos/builtin/packages/camp/package.py b/var/spack/repos/builtin/packages/camp/package.py index 882bfec528..d2d213664a 100644 --- a/var/spack/repos/builtin/packages/camp/package.py +++ b/var/spack/repos/builtin/packages/camp/package.py @@ -19,13 +19,17 @@ class Camp(CMakePackage, CudaPackage, ROCmPackage): version('master', branch='master', submodules='True') version('0.1.0', sha256='fd4f0f2a60b82a12a1d9f943f8893dc6fe770db493f8fae5ef6f7d0c439bebcc') - depends_on('cmake@3.8:', type='build') - depends_on('cmake@3.9:', type='build', when="+cuda") + # TODO: figure out gtest dependency and then set this default True. + variant('tests', default=False, description='Build tests') + + depends_on('blt', type='build') + depends_on('blt@0.3.7:', type='build', when='+rocm') def cmake_args(self): spec = self.spec options = [] + options.append('-DBLT_SOURCE_DIR={0}'.format(spec['blt'].prefix)) if '+cuda' in spec: options.extend([ @@ -55,6 +59,6 @@ def cmake_args(self): options.append('-DENABLE_HIP=OFF') options.append('-DENABLE_TESTS={0}'.format( - "On" if self.run_tests else "Off")) + 'ON' if '+tests' in spec else 'OFF')) return options diff --git a/var/spack/repos/builtin/packages/care/package.py b/var/spack/repos/builtin/packages/care/package.py new file mode 100644 index 0000000000..7c3b6a6391 --- /dev/null +++ b/var/spack/repos/builtin/packages/care/package.py @@ -0,0 +1,136 @@ +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +from spack import * + + +class Care(CMakePackage, CudaPackage, ROCmPackage): + """ + Algorithms for chai managed arrays. + """ + + homepage = "https://github.com/LLNL/CARE" + git = "https://github.com/LLNL/CARE.git" + + version('develop', branch='develop', submodules='True') + version('master', branch='main', submodules='True') + version('0.3.0', tag='v0.3.0', submodules='True') + version('0.2.0', tag='v0.2.0', submodules='True') + + variant('openmp', default=False, description='Build Shared Libs') + variant('implicit_conversions', default=True, description='Enable implicit' + 'conversions to/from raw pointers') + variant('benchmarks', default=True, description='Build benchmarks.') + variant('examples', default=True, description='Build examples.') + variant('docs', default=False, description='Build documentation') + # TODO: figure out gtest dependency and then set this default True + # and remove the +tests conflict below. + variant('tests', default=False, description='Build tests') + + depends_on('blt', type='build') + depends_on('blt@0.3.7:', type='build', when='+rocm') + + depends_on('camp') + depends_on('umpire@develop') + depends_on('raja@develop') + depends_on('chai@develop+enable_pick~benchmarks') + + # WARNING: this package currently only supports an internal cub + # package. This will cause a race condition if compiled with another + # package that uses cub. TODO: have all packages point to the same external + # cub package. + depends_on('camp+cuda', when='+cuda') + depends_on('umpire+cuda', when='+cuda') + depends_on('raja+cuda~openmp', when='+cuda') + depends_on('chai+cuda', when='+cuda') + + # variants +rocm and amdgpu_targets are not automatically passed to + # dependencies, so do it manually. + depends_on('camp+rocm', when='+rocm') + depends_on('umpire+rocm', when='+rocm') + depends_on('raja+rocm~openmp', when='+rocm') + depends_on('chai+rocm', when='+rocm') + for val in ROCmPackage.amdgpu_targets: + depends_on('camp amdgpu_target=%s' % val, when='amdgpu_target=%s' % val) + depends_on('umpire amdgpu_target=%s' % val, when='amdgpu_target=%s' % val) + depends_on('raja amdgpu_target=%s' % val, when='amdgpu_target=%s' % val) + depends_on('chai amdgpu_target=%s' % val, when='amdgpu_target=%s' % val) + + conflicts('+openmp', when='+rocm') + conflicts('+openmp', when='+cuda') + + def cmake_args(self): + spec = self.spec + + options = [] + options.append('-DBLT_SOURCE_DIR={0}'.format(spec['blt'].prefix)) + + if '+cuda' in spec: + options.extend([ + '-DENABLE_CUDA=ON', + '-DCUDA_TOOLKIT_ROOT_DIR=' + spec['cuda'].prefix]) + + if not spec.satisfies('cuda_arch=none'): + cuda_arch = spec.variants['cuda_arch'].value + options.append('-DCUDA_ARCH=sm_{0}'.format(cuda_arch[0])) + flag = '-arch sm_{0}'.format(cuda_arch[0]) + options.append('-DCMAKE_CUDA_FLAGS:STRING={0}'.format(flag)) + else: + options.append('-DENABLE_CUDA=OFF') + + if '+rocm' in spec: + options.extend([ + '-DENABLE_HIP=ON', + '-DHIP_ROOT_DIR={0}'.format(spec['hip'].prefix)]) + + archs = self.spec.variants['amdgpu_target'].value + if archs != 'none': + arch_str = ",".join(archs) + options.append( + '-DHIP_HIPCC_FLAGS=--amdgpu-target={0}'.format(arch_str) + ) + else: + options.append('-DENABLE_HIP=OFF') + + options.append('-DCARE_ENABLE_IMPLICIT_CONVERSIONS={0}'.format( + 'ON' if '+implicit_conversions' in spec else 'OFF')) + + options.append('-DCAMP_DIR:PATH=' + + spec['camp'].prefix.share.camp.cmake) + options.append('-DUMPIRE_DIR:PATH=' + + spec['umpire'].prefix.share.umpire.cmake) + options.append('-DRAJA_DIR:PATH=' + + spec['raja'].prefix.share.raja.cmake) + options.append('-DCHAI_DIR:PATH=' + + spec['chai'].prefix.share.chai.cmake) + + options.append('-DCARE_ENABLE_TESTS={0}'.format( + 'ON' if '+tests' in spec else 'OFF')) + # For tests to work, we also need BLT_ENABLE_TESTS to be on. + # This will take care of the gtest dependency. CARE developers should + # consider consolidating these flags in the future. + options.append('-DBLT_ENABLE_TESTS={0}'.format( + 'ON' if '+tests' in spec else 'OFF')) + + # There are both CARE_ENABLE_* and ENABLE_* variables in here because + # one controls the BLT infrastructure and the other controls the CARE + # infrastructure. The goal is to just be able to use the CARE_ENABLE_* + # variables, but CARE isn't set up correctly for that yet. + options.append('-DENABLE_BENCHMARKS={0}'.format( + 'ON' if '+benchmarks' in spec else 'OFF')) + options.append('-DCARE_ENABLE_BENCHMARKS={0}'.format( + 'ON' if '+benchmarks' in spec else 'OFF')) + + options.append('-DENABLE_EXAMPLES={0}'.format( + 'ON' if '+examples' in spec else 'OFF')) + options.append('-DCARE_ENABLE_EXAMPLES={0}'.format( + 'ON' if '+examples' in spec else 'OFF')) + + options.append('-DENABLE_DOCS={0}'.format( + 'ON' if '+docs' in spec else 'OFF')) + options.append('-DCARE_ENABLE_DOCS={0}'.format( + 'ON' if '+docs' in spec else 'OFF')) + + return options diff --git a/var/spack/repos/builtin/packages/chai/package.py b/var/spack/repos/builtin/packages/chai/package.py index 6fd33dea8b..2c30bb9bb6 100644 --- a/var/spack/repos/builtin/packages/chai/package.py +++ b/var/spack/repos/builtin/packages/chai/package.py @@ -23,16 +23,22 @@ class Chai(CMakePackage, CudaPackage, ROCmPackage): version('1.1.0', tag='v1.1.0', submodules='True') version('1.0', tag='v1.0', submodules='True') + variant('enable_pick', default=False, description='Enable pick method') variant('shared', default=True, description='Build Shared Libs') variant('raja', default=False, description='Build plugin for RAJA') variant('benchmarks', default=True, description='Build benchmarks.') variant('examples', default=True, description='Build examples.') + # TODO: figure out gtest dependency and then set this default True + # and remove the +tests conflict below. + variant('tests', default=False, description='Build tests') depends_on('cmake@3.8:', type='build') + depends_on('cmake@3.9:', type='build', when="+cuda") + depends_on('blt', type='build') + depends_on('blt@0.3.7:', type='build', when='+rocm') depends_on('umpire') depends_on('raja', when="+raja") - depends_on('cmake@3.9:', type='build', when="+cuda") depends_on('umpire+cuda', when="+cuda") depends_on('raja+cuda', when="+raja+cuda") @@ -44,10 +50,13 @@ class Chai(CMakePackage, CudaPackage, ROCmPackage): depends_on('umpire amdgpu_target=%s' % val, when='amdgpu_target=%s' % val) depends_on('raja amdgpu_target=%s' % val, when='+raja amdgpu_target=%s' % val) + conflicts('+benchmarks', when='~tests') + def cmake_args(self): spec = self.spec options = [] + options.append('-DBLT_SOURCE_DIR={0}'.format(spec['blt'].prefix)) if '+cuda' in spec: options.extend([ @@ -80,18 +89,14 @@ def cmake_args(self): options.extend(['-DENABLE_RAJA_PLUGIN=ON', '-DRAJA_DIR=' + spec['raja'].prefix]) + options.append('-DENABLE_PICK={0}'.format( + 'ON' if '+enable_pick' in spec else 'OFF')) + options.append('-Dumpire_DIR:PATH=' + spec['umpire'].prefix.share.umpire.cmake) options.append('-DENABLE_TESTS={0}'.format( - 'ON' if self.run_tests else 'OFF')) - - # give clear error for conflict between self.run_tests and - # benchmarks variant. - if not self.run_tests and '+benchmarks' in spec: - raise InstallError( - 'ENABLE_BENCHMARKS requires ENABLE_TESTS to be ON' - ) + 'ON' if '+tests' in spec else 'OFF')) options.append('-DENABLE_BENCHMARKS={0}'.format( 'ON' if '+benchmarks' in spec else 'OFF')) diff --git a/var/spack/repos/builtin/packages/raja/package.py b/var/spack/repos/builtin/packages/raja/package.py index c2cdf5a607..95af9c750b 100644 --- a/var/spack/repos/builtin/packages/raja/package.py +++ b/var/spack/repos/builtin/packages/raja/package.py @@ -32,16 +32,31 @@ class Raja(CMakePackage, CudaPackage, ROCmPackage): variant('shared', default=True, description='Build Shared Libs') variant('examples', default=True, description='Build examples.') variant('exercises', default=True, description='Build exercises.') + # TODO: figure out gtest dependency and then set this default True + # and remove the +tests conflict below. + variant('tests', default=False, description='Build tests') + + depends_on('blt', type='build') + depends_on('blt@0.3.7:', type='build', when='+rocm') + + depends_on('camp') + depends_on('camp+cuda', when='+cuda') + + # variants +rocm and amdgpu_targets are not automatically passed to + # dependencies, so do it manually. + depends_on('camp+rocm', when='+rocm') + for val in ROCmPackage.amdgpu_targets: + depends_on('camp amdgpu_target=%s' % val, when='amdgpu_target=%s' % val) conflicts('+openmp', when='+rocm') - depends_on('cmake@3.8:', type='build') - depends_on('cmake@3.9:', when='+cuda', type='build') - def cmake_args(self): spec = self.spec options = [] + + options.append('-DBLT_SOURCE_DIR={0}'.format(spec['blt'].prefix)) + options.append('-DENABLE_OPENMP={0}'.format( 'ON' if '+openmp' in spec else 'OFF')) @@ -84,7 +99,8 @@ def cmake_args(self): if self.spec.satisfies('%clang target=ppc64le:') or not self.run_tests: options.append('-DENABLE_TESTS=OFF') else: - options.append('-DENABLE_TESTS=ON') + options.append('-DENABLE_TESTS={0}'.format( + 'ON' if '+tests' in spec else 'OFF')) return options diff --git a/var/spack/repos/builtin/packages/umpire/package.py b/var/spack/repos/builtin/packages/umpire/package.py index 8b035e59df..ffb4cdbae1 100644 --- a/var/spack/repos/builtin/packages/umpire/package.py +++ b/var/spack/repos/builtin/packages/umpire/package.py @@ -61,6 +61,7 @@ class Umpire(CMakePackage, CudaPackage, ROCmPackage): depends_on('cmake@3.9:', when='+cuda', type='build') depends_on('blt', type='build') + depends_on('blt@0.3.7:', type='build', when='+rocm') # variants +rocm and amdgpu_targets are not automatically passed to # dependencies, so do it manually.