LBANN: add support for new python front end (#11395)
* LBANN: add python dependency and required python library dependencies * LBANN: add version 0.99 as alias for develop branch * LBANN: build Conduit library by default * LBANN: CUB is included with Hydrogen for later versions * New package: py-graphviz (dependency for LBANN) * New package: py-texttable (dependency for LBANN) * New package: py-onnx (dependency for LBANN) * New package: py-typing-extensions (dependency for py-onnx) * Conduit: add version 0.4.0 * NCCL: Add versions 2.4.6-1 and 2.4.2-1 * Protobuf: Add versions 3.7.1 and 3.7.0 * py-protobuf: add version 3.7.1 * py-typing: add version 3.6.4 * cuDNN: add versions. The source differs depending on the architecture, so the versions include a suffix to encode this. For example this adds 7.5.1-10.1-x86_64 as well as 7.5.1-10.1-ppc64le * cuDNN: Newer versions require newer versions of CUDA * Hydrogen: add configure options for CUB and Aluminum
This commit is contained in:
Brian Van Essen
Peter Scheibel
parent
7ff4105ab5
commit
92040a475f
12 changed files with 176 additions and 16 deletions
|
|
@ -38,6 +38,7 @@ class Conduit(Package):
|
|||
git = "https://github.com/LLNL/conduit.git"
|
||||
|
||||
version('master', branch='master', submodules=True, preferred=True)
|
||||
version('0.4.0', sha256='c228e6f0ce5a9c0ffb98e0b3d886f2758ace1a4b40d00f3f118542c0747c1f52')
|
||||
version('0.3.1', 'b98d1476199a46bde197220cd9cde042')
|
||||
version('0.3.0', '6396f1d1ca16594d7c66d4535d4f898e')
|
||||
# note: checksums on github automatic release source tars changed ~9/17
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@
|
|||
#
|
||||
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
|
||||
|
||||
import os
|
||||
from spack import *
|
||||
|
||||
|
||||
|
|
@ -12,16 +13,42 @@ class Cudnn(Package):
|
|||
|
||||
homepage = "https://developer.nvidia.com/cudnn"
|
||||
|
||||
version('7.3', '72666d3532850752612706601258a0b2',
|
||||
version('7.5.1-10.1-x86_64', '24017f4a56159d48fd5a31c8a930167b',
|
||||
url='https://developer.download.nvidia.com/compute/redist/cudnn/v7.5.1/cudnn-10.1-linux-x64-v7.5.1.10.tgz')
|
||||
version('7.5.1-10.1-ppc64le', 'ec0993be21e0998fc12116e4c46bce02',
|
||||
url='https://developer.download.nvidia.com/compute/redist/cudnn/v7.5.1/cudnn-10.1-linux-ppc64le-v7.5.1.10.tgz')
|
||||
version('7.5.1-10.0-x86_64', '703ed4be4d242ff4bc0ca48aaf2029bc',
|
||||
url='https://developer.download.nvidia.com/compute/redist/cudnn/v7.5.1/cudnn-10.0-linux-x64-v7.5.1.10.tgz')
|
||||
version('7.5.1-10.0-ppc64le', '8348cbab01970b75836d945d8a75915e',
|
||||
url='https://developer.download.nvidia.com/compute/redist/cudnn/v7.5.1/cudnn-10.0-linux-ppc64le-v7.5.1.10.tgz')
|
||||
version('7.5.0-10.1-x86_64', '4edc7fcfeada9f2363e97f9875154e8f',
|
||||
url='https://developer.download.nvidia.com/compute/redist/cudnn/v7.5.0/cudnn-10.1-linux-x64-v7.5.0.56.tgz')
|
||||
version('7.5.0-10.1-ppc64le', 'b927bc91d93182cb2606e51c9eeee908',
|
||||
url='https://developer.download.nvidia.com/compute/redist/cudnn/v7.5.0/cudnn-10.1-linux-ppc64le-v7.5.0.56.tgz')
|
||||
version('7.5.0-10.0-x86_64', 'b05ca261ffee1e08b496bf8a57790ee8',
|
||||
url='https://developer.download.nvidia.com/compute/redist/cudnn/v7.5.0/cudnn-10.0-linux-x64-v7.5.0.56.tgz')
|
||||
version('7.5.0-10.0-ppc64le', 'e9d2fcdb5340df2408fa08026a2a3e1c',
|
||||
url='https://developer.download.nvidia.com/compute/redist/cudnn/v7.5.0/cudnn-10.0-linux-ppc64le-v7.5.0.56.tgz')
|
||||
version('7.3.0', '72666d3532850752612706601258a0b2',
|
||||
url='http://developer.download.nvidia.com/compute/redist/cudnn/v7.3.0/cudnn-9.0-linux-x64-v7.3.0.29.tgz')
|
||||
version('7.2', '17e010153a4a95bf9c2df2a3c3ceea63',
|
||||
version('7.2.1', '17e010153a4a95bf9c2df2a3c3ceea63',
|
||||
url='http://developer.download.nvidia.com/compute/redist/cudnn/v7.2.1/cudnn-9.0-linux-x64-v7.2.1.38.tgz')
|
||||
version('6.0', 'a08ca487f88774e39eb6b0ef6507451d',
|
||||
url='http://developer.download.nvidia.com/compute/redist/cudnn/v6.0/cudnn-8.0-linux-x64-v6.0.tgz')
|
||||
version('5.1', '406f4ac7f7ee8aa9e41304c143461a69',
|
||||
url='http://developer.download.nvidia.com/compute/redist/cudnn/v5.1/cudnn-8.0-linux-x64-v5.1.tgz')
|
||||
|
||||
depends_on('cuda@8:')
|
||||
depends_on('cuda@8:', when='@5.1:@7')
|
||||
depends_on('cuda@9:', when='@7.2:@7.4')
|
||||
depends_on('cuda@10:', when='@7.5.0-10.0-ppc64le,7.5.0-10.0-x86_64,7.5.1-10.0-ppc64le,7.5.1-10.0-x86_64')
|
||||
depends_on('cuda@10.1:', when='@7.5.0-10.1-ppc64le,7.5.0-10.1-x86_64,7.5.1-10.1-ppc64le,7.5.1-10.1-x86_64')
|
||||
|
||||
def install(self, spec, prefix):
|
||||
install_tree('.', prefix)
|
||||
|
||||
if 'target=ppc64le platform=linux' in spec:
|
||||
symlink(os.path.join(prefix, 'targets', 'ppc64le-linux', 'lib'),
|
||||
prefix.lib)
|
||||
symlink(
|
||||
os.path.join(prefix, 'targets', 'ppc64le-linux', 'include'),
|
||||
prefix.include)
|
||||
|
|
|
|||
|
|
@ -117,6 +117,8 @@ def cmake_args(self):
|
|||
'-DHydrogen_USE_64BIT_BLAS_INTS:BOOL=%s' % ('+int64_blas' in spec),
|
||||
'-DHydrogen_ENABLE_MPC:BOOL=%s' % ('+mpfr' in spec),
|
||||
'-DHydrogen_GENERAL_LAPACK_FALLBACK=ON',
|
||||
'-DHydrogen_ENABLE_ALUMINUM=%s' % ('+al' in spec),
|
||||
'-DHydrogen_ENABLE_CUB=%s' % ('+cuda' in spec),
|
||||
'-DHydrogen_ENABLE_CUDA=%s' % ('+cuda' in spec),
|
||||
'-DHydrogen_ENABLE_TESTING=%s' % ('+test' in spec),
|
||||
]
|
||||
|
|
|
|||
|
|
@ -18,6 +18,7 @@ class Lbann(CMakePackage):
|
|||
git = "https://github.com/LLNL/lbann.git"
|
||||
|
||||
version('develop', branch='develop')
|
||||
version('0.99', branch='develop')
|
||||
version('0.98.1', sha256='9a2da8f41cd8bf17d1845edf9de6d60f781204ebd37bffba96d8872036c10c66')
|
||||
version('0.98', sha256='8d64b9ac0f1d60db553efa4e657f5ea87e790afe65336117267e9c7ae6f68239')
|
||||
version('0.97.1', sha256='2f2756126ac8bb993202cf532d72c4d4044e877f4d52de9fdf70d0babd500ce4')
|
||||
|
|
@ -40,10 +41,14 @@ class Lbann(CMakePackage):
|
|||
description='The build type to build',
|
||||
values=('Debug', 'Release'))
|
||||
variant('al', default=True, description='Builds with support for Aluminum Library')
|
||||
variant('conduit', default=False, description='Builds with support for Conduit Library')
|
||||
variant('conduit', default=True,
|
||||
description='Builds with support for Conduit Library ' +
|
||||
'(note that for v0.99 conduit is required)')
|
||||
variant('vtune', default=False, description='Builds with support for Intel VTune')
|
||||
variant('docs', default=False, description='Builds with support for building documentation')
|
||||
|
||||
conflicts('@:0.90,0.99:', when='~conduit')
|
||||
|
||||
# It seems that there is a need for one statement per version bounds
|
||||
depends_on('hydrogen +openmp_blas +shared +int64', when='@:0.90,0.95: ~al')
|
||||
depends_on('hydrogen +openmp_blas +shared +int64 +al', when='@:0.90,0.95: +al')
|
||||
|
|
@ -74,25 +79,39 @@ class Lbann(CMakePackage):
|
|||
|
||||
depends_on('cuda', when='+gpu')
|
||||
depends_on('cudnn', when='+gpu')
|
||||
depends_on('cub', when='+gpu')
|
||||
depends_on('cub', when='@0.94:0.98.2 +gpu')
|
||||
depends_on('mpi')
|
||||
depends_on('hwloc')
|
||||
|
||||
# LBANN wraps OpenCV calls in OpenMP parallel loops, build without OpenMP
|
||||
# Additionally disable video related options, they incorrectly link in a
|
||||
# bad OpenMP library when building with clang or Intel compilers
|
||||
# Note that for Power systems we want the environment to add +powerpc +vsx
|
||||
depends_on('opencv@3.2.0: +core +highgui +imgproc +jpeg +png +tiff +zlib '
|
||||
'+fast-math +powerpc +vsx ~calib3d ~cuda ~dnn ~eigen'
|
||||
'+fast-math ~calib3d ~cuda ~dnn ~eigen'
|
||||
'~features2d ~flann ~gtk ~ipp ~ipp_iw ~jasper ~java ~lapack ~ml'
|
||||
'~openmp ~opencl ~opencl_svm ~openclamdblas ~openclamdfft'
|
||||
'~pthreads_pf ~python ~qt ~stitching ~superres ~ts ~video'
|
||||
'~videostab ~videoio ~vtk', when='+opencv')
|
||||
|
||||
depends_on('protobuf@3.0.2: build_type=Release')
|
||||
depends_on('protobuf@3.6.1: build_type=Release')
|
||||
depends_on('cnpy')
|
||||
depends_on('nccl', when='+gpu +nccl')
|
||||
depends_on('nccl', when='@0.94:0.98.2 +gpu +nccl')
|
||||
|
||||
depends_on('conduit@master +hdf5', when='+conduit')
|
||||
depends_on('conduit@master +hdf5', when='@0.94:0.99 +conduit')
|
||||
depends_on('conduit@master +hdf5', when='@:0.90,0.99:')
|
||||
|
||||
depends_on('python@3: +shared', type=('build', 'run'), when='@:0.90,0.99:')
|
||||
extends("python")
|
||||
depends_on('py-setuptools', type='build')
|
||||
depends_on('py-argparse', type='run', when='@:0.90,0.99:')
|
||||
depends_on('py-configparser', type='run', when='@:0.90,0.99:')
|
||||
depends_on('py-graphviz@0.10.1:', type='run', when='@:0.90,0.99:')
|
||||
depends_on('py-matplotlib@3.0.0:', type='run', when='@:0.90,0.99:')
|
||||
depends_on('py-numpy@1.16.0:', type=('build', 'run'), when='@:0.90,0.99:')
|
||||
depends_on('py-onnx@1.3.0:', type='run', when='@:0.90,0.99:')
|
||||
depends_on('py-pandas@0.24.1:', type='run', when='@:0.90,0.99:')
|
||||
depends_on('py-texttable@1.4.0:', type='run', when='@:0.90,0.99:')
|
||||
|
||||
depends_on('py-breathe', type='build', when='+docs')
|
||||
depends_on('py-m2r', type='build', when='+docs')
|
||||
|
|
@ -128,7 +147,6 @@ def cmake_args(self):
|
|||
'-DLBANN_WITH_CONDUIT:BOOL=%s' % ('+conduit' in spec),
|
||||
'-DLBANN_WITH_CUDA:BOOL=%s' % ('+gpu' in spec),
|
||||
'-DLBANN_WITH_CUDNN:BOOL=%s' % ('+gpu' in spec),
|
||||
'-DLBANN_WITH_NCCL:BOOL=%s' % ('+gpu +nccl' in spec),
|
||||
'-DLBANN_WITH_SOFTMAX_CUDA:BOOL=%s' % ('+gpu' in spec),
|
||||
'-DLBANN_SEQUENTIAL_INITIALIZATION:BOOL=%s' %
|
||||
('+seq_init' in spec),
|
||||
|
|
@ -146,6 +164,9 @@ def cmake_args(self):
|
|||
'-DElemental_DIR={0}/CMake/elemental'.format(
|
||||
spec['elemental'].prefix)])
|
||||
|
||||
if self.spec.satisfies('@0.94:0.98.2'):
|
||||
args.extend(['-DLBANN_WITH_NCCL:BOOL=%s' % ('+gpu +nccl' in spec)])
|
||||
|
||||
if '+vtune' in spec:
|
||||
args.extend(['-DVTUNE_DIR={0}'.format(spec['vtune'].prefix)])
|
||||
|
||||
|
|
@ -179,12 +200,13 @@ def cmake_args(self):
|
|||
args.extend([
|
||||
'-DcuDNN_DIR={0}'.format(
|
||||
spec['cudnn'].prefix)])
|
||||
args.extend(['-DCUB_DIR={0}'.format(
|
||||
spec['cub'].prefix)])
|
||||
if '+nccl' in spec:
|
||||
args.extend([
|
||||
'-DNCCL_DIR={0}'.format(
|
||||
spec['nccl'].prefix)])
|
||||
if self.spec.satisfies('@0.94:0.98.2'):
|
||||
args.extend(['-DCUB_DIR={0}'.format(
|
||||
spec['cub'].prefix)])
|
||||
if '+nccl' in spec:
|
||||
args.extend([
|
||||
'-DNCCL_DIR={0}'.format(
|
||||
spec['nccl'].prefix)])
|
||||
|
||||
return args
|
||||
|
||||
|
|
|
|||
|
|
@ -12,6 +12,8 @@ class Nccl(MakefilePackage):
|
|||
homepage = "https://github.com/NVIDIA/nccl"
|
||||
url = "https://github.com/NVIDIA/nccl/archive/v1.3.4-1.tar.gz"
|
||||
|
||||
version('2.4.6-1', sha256='ea4421061a7b9c454f2e088f68bfdbbcefab80ce81cafc70ee6c7742b1439591')
|
||||
version('2.4.2-1', sha256='e3dd04b22eb541394bd818e5f78ac23a09cc549690d5d55d6fccc1a36155385a')
|
||||
version('2.3.7-1', sha256='e6eff80d9d2db13c61f8452e1400ca2f098d2dfe42857cb23413ce081c5b9e9b')
|
||||
version('2.3.5-5', sha256='bac9950b4d3980c25baa8e3e4541d2dfb4d21edf32ad3b89022d04920357142f')
|
||||
version('2.2', '5b9ce7fbdce0fde68e0f66318e6ff422')
|
||||
|
|
|
|||
|
|
@ -15,6 +15,8 @@ class Protobuf(CMakePackage):
|
|||
url = "https://github.com/protocolbuffers/protobuf/archive/v3.2.0.tar.gz"
|
||||
root_cmakelists_dir = "cmake"
|
||||
|
||||
version('3.7.1', sha256='f1748989842b46fa208b2a6e4e2785133cfcc3e4d43c17fecb023733f0f5443f')
|
||||
version('3.7.0', sha256='a19dcfe9d156ae45d209b15e0faed5c7b5f109b6117bfc1974b6a7b98a850320')
|
||||
version('3.6.1', sha256='3d4e589d81b2006ca603c1ab712c9715a76227293032d05b26fca603f90b3f5b')
|
||||
version('3.5.2', 'ff6742018c172c66ecc627029ad54280')
|
||||
version('3.5.1.1', '5005003ae6b94773c4bbca87a644b131')
|
||||
|
|
|
|||
33
var/spack/repos/builtin/packages/py-graphviz/package.py
Normal file
33
var/spack/repos/builtin/packages/py-graphviz/package.py
Normal file
|
|
@ -0,0 +1,33 @@
|
|||
# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other
|
||||
# Spack Project Developers. See the top-level COPYRIGHT file for details.
|
||||
#
|
||||
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
|
||||
|
||||
|
||||
from spack import *
|
||||
|
||||
|
||||
class PyGraphviz(PythonPackage):
|
||||
"""Simple Python interface for Graphviz"""
|
||||
|
||||
homepage = "https://github.com/xflr6/graphviz"
|
||||
url = "https://pypi.io/packages/source/g/graphviz/graphviz-0.10.1.zip"
|
||||
|
||||
version('0.10.1', sha256='d311be4fddfe832a56986ac5e1d6e8715d7fcb0208560da79d1bb0f72abef41f')
|
||||
|
||||
variant('dev', default=False, description='development mode')
|
||||
variant('docs', default=False, description='build documentation')
|
||||
|
||||
depends_on('python@2.7:2.8,3.4:', type=('build', 'run'))
|
||||
depends_on('py-setuptools', type='build')
|
||||
depends_on('py-tox@3.0:', type=('build', 'run'), when='+dev')
|
||||
depends_on('py-flake8', type=('build', 'run'), when='+dev')
|
||||
depends_on('py-pep8-naming', type=('build', 'run'), when='+dev')
|
||||
depends_on('py-wheel', type=('build', 'run'), when='+dev')
|
||||
depends_on('py-twine', type=('build', 'run'), when='+dev')
|
||||
depends_on('py-mock@2:', type='test')
|
||||
depends_on('py-pytest@3.4:', type='test')
|
||||
depends_on('py-pytest-mock@1.8:', type='test')
|
||||
depends_on('py-pytest-cov', type='test')
|
||||
depends_on('py-sphinx@1.7:', type=('build', 'run'), when='+docs')
|
||||
depends_on('py-sphinx-rtd-theme', type=('build', 'run'), when='+docs')
|
||||
30
var/spack/repos/builtin/packages/py-onnx/package.py
Normal file
30
var/spack/repos/builtin/packages/py-onnx/package.py
Normal file
|
|
@ -0,0 +1,30 @@
|
|||
# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other
|
||||
# Spack Project Developers. See the top-level COPYRIGHT file for details.
|
||||
#
|
||||
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
|
||||
|
||||
|
||||
from spack import *
|
||||
|
||||
|
||||
class PyOnnx(PythonPackage):
|
||||
"""Open Neural Network Exchange (ONNX) is an open ecosystem that
|
||||
empowers AI developers to choose the right tools as their
|
||||
project evolves. ONNX provides an open source format for AI
|
||||
models, both deep learning and traditional ML. It defines an
|
||||
extensible computation graph model, as well as definitions of
|
||||
built-in operators and standard data types. Currently we focus
|
||||
on the capabilities needed for inferencing (scoring)."""
|
||||
|
||||
homepage = "https://github.com/onnx/onnx"
|
||||
url = "https://pypi.io/packages/source/O/Onnx/onnx-1.5.0.tar.gz"
|
||||
|
||||
version('1.5.0', sha256='1a584a4ef62a6db178c257fffb06a9d8e61b41c0a80bfd8bcd8a253d72c4b0b4')
|
||||
|
||||
depends_on('py-setuptools', type='build')
|
||||
depends_on('protobuf')
|
||||
depends_on('py-protobuf+cpp', type=('build', 'run'))
|
||||
depends_on('py-numpy', type=('build', 'run'))
|
||||
depends_on('py-six', type=('build', 'run'))
|
||||
depends_on('py-typing@3.6.4:', type=('build', 'run'))
|
||||
depends_on('py-typing-extensions@3.6.4:', type=('build', 'run'))
|
||||
|
|
@ -20,6 +20,7 @@ class PyProtobuf(PythonPackage):
|
|||
variant('cpp', default=False,
|
||||
description='Enable the cpp implementation')
|
||||
|
||||
version('3.7.1', sha256='21e395d7959551e759d604940a115c51c6347d90a475c9baf471a1a86b5604a9')
|
||||
version('3.6.1', sha256='1489b376b0f364bcc6f89519718c057eb191d7ad6f1b395ffd93d1aa45587811')
|
||||
version('3.5.2.post1', '3b60685732bd0cbdc802dfcb6071efbcf5d927ce3127c13c33ea1a8efae3aa76')
|
||||
version('3.5.2', '09879a295fd7234e523b62066223b128c5a8a88f682e3aff62fb115e4a0d8be0')
|
||||
|
|
|
|||
18
var/spack/repos/builtin/packages/py-texttable/package.py
Normal file
18
var/spack/repos/builtin/packages/py-texttable/package.py
Normal file
|
|
@ -0,0 +1,18 @@
|
|||
# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other
|
||||
# Spack Project Developers. See the top-level COPYRIGHT file for details.
|
||||
#
|
||||
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
|
||||
|
||||
|
||||
from spack import *
|
||||
|
||||
|
||||
class PyTexttable(PythonPackage):
|
||||
"""Python module for creating simple ASCII tables."""
|
||||
|
||||
homepage = "https://github.com/foutaise/texttable/"
|
||||
url = "https://pypi.io/packages/source/t/texttable/texttable-1.6.1.tar.gz"
|
||||
|
||||
version('1.6.1', sha256='2b60a5304ccfbeac80ffae7350d7c2f5d7a24e9aab5036d0f82489746419d9b2')
|
||||
|
||||
depends_on('py-setuptools', type='build')
|
||||
|
|
@ -0,0 +1,21 @@
|
|||
# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other
|
||||
# Spack Project Developers. See the top-level COPYRIGHT file for details.
|
||||
#
|
||||
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
|
||||
|
||||
from spack import *
|
||||
|
||||
|
||||
class PyTypingExtensions(PythonPackage):
|
||||
"""The typing_extensions module contains both backports of these
|
||||
changes as well as experimental types that will eventually be
|
||||
added to the typing module, such as Protocol (see PEP 544 for
|
||||
details about protocols and static duck typing)."""
|
||||
|
||||
homepage = "https://github.com/python/typing/tree/master/typing_extensions"
|
||||
url = "https://pypi.io/packages/source/t/typing_extensions/typing_extensions-3.7.2.tar.gz"
|
||||
|
||||
version('3.7.2', sha256='fb2cd053238d33a8ec939190f30cfd736c00653a85a2919415cecf7dc3d9da71')
|
||||
|
||||
depends_on('py-setuptools', type='build')
|
||||
depends_on('py-typing@3.6.4:', type=('build', 'run'))
|
||||
|
|
@ -15,6 +15,7 @@ class PyTyping(PythonPackage):
|
|||
|
||||
import_modules = ['typing']
|
||||
|
||||
version('3.6.4', '5b2ade08d83be488f17b5fe587c27c74')
|
||||
version('3.6.1', '3fec97415bae6f742fb3c3013dedeb89')
|
||||
|
||||
# You need Python 2.7 or 3.3+ to install the typing package
|
||||
|
|
|
|||
Loading…
Reference in a new issue