CachedCMakePackage: add CUDA/HIP options and improve independent builds (#37592)
* Add CMake options for building with CUDA/HIP support to CachedCMakePackages (intended to reduce duplication across packages building with +hip/+cuda and using CachedCMakePackage) * Define generic variables like CMAKE_PREFIX_PATH for CachedCMakePackages (so that a user may invoke "cmake" themselves without needing to setthem on the command line). * Make `lbann` a CachedCMakePackage. Co-authored-by: Chris White <white238@llnl.gov>
This commit is contained in:
Brian Van Essen
GitHub
parent
042dc2e1d8
commit
543b697df1
2 changed files with 212 additions and 186 deletions
|
|
@ -2,6 +2,7 @@
|
||||||
# Spack Project Developers. See the top-level COPYRIGHT file for details.
|
# Spack Project Developers. See the top-level COPYRIGHT file for details.
|
||||||
#
|
#
|
||||||
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
|
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
|
||||||
|
import collections.abc
|
||||||
import os
|
import os
|
||||||
from typing import Tuple
|
from typing import Tuple
|
||||||
|
|
||||||
|
|
@ -13,21 +14,24 @@
|
||||||
from .cmake import CMakeBuilder, CMakePackage
|
from .cmake import CMakeBuilder, CMakePackage
|
||||||
|
|
||||||
|
|
||||||
def cmake_cache_path(name, value, comment=""):
|
def cmake_cache_path(name, value, comment="", force=False):
|
||||||
"""Generate a string for a cmake cache variable"""
|
"""Generate a string for a cmake cache variable"""
|
||||||
return 'set({0} "{1}" CACHE PATH "{2}")\n'.format(name, value, comment)
|
force_str = " FORCE" if force else ""
|
||||||
|
return 'set({0} "{1}" CACHE PATH "{2}"{3})\n'.format(name, value, comment, force_str)
|
||||||
|
|
||||||
|
|
||||||
def cmake_cache_string(name, value, comment=""):
|
def cmake_cache_string(name, value, comment="", force=False):
|
||||||
"""Generate a string for a cmake cache variable"""
|
"""Generate a string for a cmake cache variable"""
|
||||||
return 'set({0} "{1}" CACHE STRING "{2}")\n'.format(name, value, comment)
|
force_str = " FORCE" if force else ""
|
||||||
|
return 'set({0} "{1}" CACHE STRING "{2}"{3})\n'.format(name, value, comment, force_str)
|
||||||
|
|
||||||
|
|
||||||
def cmake_cache_option(name, boolean_value, comment=""):
|
def cmake_cache_option(name, boolean_value, comment="", force=False):
|
||||||
"""Generate a string for a cmake configuration option"""
|
"""Generate a string for a cmake configuration option"""
|
||||||
|
|
||||||
value = "ON" if boolean_value else "OFF"
|
value = "ON" if boolean_value else "OFF"
|
||||||
return 'set({0} {1} CACHE BOOL "{2}")\n'.format(name, value, comment)
|
force_str = " FORCE" if force else ""
|
||||||
|
return 'set({0} {1} CACHE BOOL "{2}"{3})\n'.format(name, value, comment, force_str)
|
||||||
|
|
||||||
|
|
||||||
class CachedCMakeBuilder(CMakeBuilder):
|
class CachedCMakeBuilder(CMakeBuilder):
|
||||||
|
|
@ -63,6 +67,34 @@ def cache_name(self):
|
||||||
def cache_path(self):
|
def cache_path(self):
|
||||||
return os.path.join(self.pkg.stage.source_path, self.cache_name)
|
return os.path.join(self.pkg.stage.source_path, self.cache_name)
|
||||||
|
|
||||||
|
# Implement a version of the define_from_variant for Cached packages
|
||||||
|
def define_cmake_cache_from_variant(self, cmake_var, variant=None, comment=""):
|
||||||
|
"""Return a Cached CMake field from the given variant's value.
|
||||||
|
See define_from_variant in lib/spack/spack/build_systems/cmake.py package
|
||||||
|
"""
|
||||||
|
|
||||||
|
if variant is None:
|
||||||
|
variant = cmake_var.lower()
|
||||||
|
|
||||||
|
if variant not in self.pkg.variants:
|
||||||
|
raise KeyError('"{0}" is not a variant of "{1}"'.format(variant, self.pkg.name))
|
||||||
|
|
||||||
|
if variant not in self.pkg.spec.variants:
|
||||||
|
return ""
|
||||||
|
|
||||||
|
value = self.pkg.spec.variants[variant].value
|
||||||
|
field = None
|
||||||
|
if isinstance(value, bool):
|
||||||
|
field = cmake_cache_option(cmake_var, value, comment)
|
||||||
|
else:
|
||||||
|
if isinstance(value, collections.abc.Sequence) and not isinstance(value, str):
|
||||||
|
value = ";".join(str(v) for v in value)
|
||||||
|
else:
|
||||||
|
value = str(value)
|
||||||
|
field = cmake_cache_string(cmake_var, value, comment)
|
||||||
|
|
||||||
|
return field
|
||||||
|
|
||||||
def initconfig_compiler_entries(self):
|
def initconfig_compiler_entries(self):
|
||||||
# This will tell cmake to use the Spack compiler wrappers when run
|
# This will tell cmake to use the Spack compiler wrappers when run
|
||||||
# through Spack, but use the underlying compiler when run outside of
|
# through Spack, but use the underlying compiler when run outside of
|
||||||
|
|
@ -130,6 +162,17 @@ def initconfig_compiler_entries(self):
|
||||||
libs_string = libs_format_string.format(lang)
|
libs_string = libs_format_string.format(lang)
|
||||||
entries.append(cmake_cache_string(libs_string, libs_flags))
|
entries.append(cmake_cache_string(libs_string, libs_flags))
|
||||||
|
|
||||||
|
# Set the generator in the cached config
|
||||||
|
if self.spec.satisfies("generator=make"):
|
||||||
|
entries.append(cmake_cache_string("CMAKE_GENERATOR", "Unix Makefiles"))
|
||||||
|
if self.spec.satisfies("generator=ninja"):
|
||||||
|
entries.append(cmake_cache_string("CMAKE_GENERATOR", "Ninja"))
|
||||||
|
entries.append(
|
||||||
|
cmake_cache_string(
|
||||||
|
"CMAKE_MAKE_PROGRAM", "{0}/ninja".format(spec["ninja"].prefix.bin)
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
return entries
|
return entries
|
||||||
|
|
||||||
def initconfig_mpi_entries(self):
|
def initconfig_mpi_entries(self):
|
||||||
|
|
@ -195,26 +238,57 @@ def initconfig_hardware_entries(self):
|
||||||
"#------------------{0}\n".format("-" * 60),
|
"#------------------{0}\n".format("-" * 60),
|
||||||
]
|
]
|
||||||
|
|
||||||
|
# Provide standard CMake arguments for dependent CachedCMakePackages
|
||||||
if spec.satisfies("^cuda"):
|
if spec.satisfies("^cuda"):
|
||||||
entries.append("#------------------{0}".format("-" * 30))
|
entries.append("#------------------{0}".format("-" * 30))
|
||||||
entries.append("# Cuda")
|
entries.append("# Cuda")
|
||||||
entries.append("#------------------{0}\n".format("-" * 30))
|
entries.append("#------------------{0}\n".format("-" * 30))
|
||||||
|
|
||||||
cudatoolkitdir = spec["cuda"].prefix
|
cudatoolkitdir = spec["cuda"].prefix
|
||||||
entries.append(cmake_cache_path("CUDA_TOOLKIT_ROOT_DIR", cudatoolkitdir))
|
entries.append(cmake_cache_path("CUDAToolkit_ROOT", cudatoolkitdir))
|
||||||
cudacompiler = "${CUDA_TOOLKIT_ROOT_DIR}/bin/nvcc"
|
entries.append(cmake_cache_path("CMAKE_CUDA_COMPILER", "${CUDAToolkit_ROOT}/bin/nvcc"))
|
||||||
entries.append(cmake_cache_path("CMAKE_CUDA_COMPILER", cudacompiler))
|
|
||||||
entries.append(cmake_cache_path("CMAKE_CUDA_HOST_COMPILER", "${CMAKE_CXX_COMPILER}"))
|
entries.append(cmake_cache_path("CMAKE_CUDA_HOST_COMPILER", "${CMAKE_CXX_COMPILER}"))
|
||||||
|
# Include the deprecated CUDA_TOOLKIT_ROOT_DIR for supporting BLT packages
|
||||||
|
entries.append(cmake_cache_path("CUDA_TOOLKIT_ROOT_DIR", cudatoolkitdir))
|
||||||
|
|
||||||
|
archs = spec.variants["cuda_arch"].value
|
||||||
|
if archs != "none":
|
||||||
|
arch_str = ";".join(archs)
|
||||||
|
entries.append(
|
||||||
|
cmake_cache_string("CMAKE_CUDA_ARCHITECTURES", "{0}".format(arch_str))
|
||||||
|
)
|
||||||
|
|
||||||
|
if "+rocm" in spec:
|
||||||
|
entries.append("#------------------{0}".format("-" * 30))
|
||||||
|
entries.append("# ROCm")
|
||||||
|
entries.append("#------------------{0}\n".format("-" * 30))
|
||||||
|
|
||||||
|
# Explicitly setting HIP_ROOT_DIR may be a patch that is no longer necessary
|
||||||
|
entries.append(cmake_cache_path("HIP_ROOT_DIR", "{0}".format(spec["hip"].prefix)))
|
||||||
|
entries.append(
|
||||||
|
cmake_cache_path("HIP_CXX_COMPILER", "{0}".format(self.spec["hip"].hipcc))
|
||||||
|
)
|
||||||
|
archs = self.spec.variants["amdgpu_target"].value
|
||||||
|
if archs != "none":
|
||||||
|
arch_str = ";".join(archs)
|
||||||
|
entries.append(
|
||||||
|
cmake_cache_string("CMAKE_HIP_ARCHITECTURES", "{0}".format(arch_str))
|
||||||
|
)
|
||||||
|
entries.append(cmake_cache_string("AMDGPU_TARGETS", "{0}".format(arch_str)))
|
||||||
|
entries.append(cmake_cache_string("GPU_TARGETS", "{0}".format(arch_str)))
|
||||||
|
|
||||||
return entries
|
return entries
|
||||||
|
|
||||||
def std_initconfig_entries(self):
|
def std_initconfig_entries(self):
|
||||||
|
cmake_prefix_path_env = os.environ["CMAKE_PREFIX_PATH"]
|
||||||
|
cmake_prefix_path = cmake_prefix_path_env.replace(os.pathsep, ";")
|
||||||
return [
|
return [
|
||||||
"#------------------{0}".format("-" * 60),
|
"#------------------{0}".format("-" * 60),
|
||||||
"# !!!! This is a generated file, edit at own risk !!!!",
|
"# !!!! This is a generated file, edit at own risk !!!!",
|
||||||
"#------------------{0}".format("-" * 60),
|
"#------------------{0}".format("-" * 60),
|
||||||
"# CMake executable path: {0}".format(self.pkg.spec["cmake"].command.path),
|
"# CMake executable path: {0}".format(self.pkg.spec["cmake"].command.path),
|
||||||
"#------------------{0}\n".format("-" * 60),
|
"#------------------{0}\n".format("-" * 60),
|
||||||
|
cmake_cache_path("CMAKE_PREFIX_PATH", cmake_prefix_path),
|
||||||
]
|
]
|
||||||
|
|
||||||
def initconfig_package_entries(self):
|
def initconfig_package_entries(self):
|
||||||
|
|
|
||||||
|
|
@ -4,12 +4,13 @@
|
||||||
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
|
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
|
||||||
|
|
||||||
import os
|
import os
|
||||||
|
import socket
|
||||||
import sys
|
import sys
|
||||||
|
|
||||||
from spack.package import *
|
from spack.package import *
|
||||||
|
|
||||||
|
|
||||||
class Lbann(CMakePackage, CudaPackage, ROCmPackage):
|
class Lbann(CachedCMakePackage, CudaPackage, ROCmPackage):
|
||||||
"""LBANN: Livermore Big Artificial Neural Network Toolkit. A distributed
|
"""LBANN: Livermore Big Artificial Neural Network Toolkit. A distributed
|
||||||
memory, HPC-optimized, model and data parallel training toolkit for deep
|
memory, HPC-optimized, model and data parallel training toolkit for deep
|
||||||
neural networks.
|
neural networks.
|
||||||
|
|
@ -317,39 +318,6 @@ class Lbann(CMakePackage, CudaPackage, ROCmPackage):
|
||||||
|
|
||||||
generator("ninja")
|
generator("ninja")
|
||||||
|
|
||||||
@property
|
|
||||||
def common_config_args(self):
|
|
||||||
spec = self.spec
|
|
||||||
# Environment variables
|
|
||||||
cppflags = []
|
|
||||||
cppflags.append("-DLBANN_SET_EL_RNG")
|
|
||||||
cppflags.append("-std=c++17")
|
|
||||||
args = []
|
|
||||||
args.extend(["-DCMAKE_CXX_FLAGS=%s" % " ".join(cppflags), "-DLBANN_VERSION=spack"])
|
|
||||||
|
|
||||||
if "+numpy" in spec:
|
|
||||||
args.append("-DCNPY_DIR={0}".format(spec["cnpy"].prefix))
|
|
||||||
|
|
||||||
# Use lld high performance linker
|
|
||||||
if "+lld" in spec:
|
|
||||||
args.extend(
|
|
||||||
[
|
|
||||||
"-DCMAKE_EXE_LINKER_FLAGS=-fuse-ld=lld",
|
|
||||||
"-DCMAKE_SHARED_LINKER_FLAGS=-fuse-ld=lld",
|
|
||||||
]
|
|
||||||
)
|
|
||||||
|
|
||||||
# Use gold high performance linker
|
|
||||||
if "+gold" in spec:
|
|
||||||
args.extend(
|
|
||||||
[
|
|
||||||
"-DCMAKE_EXE_LINKER_FLAGS=-fuse-ld=gold",
|
|
||||||
"-DCMAKE_SHARED_LINKER_FLAGS=-fuse-ld=gold",
|
|
||||||
]
|
|
||||||
)
|
|
||||||
|
|
||||||
return args
|
|
||||||
|
|
||||||
def setup_build_environment(self, env):
|
def setup_build_environment(self, env):
|
||||||
if self.spec.satisfies("%apple-clang"):
|
if self.spec.satisfies("%apple-clang"):
|
||||||
env.append_flags("CPPFLAGS", self.compiler.openmp_flag)
|
env.append_flags("CPPFLAGS", self.compiler.openmp_flag)
|
||||||
|
|
@ -357,181 +325,165 @@ def setup_build_environment(self, env):
|
||||||
env.append_flags("CXXFLAGS", self.spec["llvm-openmp"].headers.include_flags)
|
env.append_flags("CXXFLAGS", self.spec["llvm-openmp"].headers.include_flags)
|
||||||
env.append_flags("LDFLAGS", self.spec["llvm-openmp"].libs.ld_flags)
|
env.append_flags("LDFLAGS", self.spec["llvm-openmp"].libs.ld_flags)
|
||||||
|
|
||||||
# Get any recent versions or non-numeric version
|
def _get_sys_type(self, spec):
|
||||||
# Note that develop > numeric and non-develop < numeric
|
sys_type = spec.architecture
|
||||||
|
if "SYS_TYPE" in env:
|
||||||
|
sys_type = env["SYS_TYPE"]
|
||||||
|
return sys_type
|
||||||
|
|
||||||
@when("@:0.90,0.94:")
|
@property
|
||||||
def cmake_args(self):
|
def cache_name(self):
|
||||||
spec = self.spec
|
hostname = socket.gethostname()
|
||||||
args = self.common_config_args
|
if "SYS_TYPE" in env:
|
||||||
args.extend(
|
# Get a hostname that has no node identifier
|
||||||
[
|
hostname = hostname.rstrip("1234567890")
|
||||||
"-DCMAKE_CXX_STANDARD=17",
|
return "LBANN_{0}_{1}-{2}-{3}@{4}.cmake".format(
|
||||||
"-DLBANN_WITH_CNPY=%s" % ("+numpy" in spec),
|
hostname,
|
||||||
"-DLBANN_DETERMINISTIC:BOOL=%s" % ("+deterministic" in spec),
|
self.spec.version,
|
||||||
"-DLBANN_WITH_HWLOC=%s" % ("+hwloc" in spec),
|
self._get_sys_type(self.spec),
|
||||||
"-DLBANN_WITH_ALUMINUM:BOOL=%s" % ("+al" in spec),
|
self.spec.compiler.name,
|
||||||
"-DLBANN_WITH_ADDRESS_SANITIZER:BOOL=%s" % ("+asan" in spec),
|
self.spec.compiler.version,
|
||||||
"-DLBANN_WITH_BOOST:BOOL=%s" % ("+boost" in spec),
|
|
||||||
"-DLBANN_WITH_CONDUIT:BOOL=%s" % ("+conduit" in spec),
|
|
||||||
"-DLBANN_WITH_NVSHMEM:BOOL=%s" % ("+nvshmem" in spec),
|
|
||||||
"-DLBANN_WITH_FFT:BOOL=%s" % ("+fft" in spec),
|
|
||||||
"-DLBANN_WITH_ONEDNN:BOOL=%s" % ("+onednn" in spec),
|
|
||||||
"-DLBANN_WITH_ONNX:BOOL=%s" % ("+onnx" in spec),
|
|
||||||
"-DLBANN_WITH_EMBEDDED_PYTHON:BOOL=%s" % ("+python" in spec),
|
|
||||||
"-DLBANN_WITH_PYTHON_FRONTEND:BOOL=%s" % ("+pfe" in spec),
|
|
||||||
"-DLBANN_WITH_ROCTRACER:BOOL=%s" % ("+rocm +distconv" in spec),
|
|
||||||
"-DLBANN_WITH_TBINF=OFF",
|
|
||||||
"-DLBANN_WITH_UNIT_TESTING:BOOL=%s" % ("+unit_tests" in spec),
|
|
||||||
"-DLBANN_WITH_VISION:BOOL=%s" % ("+vision" in spec),
|
|
||||||
"-DLBANN_WITH_VTUNE:BOOL=%s" % ("+vtune" in spec),
|
|
||||||
"-DLBANN_DATATYPE={0}".format(spec.variants["dtype"].value),
|
|
||||||
"-DCEREAL_DIR={0}".format(spec["cereal"].prefix),
|
|
||||||
# protobuf is included by py-protobuf+cpp
|
|
||||||
"-DProtobuf_DIR={0}".format(spec["protobuf"].prefix),
|
|
||||||
"-Dprotobuf_MODULE_COMPATIBLE=ON",
|
|
||||||
]
|
|
||||||
)
|
)
|
||||||
|
|
||||||
|
def initconfig_compiler_entries(self):
|
||||||
|
spec = self.spec
|
||||||
|
entries = super(Lbann, self).initconfig_compiler_entries()
|
||||||
|
entries.append(cmake_cache_string("CMAKE_CXX_STANDARD", "17"))
|
||||||
if not spec.satisfies("^cmake@3.23.0"):
|
if not spec.satisfies("^cmake@3.23.0"):
|
||||||
# There is a bug with using Ninja generator in this version
|
# There is a bug with using Ninja generator in this version
|
||||||
# of CMake
|
# of CMake
|
||||||
args.append("-DCMAKE_EXPORT_COMPILE_COMMANDS=ON")
|
entries.append(cmake_cache_option("CMAKE_EXPORT_COMPILE_COMMANDS", True))
|
||||||
|
|
||||||
|
# Use lld high performance linker
|
||||||
|
if "+lld" in spec:
|
||||||
|
entries.append(cmake_cache_string("CMAKE_EXE_LINKER_FLAGS", "-fuse-ld=lld"))
|
||||||
|
entries.append(cmake_cache_string("CMAKE_SHARED_LINKER_FLAGS", "-fuse-ld=lld"))
|
||||||
|
|
||||||
|
# Use gold high performance linker
|
||||||
|
if "+gold" in spec:
|
||||||
|
entries.append(cmake_cache_string("CMAKE_EXE_LINKER_FLAGS", "-fuse-ld=gold"))
|
||||||
|
entries.append(cmake_cache_string("CMAKE_SHARED_LINKER_FLAGS", "-fuse-ld=gold"))
|
||||||
|
|
||||||
|
return entries
|
||||||
|
|
||||||
|
def initconfig_hardware_entries(self):
|
||||||
|
spec = self.spec
|
||||||
|
entries = super(Lbann, self).initconfig_hardware_entries()
|
||||||
|
|
||||||
if "+cuda" in spec:
|
if "+cuda" in spec:
|
||||||
if self.spec.satisfies("%clang"):
|
if self.spec.satisfies("%clang"):
|
||||||
for flag in self.spec.compiler_flags["cxxflags"]:
|
for flag in self.spec.compiler_flags["cxxflags"]:
|
||||||
if "gcc-toolchain" in flag:
|
if "gcc-toolchain" in flag:
|
||||||
args.append("-DCMAKE_CUDA_FLAGS=-Xcompiler={0}".format(flag))
|
entries.append(
|
||||||
|
cmake_cache_string("CMAKE_CUDA_FLAGS", "-Xcompiler={0}".format(flag))
|
||||||
|
)
|
||||||
if spec.satisfies("^cuda@11.0:"):
|
if spec.satisfies("^cuda@11.0:"):
|
||||||
args.append("-DCMAKE_CUDA_STANDARD=17")
|
entries.append(cmake_cache_string("CMAKE_CUDA_STANDARD", "17"))
|
||||||
else:
|
else:
|
||||||
args.append("-DCMAKE_CUDA_STANDARD=14")
|
entries.append(cmake_cache_string("CMAKE_CUDA_STANDARD", "14"))
|
||||||
archs = spec.variants["cuda_arch"].value
|
|
||||||
if archs != "none":
|
entries.append(self.define_cmake_cache_from_variant("LBANN_WITH_NVPROF", "nvprof"))
|
||||||
arch_str = ";".join(archs)
|
|
||||||
args.append("-DCMAKE_CUDA_ARCHITECTURES=%s" % arch_str)
|
|
||||||
|
|
||||||
if spec.satisfies("%cce") and spec.satisfies("^cuda+allow-unsupported-compilers"):
|
if spec.satisfies("%cce") and spec.satisfies("^cuda+allow-unsupported-compilers"):
|
||||||
args.append("-DCMAKE_CUDA_FLAGS=-allow-unsupported-compiler")
|
entries.append(
|
||||||
|
cmake_cache_string("CMAKE_CUDA_FLAGS", "-allow-unsupported-compiler")
|
||||||
|
)
|
||||||
|
|
||||||
if spec.satisfies("@:0.90") or spec.satisfies("@0.95:"):
|
if "+rocm" in spec:
|
||||||
args.append("-DHydrogen_DIR={0}/CMake/hydrogen".format(spec["hydrogen"].prefix))
|
if "platform=cray" in spec:
|
||||||
elif spec.satisfies("@0.94"):
|
entries.append(cmake_cache_option("MPI_ASSUME_NO_BUILTIN_MPI", True))
|
||||||
args.append("-DElemental_DIR={0}/CMake/elemental".format(spec["elemental"].prefix))
|
|
||||||
|
|
||||||
if spec.satisfies("@0.94:0.98.2"):
|
cxxflags_str = " ".join(self.spec.compiler_flags["cxxflags"])
|
||||||
args.append("-DLBANN_WITH_NCCL:BOOL=%s" % ("+cuda +nccl" in spec))
|
entries.append(
|
||||||
|
cmake_cache_string(
|
||||||
|
"HIP_HIPCC_FLAGS",
|
||||||
|
"-g -fsized-deallocation -fPIC -std=c++17 {0}".format(cxxflags_str),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
if "+vtune" in spec:
|
return entries
|
||||||
args.append("-DVTUNE_DIR={0}".format(spec["vtune"].prefix))
|
|
||||||
|
|
||||||
if "+al" in spec:
|
def initconfig_package_entries(self):
|
||||||
args.append("-DAluminum_DIR={0}".format(spec["aluminum"].prefix))
|
spec = self.spec
|
||||||
|
entries = []
|
||||||
|
entries = [
|
||||||
|
"#------------------{0}".format("-" * 60),
|
||||||
|
"# LBANN",
|
||||||
|
"#------------------{0}\n".format("-" * 60),
|
||||||
|
]
|
||||||
|
|
||||||
if "+conduit" in spec:
|
cmake_variant_fields = [
|
||||||
args.append("-DConduit_DIR={0}".format(spec["conduit"].prefix))
|
("LBANN_WITH_CNPY", "numpy"),
|
||||||
|
("LBANN_DETERMINISTIC", "deterministic"),
|
||||||
|
("LBANN_WITH_HWLOC", "hwloc"),
|
||||||
|
("LBANN_WITH_ALUMINUM", "al"),
|
||||||
|
("LBANN_WITH_ADDRESS_SANITIZER", "asan"),
|
||||||
|
("LBANN_WITH_BOOST", "boost"),
|
||||||
|
("LBANN_WITH_CONDUIT", "conduit"),
|
||||||
|
("LBANN_WITH_NVSHMEM", "nvshmem"),
|
||||||
|
("LBANN_WITH_FFT", "fft"),
|
||||||
|
("LBANN_WITH_ONEDNN", "onednn"),
|
||||||
|
("LBANN_WITH_ONNX", "onnx"),
|
||||||
|
("LBANN_WITH_EMBEDDED_PYTHON", "python"),
|
||||||
|
("LBANN_WITH_PYTHON_FRONTEND", "pfe"),
|
||||||
|
("LBANN_WITH_UNIT_TESTING", "unit_tests"),
|
||||||
|
("LBANN_WITH_VISION", "vision"),
|
||||||
|
("LBANN_WITH_VTUNE", "vtune"),
|
||||||
|
]
|
||||||
|
|
||||||
|
for opt, val in cmake_variant_fields:
|
||||||
|
entries.append(self.define_cmake_cache_from_variant(opt, val))
|
||||||
|
|
||||||
|
entries.append(cmake_cache_option("LBANN_WITH_ROCTRACER", "+rocm +distconv" in spec))
|
||||||
|
entries.append(cmake_cache_option("LBANN_WITH_TBINF", False))
|
||||||
|
entries.append(
|
||||||
|
cmake_cache_string("LBANN_DATATYPE", "{0}".format(spec.variants["dtype"].value))
|
||||||
|
)
|
||||||
|
entries.append(cmake_cache_option("protobuf_MODULE_COMPATIBLE", True))
|
||||||
|
|
||||||
|
if spec.satisfies("^python") and "+pfe" in spec:
|
||||||
|
entries.append(
|
||||||
|
cmake_cache_path(
|
||||||
|
"LBANN_PFE_PYTHON_EXECUTABLE", "{0}/python3".format(spec["python"].prefix.bin)
|
||||||
|
)
|
||||||
|
)
|
||||||
|
entries.append(
|
||||||
|
cmake_cache_string("LBANN_PFE_PYTHONPATH", env["PYTHONPATH"])
|
||||||
|
) # do NOT need to sub ; for : because
|
||||||
|
# value will only be interpreted by
|
||||||
|
# a shell, which expects :
|
||||||
|
|
||||||
# Add support for OpenMP with external (Brew) clang
|
# Add support for OpenMP with external (Brew) clang
|
||||||
if spec.satisfies("%clang platform=darwin"):
|
if spec.satisfies("%clang platform=darwin"):
|
||||||
clang = self.compiler.cc
|
clang = self.compiler.cc
|
||||||
clang_bin = os.path.dirname(clang)
|
clang_bin = os.path.dirname(clang)
|
||||||
clang_root = os.path.dirname(clang_bin)
|
clang_root = os.path.dirname(clang_bin)
|
||||||
args.extend(
|
entries.append(cmake_cache_string("OpenMP_CXX_FLAGS", "-fopenmp=libomp"))
|
||||||
[
|
entries.append(cmake_cache_string("OpenMP_CXX_LIB_NAMES", "libomp"))
|
||||||
"-DOpenMP_CXX_FLAGS=-fopenmp=libomp",
|
entries.append(
|
||||||
"-DOpenMP_CXX_LIB_NAMES=libomp",
|
cmake_cache_string(
|
||||||
"-DOpenMP_libomp_LIBRARY={0}/lib/libomp.dylib".format(clang_root),
|
"OpenMP_libomp_LIBRARY", "{0}/lib/libomp.dylib".format(clang_root)
|
||||||
]
|
)
|
||||||
)
|
)
|
||||||
|
|
||||||
if "+vision" in spec:
|
entries.append(self.define_cmake_cache_from_variant("LBANN_WITH_DIHYDROGEN", "dihydrogen"))
|
||||||
args.append("-DOpenCV_DIR:STRING={0}".format(spec["opencv"].prefix))
|
entries.append(self.define_cmake_cache_from_variant("LBANN_WITH_DISTCONV", "distconv"))
|
||||||
|
|
||||||
if "+cuda" in spec:
|
|
||||||
args.append("-DCUDA_TOOLKIT_ROOT_DIR={0}".format(spec["cuda"].prefix))
|
|
||||||
args.append("-DcuDNN_DIR={0}".format(spec["cudnn"].prefix))
|
|
||||||
if spec.satisfies("@0.94:0.98.2"):
|
|
||||||
if spec.satisfies("^cuda@:10"):
|
|
||||||
args.append("-DCUB_DIR={0}".format(spec["cub"].prefix))
|
|
||||||
if "+nccl" in spec:
|
|
||||||
args.append("-DNCCL_DIR={0}".format(spec["nccl"].prefix))
|
|
||||||
args.append("-DLBANN_WITH_NVPROF:BOOL=%s" % ("+nvprof" in spec))
|
|
||||||
|
|
||||||
if spec.satisfies("@:0.90") or spec.satisfies("@0.100:"):
|
|
||||||
args.append("-DLBANN_WITH_DIHYDROGEN:BOOL=%s" % ("+dihydrogen" in spec))
|
|
||||||
|
|
||||||
if spec.satisfies("@:0.90") or spec.satisfies("@0.101:"):
|
|
||||||
args.append("-DLBANN_WITH_DISTCONV:BOOL=%s" % ("+distconv" in spec))
|
|
||||||
|
|
||||||
if "+rocm" in spec:
|
|
||||||
args.extend(
|
|
||||||
[
|
|
||||||
"-DHIP_ROOT_DIR={0}".format(spec["hip"].prefix),
|
|
||||||
"-DHIP_CXX_COMPILER={0}".format(self.spec["hip"].hipcc),
|
|
||||||
]
|
|
||||||
)
|
|
||||||
if "platform=cray" in spec:
|
|
||||||
args.extend(["-DMPI_ASSUME_NO_BUILTIN_MPI=ON"])
|
|
||||||
archs = self.spec.variants["amdgpu_target"].value
|
|
||||||
if archs != "none":
|
|
||||||
arch_str = ",".join(archs)
|
|
||||||
cxxflags_str = " ".join(self.spec.compiler_flags["cxxflags"])
|
|
||||||
args.append(
|
|
||||||
"-DHIP_HIPCC_FLAGS=--amdgpu-target={0}"
|
|
||||||
" -g -fsized-deallocation -fPIC -std=c++17 {1}".format(arch_str, cxxflags_str)
|
|
||||||
)
|
|
||||||
args.extend(
|
|
||||||
[
|
|
||||||
"-DCMAKE_HIP_ARCHITECTURES=%s" % arch_str,
|
|
||||||
"-DAMDGPU_TARGETS=%s" % arch_str,
|
|
||||||
"-DGPU_TARGETS=%s" % arch_str,
|
|
||||||
]
|
|
||||||
)
|
|
||||||
|
|
||||||
# IF IBM ESSL is used it needs help finding the proper LAPACK libraries
|
# IF IBM ESSL is used it needs help finding the proper LAPACK libraries
|
||||||
if self.spec.satisfies("^essl"):
|
if self.spec.satisfies("^essl"):
|
||||||
args.extend(
|
entries.append(
|
||||||
[
|
cmake_cache_string(
|
||||||
"-DLAPACK_LIBRARIES=%s;-llapack;-lblas"
|
"LAPACK_LIBRARIES",
|
||||||
|
"%s;-llapack;-lblas"
|
||||||
% ";".join("-l{0}".format(lib) for lib in self.spec["essl"].libs.names),
|
% ";".join("-l{0}".format(lib) for lib in self.spec["essl"].libs.names),
|
||||||
"-DBLAS_LIBRARIES=%s;-lblas"
|
)
|
||||||
|
)
|
||||||
|
entries.append(
|
||||||
|
cmake_cache_string(
|
||||||
|
"BLAS_LIBRARIES",
|
||||||
|
"%s;-lblas"
|
||||||
% ";".join("-l{0}".format(lib) for lib in self.spec["essl"].libs.names),
|
% ";".join("-l{0}".format(lib) for lib in self.spec["essl"].libs.names),
|
||||||
]
|
)
|
||||||
)
|
)
|
||||||
|
|
||||||
return args
|
return entries
|
||||||
|
|
||||||
@when("@0.91:0.93")
|
|
||||||
def cmake_args(self):
|
|
||||||
spec = self.spec
|
|
||||||
args = self.common_config_args
|
|
||||||
args.extend(
|
|
||||||
[
|
|
||||||
"-DWITH_CUDA:BOOL=%s" % ("+cuda" in spec),
|
|
||||||
"-DWITH_CUDNN:BOOL=%s" % ("+cuda" in spec),
|
|
||||||
"-DELEMENTAL_USE_CUBLAS:BOOL=%s" % ("+cublas" in spec["elemental"]),
|
|
||||||
"-DWITH_TBINF=OFF",
|
|
||||||
"-DWITH_VTUNE=OFF",
|
|
||||||
"-DElemental_DIR={0}".format(spec["elemental"].prefix),
|
|
||||||
"-DELEMENTAL_MATH_LIBS={0}".format(spec["elemental"].libs),
|
|
||||||
"-DVERBOSE=0",
|
|
||||||
"-DLBANN_HOME=.",
|
|
||||||
]
|
|
||||||
)
|
|
||||||
|
|
||||||
if spec.variants["dtype"].value == "float":
|
|
||||||
args.append("-DDATATYPE=4")
|
|
||||||
elif spec.variants["dtype"].value == "double":
|
|
||||||
args.append("-DDATATYPE=8")
|
|
||||||
|
|
||||||
if "+vision" in spec:
|
|
||||||
args.append("-DOpenCV_DIR:STRING={0}".format(spec["opencv"].prefix))
|
|
||||||
|
|
||||||
if "+cudnn" in spec:
|
|
||||||
args.append("-DcuDNN_DIR={0}".format(spec["cudnn"].prefix))
|
|
||||||
|
|
||||||
if "+cub" in spec and spec.satisfies("^cuda@:10"):
|
|
||||||
args.append("-DCUB_DIR={0}".format(spec["cub"].prefix))
|
|
||||||
|
|
||||||
return args
|
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue