Add Intel oneAPI packages (#20411)

This creates a set of packages which all use the same script to install
components of Intel oneAPI. This includes:

* An inheritable IntelOneApiPackage which knows how to invoke the
  installation script based on which components are requested
* For components which include headers/libraries, an inheritable
  IntelOneApiLibraryPackage is provided to locate them
* Individual packages for DAL, DNN, TBB, etc.
* A package for the Intel oneAPI compilers (icx/ifx). This also includes
  icc/ifortran but these are not currently detected in this PR

lib/spack/spack/build_systems/oneapi.py

@@ -0,0 +1,80 @@
+# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+"""Common utilities for managing intel oneapi packages.
+
+"""
+
+from os.path import dirname, isdir
+
+from spack.package import Package
+from spack.util.executable import Executable
+
+from llnl.util.filesystem import find_headers, find_libraries
+
+
+class IntelOneApiPackage(Package):
+    """Base class for Intel oneAPI packages."""
+
+    homepage = 'https://software.intel.com/oneapi'
+
+    phases = ['install']
+
+    def component_info(self,
+                       dir_name,
+                       components,
+                       releases,
+                       url_name):
+        self._dir_name = dir_name
+        self._components = components
+        self._releases = releases
+        self._url_name = url_name
+
+    def url_for_version(self, version):
+        release = self._release(version)
+        return 'https://registrationcenter-download.intel.com/akdlm/irc_nas/%s/%s' % (
+            release['irc_id'], self._oneapi_file(version, release))
+
+    def install(self, spec, prefix):
+        bash = Executable('bash')
+
+        # Installer writes files in ~/intel set HOME so it goes to prefix
+        bash.add_default_env('HOME', prefix)
+
+        version = spec.versions.lowest()
+        release = self._release(version)
+        bash('./%s' % self._oneapi_file(version, release),
+             '-s', '-a', '-s', '--action', 'install',
+             '--eula', 'accept',
+             '--components',
+             self._components,
+             '--install-dir', prefix)
+
+    #
+    # Helper functions
+    #
+
+    def _release(self, version):
+        return self._releases[str(version)]
+
+    def _oneapi_file(self, version, release):
+        return 'l_%s_p_%s.%s_offline.sh' % (
+            self._url_name, version, release['build'])
+
+
+class IntelOneApiLibraryPackage(IntelOneApiPackage):
+    """Base class for Intel oneAPI library packages."""
+
+    @property
+    def headers(self):
+        include_path = '%s/%s/latest/include' % (
+            self.prefix, self._dir_name)
+        return find_headers('*', include_path, recursive=True)
+
+    @property
+    def libs(self):
+        lib_path = '%s/%s/latest/lib/intel64' % (self.prefix, self._dir_name)
+        lib_path = lib_path if isdir(lib_path) else dirname(lib_path)
+        return find_libraries('*', root=lib_path, shared=True, recursive=True)

lib/spack/spack/compilers/oneapi.py

@@ -29,13 +29,14 @@ class Oneapi(Compiler):
     PrgEnv_compiler = 'oneapi'
 
     version_argument = '--version'
-    version_regex = r'\((?:IFORT|ICC)\)|DPC\+\+ [^ ]+ [^ ]+ [^ ]+ \(([^ ]+)\)'
+    version_regex = r'(?:(?:oneAPI DPC\+\+ Compiler)|(?:ifx \(IFORT\))) (\S+)'
 
     @property
     def verbose_flag(self):
         return "-v"
 
-    required_libs = ['libirc', 'libifcore', 'libifcoremt', 'libirng']
+    required_libs = ['libirc', 'libifcore', 'libifcoremt', 'libirng',
+                     'libsvml', 'libintlc', 'libimf']
 
     @property
     def debug_flags(self):

lib/spack/spack/pkgkit.py

@@ -20,6 +20,8 @@
 from spack.build_systems.autotools import AutotoolsPackage
 from spack.build_systems.cmake import CMakePackage
 from spack.build_systems.cuda import CudaPackage
+from spack.build_systems.oneapi import IntelOneApiPackage
+from spack.build_systems.oneapi import IntelOneApiLibraryPackage
 from spack.build_systems.rocm import ROCmPackage
 from spack.build_systems.qmake import QMakePackage
 from spack.build_systems.maven import MavenPackage

lib/spack/spack/test/compilers/detection.py

@@ -152,28 +152,18 @@ def test_intel_version_detection(version_str, expected_version):
 
 
 @pytest.mark.parametrize('version_str,expected_version', [
-    (  # ICX
-        'Intel(R) oneAPI DPC++ Compiler Pro 2021.1 (2020.8.0.0827)\n'
+    (  # ICX/ICPX
+        'Intel(R) oneAPI DPC++ Compiler 2021.1 (2020.10.0.1113)\n'
         'Target: x86_64-unknown-linux-gnu\n'
         'Thread model: posix\n'
-        'InstalledDir: /soft/restricted/CNDA/sdk/\n'
-        '2020.9.15.1/oneapi/compiler/2021.1-beta09/linux/bin',
-        '2020.8.0.0827'
+        'InstalledDir: /made/up/path',
+        '2021.1'
     ),
-    (  # ICPX
-        'Intel(R) oneAPI DPC++ Compiler Pro 2021.1 (2020.8.0.0827)\n'
-        'Target: x86_64-unknown-linux-gnu\n'
-        'Thread model: posix\n'
-        'InstalledDir: /soft/restricted/CNDA/sdk/\n'
-        '2020.9.15.1/oneapi/compiler/2021.1-beta09/linux/bin',
-        '2020.8.0.0827'
+    (  # IFX
+        'ifx (IFORT) 2021.1 Beta 20201113\n'
+        'Copyright (C) 1985-2020 Intel Corporation. All rights reserved.',
+        '2021.1'
     )
-    # Detection will fail for ifx because it can't parse it from this.
-    # (  # IFX
-    #     'ifx (IFORT) 2021.1 Beta 20200827\n'
-    #     'Copyright (C) 1985-2020 Intel Corporation. All rights reserved.',
-    #     '2020.8.0.0827'
-    # )
 ])
 def test_oneapi_version_detection(version_str, expected_version):
     version = spack.compilers.oneapi.Oneapi.extract_version_from_output(

var/spack/repos/builtin/packages/intel-oneapi-ccl/package.py

@@ -0,0 +1,27 @@
+# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+
+from spack import *
+
+releases = {
+    '2021.1.1': {'irc_id': '17391', 'build': '54'}}
+
+
+class IntelOneapiCcl(IntelOneApiLibraryPackage):
+    """Intel oneAPI CCL."""
+
+    maintainers = ['rscohn2']
+
+    homepage = 'https://software.intel.com/content/www/us/en/develop/tools/oneapi/components/oneccl.html'
+
+    version('2021.1.1', sha256='de732df57a03763a286106c8b885fd60e83d17906936a8897a384b874e773f49', expand=False)
+
+    def __init__(self, spec):
+        self.component_info(dir_name='ccl',
+                            components='intel.oneapi.lin.ccl.devel',
+                            releases=releases,
+                            url_name='oneapi_ccl')
+        super(IntelOneapiCcl, self).__init__(spec)

var/spack/repos/builtin/packages/intel-oneapi-compilers/package.py

@@ -0,0 +1,62 @@
+# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+import glob
+import subprocess
+from os import path
+
+from spack import *
+
+
+releases = {'2021.1':
+            {'irc_id': '17427', 'build': '2684'}}
+
+
+class IntelOneapiCompilers(IntelOneApiPackage):
+    """Intel oneAPI compilers.
+
+    Contains icc, icpc, icx, icpx, dpcpp, ifort, ifx.
+
+    """
+
+    maintainers = ['rscohn2']
+
+    homepage = 'https://software.intel.com/content/www/us/en/develop/tools/oneapi/components/dpc-compiler.html'
+
+    version('2021.1', sha256='666b1002de3eab4b6f3770c42bcf708743ac74efeba4c05b0834095ef27a11b9', expand=False)
+
+    depends_on('patchelf', type='build')
+
+    def __init__(self, spec):
+        self.component_info(
+            dir_name='compiler',
+            components=('intel.oneapi.lin.dpcpp-cpp-compiler-pro'
+                        ':intel.oneapi.lin.ifort-compiler'),
+            releases=releases,
+            url_name='HPCKit')
+        super(IntelOneapiCompilers, self).__init__(spec)
+
+    def install(self, spec, prefix):
+        super(IntelOneapiCompilers, self).install(spec, prefix)
+        # For quick turnaround debugging, copy instead of install
+        # copytree('/opt/intel/oneapi/compiler', path.join(prefix, 'compiler'),
+        #          symlinks=True)
+        rpath_dirs = ['lib',
+                      'lib/x64',
+                      'lib/emu',
+                      'lib/oclfpga/host/linux64/lib',
+                      'lib/oclfpga/linux64/lib',
+                      'compiler/lib/intel64_lin',
+                      'compiler/lib']
+        patch_dirs = ['compiler/lib/intel64_lin',
+                      'compiler/lib/intel64',
+                      'bin']
+        eprefix = path.join(prefix, 'compiler', 'latest', 'linux')
+        rpath = ':'.join([path.join(eprefix, c) for c in rpath_dirs])
+        for pd in patch_dirs:
+            for file in glob.glob(path.join(eprefix, pd, '*')):
+                # Try to patch all files, patchelf will do nothing if
+                # file should not be patched
+                subprocess.call(['patchelf', '--set-rpath', rpath, file])

var/spack/repos/builtin/packages/intel-oneapi-dal/package.py

@@ -0,0 +1,27 @@
+# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+
+from spack import *
+
+releases = {
+    '2021.1.1': {'irc_id': '17443', 'build': '79'}}
+
+
+class IntelOneapiDal(IntelOneApiLibraryPackage):
+    """Intel oneAPI DAL."""
+
+    maintainers = ['rscohn2']
+
+    homepage = 'https://software.intel.com/content/www/us/en/develop/tools/oneapi/components/onedal.html'
+
+    version('2021.1.1', sha256='6e0e24bba462e80f0fba5a46e95cf0cca6cf17948a7753f8e396ddedd637544e', expand=False)
+
+    def __init__(self, spec):
+        self.component_info(dir_name='dal',
+                            components='intel.oneapi.lin.dal.devel',
+                            releases=releases,
+                            url_name='daal_oneapi')
+        super(IntelOneapiDal, self).__init__(spec)

var/spack/repos/builtin/packages/intel-oneapi-dnn/package.py

@@ -0,0 +1,27 @@
+# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+
+from spack import *
+
+releases = {
+    '2021.1.1': {'irc_id': '17385', 'build': '55'}}
+
+
+class IntelOneapiDnn(IntelOneApiLibraryPackage):
+    """Intel oneAPI DNN."""
+
+    maintainers = ['rscohn2']
+
+    homepage = 'https://software.intel.com/content/www/us/en/develop/tools/oneapi/components/onednn.html'
+
+    version('2021.1.1', sha256='24002c57bb8931a74057a471a5859d275516c331fd8420bee4cae90989e77dc3', expand=False)
+
+    def __init__(self, spec):
+        self.component_info(dir_name='dnn',
+                            components='intel.oneapi.lin.dnnl.devel',
+                            releases=releases,
+                            url_name='onednn')
+        super(IntelOneapiDnn, self).__init__(spec)

var/spack/repos/builtin/packages/intel-oneapi-ipp/package.py

@@ -0,0 +1,27 @@
+# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+
+from spack import *
+
+releases = {
+    '2021.1.1': {'irc_id': '17436', 'build': '47'}}
+
+
+class IntelOneapiIpp(IntelOneApiLibraryPackage):
+    """Intel oneAPI IPP."""
+
+    maintainers = ['rscohn2']
+
+    homepage = 'https://software.intel.com/content/www/us/en/develop/tools/oneapi/components/ipp.html'
+
+    version('2021.1.1', sha256='2656a3a7f1f9f1438cbdf98fd472a213c452754ef9476dd65190a7d46618ba86', expand=False)
+
+    def __init__(self, spec):
+        self.component_info(dir_name='ipp',
+                            components='intel.oneapi.lin.ipp.devel',
+                            releases=releases,
+                            url_name='ipp_oneapi')
+        super(IntelOneapiIpp, self).__init__(spec)

var/spack/repos/builtin/packages/intel-oneapi-ippcp/package.py

@@ -0,0 +1,27 @@
+# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+
+from spack import *
+
+releases = {
+    '2021.1.1': {'irc_id': '17415', 'build': '54'}}
+
+
+class IntelOneapiIppcp(IntelOneApiLibraryPackage):
+    """Intel oneAPI IPP Crypto."""
+
+    maintainers = ['rscohn2']
+
+    homepage = 'https://software.intel.com/content/www/us/en/develop/tools/oneapi/components/ipp.html'
+
+    version('2021.1.1', sha256='c0967afae22c7a223ec42542bcc702121064cd3d8f680eff36169c94f964a936', expand=False)
+
+    def __init__(self, spec):
+        self.component_info(dir_name='ippcp',
+                            components='intel.oneapi.lin.ippcp.devel',
+                            releases=releases,
+                            url_name='ippcp_oneapi')
+        super(IntelOneapiIppcp, self).__init__(spec)

var/spack/repos/builtin/packages/intel-oneapi-mkl/package.py

@@ -0,0 +1,27 @@
+# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+
+from spack import *
+
+releases = {
+    '2021.1.1': {'irc_id': '17402', 'build': '52'}}
+
+
+class IntelOneapiMkl(IntelOneApiLibraryPackage):
+    """Intel oneAPI MKL."""
+
+    maintainers = ['rscohn2']
+
+    homepage = 'https://software.intel.com/content/www/us/en/develop/tools/oneapi/components/onemkl.html'
+
+    version('2021.1.1', sha256='818b6bd9a6c116f4578cda3151da0612ec9c3ce8b2c8a64730d625ce5b13cc0c', expand=False)
+
+    def __init__(self, spec):
+        self.component_info(dir_name='mkl',
+                            components='intel.oneapi.lin.mkl.devel',
+                            releases=releases,
+                            url_name='onemkl')
+        super(IntelOneapiMkl, self).__init__(spec)

var/spack/repos/builtin/packages/intel-oneapi-mpi/package.py

@@ -0,0 +1,27 @@
+# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+
+from spack import *
+
+releases = {
+    '2021.1.1': {'irc_id': '17397', 'build': '76'}}
+
+
+class IntelOneapiMpi(IntelOneApiLibraryPackage):
+    """Intel oneAPI MPI."""
+
+    maintainers = ['rscohn2']
+
+    homepage = 'https://software.intel.com/content/www/us/en/develop/tools/oneapi/components/mpi-library.html'
+
+    version('2021.1.1', sha256='8b7693a156c6fc6269637bef586a8fd3ea6610cac2aae4e7f48c1fbb601625fe', expand=False)
+
+    def __init__(self, spec):
+        self.component_info(dir_name='mpi',
+                            components='intel.oneapi.lin.mpi.devel',
+                            releases=releases,
+                            url_name='mpi_oneapi')
+        super(IntelOneapiMpi, self).__init__(spec)

var/spack/repos/builtin/packages/intel-oneapi-tbb/package.py

@@ -0,0 +1,27 @@
+# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+
+from spack import *
+
+releases = {
+    '2021.1.1': {'irc_id': '17378', 'build': '119'}}
+
+
+class IntelOneapiTbb(IntelOneApiLibraryPackage):
+    """Intel oneAPI TBB."""
+
+    maintainers = ['rscohn2']
+
+    homepage = 'https://software.intel.com/content/www/us/en/develop/tools/oneapi/components/onetbb.html'
+
+    version('2021.1.1', sha256='535290e3910a9d906a730b24af212afa231523cf13a668d480bade5f2a01b53b', expand=False)
+
+    def __init__(self, spec):
+        self.component_info(dir_name='tbb',
+                            components='intel.oneapi.lin.tbb.devel',
+                            releases=releases,
+                            url_name='tbb_oneapi')
+        super(IntelOneapiTbb, self).__init__(spec)

var/spack/repos/builtin/packages/intel-oneapi-vpl/package.py

@@ -0,0 +1,27 @@
+# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+
+from spack import *
+
+releases = {
+    '2021.1.1': {'irc_id': '17418', 'build': '66'}}
+
+
+class IntelOneapiVpl(IntelOneApiLibraryPackage):
+    """Intel oneAPI VPL."""
+
+    maintainers = ['rscohn2']
+
+    homepage = 'https://software.intel.com/content/www/us/en/develop/tools/oneapi/components/onevpl.html'
+
+    version('2021.1.1', sha256='0fec42545b30b7bb2e4e33deb12ab27a02900f5703153d9601673a8ce43082ed', expand=False)
+
+    def __init__(self, spec):
+        self.component_info(dir_name='vpl',
+                            components='intel.oneapi.lin.vpl.devel',
+                            releases=releases,
+                            url_name='oneVPL')
+        super(IntelOneapiVpl, self).__init__(spec)