Intel software packages and license enhancements

lib/spack/spack/hooks/licensing.py

@@ -26,7 +26,7 @@
 
 import spack
 import llnl.util.tty as tty
-from llnl.util.filesystem import join_path
+from llnl.util.filesystem import join_path, mkdirp
 
 
 def pre_install(pkg):
@@ -154,6 +154,9 @@ def symlink_license(pkg):
     target = pkg.global_license_file
     for filename in pkg.license_files:
         link_name = join_path(pkg.prefix, filename)
+        license_dir = os.path.split(link_name)[0]
+        if not os.path.exists(license_dir):
+            mkdirp(license_dir)
         if os.path.exists(target):
             os.symlink(target, link_name)
             tty.msg("Added local symlink %s to global license file" %

lib/spack/spack/package.py

@@ -397,14 +397,18 @@ def __init__(self, spec):
         if self.is_extension:
             spack.repo.get(self.extendee_spec)._check_extendable()
 
+    @property
+    def global_license_dir(self):
+        """Returns the path where a global license file should be stored."""
+        spack_root = ancestor(__file__, 4)
+        return join_path(spack_root, 'etc', 'spack', 'licenses')
+
     @property
     def global_license_file(self):
         """Returns the path where a global license file should be stored."""
         if not self.license_files:
             return
-        spack_root = ancestor(__file__, 4)
+        return join_path(self.global_license_dir, self.name,
-        global_license_dir = join_path(spack_root, 'etc', 'spack', 'licenses')
-        return join_path(global_license_dir, self.name,
                          os.path.basename(self.license_files[0]))
 
     @property

var/spack/repos/builtin/packages/daal/package.py

@@ -0,0 +1,27 @@
+from spack import *
+import sys, os, re
+
+from spack.pkg.builtin.intel import IntelInstaller
+
+class Daal(IntelInstaller):
+    """Intel Data Analytics Accesleration Library.
+
+    Note: You will have to add the download file to a
+    mirror so that Spack can find it. For instructions on how to set up a
+    mirror, see http://software.llnl.gov/spack/mirrors.html"""
+
+    homepage = "https://software.intel.com/en-us/daal"
+
+    version('2016.2.181', 'aad2aa70e5599ebfe6f85b29d8719d46',
+            url="file://%s/l_daal_2016.2.181.tgz" % os.getcwd())
+    version('2016.3.210', 'ad747c0dd97dace4cad03cf2266cad28',
+            url="file://%s/l_daal_2016.3.210.tgz" % os.getcwd())
+
+    def install(self, spec, prefix):
+
+        self.intel_prefix = os.path.join(prefix, "pkg")
+        IntelInstaller.install(self, spec, prefix)
+
+        daal_dir = os.path.join(self.intel_prefix, "daal")
+        for f in os.listdir(daal_dir):
+            os.symlink(os.path.join(daal_dir, f), os.path.join(self.prefix, f))

var/spack/repos/builtin/packages/intel-parallelstudio/package.py

@@ -0,0 +1,112 @@
+from spack import *
+import sys, os, re
+
+from spack.pkg.builtin.intel import IntelInstaller, filter_pick
+
+class IntelParallelstudio(IntelInstaller):
+    """Intel Parallel Studio.
+
+    Note: You will have to add the download file to a
+    mirror so that Spack can find it. For instructions on how to set up a
+    mirror, see http://software.llnl.gov/spack/mirrors.html"""
+
+    homepage = "https://software.intel.com/en-us/intel-parallel-studio-xe"
+
+    # TODO: can also try the online installer (will download files on demand)
+    version('composer.2016.2', '1133fb831312eb519f7da897fec223fa',
+            url="file://%s/parallel_studio_xe_2016_composer_edition_update2.tgz" % os.getcwd())
+    version('professional.2016.2', '70be832f2d34c9bf596a5e99d5f2d832',
+            url="file://%s/parallel_studio_xe_2016_update2.tgz" % os.getcwd())
+    version('cluster.2016.2', '70be832f2d34c9bf596a5e99d5f2d832',
+            url="file://%s/parallel_studio_xe_2016_update2.tgz" % os.getcwd())
+    version('composer.2016.3', '3208eeabee951fc27579177b593cefe9',
+            url="file://%s/parallel_studio_xe_2016_composer_edition_update3.tgz" % os.getcwd())
+    version('professional.2016.3', 'eda19bb0d0d19709197ede58f13443f3',
+            url="file://%s/parallel_studio_xe_2016_update3.tgz" % os.getcwd())
+    version('cluster.2016.3', 'eda19bb0d0d19709197ede58f13443f3',
+            url="file://%s/parallel_studio_xe_2016_update3.tgz" % os.getcwd())
+
+    variant('rpath', default=True, description="Add rpath to .cfg files")
+    variant('all', default=False, description="Install all files associated with the requested edition")
+    variant('mpi', default=True, description="Install the Intel MPI library and ITAC tool")
+    variant('mkl', default=True, description="Install the Intel MKL library")
+    variant('daal', default=True, description="Install the Intel DAAL libraries")
+    variant('ipp', default=True, description="Install the Intel IPP libraries")
+    variant('tools', default=True, description="Install the Intel Advisor, VTune Amplifier, and Inspector tools")
+
+    provides('mpi', when='@cluster:+mpi')
+    provides('mkl', when='+mkl')
+    provides('daal', when='+daal')
+    provides('ipp', when='+ipp')
+
+    def install(self, spec, prefix):
+
+        # remove the installation DB, otherwise it will try to install into location of other Intel builds
+        try:
+            os.remove(os.path.join(os.environ["HOME"], "intel", "intel_sdp_products.db"))
+        except OSError:
+            pass # if the file does not exist
+
+        base_components = "ALL" # when in doubt, install everything
+        mpi_components = ""
+        mkl_components = ""
+        daal_components = ""
+        ipp_components = ""
+        tools_components = ""
+
+        if spec.satisfies('+all'):
+            base_components = "ALL"
+        else:
+            with open("pset/mediaconfig.xml", "r") as f:
+                lines = f.readlines()
+                all_components = []
+                for line in lines:
+                    if line.find('<Abbr>') != -1:
+                        component = line[line.find('<Abbr>') + 6:line.find('</Abbr>')]
+                        all_components.append(component)
+                base_components = filter_pick(all_components, re.compile('(comp|openmp|intel-tbb|icc|ifort|psxe|icsxe-pset)').search)
+                mpi_components = filter_pick(all_components, re.compile('(icsxe|imb|mpi|itac|intel-tc|clck)').search)
+                mkl_components = filter_pick(all_components, re.compile('(mkl)').search)
+                daal_components = filter_pick(all_components, re.compile('(daal)').search)
+                ipp_components = filter_pick(all_components, re.compile('(ipp)').search)
+                tool_components = filter_pick(all_components, re.compile('(gdb|vtune|inspector|advisor)').search)
+
+        components = base_components
+        if not spec.satisfies('+all'):
+            if spec.satisfies('+mpi') and 'cluster' in str(spec.version):
+                components += mpi_components
+            if spec.satisfies('+mkl'):
+                components += mkl_components
+            if spec.satisfies('+daal'):
+                components += daal_components
+            if spec.satisfies('+ipp'):
+                components += ipp_components
+            if spec.satisfies('+tools') and (spec.satisfies('@cluster') or spec.satisfies('@professional')):
+                components += tool_components
+
+        self.intel_components = ';'.join(components)
+        IntelInstaller.install(self, spec, prefix)
+
+        absbindir = os.path.split(os.path.realpath(os.path.join(self.prefix.bin, "icc")))[0]
+        abslibdir = os.path.split(os.path.realpath(os.path.join(self.prefix.lib, "intel64", "libimf.a")))[0]
+
+        # symlink or copy?
+        os.symlink(self.global_license_file, os.path.join(absbindir, "license.lic"))
+        if spec.satisfies('+tools') and (spec.satisfies('@cluster') or spec.satisfies('@professional')):
+            os.mkdir(os.path.join(self.prefix, "inspector_xe/licenses"))
+            os.symlink(self.global_license_file, os.path.join(self.prefix, "inspector_xe/licenses", "license.lic"))
+            os.mkdir(os.path.join(self.prefix, "advisor_xe/licenses"))
+            os.symlink(self.global_license_file, os.path.join(self.prefix, "advisor_xe/licenses", "license.lic"))
+            os.mkdir(os.path.join(self.prefix, "vtune_amplifier_xe/licenses"))
+            os.symlink(self.global_license_file, os.path.join(self.prefix, "vtune_amplifier_xe/licenses", "license.lic"))
+
+        if (spec.satisfies('+all') or spec.satisfies('+mpi')) and spec.satisfies('@cluster'):
+            os.symlink(self.global_license_file, os.path.join(self.prefix, "itac_latest", "license.lic"))
+
+        if spec.satisfies('+rpath'):
+            for compiler_command in ["icc", "icpc", "ifort"]:
+                cfgfilename = os.path.join(absbindir, "%s.cfg" %(compiler_command))
+                with open(cfgfilename, "w") as f:
+                    f.write('-Xlinker -rpath -Xlinker %s\n' %(abslibdir))
+
+        os.symlink(os.path.join(self.prefix.man, "common", "man1"), os.path.join(self.prefix.man, "man1"))

var/spack/repos/builtin/packages/intel/package.py

@@ -0,0 +1,117 @@
+from spack import *
+import sys, os, re
+
+def filter_pick(input_list, regex_filter):
+    return [l for l in input_list for m in (regex_filter(l),) if m]
+
+def unfilter_pick(input_list, regex_filter):
+    return [l for l in input_list for m in (regex_filter(l),) if not m]
+
+def get_all_components():
+    all_components = []
+    with open("pset/mediaconfig.xml", "r") as f:
+        lines = f.readlines()
+        for line in lines:
+            if line.find('<Abbr>') != -1:
+                component = line[line.find('<Abbr>') + 6:line.find('</Abbr>')]
+                all_components.append(component)
+    return all_components
+
+class IntelInstaller(Package):
+    """Base package containing common methods for installing Intel software"""
+
+    homepage = "https://software.intel.com/en-us"
+    intel_components = "ALL"
+    license_required = True
+    license_comment = '#'
+    license_files = ['Licenses/license.lic']
+    license_vars = ['INTEL_LICENSE_FILE']
+    license_url = 'https://software.intel.com/en-us/articles/intel-license-manager-faq'
+
+    @property
+    def global_license_file(self):
+        """Returns the path where a global license file should be stored."""
+        if not self.license_files:
+            return
+        return join_path(self.global_license_dir, "intel",
+                         os.path.basename(self.license_files[0]))
+
+    def install(self, spec, prefix):
+
+        # remove the installation DB, otherwise it will try to install into location of other Intel builds
+        try:
+            os.remove(os.path.join(os.environ["HOME"], "intel", "intel_sdp_products.db"))
+        except OSError:
+            pass # if the file does not exist
+
+        if not hasattr(self, "intel_prefix"):
+            self.intel_prefix = self.prefix
+
+        silent_config_filename = 'silent.cfg'
+        with open(silent_config_filename, 'w') as f:
+            f.write("""
+ACCEPT_EULA=accept
+PSET_MODE=install
+CONTINUE_WITH_INSTALLDIR_OVERWRITE=yes
+PSET_INSTALL_DIR=%s
+ACTIVATION_LICENSE_FILE=%s
+ACTIVATION_TYPE=license_file
+PHONEHOME_SEND_USAGE_DATA=no
+COMPONENTS=%s
+""" %(self.intel_prefix, self.global_license_file, self.intel_components))
+
+        install_script = which("install.sh")
+        install_script('--silent', silent_config_filename)
+
+
+class Intel(IntelInstaller):
+    """Intel Compilers.
+
+    Note: You will have to add the download file to a
+    mirror so that Spack can find it. For instructions on how to set up a
+    mirror, see http://software.llnl.gov/spack/mirrors.html"""
+
+    homepage = "https://software.intel.com/en-us/intel-parallel-studio-xe"
+
+    # TODO: can also try the online installer (will download files on demand)
+    version('16.0.2', '1133fb831312eb519f7da897fec223fa',
+            url="file://%s/parallel_studio_xe_2016_composer_edition_update2.tgz" % os.getcwd())
+    version('16.0.3', '3208eeabee951fc27579177b593cefe9',
+            url="file://%s/parallel_studio_xe_2016_composer_edition_update3.tgz" % os.getcwd())
+
+    variant('rpath', default=True, description="Add rpath to .cfg files")
+
+    def install(self, spec, prefix):
+
+        # remove the installation DB, otherwise it will try to install into location of other Intel builds
+        try:
+            os.remove(os.path.join(os.environ["HOME"], "intel", "intel_sdp_products.db"))
+        except OSError:
+            pass # if the file does not exist
+
+        components = []
+        with open("pset/mediaconfig.xml", "r") as f:
+            lines = f.readlines()
+            all_components = []
+            for line in lines:
+                if line.find('<Abbr>') != -1:
+                    component = line[line.find('<Abbr>') + 6:line.find('</Abbr>')]
+                    all_components.append(component)
+            components = filter_pick(all_components, re.compile('(comp|openmp|intel-tbb|icc|ifort|psxe|icsxe-pset)').search)
+
+        self.intel_components = ';'.join(components)
+        IntelInstaller.install(self, spec, prefix)
+
+        absbindir = os.path.split(os.path.realpath(os.path.join(self.prefix.bin, "icc")))[0]
+        abslibdir = os.path.split(os.path.realpath(os.path.join(self.prefix.lib, "intel64", "libimf.a")))[0]
+
+        # symlink or copy?
+        os.symlink(self.global_license_file, os.path.join(absbindir, "license.lic"))
+
+        if spec.satisfies('+rpath'):
+            for compiler_command in ["icc", "icpc", "ifort"]:
+                cfgfilename = os.path.join(absbindir, "%s.cfg" %(compiler_command))
+                with open(cfgfilename, "w") as f:
+                    f.write('-Xlinker -rpath -Xlinker %s\n' %(abslibdir))
+
+        os.symlink(os.path.join(self.prefix.man, "common", "man1"), os.path.join(self.prefix.man, "man1"))

var/spack/repos/builtin/packages/ipp/package.py

@@ -0,0 +1,25 @@
+from spack import *
+import sys, os, re
+
+from spack.pkg.builtin.intel import IntelInstaller
+
+class Ipp(IntelInstaller):
+    """Intel Integrated Performance Primitives.
+
+    Note: You will have to add the download file to a
+    mirror so that Spack can find it. For instructions on how to set up a
+    mirror, see http://software.llnl.gov/spack/mirrors.html"""
+
+    homepage = "https://software.intel.com/en-us/intel-ipp"
+
+    version('9.0.3.210', '0e1520dd3de7f811a6ef6ebc7aa429a3',
+            url="file://%s/l_ipp_9.0.3.210.tgz" % os.getcwd())
+
+    def install(self, spec, prefix):
+
+        self.intel_prefix = os.path.join(prefix, "pkg")
+        IntelInstaller.install(self, spec, prefix)
+
+        ipp_dir = os.path.join(self.intel_prefix, "ipp")
+        for f in os.listdir(ipp_dir):
+            os.symlink(os.path.join(ipp_dir, f), os.path.join(self.prefix, f))

var/spack/repos/builtin/packages/mkl/package.py

@@ -0,0 +1,27 @@
+from spack import *
+import os, re, sys
+
+from spack.pkg.builtin.intel import IntelInstaller
+
+class Mkl(IntelInstaller):
+    """Intel Math Kernel Library.
+
+    Note: You will have to add the download file to a
+    mirror so that Spack can find it. For instructions on how to set up a
+    mirror, see http://software.llnl.gov/spack/mirrors.html"""
+
+    homepage = "https://software.intel.com/en-us/intel-mkl"
+
+    version('11.3.2.181', '536dbd82896d6facc16de8f961d17d65',
+            url="file://%s/l_mkl_11.3.2.181.tgz" % os.getcwd())
+    version('11.3.3.210', 'f72546df27f5ebb0941b5d21fd804e34',
+            url="file://%s/l_mkl_11.3.3.210.tgz" % os.getcwd())
+
+    def install(self, spec, prefix):
+
+        self.intel_prefix = os.path.join(prefix, "pkg")
+        IntelInstaller.install(self, spec, prefix)
+
+        mkl_dir = os.path.join(self.intel_prefix, "mkl")
+        for f in os.listdir(mkl_dir):
+            os.symlink(os.path.join(mkl_dir, f), os.path.join(self.prefix, f))