From dce590fb21af844230727b283f6c8bf759ea805c Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Sat, 26 Mar 2016 19:59:02 -0700 Subject: [PATCH 1/4] Add a dso_suffix variable to build_environment - Consolidate this in one place so that we don't have to do it in every build. - Will update further once better OS support is committed. Shoudl really be an attribute of the forthcoming `Platform` class. --- lib/spack/spack/build_environment.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/lib/spack/spack/build_environment.py b/lib/spack/spack/build_environment.py index 119a255a34..640db0c1d1 100644 --- a/lib/spack/spack/build_environment.py +++ b/lib/spack/spack/build_environment.py @@ -59,6 +59,11 @@ SPACK_DEBUG_LOG_DIR = 'SPACK_DEBUG_LOG_DIR' +# Platform-specific library suffix. +dso_suffix = 'dylib' if sys.platform == 'darwin' else 'so' + + + class MakeExecutable(Executable): """Special callable executable object for make so the user can specify parallel or not on a per-invocation basis. Using @@ -246,6 +251,9 @@ def set_module_variables_for_package(pkg, module): # a Prefix object. m.prefix = pkg.prefix + # Platform-specific library suffix. + m.dso_suffix = dso_suffix + def get_rpaths(pkg): """Get a list of all the rpaths for a package.""" From cde33205827edbed988ec4548c983ff93e9445c6 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Sat, 26 Mar 2016 20:00:28 -0700 Subject: [PATCH 2/4] Add a method to find the containing directory of a library. --- lib/spack/llnl/util/filesystem.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/lib/spack/llnl/util/filesystem.py b/lib/spack/llnl/util/filesystem.py index c4665c284c..92f9ca2983 100644 --- a/lib/spack/llnl/util/filesystem.py +++ b/lib/spack/llnl/util/filesystem.py @@ -27,7 +27,7 @@ 'force_remove', 'join_path', 'ancestor', 'can_access', 'filter_file', 'FileFilter', 'change_sed_delimiter', 'is_exe', 'force_symlink', 'set_executable', 'copy_mode', 'unset_executable_mode', - 'remove_dead_links', 'remove_linked_tree'] + 'remove_dead_links', 'remove_linked_tree', 'find_library_path'] import os import sys @@ -392,3 +392,18 @@ def remove_linked_tree(path): os.unlink(path) else: shutil.rmtree(path, True) + + +def find_library_path(libname, *paths): + """Searches for a file called in each path. + + Return: + directory where the library was found, if found. None otherwise. + + """ + for path in paths: + library = join_path(path, libname) + if os.path.exists(library): + return path + return None + From 66bb71534b3ab7eaf7b27743efc0ecf9763e4982 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Sat, 26 Mar 2016 20:02:08 -0700 Subject: [PATCH 3/4] Better blas/lapack and scipy packages: more robust, added `shared` variants. - py-scipy now builds with netlib-lapack, openblas, and atlas. - started a convention for passing lib info from blas/lapack implementations. - Improved netlib-lapack: - Also build static libs when `shared` variant is enabled. - Enable CBLAS build - needed minor patch to build correctly. - Added `shared` variant to OpenBLAS. - Made OpenBLAS build properly shared and static. --- .../builtin/packages/netlib-lapack/package.py | 40 ++++++++++++-- .../builtin/packages/openblas/package.py | 52 +++++++++++++++---- .../builtin/packages/py-scipy/package.py | 12 +++-- 3 files changed, 88 insertions(+), 16 deletions(-) diff --git a/var/spack/repos/builtin/packages/netlib-lapack/package.py b/var/spack/repos/builtin/packages/netlib-lapack/package.py index c4b7ce3b04..05436332ac 100644 --- a/var/spack/repos/builtin/packages/netlib-lapack/package.py +++ b/var/spack/repos/builtin/packages/netlib-lapack/package.py @@ -31,8 +31,16 @@ class NetlibLapack(Package): depends_on('cmake') depends_on('blas', when='+external-blas') - def install(self, spec, prefix): - cmake_args = ['-DBUILD_SHARED_LIBS:BOOL=%s' % ('ON' if '+shared' in spec else 'OFF'), + + def patch(self): + # Fix cblas CMakeLists.txt -- has wrong case for subdirectory name. + filter_file('${CMAKE_CURRENT_SOURCE_DIR}/CMAKE/', + '${CMAKE_CURRENT_SOURCE_DIR}/cmake/', 'CBLAS/CMakeLists.txt', string=True) + + + def install_one(self, spec, prefix, shared): + cmake_args = ['-DBUILD_SHARED_LIBS:BOOL=%s' % ('ON' if shared else 'OFF'), + '-DCBLAS=ON', # always build CBLAS '-DCMAKE_BUILD_TYPE:STRING=%s' % ('Debug' if '+debug' in spec else 'Release'), '-DLAPACKE:BOOL=%s' % ('ON' if '+lapacke' in spec else 'OFF')] if '+external-blas' in spec: @@ -45,7 +53,33 @@ def install(self, spec, prefix): cmake_args.extend(std_cmake_args) - with working_dir('spack-build', create=True): + build_dir = 'spack-build' + ('-shared' if shared else '-static') + with working_dir(build_dir, create=True): cmake('..', *cmake_args) make() make("install") + + + def install(self, spec, prefix): + # Always build static libraries. + self.install_one(spec, prefix, False) + + # Build shared libraries if requested. + if '+shared' in spec: + self.install_one(spec, prefix, True) + + + def setup_dependent_package(self, module, dspec): + # This is WIP for a prototype interface for virtual packages. + # We can update this as more builds start depending on BLAS/LAPACK. + libdir = find_library_path('libblas.a', self.prefix.lib64, self.prefix.lib) + + self.spec.blas_static_lib = join_path(libdir, 'libblas.a') + self.spec.lapack_static_lib = join_path(libdir, 'liblapack.a') + + if '+shared' in self.spec: + self.spec.blas_shared_lib = join_path(libdir, 'libblas.%s' % dso_suffix) + self.spec.lapack_shared_lib = join_path(libdir, 'liblapack.%s' % dso_suffix) + + + diff --git a/var/spack/repos/builtin/packages/openblas/package.py b/var/spack/repos/builtin/packages/openblas/package.py index 1d10f217c4..4ec829a85b 100644 --- a/var/spack/repos/builtin/packages/openblas/package.py +++ b/var/spack/repos/builtin/packages/openblas/package.py @@ -1,5 +1,6 @@ from spack import * import sys +import os class Openblas(Package): """OpenBLAS: An optimized BLAS library""" @@ -10,29 +11,60 @@ class Openblas(Package): version('0.2.16', 'fef46ab92463bdbb1479dcec594ef6dc') version('0.2.15', 'b1190f3d3471685f17cfd1ec1d252ac9') + variant('shared', default=True, description="Build shared libraries as well as static libs.") + # virtual dependency provides('blas') provides('lapack') + def install(self, spec, prefix): - extra=[] + make_defs = ['CC=%s' % spack_cc, + 'FC=%s' % spack_fc] + + make_targets = ['libs', 'netlib'] + + # Build shared if variant is set. + if '+shared' in spec: + make_targets += ['shared'] + else: + make_defs += ['NO_SHARED=1'] + + # fix missing _dggsvd_ and _sggsvd_ if spec.satisfies('@0.2.16'): - extra.extend([ - 'BUILD_LAPACK_DEPRECATED=1' # fix missing _dggsvd_ and _sggsvd_ - ]) + make_defs += ['BUILD_LAPACK_DEPRECATED=1'] - make('libs', 'netlib', 'shared', 'CC=cc', 'FC=f77',*extra) - make("tests") - make('install', "PREFIX='%s'" % prefix) + make_args = make_defs + make_targets + make(*make_args) + + make("tests", *make_defs) + + # no quotes around prefix (spack doesn't use a shell) + make('install', "PREFIX=%s" % prefix, *make_defs) - lib_dsuffix = 'dylib' if sys.platform == 'darwin' else 'so' # Blas virtual package should provide blas.a and libblas.a with working_dir(prefix.lib): symlink('libopenblas.a', 'blas.a') symlink('libopenblas.a', 'libblas.a') - symlink('libopenblas.%s' % lib_dsuffix, 'libblas.%s' % lib_dsuffix) + if '+shared' in spec: + symlink('libopenblas.%s' % dso_suffix, 'libblas.%s' % dso_suffix) # Lapack virtual package should provide liblapack.a with working_dir(prefix.lib): symlink('libopenblas.a', 'liblapack.a') - symlink('libopenblas.%s' % lib_dsuffix, 'liblapack.%s' % lib_dsuffix) + if '+shared' in spec: + symlink('libopenblas.%s' % dso_suffix, 'liblapack.%s' % dso_suffix) + + + def setup_dependent_package(self, module, dspec): + # This is WIP for a prototype interface for virtual packages. + # We can update this as more builds start depending on BLAS/LAPACK. + libdir = find_library_path('libopenblas.a', self.prefix.lib64, self.prefix.lib) + + self.spec.blas_static_lib = join_path(libdir, 'libopenblas.a') + self.spec.lapack_static_lib = self.spec.blas_static_lib + + if '+shared' in self.spec: + self.spec.blas_shared_lib = join_path(libdir, 'libopenblas.%s' % dso_suffix) + self.spec.lapack_shared_lib = self.spec.blas_shared_lib + diff --git a/var/spack/repos/builtin/packages/py-scipy/package.py b/var/spack/repos/builtin/packages/py-scipy/package.py index c2161c90c4..4d47c641ee 100644 --- a/var/spack/repos/builtin/packages/py-scipy/package.py +++ b/var/spack/repos/builtin/packages/py-scipy/package.py @@ -11,9 +11,15 @@ class PyScipy(Package): extends('python') depends_on('py-nose') - depends_on('py-numpy') - depends_on('blas') - depends_on('lapack') + depends_on('py-numpy+blas+lapack') def install(self, spec, prefix): + if 'atlas' in spec: + # libatlas.so actually isn't always installed, but this + # seems to make the build autodetect things correctly. + env['ATLAS'] = join_path(spec['atlas'].prefix.lib, 'libatlas.' + dso_suffix) + else: + env['BLAS'] = spec['blas'].blas_shared_lib + env['LAPACK'] = spec['lapack'].lapack_shared_lib + python('setup.py', 'install', '--prefix=%s' % prefix) From e049fc2840ba81cdff2ca7edd798c5e961ae94e9 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Sun, 27 Mar 2016 11:47:20 -0700 Subject: [PATCH 4/4] Run post-install hoooks before build stage is removed. - Build will properly fail when post-install hoooks fail. - Post-install hooks have a proper working directory set now. --- lib/spack/spack/package.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index 9af3221837..c17bec4a14 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -926,6 +926,9 @@ def build_process(): install(env_path, env_install_path) dump_packages(self.spec, packages_dir) + # Run post install hooks before build stage is removed. + spack.hooks.post_install(self) + # Stop timer. self._total_time = time.time() - start_time build_time = self._total_time - self._fetch_time @@ -954,9 +957,6 @@ def build_process(): # the database, so that we don't need to re-read from file. spack.installed_db.add(self.spec, self.prefix) - # Once everything else is done, run post install hooks - spack.hooks.post_install(self) - def sanity_check_prefix(self): """This function checks whether install succeeded."""