From 7eca383b10f017666c320ffd7deb6cae83b5c26b Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Thu, 10 Mar 2016 01:12:25 -0800 Subject: [PATCH 1/9] Add sanity check paths to packages; fix #505 --- lib/spack/spack/package.py | 26 ++++++++++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index 696adaf896..02fb3e5834 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -318,6 +318,17 @@ class SomePackage(Package): """Most packages are NOT extendable. Set to True if you want extensions.""" extendable = False + """List of prefix-relative file paths. If these do not exist after + install, or if they exist but are not files, sanity checks fail. + """ + sanity_check_files = [] + + """List of prefix-relative directory paths. If these do not exist + after install, or if they exist but are not directories, sanity + checks will fail. + """ + sanity_check_dirs = [] + def __init__(self, spec): # this determines how the package should be built. @@ -909,7 +920,7 @@ def build_process(): raise e # Ensure that something was actually installed. - self._sanity_check_install() + self.sanity_check_prefix() # Copy provenance into the install directory on success log_install_path = spack.install_layout.build_log_path(self.spec) @@ -952,7 +963,18 @@ def build_process(): spack.hooks.post_install(self) - def _sanity_check_install(self): + def sanity_check_prefix(self): + """This function checks whether install succeeded.""" + def check_paths(path_list, filetype, predicate): + for path in path_list: + abs_path = os.path.join(self.prefix, path) + if not predicate(abs_path): + raise InstallError("Install failed for %s. No such %s in prefix: %s" + % (self.name, filetype, path)) + + check_paths(self.sanity_check_files, 'file', os.path.isfile) + check_paths(self.sanity_check_dirs, 'directory', os.path.isdir) + installed = set(os.listdir(self.prefix)) installed.difference_update(spack.install_layout.hidden_file_paths) if not installed: From 68d22253eca40bc4223234d6c26204f292be6a0c Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Tue, 15 Mar 2016 17:22:48 -0700 Subject: [PATCH 2/9] Add sanity checks to the libelf build. --- var/spack/repos/builtin/packages/libelf/package.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/var/spack/repos/builtin/packages/libelf/package.py b/var/spack/repos/builtin/packages/libelf/package.py index 29bc21b65c..0fcb56c164 100644 --- a/var/spack/repos/builtin/packages/libelf/package.py +++ b/var/spack/repos/builtin/packages/libelf/package.py @@ -38,6 +38,9 @@ class Libelf(Package): provides('elf') + sanity_check_files = ['include/libelf.h'] + sanity_check_dirs = ['lib'] + def install(self, spec, prefix): configure("--prefix=" + prefix, "--enable-shared", From 9c2996667433e4f69eb25cce2d74a94caeb1646c Mon Sep 17 00:00:00 2001 From: Nicolas Richart Date: Wed, 16 Mar 2016 12:57:17 +0100 Subject: [PATCH 3/9] Change urls in binutils to help 'spack checksum/versions' + adding latest version --- var/spack/repos/builtin/packages/binutils/package.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/var/spack/repos/builtin/packages/binutils/package.py b/var/spack/repos/builtin/packages/binutils/package.py index de04221e33..897539a439 100644 --- a/var/spack/repos/builtin/packages/binutils/package.py +++ b/var/spack/repos/builtin/packages/binutils/package.py @@ -4,10 +4,13 @@ class Binutils(Package): """GNU binutils, which contain the linker, assembler, objdump and others""" homepage = "http://www.gnu.org/software/binutils/" - version('2.25', 'd9f3303f802a5b6b0bb73a335ab89d66',url="ftp://ftp.gnu.org/gnu/binutils/binutils-2.25.tar.bz2") - version('2.24', 'e0f71a7b2ddab0f8612336ac81d9636b',url="ftp://ftp.gnu.org/gnu/binutils/binutils-2.24.tar.bz2") - version('2.23.2', '4f8fa651e35ef262edc01d60fb45702e',url="ftp://ftp.gnu.org/gnu/binutils/binutils-2.23.2.tar.bz2") - version('2.20.1', '2b9dc8f2b7dbd5ec5992c6e29de0b764',url="ftp://ftp.gnu.org/gnu/binutils/binutils-2.20.1.tar.bz2") + url="https://ftp.gnu.org/gnu/binutils/binutils-2.25.tar.bz2" + + version('2.26', '64146a0faa3b411ba774f47d41de239f') + version('2.25', 'd9f3303f802a5b6b0bb73a335ab89d66') + version('2.24', 'e0f71a7b2ddab0f8612336ac81d9636b') + version('2.23.2', '4f8fa651e35ef262edc01d60fb45702e') + version('2.20.1', '2b9dc8f2b7dbd5ec5992c6e29de0b764') # Add a patch that creates binutils libiberty_pic.a which is preferred by OpenSpeedShop and cbtf-krell variant('krellpatch', default=False, description="build with openspeedshop based patch.") From bb04d5cc63182b6129f9abf78db260a2dab9f506 Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Wed, 16 Mar 2016 14:27:22 -0500 Subject: [PATCH 4/9] Fix grammar in mirrors documentation --- lib/spack/docs/mirrors.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/spack/docs/mirrors.rst b/lib/spack/docs/mirrors.rst index b20fedb55f..dad04d053b 100644 --- a/lib/spack/docs/mirrors.rst +++ b/lib/spack/docs/mirrors.rst @@ -38,7 +38,7 @@ contains tarballs for each package, named after each package. .. note:: - Archives are **not** named exactly they were in the package's fetch + Archives are **not** named exactly the way they were in the package's fetch URL. They have the form ``-.``, where ```` is Spack's name for the package, ```` is the version of the tarball, and ```` is whatever format the From fa2a66db6721ef5d7b54b9cede69aa2f1b5b7531 Mon Sep 17 00:00:00 2001 From: citibeth Date: Wed, 16 Mar 2016 15:46:59 -0400 Subject: [PATCH 5/9] 1. Disabled git versions known to have vulnerabilities. 2. Added autoconf command to allow building directly from GitHub source. --- .../repos/builtin/packages/git/package.py | 24 +++++++++++++------ 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/var/spack/repos/builtin/packages/git/package.py b/var/spack/repos/builtin/packages/git/package.py index ddc5078c4d..586b6ce3c3 100644 --- a/var/spack/repos/builtin/packages/git/package.py +++ b/var/spack/repos/builtin/packages/git/package.py @@ -5,14 +5,22 @@ class Git(Package): system designed to handle everything from small to very large projects with speed and efficiency.""" homepage = "http://git-scm.com" - url = "https://www.kernel.org/pub/software/scm/git/git-2.2.1.tar.gz" + url = "https://github.com/git/git/tarball/v2.7.1" - version('2.6.3', 'b711be7628a4a2c25f38d859ee81b423') - version('2.6.2', 'da293290da69f45a86a311ad3cd43dc8') - version('2.6.1', '4c62ee9c5991fe93d99cf2a6b68397fd') - version('2.6.0', 'eb76a07148d94802a1745d759716a57e') - version('2.5.4', '3eca2390cf1fa698b48e2a233563a76b') - version('2.2.1', 'ff41fdb094eed1ec430aed8ee9b9849c') + version('2.8.0-rc2', 'c2cf9f2cc70e35f2fafbaf9258f82e4c') + version('2.7.3', 'fa1c008b56618c355a32ba4a678305f6') + version('2.7.1', 'bf0706b433a8dedd27a63a72f9a66060') + + + # See here for info on vulnerable Git versions: + # http://www.theregister.co.uk/2016/03/16/git_server_client_patch_now/ + # All the following are vulnerable + #version('2.6.3', 'b711be7628a4a2c25f38d859ee81b423') + #version('2.6.2', 'da293290da69f45a86a311ad3cd43dc8') + #version('2.6.1', '4c62ee9c5991fe93d99cf2a6b68397fd') + #version('2.6.0', 'eb76a07148d94802a1745d759716a57e') + #version('2.5.4', '3eca2390cf1fa698b48e2a233563a76b') + #version('2.2.1', 'ff41fdb094eed1ec430aed8ee9b9849c') # Git compiles with curl support by default on but if your system @@ -24,6 +32,7 @@ class Git(Package): variant("expat", default=False, description="Add the internal support of expat for https push") depends_on("openssl") + depends_on("autoconf") depends_on("curl", when="+curl") depends_on("expat", when="+expat") @@ -47,6 +56,7 @@ def install(self, spec, prefix): if '+expat' in spec: configure_args.append("--with-expat=%s" % spec['expat'].prefix) + which('autoreconf')('-i') configure(*configure_args) make() make("install") From 9e4c757f50ec98e55a74ad13fda70f12ce788891 Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Thu, 17 Mar 2016 13:45:57 -0400 Subject: [PATCH 6/9] Update curl to 7.47.1 --- var/spack/repos/builtin/packages/curl/package.py | 1 + 1 file changed, 1 insertion(+) diff --git a/var/spack/repos/builtin/packages/curl/package.py b/var/spack/repos/builtin/packages/curl/package.py index 9e684445c7..ab6305fc08 100644 --- a/var/spack/repos/builtin/packages/curl/package.py +++ b/var/spack/repos/builtin/packages/curl/package.py @@ -7,6 +7,7 @@ class Curl(Package): homepage = "http://curl.haxx.se" url = "http://curl.haxx.se/download/curl-7.46.0.tar.bz2" + version('7.47.1', '9ea3123449439bbd960cd25cf98796fb') version('7.46.0', '9979f989a2a9930d10f1b3deeabc2148') version('7.45.0', '62c1a352b28558f25ba6209214beadc8') version('7.44.0', '6b952ca00e5473b16a11f05f06aa8dae') From 90268876f7d740e5b7dcfd079168f248a337791a Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Thu, 17 Mar 2016 18:49:58 -0700 Subject: [PATCH 7/9] Change sanity_check_[file|dir] to sanity_check_is_[file|dir], per #554 - Add documentation as well. --- lib/spack/docs/packaging_guide.rst | 84 +++++++++++++++---- lib/spack/spack/package.py | 22 +++-- .../repos/builtin/packages/libelf/package.py | 3 +- 3 files changed, 84 insertions(+), 25 deletions(-) diff --git a/lib/spack/docs/packaging_guide.rst b/lib/spack/docs/packaging_guide.rst index 169899212d..c1077e4497 100644 --- a/lib/spack/docs/packaging_guide.rst +++ b/lib/spack/docs/packaging_guide.rst @@ -1559,11 +1559,11 @@ you ask for a particular spec. A user may have certain preferences for how packages should be concretized on their system. For example, one user may prefer packages built with OpenMPI and the Intel compiler. Another user may prefer -packages be built with MVAPICH and GCC. +packages be built with MVAPICH and GCC. Spack can be configured to prefer certain compilers, package versions, depends_on, and variants during concretization. -The preferred configuration can be controlled via the +The preferred configuration can be controlled via the ``~/.spack/packages.yaml`` file for user configuations, or the ``etc/spack/packages.yaml`` site configuration. @@ -1582,32 +1582,32 @@ Here's an example packages.yaml file that sets preferred packages: compiler: [gcc@4.4.7, gcc@4.6:, intel, clang, pgi] providers: mpi: [mvapich, mpich, openmpi] - + At a high level, this example is specifying how packages should be -concretized. The dyninst package should prefer using gcc 4.9 and +concretized. The dyninst package should prefer using gcc 4.9 and be built with debug options. The gperftools package should prefer version 2.2 over 2.4. Every package on the system should prefer mvapich for -its MPI and gcc 4.4.7 (except for Dyninst, which overrides this by preferring gcc 4.9). -These options are used to fill in implicit defaults. Any of them can be overwritten +its MPI and gcc 4.4.7 (except for Dyninst, which overrides this by preferring gcc 4.9). +These options are used to fill in implicit defaults. Any of them can be overwritten on the command line if explicitly requested. -Each packages.yaml file begins with the string ``packages:`` and +Each packages.yaml file begins with the string ``packages:`` and package names are specified on the next level. The special string ``all`` -applies settings to each package. Underneath each package name is -one or more components: ``compiler``, ``variants``, ``version``, -or ``providers``. Each component has an ordered list of spec +applies settings to each package. Underneath each package name is +one or more components: ``compiler``, ``variants``, ``version``, +or ``providers``. Each component has an ordered list of spec ``constraints``, with earlier entries in the list being preferred over later entries. -Sometimes a package installation may have constraints that forbid +Sometimes a package installation may have constraints that forbid the first concretization rule, in which case Spack will use the first legal concretization rule. Going back to the example, if a user -requests gperftools 2.3 or later, then Spack will install version 2.4 +requests gperftools 2.3 or later, then Spack will install version 2.4 as the 2.4 version of gperftools is preferred over 2.3. -An explicit concretization rule in the preferred section will always -take preference over unlisted concretizations. In the above example, +An explicit concretization rule in the preferred section will always +take preference over unlisted concretizations. In the above example, xlc isn't listed in the compiler list. Every listed compiler from gcc to pgi will thus be preferred over the xlc compiler. @@ -2160,6 +2160,62 @@ package, this allows us to avoid race conditions in the library's build system. +.. _sanity-checks: + +Sanity checking an intallation +-------------------------------- + +By default, Spack assumes that a build has failed if nothing is +written to the install prefix, and that it has succeeded if anything +(a file, a directory, etc.) is written to the install prefix after +``install()`` completes. + +Consider a simple autotools build like this: + +.. code-block:: python + + def install(self, spec, prefix): + configure("--prefix=" + prefix) + make() + make("install") + +If you are using using standard autotools or CMake, ``configure`` and +``make`` will not write anything to the install prefix. Only ``make +install`` writes the files, and only once the build is already +complete. Not all builds are like this. Many builds of scientific +software modify the install prefix *before* ``make install``. Builds +like this can falsely report that they were successfully installed if +an error occurs before the install is complete but after files have +been written to the ``prefix``. + + +``sanity_check_is_file`` and ``sanity_check_is_dir`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +You can optionally specify *sanity checks* to deal with this problem. +Add properties like this to your package: + +.. code-block:: python + + class MyPackage(Package): + ... + + sanity_check_is_file = ['include/libelf.h'] + sanity_check_is_dir = [lib] + + def install(self, spec, prefix): + configure("--prefix=" + prefix) + make() + make("install") + +Now, after ``install()`` runs, Spack will check whether +``$prefix/include/libelf.h`` exists and is a file, and whether +``$prefix/lib`` exists and is a directory. If the checks fail, then +the build will fail and the install prefix will be removed. If they +succeed, Spack considers the build succeeful and keeps the prefix in +place. + + .. _file-manipulation: File manipulation functions diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index 02fb3e5834..291056a7b9 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -318,16 +318,17 @@ class SomePackage(Package): """Most packages are NOT extendable. Set to True if you want extensions.""" extendable = False - """List of prefix-relative file paths. If these do not exist after - install, or if they exist but are not files, sanity checks fail. + """List of prefix-relative file paths (or a single path). If these do + not exist after install, or if they exist but are not files, + sanity checks fail. """ - sanity_check_files = [] + sanity_check_is_file = [] - """List of prefix-relative directory paths. If these do not exist - after install, or if they exist but are not directories, sanity - checks will fail. + """List of prefix-relative directory paths (or a single path). If + these do not exist after install, or if they exist but are not + directories, sanity checks will fail. """ - sanity_check_dirs = [] + sanity_check_is_dir = [] def __init__(self, spec): @@ -966,14 +967,17 @@ def build_process(): def sanity_check_prefix(self): """This function checks whether install succeeded.""" def check_paths(path_list, filetype, predicate): + if isinstance(path_list, basestring): + path_list = [path_list] + for path in path_list: abs_path = os.path.join(self.prefix, path) if not predicate(abs_path): raise InstallError("Install failed for %s. No such %s in prefix: %s" % (self.name, filetype, path)) - check_paths(self.sanity_check_files, 'file', os.path.isfile) - check_paths(self.sanity_check_dirs, 'directory', os.path.isdir) + check_paths(self.sanity_check_is_file, 'file', os.path.isfile) + check_paths(self.sanity_check_is_dir, 'directory', os.path.isdir) installed = set(os.listdir(self.prefix)) installed.difference_update(spack.install_layout.hidden_file_paths) diff --git a/var/spack/repos/builtin/packages/libelf/package.py b/var/spack/repos/builtin/packages/libelf/package.py index 0fcb56c164..9f16708af5 100644 --- a/var/spack/repos/builtin/packages/libelf/package.py +++ b/var/spack/repos/builtin/packages/libelf/package.py @@ -38,8 +38,7 @@ class Libelf(Package): provides('elf') - sanity_check_files = ['include/libelf.h'] - sanity_check_dirs = ['lib'] + sanity_check_is_file = 'include/libelf.h' def install(self, spec, prefix): configure("--prefix=" + prefix, From 802acb4d166a7ca3ff2db3c0a0c95a3ad8a6a81a Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Thu, 17 Mar 2016 13:43:40 -0400 Subject: [PATCH 8/9] Update PAPI to 5.4.3 --- var/spack/repos/builtin/packages/papi/package.py | 1 + 1 file changed, 1 insertion(+) diff --git a/var/spack/repos/builtin/packages/papi/package.py b/var/spack/repos/builtin/packages/papi/package.py index 910e0aa9f9..53d69e28d9 100644 --- a/var/spack/repos/builtin/packages/papi/package.py +++ b/var/spack/repos/builtin/packages/papi/package.py @@ -13,6 +13,7 @@ class Papi(Package): homepage = "http://icl.cs.utk.edu/papi/index.html" url = "http://icl.cs.utk.edu/projects/papi/downloads/papi-5.4.1.tar.gz" + version('5.4.3', '3211b5a5bb389fe692370f5cf4cc2412') version('5.4.1', '9134a99219c79767a11463a76b0b01a2') version('5.3.0', '367961dd0ab426e5ae367c2713924ffb') From 1b279cd7ff29e09ef9afa1a328b964487b5b2030 Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Thu, 17 Mar 2016 13:55:45 -0400 Subject: [PATCH 9/9] Update OpenBLAS to 0.2.16 --- var/spack/repos/builtin/packages/openblas/package.py | 1 + 1 file changed, 1 insertion(+) diff --git a/var/spack/repos/builtin/packages/openblas/package.py b/var/spack/repos/builtin/packages/openblas/package.py index 3c909360a4..e16d3fe89c 100644 --- a/var/spack/repos/builtin/packages/openblas/package.py +++ b/var/spack/repos/builtin/packages/openblas/package.py @@ -5,6 +5,7 @@ class Openblas(Package): homepage = "http://www.openblas.net" url = "http://github.com/xianyi/OpenBLAS/archive/v0.2.15.tar.gz" + version('0.2.16', 'fef46ab92463bdbb1479dcec594ef6dc') version('0.2.15', 'b1190f3d3471685f17cfd1ec1d252ac9') # virtual dependency