Merge branch 'develop' of https://github.com/LLNL/spack into features/custom_modulefile_from_config

Conflicts:
	lib/spack/spack/config.py

.flake8

@@ -0,0 +1,20 @@
+# -*- conf -*-
+# flake8 settings for Spack.
+#
+# Below we describe which flake8 checks Spack ignores and what the
+# rationale is.
+#
+# Let people line things up nicely:
+# - E221: multiple spaces before operator
+# - E241: multiple spaces after ‘,’
+#
+# Spack allows wildcard imports:
+# - F403: disable wildcard import
+#
+# These are required to get the package.py files to test clean.
+# - F821: undefined name (needed for cmake, configure, etc.)
+# - F999: name name be undefined or undefined from star imports.
+#
+[flake8]
+ignore = E221,E241,F403,F821,F999
+max-line-length = 79

.style.yapf

@@ -1,3 +0,0 @@
-[style]
-based_on_style = pep8
-column_limit = 79

.travis.yml

@@ -15,20 +15,22 @@ before_install:
   # Need this for the git tests to succeed.
   - git config --global user.email "spack@example.com"
   - git config --global user.name "Test User"
+
   # Need this to be able to compute the list of changed files
   - git fetch origin develop:develop
 
 script:
+  # Regular spack setup and tests
   - . share/spack/setup-env.sh
   - spack compilers
   - spack config get compilers
   - spack install -v libdwarf
+
   # Run unit tests with code coverage
   - coverage run bin/spack test
-  # Checks if the file that have been changed are flake8 conformant
-  - CHANGED_PYTHON_FILES=`git diff develop... --name-only | perl -ne 'print if /\.py$/'`
-  - if [[ ${CHANGED_PYTHON_FILES} ]] ; then flake8 --format pylint --config flake8.ini ${CHANGED_PYTHON_FILES} ; fi
 
+  # Run flake8 code style checks.
+  - share/spack/qa/run-flake8
 
 after_success:
   - coveralls

README.md

@@ -63,6 +63,11 @@ a [pull request](https://help.github.com/articles/using-pull-requests/).
 When you send your request, make ``develop`` the destination branch on the
 [Spack repository](https://github.com/LLNL/spack).
 
+Your contribution will need to pass all the tests run by the `spack test`
+command, as well as the formatting checks in `share/spack/qa/run-flake8`.
+You should run both of these before submitting your pull request, to
+ensure that the online checks succeed.
+
 Spack is using a rough approximation of the [Git
 Flow](http://nvie.com/posts/a-successful-git-branching-model/)
 branching model.  The ``develop`` branch contains the latest

flake8.ini

@@ -1,3 +0,0 @@
-[flake8]
-ignore = W391,F403,E221,F821
-max-line-length = 79

lib/spack/docs/basic_usage.rst

@@ -1456,6 +1456,51 @@ several variants:
 
        spack deactivate -a python
 
+Filesystem requirements
+--------------------------
+
+Spack currently needs to be run from a filesystem that supports
+``flock`` locking semantics.  Nearly all local filesystems and recent
+versions of NFS support this, but parallel filesystems may be mounted
+without ``flock`` support enabled.  You can determine how your
+filesystems are mounted with ``mount -p``.  The output for a Lustre
+filesystem might look like this:
+
+.. code-block:: sh
+
+   $ mount -l | grep lscratch
+   pilsner-mds1-lnet0@o2ib100:/lsd on /p/lscratchd type lustre (rw,nosuid,noauto,_netdev,lazystatfs,flock)
+   porter-mds1-lnet0@o2ib100:/lse on /p/lscratche type lustre (rw,nosuid,noauto,_netdev,lazystatfs,flock)
+
+Note the ``flock`` option on both Lustre mounts.  If you do not see
+this or a similar option for your filesystem, you may need ot ask your
+system administrator to enable ``flock``.
+
+This issue typically manifests with the error below:
+
+.. code-block:: sh
+
+   $ ./spack find
+   Traceback (most recent call last):
+   File "./spack", line 176, in <module>
+     main()
+   File "./spack", line 154, in main
+     return_val = command(parser, args)
+   File "./spack/lib/spack/spack/cmd/find.py", line 170, in find
+     specs = set(spack.installed_db.query(**q_args))
+   File "./spack/lib/spack/spack/database.py", line 551, in query
+     with self.read_transaction():
+   File "./spack/lib/spack/spack/database.py", line 598, in __enter__
+     if self._enter() and self._acquire_fn:
+   File "./spack/lib/spack/spack/database.py", line 608, in _enter
+     return self._db.lock.acquire_read(self._timeout)
+   File "./spack/lib/spack/llnl/util/lock.py", line 103, in acquire_read
+     self._lock(fcntl.LOCK_SH, timeout)   # can raise LockError.
+   File "./spack/lib/spack/llnl/util/lock.py", line 64, in _lock
+     fcntl.lockf(self._fd, op | fcntl.LOCK_NB)
+   IOError: [Errno 38] Function not implemented
+
+A nicer error message is TBD in future versions of Spack.
 
 Getting Help
 -----------------------

lib/spack/spack/architecture.py

@@ -34,14 +34,14 @@
 
 class InvalidSysTypeError(serr.SpackError):
     def __init__(self, sys_type):
-        super(InvalidSysTypeError, self).__init__(
-            "Invalid sys_type value for Spack: " + sys_type)
+        super(InvalidSysTypeError,
+              self).__init__("Invalid sys_type value for Spack: " + sys_type)
 
 
 class NoSysTypeError(serr.SpackError):
     def __init__(self):
-        super(NoSysTypeError, self).__init__(
-            "Could not determine sys_type for this machine.")
+        super(NoSysTypeError,
+              self).__init__("Could not determine sys_type for this machine.")
 
 
 def get_sys_type_from_spack_globals():
@@ -69,15 +69,15 @@ def get_sys_type_from_platform():
 @memoized
 def sys_type():
     """Returns a SysType for the current machine."""
-    methods = [get_sys_type_from_spack_globals,
-               get_sys_type_from_environment,
+    methods = [get_sys_type_from_spack_globals, get_sys_type_from_environment,
                get_sys_type_from_platform]
 
     # search for a method that doesn't return None
     sys_type = None
     for method in methods:
         sys_type = method()
-        if sys_type: break
+        if sys_type:
+            break
 
     # Couldn't determine the sys_type for this machine.
     if sys_type is None:

lib/spack/spack/config.py

@@ -1,3 +1,4 @@
+# flake8: noqa
 ##############################################################################
 # Copyright (c) 2013-2015, Lawrence Livermore National Security, LLC.
 # Produced at the Lawrence Livermore National Laboratory.
@@ -401,13 +402,14 @@ def set_pp_defaults(validator, properties, instance, schema):
             yield err
 
     return validators.extend(validator_class, {
-        "properties" : set_defaults,
-        "patternProperties" : set_pp_defaults
+        "properties": set_defaults,
+        "patternProperties": set_pp_defaults
     })
 
 
 DefaultSettingValidator = extend_with_default(Draft4Validator)
 
+
 def validate_section(data, schema):
     """Validate data read in from a Spack YAML file.
 
@@ -442,16 +444,14 @@ def get_section_filename(self, section):
         validate_section_name(section)
         return os.path.join(self.path, "%s.yaml" % section)
 
-
     def get_section(self, section):
-        if not section in self.sections:
+        if section not in self.sections:
             path   = self.get_section_filename(section)
             schema = section_schemas[section]
             data   = _read_config_file(path, schema)
             self.sections[section] = data
         return self.sections[section]
 
-
     def write_section(self, section):
         filename = self.get_section_filename(section)
         data = self.get_section(section)
@@ -465,7 +465,6 @@ def write_section(self, section):
         except (yaml.YAMLError, IOError) as e:
             raise ConfigFileError("Error writing to config file: '%s'" % str(e))
 
-
     def clear(self):
         """Empty cached config information."""
         self.sections = {}
@@ -571,7 +570,7 @@ def they_are(t):
     # Source dict is merged into dest.
     elif they_are(dict):
         for sk, sv in source.iteritems():
-            if not sk in dest:
+            if sk not in dest:
                 dest[sk] = copy.copy(sv)
             else:
                 dest[sk] = _merge_yaml(dest[sk], source[sk])
@@ -640,6 +639,9 @@ def update_config(section, update_data, scope=None):
     # read in the config to ensure we've got current data
     configuration = get_config(section)
 
+    if isinstance(update_data, list):
+        configuration = update_data
+    else:
         configuration.update(update_data)
 
     # read only the requested section's data.
@@ -682,16 +684,20 @@ def spec_externals(spec):
 def is_spec_buildable(spec):
     """Return true if the spec pkgspec is configured as buildable"""
     allpkgs = get_config('packages')
-    name = spec.name
-    if not spec.name in allpkgs:
+    if spec.name not in allpkgs:
         return True
-    if not 'buildable' in allpkgs[spec.name]:
+    if 'buildable' not in allpkgs[spec.name]:
         return True
     return allpkgs[spec.name]['buildable']
 
 
-class ConfigError(SpackError): pass
-class ConfigFileError(ConfigError): pass
+class ConfigError(SpackError):
+    pass
+
+
+class ConfigFileError(ConfigError):
+    pass
+
 
 def get_path(path, data):
     if path:
@@ -699,6 +705,7 @@ def get_path(path, data):
     else:
         return data
 
+
 class ConfigFormatError(ConfigError):
     """Raised when a configuration format does not match its schema."""
     def __init__(self, validation_error, data):
@@ -733,5 +740,6 @@ def __init__(self, validation_error, data):
         message = '%s: %s' % (location, validation_error.message)
         super(ConfigError, self).__init__(message)
 
+
 class ConfigSanityError(ConfigFormatError):
     """Same as ConfigFormatError, raised when config is written by Spack."""

lib/spack/spack/test/config.py

@@ -72,6 +72,10 @@
     }
 }
 
+# Some Sample repo data
+repos_low = [ "/some/path" ]
+repos_high = [ "/some/other/path" ]
+
 class ConfigTest(MockPackagesTest):
 
     def setUp(self):
@@ -95,6 +99,12 @@ def check_config(self, comps, arch, *compiler_names):
                 actual = config[arch][key][c]
                 self.assertEqual(expected, actual)
 
+    def test_write_list_in_memory(self):
+        spack.config.update_config('repos', repos_low, 'test_low_priority')
+        spack.config.update_config('repos', repos_high, 'test_high_priority')
+        config = spack.config.get_config('repos')
+        self.assertEqual(config, repos_high+repos_low)
+
     def test_write_key_in_memory(self):
         # Write b_comps "on top of" a_comps.
         spack.config.update_config('compilers', a_comps, 'test_low_priority')

share/spack/qa/run-flake8

@@ -0,0 +1,55 @@
+#!/bin/bash
+#
+# This script runs source code style checks on Spack.
+#
+# It should be executed from the top-level directory of the repo,
+# e.g.:
+#
+#    share/spack/qa/run-flake8
+#
+# To run it, you'll need to have the Python flake8 installed locally.
+#
+PYTHONPATH=./lib/spack:$PYTHONPATH
+
+flake8="$(which flake8)"
+if [[ ! $flake8 ]]; then
+    echo "ERROR: flake8 is required to run this script."
+    exit 1
+fi
+
+# Check if changed files are flake8 conformant [framework]
+changed=$(git diff --name-only develop... | grep '.py$')
+
+# Exempt url lines in changed packages from overlong line errors.
+for file in $changed; do
+    if [[ $file = *package.py ]]; then
+        perl -i~ -pe 's/^(\s*url\s*=.*)$/\1  # NOQA: ignore=E501/' $file;
+    fi
+done
+
+return_code=0
+if [[ $changed ]]; then
+    echo =======================================================
+    echo  flake8: running flake8 code checks on spack.
+    echo
+    echo  Modified files:
+    echo  $changed | perl -pe 's/^/  /;s/ +/\n  /g'
+    echo =======================================================
+    if flake8 --format pylint $changed; then
+        echo "Flake8 checks were clean."
+    else
+        echo "Flake8 found errors."
+        return_code=1
+    fi
+else
+    echo No core framework files modified.
+fi
+
+# Restore original package files after modifying them.
+for file in $changed; do
+    if [[ $file = *package.py ]]; then
+        mv "${file}~" "${file}"
+    fi
+done
+
+exit $return_code

var/spack/repos/builtin/packages/ImageMagick/package.py

@@ -1,10 +1,11 @@
 from spack import *
 
+
 class Imagemagick(Package):
     """ImageMagick is a image processing library"""
     homepage = "http://www.imagemagic.org"
 
-    #-------------------------------------------------------------------------
+    # -------------------------------------------------------------------------
     # ImageMagick does not keep around anything but *-10 versions, so
     # this URL may change.  If you want the bleeding edge, you can
     # uncomment it and see if it works but you may need to try to
@@ -17,14 +18,16 @@ class Imagemagick(Package):
     # version('6.9.0-6', 'c1bce7396c22995b8bdb56b7797b4a1b',
     # url="http://www.imagemagick.org/download/ImageMagick-6.9.0-6.tar.bz2")
 
-    #-------------------------------------------------------------------------
+    # -------------------------------------------------------------------------
     # *-10 versions are archived, so these versions should fetch reliably.
     # -------------------------------------------------------------------------
-    version('6.8.9-10', 'aa050bf9785e571c956c111377bbf57c',
+    version(
+        '6.8.9-10',
+        'aa050bf9785e571c956c111377bbf57c',
         url="http://sourceforge.net/projects/imagemagick/files/old-sources/6.x/6.8/ImageMagick-6.8.9-10.tar.gz/download")
 
-    depends_on('libtool')
     depends_on('jpeg')
+    depends_on('libtool')
     depends_on('libpng')
     depends_on('freetype')
     depends_on('fontconfig')
@@ -32,6 +35,5 @@ class Imagemagick(Package):
 
     def install(self, spec, prefix):
         configure("--prefix=%s" % prefix)
-
         make()
         make("install")

var/spack/repos/builtin/packages/Mitos/package.py

@@ -1,5 +1,6 @@
 from spack import *
 
+
 class Mitos(Package):
     """Mitos is a library and a tool for collecting sampled memory
     performance data to view with MemAxes"""
@@ -11,9 +12,7 @@ class Mitos(Package):
             git='https://github.com/llnl/Mitos.git',
             commit='8cb143a2e8c00353ff531a781a9ca0992b0aaa3d')
 
-    version('0.9.1',
-            git='https://github.com/llnl/Mitos.git',
-            tag='v0.9.1')
+    version('0.9.1', git='https://github.com/llnl/Mitos.git', tag='v0.9.1')
 
     depends_on('dyninst@8.2.1:')
     depends_on('hwloc')

var/spack/repos/builtin/packages/astyle/package.py

@@ -1,8 +1,11 @@
 from spack import *
-import os
+
 
 class Astyle(Package):
-    """A Free, Fast, and Small Automatic Formatter for C, C++, C++/CLI, Objective-C, C#, and Java Source Code."""
+    """
+    A Free, Fast, and Small Automatic Formatter for C, C++, C++/CLI,
+    Objective-C, C#, and Java Source Code.
+    """
     homepage = "http://astyle.sourceforge.net/"
     url      = "http://downloads.sourceforge.net/project/astyle/astyle/astyle%202.04/astyle_2.04_linux.tar.gz"
 
@@ -11,8 +14,15 @@ class Astyle(Package):
     def install(self, spec, prefix):
 
         with working_dir('src'):
+            # we need to edit the makefile in place to set compiler:
+            make_file = join_path(self.stage.source_path,
+                                  'build', 'gcc', 'Makefile')
+            filter_file(r'^CXX\s*=.*', 'CXX=%s'.format(spack_cxx), make_file)
+
             make('-f',
-                join_path(self.stage.source_path,'build','clang','Makefile'),
+                 make_file,
                  parallel=False)
+
             mkdirp(self.prefix.bin)
-            install(join_path(self.stage.source_path, 'src','bin','astyle'), self.prefix.bin)
+            install(join_path(self.stage.source_path, 'src', 'bin', 'astyle'),
+                    self.prefix.bin)

var/spack/repos/builtin/packages/py-astroid/package.py

@@ -0,0 +1,22 @@
+from spack import depends_on, extends, version
+from spack import Package
+
+
+class PyAstroid(Package):
+    homepage = "https://www.astroid.org/"
+    url      = "https://github.com/PyCQA/astroid/archive/astroid-1.4.5.tar.gz"
+
+    version('1.4.5', '7adfc55809908297ef430efe4ea20ac3')
+    version('1.4.4', '8ae6f63f6a2b260bb7f647dafccbc796')
+    version('1.4.3', '4647159de7d4d0c4b1de23ecbfb8e246')
+    version('1.4.2', '677f7965840f375af51b0e86403bee6a')
+    version('1.4.1', 'ed70bfed5e4b25be4292e7fe72da2c02')
+
+    extends('python')
+    depends_on('py-logilab-common')
+    depends_on('py-setuptools')
+    depends_on('py-six')
+
+    def install(self, spec, prefix):
+        python('setup.py', 'install', '--prefix=%s' % prefix)
+

var/spack/repos/builtin/packages/py-genshi/package.py

@@ -0,0 +1,18 @@
+from spack import version, extends, depends_on
+from spack import Package
+
+
+class PyGenshi(Package):
+    """Python toolkit for generation of output for the web"""
+    homepage = "https://genshi.edgewall.org/"
+    url      = "http://ftp.edgewall.com/pub/genshi/Genshi-0.7.tar.gz"
+
+    version('0.7', '54e64dd69da3ec961f86e686e0848a82')
+    version('0.6.1', '372c368c8931110b0a521fa6091742d7')
+    version('0.6', '604e8b23b4697655d36a69c2d8ef7187')
+
+    extends("python")
+    depends_on("py-setuptools")
+
+    def install(self, spec, prefix):
+        python('setup.py', 'install', '--prefix=%s' % prefix)

var/spack/repos/builtin/packages/py-jinja2/package.py

@@ -0,0 +1,27 @@
+from spack import depends_on, extends, version
+from spack import Package
+
+
+class PyJinja2(Package):
+    """
+    Jinja2 is a template engine written in pure Python. It provides
+    a Django inspired non-XML syntax but supports inline expressions
+    and an optional sandboxed environment.
+    """
+
+    homepage = "http://jinja.pocoo.org/"
+    url      = "https://github.com/pallets/jinja/archive/2.8.tar.gz"
+
+    version('2.8', '4114200650d7630594e3bc70af23f59e')
+    version('2.7.3', '55b87bdc8e585b8b5b86734eefce2621')
+    version('2.7.2', '8e8f226809ae6363009b9296e30adf30')
+    version('2.7.1', '69b6675553c81b1087f95cae7f2179bb')
+    version('2.7', 'ec70433f325051dcedacbb2465028a35')
+
+    extends("python")
+    depends_on("py-setuptools")
+    depends_on("py-markupsafe")
+
+    def install(self, spec, prefix):
+        python('setup.py', 'install', '--prefix=%s' % prefix)
+

var/spack/repos/builtin/packages/py-logilab-common/package.py

@@ -0,0 +1,18 @@
+from spack import depends_on, extends, version
+from spack import Package
+
+
+class PyLogilabCommon(Package):
+    """Common modules used by Logilab projects"""
+    homepage = "https://www.logilab.org/project/logilab-common"
+    url      = "https://pypi.python.org/packages/a7/31/1650d23e44794d46935d82b86e73454cc83b814cbe1365260ccce8a2f4c6/logilab-common-1.2.0.tar.gz"
+
+    version('1.2.0', 'f7b51351b7bfe052746fa04c03253c0b')
+
+    extends("python")
+    depends_on("py-setuptools")
+    depends_on("py-six")
+
+    def install(self, spec, prefix):
+        python('setup.py', 'install', '--prefix=%s' % prefix)
+

var/spack/repos/builtin/packages/py-markupsafe/package.py

@@ -0,0 +1,27 @@
+from spack import depends_on, extends, version
+from spack import Package
+
+
+class PyMarkupsafe(Package):
+    """
+    MarkupSafe is a library for Python that implements a unicode
+    string that is aware of HTML escaping rules and can be used
+    to implement automatic string escaping. It is used by Jinja 2,
+    the Mako templating engine, the Pylons web framework and many more.
+    """
+
+    homepage = "http://www.pocoo.org/projects/markupsafe/"
+    url      = "https://github.com/pallets/markupsafe/archive/0.23.tar.gz"
+
+    version('0.23', '1a0dadc95169832367c9dcf142155cde')
+    version('0.22', '7a2ac7427b58def567628d06dc328396')
+    version('0.21', 'aebcd93ee05269773c8b80bb6c86fc2f')
+    version('0.20', '0c1fef97c8fd6a986d708f08d7f84a02')
+    version('0.19', '64b05361adb92c11839fc470e308c593')
+
+    extends("python")
+    depends_on("py-setuptools")
+
+    def install(self, spec, prefix):
+        python('setup.py', 'install', '--prefix=%s' % prefix)
+

var/spack/repos/builtin/packages/py-mistune/package.py

@@ -0,0 +1,22 @@
+from spack import depends_on, extends, version
+from spack import Package
+
+
+class PyMistune(Package):
+    """
+    Python markdown parser
+    """
+    homepage = "http://mistune.readthedocs.org/en/latest/"
+    url      = "https://github.com/lepture/mistune/archive/v0.7.1.tar.gz"
+
+    version('0.7.1', '0d9c29700c670790c5b2471070d32ec2')
+    version('0.7', '77750ae8b8d0d584894224a7e0c0523a')
+    version('0.6', 'd4f3d4f28a69e715f82b591d5dacf9a6')
+    version('0.5.1', '1c6cfce28a4aa90cf125217cd6c6fe6c')
+    version('0.5', '997736554f1f95eea78c66ae339b5722')
+
+    extends('python')
+    depends_on('py-setuptools')
+
+    def install(self, spec, prefix):
+        python('setup.py', 'install', '--prefix=%s' % prefix)

var/spack/repos/builtin/packages/py-prettytable/package.py

@@ -0,0 +1,20 @@
+from spack import depends_on, extends, version
+from spack import Package
+
+
+class PyPrettytable(Package):
+    """
+    PrettyTable is a simple Python library designed to make
+    it quick and easy to represent tabular data in visually
+    appealing ASCII tables
+    """
+    homepage = "https://code.google.com/archive/p/prettytable/"
+    url      = "https://pypi.python.org/packages/e0/a1/36203205f77ccf98f3c6cf17cf068c972e6458d7e58509ca66da949ca347/prettytable-0.7.2.tar.gz"
+
+    version('0.7.2', 'a6b80afeef286ce66733d54a0296b13b')
+
+    extends("python")
+    depends_on("py-setuptools")
+
+    def install(self, spec, prefix):
+        python('setup.py', 'install', '--prefix=%s' % prefix)

var/spack/repos/builtin/packages/py-py2neo/package.py

@@ -0,0 +1,22 @@
+from spack import depends_on, extends, version
+from spack import Package
+
+
+class PyPy2neo(Package):
+    """FIXME: put a proper description of your package here."""
+    # FIXME: add a proper url for your package's homepage here.
+    homepage = "http://www.example.com"
+    url      = "https://github.com/nigelsmall/py2neo/archive/py2neo-2.0.8.tar.gz"
+
+    version('2.0.8', 'e3ec5172a9e006515ef4155688a05a55')
+    version('2.0.7', '4cfbc5b7dfd7757f3d2e324805faa639')
+    version('2.0.6', '53e4cdb1a95fbae501c66e541d5f4929')
+    version('2.0.5', '143b1f9c0aa22faf170c1b9f84c7343b')
+    version('2.0.4', 'b3f7efd3344dc3f66db4eda11e5899f7')
+
+    depends_on("py-setuptools")
+    extends("python")
+
+    def install(self, spec, prefix):
+        python('setup.py', 'install', '--prefix=%s' % prefix)
+

var/spack/repos/builtin/packages/py-storm/package.py

@@ -0,0 +1,16 @@
+from spack import depends_on, extends, version
+from spack import Package
+
+
+class PyStorm(Package):
+    """Storm is an object-relational mapper (ORM) for Python"""
+    homepage = "https://storm.canonical.com/"
+    url      = "https://launchpad.net/storm/trunk/0.20/+download/storm-0.20.tar.gz"
+
+    version('0.20', '8628503141f0f06c0749d607ac09b9c7')
+
+    extends('python')
+    depends_on('py-setuptools')
+
+    def install(self, spec, prefix):
+        python('setup.py', 'install', '--prefix=%s' % prefix)