Add support for configuration files. Fix SPACK-24.

.gitignore

@@ -4,3 +4,4 @@
 *~
 .DS_Store
 .idea
+/etc/spackconfig

lib/spack/spack/cmd/config.py

@@ -0,0 +1,77 @@
+##############################################################################
+# Copyright (c) 2013, Lawrence Livermore National Security, LLC.
+# Produced at the Lawrence Livermore National Laboratory.
+#
+# This file is part of Spack.
+# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
+# LLNL-CODE-647188
+#
+# For details, see https://scalability-llnl.github.io/spack
+# Please also see the LICENSE file for our notice and the LGPL.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License (as published by
+# the Free Software Foundation) version 2.1 dated February 1999.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and
+# conditions of the GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with this program; if not, write to the Free Software Foundation,
+# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+##############################################################################
+import sys
+import argparse
+
+import llnl.util.tty as tty
+
+import spack.config
+
+description = "Get and set configuration options."
+
+def setup_parser(subparser):
+    scope_group = subparser.add_mutually_exclusive_group()
+
+    # File scope
+    scope_group.add_argument(
+        '--user', action='store_const', const='user', dest='scope',
+        help="Use config file in user home directory (default).")
+    scope_group.add_argument(
+        '--site', action='store_const', const='site', dest='scope',
+        help="Use config file in spack prefix.")
+
+    # Get (vs. default set)
+    subparser.add_argument(
+        '--get', action='store_true', dest='get',
+        help="Get the value associated with a key.")
+
+    # positional arguments (value is only used on set)
+    subparser.add_argument(
+        'key', help="Get the value associated with KEY")
+    subparser.add_argument(
+        'value', nargs='?', default=None,
+        help="Value to associate with key")
+
+
+def config(parser, args):
+    key, value = args.key, args.value
+
+    # If we're writing need to do a few checks.
+    if not args.get:
+        # Default scope for writing is user scope.
+        if not args.scope:
+            args.scope = 'user'
+
+        if args.value is None:
+            tty.die("No value for '%s'.  " % args.key
+                    + "Spack config requires a key and a value.")
+
+    config = spack.config.get_config(args.scope)
+
+    if args.get:
+        print config.get_value(key)
+    else:
+        config.set_value(key, value)
+        config.write()

lib/spack/spack/config.py

@@ -0,0 +1,449 @@
+##############################################################################
+# Copyright (c) 2013, Lawrence Livermore National Security, LLC.
+# Produced at the Lawrence Livermore National Laboratory.
+#
+# This file is part of Spack.
+# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
+# LLNL-CODE-647188
+#
+# For details, see https://scalability-llnl.github.io/spack
+# Please also see the LICENSE file for our notice and the LGPL.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License (as published by
+# the Free Software Foundation) version 2.1 dated February 1999.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and
+# conditions of the GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with this program; if not, write to the Free Software Foundation,
+# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+##############################################################################
+"""This module implements Spack's configuration file handling.
+
+Configuration file scopes
+===============================
+
+When Spack runs, it pulls configuration data from several config
+files, much like bash shells.  In Spack, there are two configuration
+scopes:
+
+ 1. ``site``: Spack loads site-wide configuration options from
+   ``$(prefix)/etc/spackconfig``.
+
+ 2. ``user``: Spack next loads per-user configuration options from
+    ~/.spackconfig.
+
+If user options have the same names as site options, the user options
+take precedence.
+
+
+Configuration file format
+===============================
+
+Configuration files are formatted using .gitconfig syntax, which is
+much like Windows .INI format.  This format is implemented by Python's
+ConfigParser class, and it's easy to read and versatile.
+
+The file is divided into sections, like this ``compiler`` section::
+
+     [compiler]
+         cc = /usr/bin/gcc
+
+In each section there are options (cc), and each option has a value
+(/usr/bin/gcc).
+
+Borrowing from git, we also allow named sections, e.g.:
+
+     [compiler "gcc@4.7.3"]
+         cc = /usr/bin/gcc
+
+This is a compiler section, but it's for the specific compiler,
+``gcc@4.7.3``.  ``gcc@4.7.3`` is the name.
+
+
+Keys
+===============================
+
+Together, the section, name, and option, separated by periods, are
+called a ``key``.  Keys can be used on the command line to set
+configuration options explicitly (this is also borrowed from git).
+
+For example, to change the C compiler used by gcc@4.7.3, you could do
+this:
+
+    spack config compiler.gcc@4.7.3.cc /usr/local/bin/gcc
+
+That will create a named compiler section in the user's .spackconfig
+like the one shown above.
+"""
+import os
+import re
+import inspect
+from collections import OrderedDict
+import ConfigParser as cp
+
+from llnl.util.lang import memoized
+
+import spack
+import spack.error
+
+__all__ = [
+    'SpackConfigParser', 'get_config', 'SpackConfigurationError',
+    'InvalidConfigurationScopeError', 'InvalidSectionNameError',
+    'ReadOnlySpackConfigError', 'ConfigParserError', 'NoOptionError',
+    'NoSectionError']
+
+_named_section_re = r'([^ ]+) "([^"]+)"'
+
+"""Names of scopes and their corresponding configuration files."""
+_scopes = OrderedDict({
+    'site' : os.path.join(spack.etc_path, 'spackconfig'),
+    'user' : os.path.expanduser('~/.spackconfig')
+})
+
+_field_regex = r'^([\w-]*)'        \
+               r'(?:\.(.*(?=.)))?' \
+               r'(?:\.([\w-]+))?$'
+
+_section_regex = r'^([\w-]*)\s*' \
+                 r'\"([^"]*\)\"$'
+
+
+def get_config(scope=None):
+    """Get a Spack configuration object, which can be used to set options.
+
+       With no arguments, this returns a SpackConfigParser with config
+       options loaded from all config files.  This is how client code
+       should read Spack configuration options.
+
+       Optionally, a scope parameter can be provided.  Valid scopes
+       are ``site`` and ``user``.  If a scope is provided, only the
+       options from that scope's configuration file are loaded.  The
+       caller can set or unset options, then call ``write()`` on the
+       config object to write it back out to the original config file.
+    """
+    if scope is None:
+        return SpackConfigParser()
+    elif scope not in _scopes:
+        raise UnknownConfigurationScopeError(scope)
+    else:
+        return SpackConfigParser(_scopes[scope])
+
+
+def _parse_key(key):
+    """Return the section, name, and option the field describes.
+       Values are returned in a 3-tuple.
+
+       e.g.:
+       The field name ``compiler.gcc@4.7.3.cc`` refers to the 'cc' key
+       in a section that looks like this:
+
+          [compiler "gcc@4.7.3"]
+              cc = /usr/local/bin/gcc
+
+       * The section is ``compiler``
+       * The name is ``gcc@4.7.3``
+       * The key is ``cc``
+    """
+    match = re.search(_field_regex, key)
+    if match:
+        return match.groups()
+    else:
+        raise InvalidSectionNameError(key)
+
+
+def _make_section_name(section, name):
+    if not name:
+        return section
+    return '%s "%s"' % (section, name)
+
+
+def _autokey(fun):
+    """Allow a function to be called with a string key like
+       'compiler.gcc.cc', or with the section, name, and option
+       separated. Function should take at least three args, e.g.:
+
+           fun(self, section, name, option, [...])
+
+       This will allow the function above to be called normally or
+       with a string key, e.g.:
+
+           fun(self, key, [...])
+    """
+    argspec = inspect.getargspec(fun)
+    fun_nargs = len(argspec[0])
+
+    def string_key_func(*args):
+        nargs = len(args)
+        if nargs == fun_nargs - 2:
+            section, name, option = _parse_key(args[1])
+            return fun(args[0], section, name, option, *args[2:])
+
+        elif nargs == fun_nargs:
+            return fun(*args)
+
+        else:
+            raise TypeError(
+                "%s takes %d or %d args (found %d)."
+                % (fun.__name__, fun_nargs - 2, fun_nargs, len(args)))
+    return string_key_func
+
+
+
+class SpackConfigParser(cp.RawConfigParser):
+    """Slightly modified from Python's raw config file parser to accept
+    leading whitespace.
+    """
+    # Slightly modified Python option expression. This one allows
+    # leading whitespace.
+    OPTCRE = re.compile(
+        r'\s*(?P<option>[^:=\s][^:=]*)'  # allow leading whitespace
+        r'\s*(?P<vi>[:=])\s*'
+        r'(?P<value>.*)$'
+        )
+
+    def __init__(self, file_or_files=None):
+        cp.RawConfigParser.__init__(self, dict_type=OrderedDict)
+
+        if not file_or_files:
+            file_or_files = [path for path in _scopes.values()]
+
+        if isinstance(file_or_files, basestring):
+            self.read([file_or_files])
+            self.filename = file_or_files
+
+        else:
+            self.read(file_or_files)
+            self.filename = None
+
+
+    @_autokey
+    def set_value(self, section, name, option, value):
+        """Set the value for a key.  If the key is in a section or named
+           section that does not yet exist, add that section.
+        """
+        sn = _make_section_name(section, name)
+        if not self.has_section(sn):
+            self.add_section(sn)
+        self.set(sn, option, value)
+
+
+    @_autokey
+    def get_value(self, section, name, option):
+        """Get the value for a key.  Raises NoOptionError or NoSectionError if
+           the key is not present."""
+        sn = _make_section_name(section, name)
+        try:
+            return self.get(sn, option)
+
+        except cp.NoOptionError, e:  raise NoOptionError(e)
+        except cp.NoSectionError, e: raise NoSectionError(e)
+        except cp.Error, e:          raise ConfigParserError(e)
+
+
+    @_autokey
+    def has_value(self, section, name, option):
+        """Return whether the configuration file has a value for a
+           particular key."""
+        sn = _make_section_name(section, name)
+        return self.has_option(sn, option)
+
+
+    def get_section_names(self, sectype):
+        """Get all named sections with the specified type.
+           A named section looks like this:
+
+               [compiler "gcc@4.7"]
+
+           Names of sections are returned as a list, e.g.:
+
+               ['gcc@4.7', 'intel@12.3', 'pgi@4.2']
+
+           You can get items in the sections like this:
+        """
+        sections = []
+        for secname in self.sections():
+            match = re.match(_named_section_re, secname)
+            if match:
+                t, name = match.groups()
+                if t == sectype:
+                    sections.append(name)
+        return sections
+
+
+    def write(self, path_or_fp=None):
+        """Write this configuration out to a file.
+
+           If called with no arguments, this will write the
+           configuration out to the file from which it was read.  If
+           this config was read from multiple files, e.g. site
+           configuration and then user configuration, write will
+           simply raise an error.
+
+           If called with a path or file object, this will write the
+           configuration out to the supplied path or file object.
+        """
+        if path_or_fp is None:
+            if not self.filename:
+                raise ReadOnlySpackConfigError()
+            path_or_fp = self.filename
+
+        if isinstance(path_or_fp, basestring):
+            path_or_fp = open(path_or_fp, 'w')
+
+        self._write(path_or_fp)
+
+
+    def _read(self, fp, fpname):
+        """This is a copy of Python 2.7's _read() method, with support for
+           continuation lines removed.
+        """
+        cursect = None                        # None, or a dictionary
+        optname = None
+        lineno = 0
+        e = None                              # None, or an exception
+        while True:
+            line = fp.readline()
+            if not line:
+                break
+            lineno = lineno + 1
+            # comment or blank line?
+            if line.strip() == '' or line[0] in '#;':
+                continue
+            if line.split(None, 1)[0].lower() == 'rem' and line[0] in "rR":
+                # no leading whitespace
+                continue
+            # a section header or option header?
+            else:
+                # is it a section header?
+                mo = self.SECTCRE.match(line)
+                if mo:
+                    sectname = mo.group('header')
+                    if sectname in self._sections:
+                        cursect = self._sections[sectname]
+                    elif sectname == cp.DEFAULTSECT:
+                        cursect = self._defaults
+                    else:
+                        cursect = self._dict()
+                        cursect['__name__'] = sectname
+                        self._sections[sectname] = cursect
+                    # So sections can't start with a continuation line
+                    optname = None
+                # no section header in the file?
+                elif cursect is None:
+                    raise cp.MissingSectionHeaderError(fpname, lineno, line)
+                # an option line?
+                else:
+                    mo = self._optcre.match(line)
+                    if mo:
+                        optname, vi, optval = mo.group('option', 'vi', 'value')
+                        optname = self.optionxform(optname.rstrip())
+                        # This check is fine because the OPTCRE cannot
+                        # match if it would set optval to None
+                        if optval is not None:
+                            if vi in ('=', ':') and ';' in optval:
+                                # ';' is a comment delimiter only if it follows
+                                # a spacing character
+                                pos = optval.find(';')
+                                if pos != -1 and optval[pos-1].isspace():
+                                    optval = optval[:pos]
+                            optval = optval.strip()
+                            # allow empty values
+                            if optval == '""':
+                                optval = ''
+                            cursect[optname] = [optval]
+                        else:
+                            # valueless option handling
+                            cursect[optname] = optval
+                    else:
+                        # a non-fatal parsing error occurred.  set up the
+                        # exception but keep going. the exception will be
+                        # raised at the end of the file and will contain a
+                        # list of all bogus lines
+                        if not e:
+                            e = cp.ParsingError(fpname)
+                        e.append(lineno, repr(line))
+        # if any parsing errors occurred, raise an exception
+        if e:
+            raise e
+
+        # join the multi-line values collected while reading
+        all_sections = [self._defaults]
+        all_sections.extend(self._sections.values())
+        for options in all_sections:
+            for name, val in options.items():
+                if isinstance(val, list):
+                    options[name] = '\n'.join(val)
+
+
+    def _write(self, fp):
+        """Write an .ini-format representation of the configuration state.
+
+           This is taken from the default Python 2.7 source.  It writes 4
+           spaces at the beginning of lines instead of no leading space.
+        """
+        if self._defaults:
+            fp.write("[%s]\n" % cp.DEFAULTSECT)
+            for (key, value) in self._defaults.items():
+                fp.write("    %s = %s\n" % (key, str(value).replace('\n', '\n\t')))
+            fp.write("\n")
+        for section in self._sections:
+            # Allow leading whitespace
+            fp.write("[%s]\n" % section)
+            for (key, value) in self._sections[section].items():
+                if key == "__name__":
+                    continue
+                if (value is not None) or (self._optcre == self.OPTCRE):
+                    key = " = ".join((key, str(value).replace('\n', '\n\t')))
+                fp.write("    %s\n" % (key))
+            fp.write("\n")
+
+
+
+class SpackConfigurationError(spack.error.SpackError):
+    def __init__(self, *args):
+        super(SpackConfigurationError, self).__init__(*args)
+
+
+class InvalidConfigurationScopeError(SpackConfigurationError):
+    def __init__(self, scope):
+        super(InvalidConfigurationScopeError, self).__init__(
+            "Invalid configuration scope: '%s'" % scope,
+            "Options are: %s" % ", ".join(*_scopes.values()))
+
+
+class InvalidSectionNameError(SpackConfigurationError):
+    """Raised when the name for a section is invalid."""
+    def __init__(self, name):
+        super(InvalidSectionNameError, self).__init__(
+            "Invalid section specifier: '%s'" % name)
+
+
+class ReadOnlySpackConfigError(SpackConfigurationError):
+    """Raised when user attempts to write to a config read from multiple files."""
+    def __init__(self):
+        super(ReadOnlySpackConfigError, self).__init__(
+            "Can only write to a single-file SpackConfigParser")
+
+
+class ConfigParserError(SpackConfigurationError):
+    """Wrapper for the Python ConfigParser's errors"""
+    def __init__(self, error):
+        super(ConfigParserError, self).__init__(str(error))
+        self.error = error
+
+
+class NoOptionError(ConfigParserError):
+    """Wrapper for ConfigParser NoOptionError"""
+    def __init__(self, error):
+        super(NoOptionError, self).__init__(error)
+
+
+class NoSectionError(ConfigParserError):
+    """Wrapper for ConfigParser NoOptionError"""
+    def __init__(self, error):
+        super(NoSectionError, self).__init__(error)

lib/spack/spack/test/__init__.py

@@ -44,7 +44,8 @@
               'concretize',
               'multimethod',
               'install',
-              'package_sanity']
+              'package_sanity',
+              'config']
 
 
 def list_tests():

lib/spack/spack/test/config.py

@@ -0,0 +1,69 @@
+##############################################################################
+# Copyright (c) 2013, Lawrence Livermore National Security, LLC.
+# Produced at the Lawrence Livermore National Laboratory.
+#
+# This file is part of Spack.
+# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
+# LLNL-CODE-647188
+#
+# For details, see https://scalability-llnl.github.io/spack
+# Please also see the LICENSE file for our notice and the LGPL.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License (as published by
+# the Free Software Foundation) version 2.1 dated February 1999.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and
+# conditions of the GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with this program; if not, write to the Free Software Foundation,
+# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+##############################################################################
+import unittest
+import shutil
+import os
+from tempfile import mkdtemp
+
+from spack.config import *
+
+
+class ConfigTest(unittest.TestCase):
+
+    @classmethod
+    def setUp(cls):
+        cls.tmp_dir = mkdtemp('.tmp', 'spack-config-test-')
+
+
+    @classmethod
+    def tearDown(cls):
+        shutil.rmtree(cls.tmp_dir, True)
+
+
+    def get_path(self):
+        return os.path.join(ConfigTest.tmp_dir, "spackconfig")
+
+
+    def test_write_key(self):
+        config = SpackConfigParser(self.get_path())
+        config.set_value('compiler.cc',  'a')
+        config.set_value('compiler.cxx', 'b')
+        config.set_value('compiler', 'gcc@4.7.3', 'cc',  'c')
+        config.set_value('compiler', 'gcc@4.7.3', 'cxx', 'd')
+        config.write()
+
+        config = SpackConfigParser(self.get_path())
+
+        self.assertEqual(config.get_value('compiler.cc'),  'a')
+        self.assertEqual(config.get_value('compiler.cxx'), 'b')
+        self.assertEqual(config.get_value('compiler', 'gcc@4.7.3', 'cc'), 'c')
+        self.assertEqual(config.get_value('compiler', 'gcc@4.7.3', 'cxx'), 'd')
+
+        self.assertEqual(config.get_value('compiler', None, 'cc'),  'a')
+        self.assertEqual(config.get_value('compiler', None, 'cxx'), 'b')
+        self.assertEqual(config.get_value('compiler.gcc@4.7.3.cc'), 'c')
+        self.assertEqual(config.get_value('compiler.gcc@4.7.3.cxx'), 'd')
+
+        self.assertRaises(NoOptionError, config.get_value, 'compiler', None, 'fc')