Add packagerepos to spack, allowing for creating multiple package repositories.

lib/spack/spack/__init__.py

@@ -62,6 +62,12 @@
 mock_site_config = join_path(mock_config_path, "site_spackconfig")
 mock_user_config = join_path(mock_config_path, "user_spackconfig")
 
+#
+# Setup the spack.repos namespace
+#
+from spack.repo_loader import RepoNamespace
+repos = RepoNamespace()
+
 #
 # This controls how spack lays out install prefixes and
 # stage directories.

lib/spack/spack/build_environment.py

@@ -157,7 +157,7 @@ def set_build_environment_variables(pkg):
     path_set("PKG_CONFIG_PATH", pkg_config_dirs)
 
 
-def set_module_variables_for_package(pkg):
+def set_module_variables_for_package(pkg, m):
     """Populate the module scope of install() with some useful functions.
        This makes things easier for package writers.
     """
@@ -228,11 +228,32 @@ def get_rpaths(pkg):
     return rpaths
 
 
+def parent_class_modules(cls):
+    """Get list of super class modules that are all descend from spack.Package"""
+    if not issubclass(cls, spack.Package) or issubclass(spack.Package, cls):
+        return []
+    result = []
+    module = sys.modules.get(cls.__module__)
+    if module:
+        result = [ module ]
+    for c in cls.__bases__:
+        result.extend(parent_class_modules(c))
+    return result
+
+
 def setup_package(pkg):
     """Execute all environment setup routines."""
     set_compiler_environment_variables(pkg)
     set_build_environment_variables(pkg)
-    set_module_variables_for_package(pkg)
+
+    # If a user makes their own package repo, e.g.
+    # spack.repos.mystuff.libelf.Libelf, and they inherit from
+    # an existing class like spack.repos.original.libelf.Libelf,
+    # then set the module variables for both classes so the
+    # parent class can still use them if it gets called.
+    modules = parent_class_modules(pkg.__class__)
+    for mod in modules:
+        set_module_variables_for_package(pkg, mod)
 
     # Allow dependencies to set up environment as well.
     for dep_spec in pkg.spec.traverse(root=False):

lib/spack/spack/cmd/create.py

@@ -93,6 +93,9 @@ def setup_parser(subparser):
     subparser.add_argument(
         '-n', '--name', dest='alternate_name', default=None,
         help="Override the autodetected name for the created package.")
+    subparser.add_argument(
+        '-p', '--package-repo', dest='package_repo', default=None,
+        help="Create the package in the specified packagerepo.")
     subparser.add_argument(
         '-f', '--force', action='store_true', dest='force',
         help="Overwrite any existing package file with the same name.")
@@ -160,12 +163,21 @@ def create(parser, args):
             tty.die("Couldn't guess a name for this package. Try running:", "",
                     "spack create --name <name> <url>")
 
+    package_repo = args.package_repo
+
     if not valid_module_name(name):
         tty.die("Package name can only contain A-Z, a-z, 0-9, '_' and '-'")
 
     tty.msg("This looks like a URL for %s version %s." % (name, version))
     tty.msg("Creating template for package %s" % name)
 
+    # Create a directory for the new package.
+    pkg_path = spack.db.filename_for_package_name(name, package_repo)
+    if os.path.exists(pkg_path) and not args.force:
+        tty.die("%s already exists." % pkg_path)
+    else:
+        mkdirp(os.path.dirname(pkg_path))
+
     versions = spack.package.find_versions_of_archive(url)
     rkeys = sorted(versions.keys(), reverse=True)
     versions = OrderedDict(zip(rkeys, (versions[v] for v in rkeys)))

lib/spack/spack/cmd/packagerepo.py

@@ -0,0 +1,85 @@
+##############################################################################
+# Copyright (c) 2013, Lawrence Livermore National Security, LLC.
+# Produced at the Lawrence Livermore National Laboratory.
+#
+# This file is part of Spack.
+# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
+# LLNL-CODE-647188
+#
+# For details, see https://scalability-llnl.github.io/spack
+# Please also see the LICENSE file for our notice and the LGPL.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License (as published by
+# the Free Software Foundation) version 2.1 dated February 1999.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and
+# conditions of the GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with this program; if not, write to the Free Software Foundation,
+# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+##############################################################################
+from external import argparse
+
+import llnl.util.tty as tty
+from llnl.util.tty.color import colorize
+from llnl.util.tty.colify import colify
+from llnl.util.lang import index_by
+
+import spack.spec
+import spack.config
+from spack.util.environment import get_path
+
+description = "Manage package sources"
+
+def setup_parser(subparser):
+    sp = subparser.add_subparsers(
+        metavar='SUBCOMMAND', dest='packagerepo_command')
+
+    add_parser = sp.add_parser('add', help=packagerepo_add.__doc__)
+    add_parser.add_argument('directory', help="Directory containing the packages.")
+    
+    remove_parser = sp.add_parser('remove', help=packagerepo_remove.__doc__)
+    remove_parser.add_argument('name')
+
+    list_parser = sp.add_parser('list', help=packagerepo_list.__doc__)
+
+
+def packagerepo_add(args):
+    """Add package sources to the Spack configuration."""
+    config = spack.config.get_config()
+    user_config = spack.config.get_config('user')
+    orig = None
+    if config.has_value('packagerepo', '', 'directories'):
+        orig = config.get_value('packagerepo', '', 'directories')
+    if orig and args.directory in orig.split(':'):
+        tty.die('Repo directory %s already exists in the repo list' % args.directory)
+
+    newsetting = orig + ':' + args.directory if orig else args.directory
+    user_config.set_value('packagerepo', '', 'directories', newsetting)
+    user_config.write()
+
+
+def packagerepo_remove(args):
+    """Remove a package source from the Spack configuration"""
+    pass
+
+
+def packagerepo_list(args):
+    """List package sources and their mnemoics"""
+    root_names = spack.db.repos
+    max_len = max(len(s[0]) for s in root_names)
+    fmt = "%%-%ds%%s" % (max_len + 4)
+    for root in root_names:
+        print fmt % (root[0], root[1])
+        
+    
+
+def packagerepo(parser, args):
+    action = { 'add'    : packagerepo_add,
+               'remove' : packagerepo_remove,
+               'list'   : packagerepo_list }
+    action[args.packagerepo_command](args)

lib/spack/spack/packages.py

@@ -23,10 +23,14 @@
 # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
 ##############################################################################
 import os
+import exceptions
 import sys
 import inspect
 import glob
 import imp
+import spack.config
+import re
+from contextlib import closing
 
 import llnl.util.tty as tty
 from llnl.util.filesystem import join_path
@@ -36,13 +40,11 @@
 import spack.spec
 from spack.virtual import ProviderIndex
 from spack.util.naming import mod_to_class, validate_module_name
+from sets import Set
+from spack.repo_loader import RepoLoader, imported_packages_module, package_file_name
 
-# Name of module under which packages are imported
+# Filename for package repo names
-_imported_packages_module = 'spack.packages'
+_packagerepo_filename = 'reponame'
-
-# Name of the package file inside a package directory
-_package_file_name = 'package.py'
-
 
 def _autospec(function):
     """Decorator that automatically converts the argument of a single-arg
@@ -55,13 +57,57 @@ def converter(self, spec_like, **kwargs):
 
 
 class PackageDB(object):
-    def __init__(self, root):
+    def __init__(self, default_root):
         """Construct a new package database from a root directory."""
-        self.root = root
+
+        #Collect the repos from the config file and read their names from the file system
+        repo_dirs = self._repo_list_from_config()
+        repo_dirs.append(default_root)
+        self.repos = [(self._read_reponame_from_directory(dir), dir) for dir in repo_dirs]
+
+        # Check for duplicate repo names
+        s = set()
+        dups = set(r for r in self.repos if r[0] in s or s.add(r[0]))
+        if dups:
+            reponame = list(dups)[0][0]
+            dir1 = list(dups)[0][1]
+            dir2 = dict(s)[reponame]
+            tty.die("Package repo %s in directory %s has the same name as the "
+                      "repo in directory %s" % 
+                      (reponame, dir1, dir2))
+
+        # For each repo, create a RepoLoader
+        self.repo_loaders = dict([(r[0], RepoLoader(r[0], r[1])) for r in self.repos])
+        
         self.instances = {}
         self.provider_index = None
 
 
+    def _read_reponame_from_directory(self, dir):
+        """For a packagerepo directory, read the repo name from the dir/reponame file"""
+        path = os.path.join(dir, 'reponame')
+
+        try:            
+            with closing(open(path, 'r')) as reponame_file:
+                name = reponame_file.read().lstrip().rstrip()                
+                if not re.match(r'[a-zA-Z][a-zA-Z0-9]+', name):
+                    tty.die("Package repo name '%s', read from %s, is an invalid name. "
+                            "Repo names must began with a letter and only contain letters "
+                            "and numbers." % (name, path)) 
+                return name
+        except exceptions.IOError, e:
+            tty.die("Could not read from package repo name file %s" % path)
+
+
+
+    def _repo_list_from_config(self):
+        """Read through the spackconfig and return the list of packagerepo directories"""
+        config = spack.config.get_config()
+        if not config.has_option('packagerepo', 'directories'): return []
+        dir_string = config.get('packagerepo', 'directories')
+        return dir_string.split(':')
+
+    
     @_autospec
     def get(self, spec, **kwargs):
         if spec.virtual:
@@ -130,13 +176,33 @@ def installed_extensions_for(self, extendee_spec):
                 # catching exceptions.
 
 
-    def dirname_for_package_name(self, pkg_name):
+    def repo_for_package_name(self, pkg_name, packagerepo_name=None):
+        """Find the dirname for a package and the packagerepo it came from
+           if packagerepo_name is not None, then search for the package in the
+           specified packagerepo"""
+        #Look for an existing package under any matching packagerepos
+        roots = [pkgrepo for pkgrepo in self.repos
+                 if not packagerepo_name or packagerepo_name == pkgrepo[0]]
+
+        if not roots:
+            tty.die("Package repo %s does not exist" % packagerepo_name)
+
+        for pkgrepo in roots:
+            path = join_path(pkgrepo[1], pkg_name)
+            if os.path.exists(path):
+                return (pkgrepo[0], path)
+        
+        repo_to_add_to = roots[-1]
+        return (repo_to_add_to[0], join_path(repo_to_add_to[1], pkg_name))
+
+
+    def dirname_for_package_name(self, pkg_name, packagerepo_name=None):
         """Get the directory name for a particular package.  This is the
            directory that contains its package.py file."""
-        return join_path(self.root, pkg_name)
+        return self.repo_for_package_name(pkg_name, packagerepo_name)[1]
 
 
-    def filename_for_package_name(self, pkg_name):
+    def filename_for_package_name(self, pkg_name, packagerepo_name=None):
         """Get the filename for the module we should load for a particular
            package.  Packages for a pacakge DB live in
            ``$root/<package_name>/package.py``
@@ -144,10 +210,15 @@ def filename_for_package_name(self, pkg_name):
            This will return a proper package.py path even if the
            package doesn't exist yet, so callers will need to ensure
            the package exists before importing.
+
+           If a packagerepo is specified, then return existing
+           or new paths in the specified packagerepo directory.  If no
+           package repo is supplied, return an existing path from any
+           package repo, and new paths in the default package repo.
         """
         validate_module_name(pkg_name)
-        pkg_dir = self.dirname_for_package_name(pkg_name)
+        pkg_dir = self.dirname_for_package_name(pkg_name, packagerepo_name)
-        return join_path(pkg_dir, _package_file_name)
+        return join_path(pkg_dir, package_file_name)
 
 
     def installed_package_specs(self):
@@ -176,13 +247,18 @@ def installed_known_package_specs(self):
     @memoized
     def all_package_names(self):
         """Generator function for all packages.  This looks for
-           ``<pkg_name>/package.py`` files within the root direcotry"""
+           ``<pkg_name>/package.py`` files within the repo direcotories"""
-        all_package_names = []
+        all_packages = Set()
-        for pkg_name in os.listdir(self.root):
+        for repo in self.repos:
-            pkg_dir  = join_path(self.root, pkg_name)
+            dir = repo[1]
-            pkg_file = join_path(pkg_dir, _package_file_name)
+            if not os.path.isdir(dir):
+                continue
+            for pkg_name in os.listdir(dir):
+                pkg_dir  = join_path(dir, pkg_name)
+                pkg_file = join_path(pkg_dir, package_file_name)
                 if os.path.isfile(pkg_file):
-                all_package_names.append(pkg_name)
+                    all_packages.add(pkg_name)
+        all_package_names = list(all_packages)
         all_package_names.sort()        
         return all_package_names
 
@@ -200,34 +276,13 @@ def exists(self, pkg_name):
 
     @memoized
     def get_class_for_package_name(self, pkg_name):
-        """Get an instance of the class for a particular package.
+        """Get an instance of the class for a particular package."""
+        repo = self.repo_for_package_name(pkg_name)
+        module_name = imported_packages_module + '.' + repo[0] + '.' + pkg_name        
 
-           This method uses Python's ``imp`` package to load python
+        module = self.repo_loaders[repo[0]].get_module(pkg_name)
-           source from a Spack package's ``package.py`` file.  A
-           normal python import would only load each package once, but
-           because we do this dynamically, the method needs to be
-           memoized to ensure there is only ONE package class
-           instance, per package, per database.
-        """
-        file_path = self.filename_for_package_name(pkg_name)
-
-        if os.path.exists(file_path):
-            if not os.path.isfile(file_path):
-                tty.die("Something's wrong. '%s' is not a file!" % file_path)
-            if not os.access(file_path, os.R_OK):
-                tty.die("Cannot read '%s'!" % file_path)
-        else:
-            raise UnknownPackageError(pkg_name)
 
         class_name = mod_to_class(pkg_name)
-        try:
-            module_name = _imported_packages_module + '.' + pkg_name
-            module = imp.load_source(module_name, file_path)
-
-        except ImportError, e:
-            tty.die("Error while importing %s from %s:\n%s" % (
-                pkg_name, file_path, e.message))
-
         cls = getattr(module, class_name)
         if not inspect.isclass(cls):
             tty.die("%s.%s is not a class" % (pkg_name, class_name))

lib/spack/spack/repo_loader.py

@@ -0,0 +1,115 @@
+import spack
+import spack.repos
+import re
+import types
+from llnl.util.lang import *
+
+# Name of module under which packages are imported
+imported_packages_module = 'spack.repos'
+
+# Name of the package file inside a package directory
+package_file_name = 'package.py'
+
+import sys
+class LazyLoader:
+    """The LazyLoader handles cases when repo modules or classes
+       are imported.  It watches for 'spack.repos.*' loads, then 
+       redirects the load to the appropriate module."""
+    def find_module(self, fullname, pathname):
+        if not fullname.startswith(imported_packages_module):
+            return None
+        partial_name = fullname[len(imported_packages_module)+1:]
+        repo = partial_name.split('.')[0]        
+        module = partial_name.split('.')[1]
+        repo_loader = spack.db.repo_loaders.get(repo)
+        if repo_loader:
+            try:
+                self.mod = repo_loader.get_module(module)
+                return self
+            except (ImportError, spack.packages.UnknownPackageError):
+                return None
+
+    def load_module(self, fullname):
+        return self.mod
+
+sys.meta_path.append(LazyLoader())
+
+_reponames = {}
+class RepoNamespace(types.ModuleType):
+    """The RepoNamespace holds the repository namespaces under
+       spack.repos.  For example, when accessing spack.repos.original
+       this class will use __getattr__ to translate the 'original'
+       into one of spack's known repositories"""
+    def __init__(self):
+        import sys
+        sys.modules[imported_packages_module] = self
+        
+    def __getattr__(self, name):
+        if name in _reponames:
+            return _reponames[name]
+        raise AttributeError
+
+    @property
+    def __file__(self):
+        return None
+
+    @property
+    def __path__(self):
+        return []
+
+
+class RepoLoader(types.ModuleType):
+    """Each RepoLoader is associated with a repository, and the RepoLoader is
+       responsible for loading packages out of that repository.  For example,
+       a RepoLoader may be responsible for spack.repos.original, and when someone
+       references spack.repos.original.libelf that RepoLoader will load the 
+       libelf package."""
+    def __init__(self, reponame, repopath):
+        self.path = repopath
+        self.reponame = reponame
+        self.module_name = imported_packages_module + '.' + reponame
+        if not reponame in _reponames:
+            _reponames[reponame] = self
+        spack.repos.add_repo(reponame, self)
+
+        import sys
+        sys.modules[self.module_name] = self
+
+
+    @property
+    def __path__(self):
+        return [ self.path ]
+
+
+    def __getattr__(self, name):
+        if name[0] == '_':
+            raise AttributeError
+        return self.get_module(name)
+
+
+    @memoized
+    def get_module(self, pkg_name):
+        import os
+        import imp
+        import llnl.util.tty as tty
+
+        file_path = os.path.join(self.path, pkg_name, package_file_name)
+        if os.path.exists(file_path):
+            if not os.path.isfile(file_path):
+                tty.die("Something's wrong. '%s' is not a file!" % file_path)
+            if not os.access(file_path, os.R_OK):
+                tty.die("Cannot read '%s'!" % file_path)
+        else:
+            raise spack.packages.UnknownPackageError(pkg_name)
+
+        try:
+            module_name = imported_packages_module + '.' + self.reponame + '.' + pkg_name
+            module = imp.load_source(module_name, file_path)
+
+        except ImportError, e:
+            tty.die("Error while importing %s from %s:\n%s" % (
+                pkg_name, file_path, e.message))
+
+        return module
+
+

var/spack/packages/reponame

@@ -0,0 +1 @@
+original