hlrs-software-stack/spack
7
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Add packagerepos to spack, allowing for creating multiple package repositories.

Browse source
This commit is contained in:
Matthew LeGendre 2015-04-20 10:38:18 -07:00 committed by Todd Gamblin
parent c8f65c1530
commit c7b8d09c7f
7 changed files with 343 additions and 48 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
lib/spack/spack/__init__.py
View file

@ -62,6 +62,12 @@
mock_site_config = join_path(mock_config_path, "site_spackconfig")
mock_user_config = join_path(mock_config_path, "user_spackconfig")
#
# Setup the spack.repos namespace
#
from spack.repo_loader import RepoNamespace
repos = RepoNamespace()
#
# This controls how spack lays out install prefixes and
# stage directories.

25
lib/spack/spack/build_environment.py
View file

@ -157,7 +157,7 @@ def set_build_environment_variables(pkg):
path_set("PKG_CONFIG_PATH", pkg_config_dirs)
def set_module_variables_for_package(pkg):
def set_module_variables_for_package(pkg, m):
"""Populate the module scope of install() with some useful functions.
This makes things easier for package writers.
"""
@ -228,11 +228,32 @@ def get_rpaths(pkg):
return rpaths
def parent_class_modules(cls):
"""Get list of super class modules that are all descend from spack.Package"""
if not issubclass(cls, spack.Package) or issubclass(spack.Package, cls):
return []
result = []
module = sys.modules.get(cls.__module__)
if module:
result = [ module ]
for c in cls.__bases__:
result.extend(parent_class_modules(c))
return result
def setup_package(pkg):
"""Execute all environment setup routines."""
set_compiler_environment_variables(pkg)
set_build_environment_variables(pkg)
set_module_variables_for_package(pkg)
# If a user makes their own package repo, e.g.
# spack.repos.mystuff.libelf.Libelf, and they inherit from
# an existing class like spack.repos.original.libelf.Libelf,
# then set the module variables for both classes so the
# parent class can still use them if it gets called.
modules = parent_class_modules(pkg.__class__)
for mod in modules:
set_module_variables_for_package(pkg, mod)
# Allow dependencies to set up environment as well.
for dep_spec in pkg.spec.traverse(root=False):

12
lib/spack/spack/cmd/create.py
View file

@ -93,6 +93,9 @@ def setup_parser(subparser):
subparser.add_argument(
'-n', '--name', dest='alternate_name', default=None,
help="Override the autodetected name for the created package.")
subparser.add_argument(
'-p', '--package-repo', dest='package_repo', default=None,
help="Create the package in the specified packagerepo.")
subparser.add_argument(
'-f', '--force', action='store_true', dest='force',
help="Overwrite any existing package file with the same name.")
@ -160,12 +163,21 @@ def create(parser, args):
tty.die("Couldn't guess a name for this package. Try running:", "",
"spack create --name <name> <url>")
package_repo = args.package_repo
if not valid_module_name(name):
tty.die("Package name can only contain A-Z, a-z, 0-9, '_' and '-'")
tty.msg("This looks like a URL for %s version %s." % (name, version))
tty.msg("Creating template for package %s" % name)
# Create a directory for the new package.
pkg_path = spack.db.filename_for_package_name(name, package_repo)
if os.path.exists(pkg_path) and not args.force:
tty.die("%s already exists." % pkg_path)
else:
mkdirp(os.path.dirname(pkg_path))
versions = spack.package.find_versions_of_archive(url)
rkeys = sorted(versions.keys(), reverse=True)
versions = OrderedDict(zip(rkeys, (versions[v] for v in rkeys)))

85
lib/spack/spack/cmd/packagerepo.py Normal file
View file

@ -0,0 +1,85 @@
##############################################################################
# Copyright (c) 2013, Lawrence Livermore National Security, LLC.
# Produced at the Lawrence Livermore National Laboratory.
#
# This file is part of Spack.
# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
# LLNL-CODE-647188
#
# For details, see https://scalability-llnl.github.io/spack
# Please also see the LICENSE file for our notice and the LGPL.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License (as published by
# the Free Software Foundation) version 2.1 dated February 1999.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and
# conditions of the GNU General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with this program; if not, write to the Free Software Foundation,
# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
##############################################################################
from external import argparse
import llnl.util.tty as tty
from llnl.util.tty.color import colorize
from llnl.util.tty.colify import colify
from llnl.util.lang import index_by
import spack.spec
import spack.config
from spack.util.environment import get_path
description = "Manage package sources"
def setup_parser(subparser):
sp = subparser.add_subparsers(
metavar='SUBCOMMAND', dest='packagerepo_command')
add_parser = sp.add_parser('add', help=packagerepo_add.__doc__)
add_parser.add_argument('directory', help="Directory containing the packages.")
remove_parser = sp.add_parser('remove', help=packagerepo_remove.__doc__)
remove_parser.add_argument('name')
list_parser = sp.add_parser('list', help=packagerepo_list.__doc__)
def packagerepo_add(args):
"""Add package sources to the Spack configuration."""
config = spack.config.get_config()
user_config = spack.config.get_config('user')
orig = None
if config.has_value('packagerepo', '', 'directories'):
orig = config.get_value('packagerepo', '', 'directories')
if orig and args.directory in orig.split(':'):
tty.die('Repo directory %s already exists in the repo list' % args.directory)
newsetting = orig + ':' + args.directory if orig else args.directory
user_config.set_value('packagerepo', '', 'directories', newsetting)
user_config.write()
def packagerepo_remove(args):
"""Remove a package source from the Spack configuration"""
pass
def packagerepo_list(args):
"""List package sources and their mnemoics"""
root_names = spack.db.repos
max_len = max(len(s[0]) for s in root_names)
fmt = "%%-%ds%%s" % (max_len + 4)
for root in root_names:
print fmt % (root[0], root[1])
def packagerepo(parser, args):
action = { 'add' : packagerepo_add,
'remove' : packagerepo_remove,
'list' : packagerepo_list }
action[args.packagerepo_command](args)

143
lib/spack/spack/packages.py
View file

@ -23,10 +23,14 @@
# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
##############################################################################
import os
import exceptions
import sys
import inspect
import glob
import imp
import spack.config
import re
from contextlib import closing
import llnl.util.tty as tty
from llnl.util.filesystem import join_path
@ -36,13 +40,11 @@
import spack.spec
from spack.virtual import ProviderIndex
from spack.util.naming import mod_to_class, validate_module_name
from sets import Set
from spack.repo_loader import RepoLoader, imported_packages_module, package_file_name
# Name of module under which packages are imported
_imported_packages_module = 'spack.packages'
# Name of the package file inside a package directory
_package_file_name = 'package.py'
# Filename for package repo names
_packagerepo_filename = 'reponame'
def _autospec(function):
"""Decorator that automatically converts the argument of a single-arg
@ -55,13 +57,57 @@ def converter(self, spec_like, **kwargs):
class PackageDB(object):
def __init__(self, root):
def __init__(self, default_root):
"""Construct a new package database from a root directory."""
self.root = root
#Collect the repos from the config file and read their names from the file system
repo_dirs = self._repo_list_from_config()
repo_dirs.append(default_root)
self.repos = [(self._read_reponame_from_directory(dir), dir) for dir in repo_dirs]
# Check for duplicate repo names
s = set()
dups = set(r for r in self.repos if r[0] in s or s.add(r[0]))
if dups:
reponame = list(dups)[0][0]
dir1 = list(dups)[0][1]
dir2 = dict(s)[reponame]
tty.die("Package repo %s in directory %s has the same name as the "
"repo in directory %s" %
(reponame, dir1, dir2))
# For each repo, create a RepoLoader
self.repo_loaders = dict([(r[0], RepoLoader(r[0], r[1])) for r in self.repos])
self.instances = {}
self.provider_index = None
def _read_reponame_from_directory(self, dir):
"""For a packagerepo directory, read the repo name from the dir/reponame file"""
path = os.path.join(dir, 'reponame')
try:
with closing(open(path, 'r')) as reponame_file:
name = reponame_file.read().lstrip().rstrip()
if not re.match(r'[a-zA-Z][a-zA-Z0-9]+', name):
tty.die("Package repo name '%s', read from %s, is an invalid name. "
"Repo names must began with a letter and only contain letters "
"and numbers." % (name, path))
return name
except exceptions.IOError, e:
tty.die("Could not read from package repo name file %s" % path)
def _repo_list_from_config(self):
"""Read through the spackconfig and return the list of packagerepo directories"""
config = spack.config.get_config()
if not config.has_option('packagerepo', 'directories'): return []
dir_string = config.get('packagerepo', 'directories')
return dir_string.split(':')
@_autospec
def get(self, spec, **kwargs):
if spec.virtual:
@ -130,13 +176,33 @@ def installed_extensions_for(self, extendee_spec):
# catching exceptions.
def dirname_for_package_name(self, pkg_name):
def repo_for_package_name(self, pkg_name, packagerepo_name=None):
"""Find the dirname for a package and the packagerepo it came from
if packagerepo_name is not None, then search for the package in the
specified packagerepo"""
#Look for an existing package under any matching packagerepos
roots = [pkgrepo for pkgrepo in self.repos
if not packagerepo_name or packagerepo_name == pkgrepo[0]]
if not roots:
tty.die("Package repo %s does not exist" % packagerepo_name)
for pkgrepo in roots:
path = join_path(pkgrepo[1], pkg_name)
if os.path.exists(path):
return (pkgrepo[0], path)
repo_to_add_to = roots[-1]
return (repo_to_add_to[0], join_path(repo_to_add_to[1], pkg_name))
def dirname_for_package_name(self, pkg_name, packagerepo_name=None):
"""Get the directory name for a particular package. This is the
directory that contains its package.py file."""
return join_path(self.root, pkg_name)
return self.repo_for_package_name(pkg_name, packagerepo_name)[1]
def filename_for_package_name(self, pkg_name):
def filename_for_package_name(self, pkg_name, packagerepo_name=None):
"""Get the filename for the module we should load for a particular
package. Packages for a pacakge DB live in
``$root/<package_name>/package.py``
@ -144,10 +210,15 @@ def filename_for_package_name(self, pkg_name):
This will return a proper package.py path even if the
package doesn't exist yet, so callers will need to ensure
the package exists before importing.
If a packagerepo is specified, then return existing
or new paths in the specified packagerepo directory. If no
package repo is supplied, return an existing path from any
package repo, and new paths in the default package repo.
"""
validate_module_name(pkg_name)
pkg_dir = self.dirname_for_package_name(pkg_name)
return join_path(pkg_dir, _package_file_name)
pkg_dir = self.dirname_for_package_name(pkg_name, packagerepo_name)
return join_path(pkg_dir, package_file_name)
def installed_package_specs(self):
@ -176,13 +247,18 @@ def installed_known_package_specs(self):
@memoized
def all_package_names(self):
"""Generator function for all packages. This looks for
``<pkg_name>/package.py`` files within the root direcotry"""
all_package_names = []
for pkg_name in os.listdir(self.root):
pkg_dir = join_path(self.root, pkg_name)
pkg_file = join_path(pkg_dir, _package_file_name)
``<pkg_name>/package.py`` files within the repo direcotories"""
all_packages = Set()
for repo in self.repos:
dir = repo[1]
if not os.path.isdir(dir):
continue
for pkg_name in os.listdir(dir):
pkg_dir = join_path(dir, pkg_name)
pkg_file = join_path(pkg_dir, package_file_name)
if os.path.isfile(pkg_file):
all_package_names.append(pkg_name)
all_packages.add(pkg_name)
all_package_names = list(all_packages)
all_package_names.sort()
return all_package_names
@ -200,34 +276,13 @@ def exists(self, pkg_name):
@memoized
def get_class_for_package_name(self, pkg_name):
"""Get an instance of the class for a particular package.
"""Get an instance of the class for a particular package."""
repo = self.repo_for_package_name(pkg_name)
module_name = imported_packages_module + '.' + repo[0] + '.' + pkg_name
This method uses Python's ``imp`` package to load python
source from a Spack package's ``package.py`` file. A
normal python import would only load each package once, but
because we do this dynamically, the method needs to be
memoized to ensure there is only ONE package class
instance, per package, per database.
"""
file_path = self.filename_for_package_name(pkg_name)
if os.path.exists(file_path):
if not os.path.isfile(file_path):
tty.die("Something's wrong. '%s' is not a file!" % file_path)
if not os.access(file_path, os.R_OK):
tty.die("Cannot read '%s'!" % file_path)
else:
raise UnknownPackageError(pkg_name)
module = self.repo_loaders[repo[0]].get_module(pkg_name)
class_name = mod_to_class(pkg_name)
try:
module_name = _imported_packages_module + '.' + pkg_name
module = imp.load_source(module_name, file_path)
except ImportError, e:
tty.die("Error while importing %s from %s:\n%s" % (
pkg_name, file_path, e.message))
cls = getattr(module, class_name)
if not inspect.isclass(cls):
tty.die("%s.%s is not a class" % (pkg_name, class_name))

115
lib/spack/spack/repo_loader.py Normal file
View file

@ -0,0 +1,115 @@
import spack
import spack.repos
import re
import types
from llnl.util.lang import *
# Name of module under which packages are imported
imported_packages_module = 'spack.repos'
# Name of the package file inside a package directory
package_file_name = 'package.py'
import sys
class LazyLoader:
"""The LazyLoader handles cases when repo modules or classes
are imported. It watches for 'spack.repos.*' loads, then
redirects the load to the appropriate module."""
def find_module(self, fullname, pathname):
if not fullname.startswith(imported_packages_module):
return None
partial_name = fullname[len(imported_packages_module)+1:]
repo = partial_name.split('.')[0]
module = partial_name.split('.')[1]
repo_loader = spack.db.repo_loaders.get(repo)
if repo_loader:
try:
self.mod = repo_loader.get_module(module)
return self
except (ImportError, spack.packages.UnknownPackageError):
return None
def load_module(self, fullname):
return self.mod
sys.meta_path.append(LazyLoader())
_reponames = {}
class RepoNamespace(types.ModuleType):
"""The RepoNamespace holds the repository namespaces under
spack.repos. For example, when accessing spack.repos.original
this class will use __getattr__ to translate the 'original'
into one of spack's known repositories"""
def __init__(self):
import sys
sys.modules[imported_packages_module] = self
def __getattr__(self, name):
if name in _reponames:
return _reponames[name]
raise AttributeError
@property
def __file__(self):
return None
@property
def __path__(self):
return []
class RepoLoader(types.ModuleType):
"""Each RepoLoader is associated with a repository, and the RepoLoader is
responsible for loading packages out of that repository. For example,
a RepoLoader may be responsible for spack.repos.original, and when someone
references spack.repos.original.libelf that RepoLoader will load the
libelf package."""
def __init__(self, reponame, repopath):
self.path = repopath
self.reponame = reponame
self.module_name = imported_packages_module + '.' + reponame
if not reponame in _reponames:
_reponames[reponame] = self
spack.repos.add_repo(reponame, self)
import sys
sys.modules[self.module_name] = self
@property
def __path__(self):
return [ self.path ]
def __getattr__(self, name):
if name[0] == '_':
raise AttributeError
return self.get_module(name)
@memoized
def get_module(self, pkg_name):
import os
import imp
import llnl.util.tty as tty
file_path = os.path.join(self.path, pkg_name, package_file_name)
if os.path.exists(file_path):
if not os.path.isfile(file_path):
tty.die("Something's wrong. '%s' is not a file!" % file_path)
if not os.access(file_path, os.R_OK):
tty.die("Cannot read '%s'!" % file_path)
else:
raise spack.packages.UnknownPackageError(pkg_name)
try:
module_name = imported_packages_module + '.' + self.reponame + '.' + pkg_name
module = imp.load_source(module_name, file_path)
except ImportError, e:
tty.die("Error while importing %s from %s:\n%s" % (
pkg_name, file_path, e.message))
return module

1
var/spack/packages/reponame Normal file
View file

@ -0,0 +1 @@
original