From 360b307f683e146151a65e1d788ce1d154c47ace Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Mon, 24 Aug 2015 09:14:16 -0700 Subject: [PATCH] Save progress. import gov.llnl.spack.mpich works. --- lib/spack/llnl/util/tty/colify.py | 7 + lib/spack/spack/__init__.py | 12 +- lib/spack/spack/cmd/repo.py | 17 +- lib/spack/spack/config.py | 4 +- lib/spack/spack/packages.py | 568 ++++++++++++------- lib/spack/spack/repo_loader.py | 22 +- lib/spack/spack/spec.py | 2 +- lib/spack/spack/test/directory_layout.py | 9 +- lib/spack/spack/test/mock_packages_test.py | 12 +- lib/spack/spack/test/package_sanity.py | 6 +- lib/spack/spack/test/packages.py | 6 +- lib/spack/spack/util/naming.py | 31 + var/spack/mock_packages/_repo.yaml | 2 + var/spack/mock_packages/repo.yaml | 2 - var/spack/packages/{repo.yaml => _repo.yaml} | 0 15 files changed, 452 insertions(+), 248 deletions(-) create mode 100644 var/spack/mock_packages/_repo.yaml delete mode 100644 var/spack/mock_packages/repo.yaml rename var/spack/packages/{repo.yaml => _repo.yaml} (100%) diff --git a/lib/spack/llnl/util/tty/colify.py b/lib/spack/llnl/util/tty/colify.py index 66c52c3968..acf64c1e13 100644 --- a/lib/spack/llnl/util/tty/colify.py +++ b/lib/spack/llnl/util/tty/colify.py @@ -220,6 +220,13 @@ def colify(elts, **options): def colify_table(table, **options): + """Version of colify() for data expressed in rows, (list of lists). + + Same as regular colify but takes a list of lists, where each + sub-list must be the same length, and each is interpreted as a + row in a table. Regular colify displays a sequential list of + values in columns. + """ if table is None: raise TypeError("Can't call colify_table on NoneType") elif not table or not table[0]: diff --git a/lib/spack/spack/__init__.py b/lib/spack/spack/__init__.py index 09bc9ca52a..71e3ac3715 100644 --- a/lib/spack/spack/__init__.py +++ b/lib/spack/spack/__init__.py @@ -23,8 +23,10 @@ # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## import os +import sys import tempfile from llnl.util.filesystem import * +import llnl.util.tty as tty # This lives in $prefix/lib/spack/spack/__file__ prefix = ancestor(__file__, 4) @@ -42,6 +44,7 @@ hooks_path = join_path(module_path, "hooks") var_path = join_path(prefix, "var", "spack") stage_path = join_path(var_path, "stage") +packages_path = join_path(var_path, "packages") opt_path = join_path(prefix, "opt") install_path = join_path(opt_path, "spack") share_path = join_path(prefix, "share", "spack") @@ -55,9 +58,12 @@ # # Set up the default packages database. # -from spack.packages import PackageDB -packages_path = join_path(var_path, "packages") -db = PackageDB() +import spack.packages +_repo_paths = spack.config.get_repos_config() +if not _repo_paths: + tty.die("Spack configuration contains no package repositories.") +db = spack.packages.PackageFinder(*_repo_paths) +sys.meta_path.append(db) # # Paths to mock files for testing. diff --git a/lib/spack/spack/cmd/repo.py b/lib/spack/spack/cmd/repo.py index 1261c7ada9..e290f60b7b 100644 --- a/lib/spack/spack/cmd/repo.py +++ b/lib/spack/spack/cmd/repo.py @@ -32,7 +32,7 @@ import spack.spec import spack.config from spack.util.environment import get_path -from spack.packages import repo_config +from spack.packages import repo_config_filename import os import exceptions @@ -50,13 +50,8 @@ def setup_parser(subparser): create_parser = sp.add_parser('create', help=repo_create.__doc__) create_parser.add_argument('directory', help="Directory containing the packages.") create_parser.add_argument('name', help="Name of new package repository.") -<<<<<<< HEAD:lib/spack/spack/cmd/packagerepo.py - - remove_parser = sp.add_parser('remove', help=packagerepo_remove.__doc__) -======= remove_parser = sp.add_parser('remove', help=repo_remove.__doc__) ->>>>>>> Save changes to external repo integration:lib/spack/spack/cmd/repo.py remove_parser.add_argument('name') list_parser = sp.add_parser('list', help=repo_list.__doc__) @@ -81,7 +76,7 @@ def repo_add(args): """Add package sources to the Spack configuration.""" if not add_to_config(args.directory): tty.die('Repo directory %s already exists in the repo list' % dir) - + def repo_create(args): """Create a new package repo at a directory and name""" @@ -95,13 +90,13 @@ def repo_create(args): mkdirp(dir) except exceptions.OSError, e: tty.die('Failed to create new directory %s' % dir) - path = os.path.join(dir, repo_config) + path = os.path.join(dir, repo_config_filename) try: with closing(open(path, 'w')) as repofile: repofile.write(name + '\n') except exceptions.IOError, e: tty.die('Could not create new file %s' % path) - + if not add_to_config(args.directory): tty.warn('Repo directory %s already exists in the repo list' % dir) @@ -118,8 +113,8 @@ def repo_list(args): fmt = "%%-%ds%%s" % (max_len + 4) for root in root_names: print fmt % (root[0], root[1]) - - + + def repo(parser, args): action = { 'add' : repo_add, diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py index dc59f9a5a3..66da91f629 100644 --- a/lib/spack/spack/config.py +++ b/lib/spack/spack/config.py @@ -269,7 +269,9 @@ def get_repos_config(): config = get_config() if 'repos' not in config: return [] - return config['repos'] + + repo_list = config['repos'] + return [substitute_spack_prefix(repo) for repo in repo_list] def get_mirror_config(): diff --git a/lib/spack/spack/packages.py b/lib/spack/spack/packages.py index c414234386..df54b12324 100644 --- a/lib/spack/spack/packages.py +++ b/lib/spack/spack/packages.py @@ -28,7 +28,6 @@ import inspect import glob import imp -import spack.config import re import itertools import traceback @@ -41,149 +40,327 @@ import spack.error import spack.spec from spack.virtual import ProviderIndex -from spack.util.naming import mod_to_class, validate_module_name -from sets import Set -from spack.repo_loader import RepoLoader, imported_packages_module, package_file_name +from spack.util.naming import * # Filename for package repo names -repo_config = 'repo.yaml' +repo_config_filename = '_repo.yaml' + +# Filename for packages in repos. +package_file_name = 'package.py' def _autospec(function): """Decorator that automatically converts the argument of a single-arg function to a Spec.""" - def converter(self, spec_like, **kwargs): + def converter(self, spec_like, *args, **kwargs): if not isinstance(spec_like, spack.spec.Spec): spec_like = spack.spec.Spec(spec_like) - return function(self, spec_like, **kwargs) + return function(self, spec_like, *args, **kwargs) return converter -def sliding_window(seq, n): - it = iter(seq) - result = tuple(itertools.islice(it, n)) - if len(result) == n: - yield result - for elem in it: - result = result[1:] + (elem,) - yield result +class NamespaceTrie(object): + def __init__(self): + self._elements = {} -class PackageDB(object): + def __setitem__(self, namespace, repo): + parts = namespace.split('.') + cur = self._elements + for p in parts[:-1]: + if p not in cur: + cur[p] = {} + cur = cur[p] + + cur[parts[-1]] = repo + + + def __getitem__(self, namespace): + parts = namespace.split('.') + cur = self._elements + for p in parts: + if p not in cur: + raise KeyError("Can't find namespace %s in trie" % namespace) + cur = cur[p] + return cur + + + def __contains__(self, namespace): + parts = namespace.split('.') + cur = self._elements + for p in parts: + if not isinstance(cur, dict): + return False + if p not in cur: + return False + cur = cur[p] + return True + + + +class PackageFinder(object): + """A PackageFinder is a wrapper around a list of PackageDBs. + + It functions exactly like a PackageDB, but it operates on the + combined results of the PackageDBs in its list instead of on a + single package repository. + """ def __init__(self, *repo_dirs): - """Construct a new package database from a list of directories. - - Args: - repo_dirs List of directories containing packages. - - If ``repo_dirs`` is empty, gets repository list from Spack configuration. - """ - if not repo_dirs: - repo_dirs = spack.config.get_repos_config() - if not repo_dirs: - tty.die("Spack configuration contains no package repositories.") - - # Collect the repos from the config file and read their names - # from the file system - repo_dirs = [spack.config.substitute_spack_prefix(rd) for rd in repo_dirs] - self.repos = [] - for rdir in repo_dirs: - rname = self._read_reponame_from_directory(rdir) - if rname: - self.repos.append((self._read_reponame_from_directory(rdir), rdir)) + self.by_namespace = NamespaceTrie() + self.by_path = {} + + for root in repo_dirs: + repo = PackageDB(root) + self.put_last(repo) - by_path = sorted(self.repos, key=lambda r:r[1]) - by_name = sorted(self.repos, key=lambda r:r[0]) + def _check_repo(self, repo): + if repo.root in self.by_path: + raise DuplicateRepoError("Package repos are the same", + repo, self.by_path[repo.root]) - for r1, r2 in by_path: - if r1[1] == r2[1]: - tty.die("Package repos are the same:", - " %20s %s" % r1, " %20s %s" % r2) - - for r1, r2 in by_name: - if r1[0] == r2[0]: - tty.die("Package repos cannot have the same name:", - " %20s %s" % r1, " %20s %s" % r2) - - # For each repo, create a RepoLoader - self.repo_loaders = dict((name, RepoLoader(name, path)) - for name, path in self.repos) - - self.instances = {} - self.provider_index = None + if repo.namespace in self.by_namespace: + tty.error("Package repos cannot have the same name", + repo, self.by_namespace[repo.namespace]) - def _read_reponame_from_directory(self, dir): - """For a packagerepo directory, read the repo name from the - $root/repo.yaml file""" - path = os.path.join(dir, repo_config) + def _add(self, repo): + self._check_repo(repo) + self.by_namespace[repo.namespace] = repo + self.by_path[repo.root] = repo - try: - with open(path) as reponame_file: - yaml_data = yaml.load(reponame_file) - if (not yaml_data or - 'repo' not in yaml_data or - 'namespace' not in yaml_data['repo']): - tty.die("Invalid %s in %s" % (repo_config, dir)) + def put_first(self, repo): + self._add(repo) + self.repos.insert(0, repo) - name = yaml_data['repo']['namespace'] - if not re.match(r'[a-zA-Z][a-zA-Z0-9_.]+', name): - tty.die( - "Package repo name '%s', read from %s, is an invalid name. " - "Repo names must began with a letter and only contain " - "letters and numbers." % (name, path)) + + def put_last(self, repo): + self._add(repo) + self.repos.append(repo) + + + def remove(self, repo): + if repo in self.repos: + self.repos.remove(repo) + + + def swap(self, other): + repos = self.repos + by_namespace = self.by_namespace + by_path = self.by_path + + self.repos = other.repos + self.by_namespace = other.by_namespace + self.by_pah = other.by_path + + other.repos = repos + other.by_namespace = by_namespace + other.by_path = by_path + + + def all_package_names(self): + all_pkgs = set() + for repo in self.repos: + all_pkgs.update(set(repo.all_package_names())) + return all_pkgs + + + def all_packages(self): + for name in self.all_package_names(): + yield self.get(name) + + + def providers_for(self, vpkg_name): + # TODO: USE MORE THAN FIRST REPO + return self.repos[0].providers_for(vpkg_name) + + + def _get_spack_pkg_name(self, repo, py_module_name): + """Allow users to import Spack packages using legal Python identifiers. + + A python identifier might map to many different Spack package + names due to hyphen/underscore ambiguity. + + Easy example: + num3proxy -> 3proxy + + Ambiguous: + foo_bar -> foo_bar, foo-bar + + More ambiguous: + foo_bar_baz -> foo_bar_baz, foo-bar-baz, foo_bar-baz, foo-bar_baz + """ + if py_module_name in repo: + return py_module_name + + options = possible_spack_module_names(py_module_name) + options.remove(py_module_name) + for name in options: + if name in repo: return name - except exceptions.IOError, e: - tty.die("Error reading %s when opening %s" % (repo_config, dir)) + + return None + + + def find_module(self, fullname, path=None): + if fullname in self.by_namespace: + return self + + namespace, dot, module_name = fullname.rpartition('.') + if namespace not in self.by_namespace: + return None + + repo = self.by_namespace[namespace] + name = self._get_spack_pkg_name(repo, module_name) + if not name: + return None + + return self + + + def load_module(self, fullname): + if fullname in sys.modules: + return sys.modules[fullname] + + if fullname in self.by_namespace: + ns = self.by_namespace[fullname] + module = imp.new_module(fullname) + module.__file__ = "" + module.__path__ = [] + module.__package__ = fullname + + else: + namespace, dot, module_name = fullname.rpartition('.') + if namespace not in self.by_namespace: + raise ImportError( + "No Spack repository with namespace %s" % namespace) + + repo = self.by_namespace[namespace] + name = self._get_spack_pkg_name(repo, module_name) + if not name: + raise ImportError( + "No module %s in Spack repository %s" % (module_name, repo)) + + fullname = namespace + '.' + name + file_path = os.path.join(repo.root, name, package_file_name) + module = imp.load_source(fullname, file_path) + module.__package__ = namespace + + module.__loader__ = self + sys.modules[fullname] = module + return module @_autospec - def get(self, spec, **kwargs): + def get(self, spec, new=False): + for repo in self.repos: + if spec.name in repo: + return repo.get(spec, new) + raise UnknownPackageError(spec.name) + + + def get_repo(self, namespace): + if namespace in self.by_namespace: + repo = self.by_namespace[namespace] + if isinstance(repo, PackageDB): + return repo + return None + + + def exists(self, pkg_name): + return any(repo.exists(pkg_name) for repo in self.repos) + + + def __contains__(self, pkg_name): + return self.exists(pkg_name) + + + +class PackageDB(object): + """Class representing a package repository in the filesystem. + + Each package repository must have a top-level configuration file + called `_repo.yaml`. + + Currently, `_repo.yaml` this must define: + + `namespace`: + A Python namespace where the repository's packages should live. + + """ + def __init__(self, root): + """Instantiate a package repository from a filesystem path.""" + # Root directory, containing _repo.yaml and package dirs + self.root = root + + # Config file in /_repo.yaml + self.config_file = os.path.join(self.root, repo_config_filename) + + # Read configuration from _repo.yaml + config = self._read_config() + if not 'namespace' in config: + tty.die('Package repo in %s must define a namespace in %s.' + % (self.root, repo_config_filename)) + + # Check namespace in the repository configuration. + self.namespace = config['namespace'] + if not re.match(r'[a-zA-Z][a-zA-Z0-9_.]+', self.namespace): + tty.die(("Invalid namespace '%s' in '%s'. Namespaces must be " + "valid python identifiers separated by '.'") + % (self.namespace, self.root)) + + # These are internal cache variables. + self._instances = {} + self._provider_index = None + + + def _read_config(self): + """Check for a YAML config file in this db's root directory.""" + try: + with open(self.config_file) as reponame_file: + yaml_data = yaml.load(reponame_file) + + if (not yaml_data or 'repo' not in yaml_data or + not isinstance(yaml_data['repo'], dict)): + tty.die("Invalid %s in repository %s" + % (repo_config_filename, self.root)) + + return yaml_data['repo'] + + except exceptions.IOError, e: + tty.die("Error reading %s when opening %s" + % (self.config_file, self.root)) + + + @_autospec + def get(self, spec, new=False): if spec.virtual: raise UnknownPackageError(spec.name) - if kwargs.get('new', False): - if spec in self.instances: - del self.instances[spec] + if new: + if spec in self._instances: + del self._instances[spec] - if not spec in self.instances: + if not spec in self._instances: package_class = self.get_class_for_package_name(spec.name, spec.repo) try: copy = spec.copy() - self.instances[copy] = package_class(copy) + self._instances[copy] = package_class(copy) except Exception, e: if spack.debug: sys.excepthook(*sys.exc_info()) raise FailedConstructorError(spec.name, *sys.exc_info()) - return self.instances[spec] - - - @_autospec - def delete(self, spec): - """Force a package to be recreated.""" - del self.instances[spec] - - - def purge(self): - """Clear entire package instance cache.""" - self.instances.clear() - - - @_autospec - def get_installed(self, spec): - """Get all the installed specs that satisfy the provided spec constraint.""" - return [s for s in self.installed_package_specs() if s.satisfies(spec)] + return self._instances[spec] @_autospec def providers_for(self, vpkg_spec): - if self.provider_index is None: - self.provider_index = ProviderIndex(self.all_package_names()) + if self._provider_index is None: + self._provider_index = ProviderIndex(self.all_package_names()) - providers = self.provider_index.providers_for(vpkg_spec) + providers = self._provider_index.providers_for(vpkg_spec) if not providers: raise UnknownPackageError(vpkg_spec.name) return providers @@ -194,6 +371,97 @@ def extensions_for(self, extendee_spec): return [p for p in self.all_packages() if p.extends(extendee_spec)] + def dirname_for_package_name(self, pkg_name): + """Get the directory name for a particular package. This is the + directory that contains its package.py file.""" + return join_path(self.root, pkg_name) + + + def filename_for_package_name(self, pkg_name): + """Get the filename for the module we should load for a particular + package. Packages for a pacakge DB live in + ``$root//package.py`` + + This will return a proper package.py path even if the + package doesn't exist yet, so callers will need to ensure + the package exists before importing. + """ + validate_module_name(pkg_name) + pkg_dir = self.dirname_for_package_name(pkg_name) + return join_path(pkg_dir, package_file_name) + + + @memoized + def all_package_names(self): + """Generator function for all packages. This looks for + ``/package.py`` files within the repo direcotories""" + all_package_names = [] + + for pkg_name in os.listdir(self.root): + pkg_dir = join_path(self.root, pkg_name) + pkg_file = join_path(pkg_dir, package_file_name) + if os.path.isfile(pkg_file): + all_package_names.append(pkg_name) + + return sorted(all_package_names) + + + def all_packages(self): + for name in self.all_package_names(): + yield self.get(name) + + + @memoized + def exists(self, pkg_name): + """Whether a package with the supplied name exists.""" + return os.path.exists(self.filename_for_package_name(pkg_name)) + + + @memoized + def get_class_for_package_name(self, pkg_name, reponame = None): + """Get an instance of the class for a particular package.""" + file_path = self.filename_for_package_name(pkg_name) + + if os.path.exists(file_path): + if not os.path.isfile(file_path): + tty.die("Something's wrong. '%s' is not a file!" % file_path) + if not os.access(file_path, os.R_OK): + tty.die("Cannot read '%s'!" % file_path) + else: + raise UnknownPackageError(pkg_name, self.namespace) + + class_name = mod_to_class(pkg_name) + module = __import__(self.namespace + '.' + pkg_name, fromlist=[class_name]) + cls = getattr(module, class_name) + if not inspect.isclass(cls): + tty.die("%s.%s is not a class" % (pkg_name, class_name)) + + return cls + + + def __str__(self): + return "" % (self.namespace, self.root) + + + def __repr__(self): + return self.__str__() + + + def __contains__(self, pkg_name): + return self.exists(pkg_name) + + + # + # Below functions deal with installed packages, and should be + # moved to some other part of Spack (conbine with + # directory_layout?) + # + @_autospec + def get_installed(self, spec): + """Get all the installed specs that satisfy the provided spec constraint.""" + return [s for s in self.installed_package_specs() if s.satisfies(spec)] + + @_autospec def installed_extensions_for(self, extendee_spec): for s in self.installed_package_specs(): @@ -203,53 +471,6 @@ def installed_extensions_for(self, extendee_spec): except UnknownPackageError, e: # Skip packages we know nothing about continue - # TODO: add some conditional way to do this instead of - # catching exceptions. - - - def repo_for_package_name(self, pkg_name, packagerepo_name=None): - """Find the dirname for a package and the packagerepo it came from - if packagerepo_name is not None, then search for the package in the - specified packagerepo""" - #Look for an existing package under any matching packagerepos - roots = [pkgrepo for pkgrepo in self.repos - if not packagerepo_name or packagerepo_name == pkgrepo[0]] - - if not roots: - tty.die("Package repo %s does not exist" % packagerepo_name) - - for pkgrepo in roots: - path = join_path(pkgrepo[1], pkg_name) - if os.path.exists(path): - return (pkgrepo[0], path) - - repo_to_add_to = roots[-1] - return (repo_to_add_to[0], join_path(repo_to_add_to[1], pkg_name)) - - - def dirname_for_package_name(self, pkg_name, packagerepo_name=None): - """Get the directory name for a particular package. This is the - directory that contains its package.py file.""" - return self.repo_for_package_name(pkg_name, packagerepo_name)[1] - - - def filename_for_package_name(self, pkg_name, packagerepo_name=None): - """Get the filename for the module we should load for a particular - package. Packages for a pacakge DB live in - ``$root//package.py`` - - This will return a proper package.py path even if the - package doesn't exist yet, so callers will need to ensure - the package exists before importing. - - If a packagerepo is specified, then return existing - or new paths in the specified packagerepo directory. If no - package repo is supplied, return an existing path from any - package repo, and new paths in the default package repo. - """ - validate_module_name(pkg_name) - pkg_dir = self.dirname_for_package_name(pkg_name, packagerepo_name) - return join_path(pkg_dir, package_file_name) def installed_package_specs(self): @@ -275,52 +496,6 @@ def installed_known_package_specs(self): yield spec - @memoized - def all_package_names(self): - """Generator function for all packages. This looks for - ``/package.py`` files within the repo direcotories""" - all_packages = Set() - for repo in self.repos: - dir = repo[1] - if not os.path.isdir(dir): - continue - for pkg_name in os.listdir(dir): - pkg_dir = join_path(dir, pkg_name) - pkg_file = join_path(pkg_dir, package_file_name) - if os.path.isfile(pkg_file): - all_packages.add(pkg_name) - all_package_names = list(all_packages) - all_package_names.sort() - return all_package_names - - - def all_packages(self): - for name in self.all_package_names(): - yield self.get(name) - - - @memoized - def exists(self, pkg_name): - """Whether a package with the supplied name exists .""" - return os.path.exists(self.filename_for_package_name(pkg_name)) - - - @memoized - def get_class_for_package_name(self, pkg_name, reponame = None): - """Get an instance of the class for a particular package.""" - (reponame, repodir) = self.repo_for_package_name(pkg_name, reponame) - module_name = imported_packages_module + '.' + reponame + '.' + pkg_name - - module = self.repo_loaders[reponame].get_module(pkg_name) - - class_name = mod_to_class(pkg_name) - cls = getattr(module, class_name) - if not inspect.isclass(cls): - tty.die("%s.%s is not a class" % (pkg_name, class_name)) - - return cls - - class UnknownPackageError(spack.error.SpackError): """Raised when we encounter a package spack doesn't have.""" def __init__(self, name, repo=None): @@ -333,6 +508,13 @@ def __init__(self, name, repo=None): self.name = name +class DuplicateRepoError(spack.error.SpackError): + """Raised when duplicate repos are added to a PackageFinder.""" + def __init__(self, msg, repo1, repo2): + super(UnknownPackageError, self).__init__( + "%s: %s, %s" % (msg, repo1, repo2)) + + class FailedConstructorError(spack.error.SpackError): """Raised when a package's class constructor fails.""" def __init__(self, name, exc_type, exc_obj, exc_tb): diff --git a/lib/spack/spack/repo_loader.py b/lib/spack/spack/repo_loader.py index 92da1cf709..441011cf98 100644 --- a/lib/spack/spack/repo_loader.py +++ b/lib/spack/spack/repo_loader.py @@ -12,7 +12,6 @@ # Name of the package file inside a package directory package_file_name = 'package.py' -import sys class LazyLoader: """The LazyLoader handles cases when repo modules or classes are imported. It watches for 'spack.repos.*' loads, then @@ -21,15 +20,6 @@ def find_module(self, fullname, pathname): if not fullname.startswith(imported_packages_module): return None - print "HERE ===" - print - for line in traceback.format_stack(): - print " ", line.strip() - print - print "full: ", fullname - print "path: ", pathname - print - partial_name = fullname[len(imported_packages_module)+1:] print "partial: ", partial_name @@ -50,7 +40,7 @@ def find_module(self, fullname, pathname): def load_module(self, fullname): return self.mod -sys.meta_path.append(LazyLoader()) +#sys.meta_path.append(LazyLoader()) _reponames = {} class RepoNamespace(types.ModuleType): @@ -59,7 +49,6 @@ class RepoNamespace(types.ModuleType): this class will use __getattr__ to translate the 'original' into one of spack's known repositories""" def __init__(self): - import sys sys.modules[imported_packages_module] = self def __getattr__(self, name): @@ -89,7 +78,6 @@ def __init__(self, reponame, repopath): if not reponame in _reponames: _reponames[reponame] = self - import sys sys.modules[self.module_name] = self @@ -110,14 +98,6 @@ def get_module(self, pkg_name): import imp import llnl.util.tty as tty - file_path = os.path.join(self.path, pkg_name, package_file_name) - if os.path.exists(file_path): - if not os.path.isfile(file_path): - tty.die("Something's wrong. '%s' is not a file!" % file_path) - if not os.access(file_path, os.R_OK): - tty.die("Cannot read '%s'!" % file_path) - else: - raise spack.packages.UnknownPackageError(pkg_name, self.reponame if self.reponame != 'original' else None) try: module_name = imported_packages_module + '.' + self.reponame + '.' + pkg_name diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index 972ba9ccbb..1666457502 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -1714,7 +1714,7 @@ def spec(self): spec_repo = lst[-2] else: spec_name = self.token.value - (spec_repo, repodir) = spack.db.repo_for_package_name(spec_name) + spec_repo = 'gov.llnl.spack' self.check_identifier(spec_name) diff --git a/lib/spack/spack/test/directory_layout.py b/lib/spack/spack/test/directory_layout.py index b3ad8efec4..55b3f0b18f 100644 --- a/lib/spack/spack/test/directory_layout.py +++ b/lib/spack/spack/test/directory_layout.py @@ -34,7 +34,7 @@ import spack from spack.spec import Spec -from spack.packages import PackageDB +from spack.packages import PackageFinder from spack.directory_layout import YamlDirectoryLayout # number of packages to test (to reduce test time) @@ -123,7 +123,7 @@ def test_handle_unknown_package(self): information about installed packages' specs to uninstall or query them again if the package goes away. """ - mock_db = PackageDB(spack.mock_packages_path) + mock_db = PackageFinder(spack.mock_packages_path) not_in_mock = set.difference( set(spack.db.all_package_names()), @@ -145,8 +145,7 @@ def test_handle_unknown_package(self): self.layout.create_install_directory(spec) installed_specs[spec] = self.layout.path_for_spec(spec) - tmp = spack.db - spack.db = mock_db + spack.db.swap(mock_db) # Now check that even without the package files, we know # enough to read a spec from the spec file. @@ -161,7 +160,7 @@ def test_handle_unknown_package(self): self.assertTrue(spec.eq_dag(spec_from_file)) self.assertEqual(spec.dag_hash(), spec_from_file.dag_hash()) - spack.db = tmp + spack.db.swap(mock_db) def test_find(self): diff --git a/lib/spack/spack/test/mock_packages_test.py b/lib/spack/spack/test/mock_packages_test.py index 00f81114af..1f46d65790 100644 --- a/lib/spack/spack/test/mock_packages_test.py +++ b/lib/spack/spack/test/mock_packages_test.py @@ -22,11 +22,12 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## +import sys import unittest import spack import spack.config -from spack.packages import PackageDB +from spack.packages import PackageFinder from spack.spec import Spec @@ -43,8 +44,8 @@ def initmock(self): # Use the mock packages database for these tests. This allows # us to set up contrived packages that don't interfere with # real ones. - self.real_db = spack.db - spack.db = PackageDB(spack.mock_packages_path) + self.db = PackageFinder(spack.mock_packages_path) + spack.db.swap(self.db) spack.config.clear_config_caches() self.real_scopes = spack.config.config_scopes @@ -55,7 +56,8 @@ def initmock(self): def cleanmock(self): """Restore the real packages path after any test.""" - spack.db = self.real_db + spack.db.swap(self.db) + spack.config.config_scopes = self.real_scopes spack.config.clear_config_caches() @@ -66,5 +68,3 @@ def setUp(self): def tearDown(self): self.cleanmock() - - diff --git a/lib/spack/spack/test/package_sanity.py b/lib/spack/spack/test/package_sanity.py index 6222e7b5f8..70b5d6a478 100644 --- a/lib/spack/spack/test/package_sanity.py +++ b/lib/spack/spack/test/package_sanity.py @@ -47,10 +47,10 @@ def test_get_all_packages(self): def test_get_all_mock_packages(self): """Get the mock packages once each too.""" - tmp = spack.db - spack.db = PackageDB(spack.mock_packages_path) + db = PackageFinder(spack.mock_packages_path) + spack.db.swap(db) self.check_db() - spack.db = tmp + spack.db.swap(db) def test_url_versions(self): diff --git a/lib/spack/spack/test/packages.py b/lib/spack/spack/test/packages.py index a8183cf6a6..42bd91ec5c 100644 --- a/lib/spack/spack/test/packages.py +++ b/lib/spack/spack/test/packages.py @@ -44,7 +44,8 @@ def test_package_name(self): def test_package_filename(self): - filename = spack.db.filename_for_package_name('mpich') + repo = spack.db.get_repo('gov.llnl.spack.mock') + filename = repo.filename_for_package_name('mpich') self.assertEqual(filename, join_path(spack.mock_packages_path, 'mpich', 'package.py')) @@ -54,7 +55,8 @@ def test_package_name(self): def test_nonexisting_package_filename(self): - filename = spack.db.filename_for_package_name('some-nonexisting-package') + repo = spack.db.get_repo('gov.llnl.spack.mock') + filename = repo.filename_for_package_name('some-nonexisting-package') self.assertEqual(filename, join_path(spack.mock_packages_path, 'some-nonexisting-package', 'package.py')) diff --git a/lib/spack/spack/util/naming.py b/lib/spack/spack/util/naming.py index 782afbd4bb..a7b6e2b436 100644 --- a/lib/spack/spack/util/naming.py +++ b/lib/spack/spack/util/naming.py @@ -1,10 +1,14 @@ # Need this because of spack.util.string from __future__ import absolute_import import string +import itertools import re import spack +__all__ = ['mod_to_class', 'spack_module_to_python_module', 'valid_module_name', + 'validate_module_name', 'possible_spack_module_names'] + # Valid module names can contain '-' but can't start with it. _valid_module_re = r'^\w[\w-]*$' @@ -42,6 +46,33 @@ def mod_to_class(mod_name): return class_name +def spack_module_to_python_module(mod_name): + """Given a Spack module name, returns the name by which it can be + imported in Python. + """ + if re.match(r'[0-9]', mod_name): + mod_name = 'num' + mod_name + + return mod_name.replace('-', '_') + + +def possible_spack_module_names(python_mod_name): + """Given a Python module name, return a list of all possible spack module + names that could correspond to it.""" + mod_name = re.sub(r'^num(\d)', r'\1', python_mod_name) + + parts = re.split(r'(_)', mod_name) + options = [['_', '-']] * mod_name.count('_') + + results = [] + for subs in itertools.product(*options): + s = list(parts) + s[1::2] = subs + results.append(''.join(s)) + + return results + + def valid_module_name(mod_name): """Return whether the mod_name is valid for use in Spack.""" return bool(re.match(_valid_module_re, mod_name)) diff --git a/var/spack/mock_packages/_repo.yaml b/var/spack/mock_packages/_repo.yaml new file mode 100644 index 0000000000..b97b978de3 --- /dev/null +++ b/var/spack/mock_packages/_repo.yaml @@ -0,0 +1,2 @@ +repo: + namespace: gov.llnl.spack.mock diff --git a/var/spack/mock_packages/repo.yaml b/var/spack/mock_packages/repo.yaml deleted file mode 100644 index d065896006..0000000000 --- a/var/spack/mock_packages/repo.yaml +++ /dev/null @@ -1,2 +0,0 @@ -repo: - namespace: mock diff --git a/var/spack/packages/repo.yaml b/var/spack/packages/_repo.yaml similarity index 100% rename from var/spack/packages/repo.yaml rename to var/spack/packages/_repo.yaml