hlrs-software-stack/spack
7
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Eliminated all calls that relied on finding all packages in the opt directory

Browse source 
Replaced them all with references to the database

Implemented caching in the database. The database now only re-reads data
if the database file exists and was changed since this file last wrote to it.

Added the installed_db field to the spack instance

Left the call to all_specs from testdirectory_layout.py for now.
This commit is contained in:
Gregory Becker 2015-08-21 16:42:12 -07:00
parent 55f68bb2b0
commit fb1874165b
11 changed files with 122 additions and 96 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
lib/spack/spack/__init__.py
View file

@ -53,6 +53,12 @@
packages_path = join_path(var_path, "packages")
db = PackageDB(packages_path)
#
# Set up the installed packages database
#
from spack.database import Database
installed_db = Database(install_path)
#
# Paths to mock files for testing.
#

2
lib/spack/spack/cmd/__init__.py
View file

@ -124,7 +124,7 @@ def elide_list(line_list, max_num=10):
def disambiguate_spec(spec):
matching_specs = spack.db.get_installed(spec)
matching_specs = spack.installed_db.get_installed(spec)
if not matching_specs:
tty.die("Spec '%s' matches no installed packages." % spec)

2
lib/spack/spack/cmd/deactivate.py
View file

@ -54,7 +54,7 @@ def deactivate(parser, args):
if args.all:
if pkg.extendable:
tty.msg("Deactivating all extensions of %s" % pkg.spec.short_spec)
ext_pkgs = spack.db.installed_extensions_for(spec)
ext_pkgs = spack.installed_db.installed_extensions_for(spec)
for ext_pkg in ext_pkgs:
ext_pkg.spec.normalize()

2
lib/spack/spack/cmd/extensions.py
View file

@ -80,7 +80,7 @@ def extensions(parser, args):
colify(ext.name for ext in extensions)
# List specs of installed extensions.
installed = [s.spec for s in spack.db.installed_extensions_for(spec)]
installed = [s.spec for s in spack.installed_db.installed_extensions_for(spec)]
print
if not installed:
tty.msg("None installed.")

4
lib/spack/spack/cmd/find.py
View file

@ -138,9 +138,9 @@ def find(parser, args):
# Get all the specs the user asked for
if not query_specs:
specs = set(spack.db.installed_package_specs())
specs = set(spack.installed_db.installed_package_specs())
else:
results = [set(spack.db.get_installed(qs)) for qs in query_specs]
results = [set(spack.installed_db.get_installed(qs)) for qs in query_specs]
specs = set.union(*results)
if not args.mode:

4
lib/spack/spack/cmd/module.py
View file

@ -65,7 +65,7 @@ def module_find(mtype, spec_array):
tty.die("You can only pass one spec.")
spec = specs[0]
specs = [s for s in spack.db.installed_package_specs() if s.satisfies(spec)]
specs = [s for s in spack.installed_db.installed_package_specs() if s.satisfies(spec)]
if len(specs) == 0:
tty.die("No installed packages match spec %s" % spec)
@ -86,7 +86,7 @@ def module_find(mtype, spec_array):
def module_refresh():
"""Regenerate all module files for installed packages known to
spack (some packages may no longer exist)."""
specs = [s for s in spack.db.installed_known_package_specs()]
specs = [s for s in spack.installed_db.installed_known_package_specs()]
for name, cls in module_types.items():
tty.msg("Regenerating %s module files." % name)

2
lib/spack/spack/cmd/uninstall.py
View file

@ -59,7 +59,7 @@ def uninstall(parser, args):
# Fail and ask user to be unambiguous if it doesn't
pkgs = []
for spec in specs:
matching_specs = spack.db.get_installed(spec)
matching_specs = spack.installed_db.get_installed(spec)
if not args.all and len(matching_specs) > 1:
tty.error("%s matches multiple packages:" % spec)
print

142
lib/spack/spack/database.py
View file

@ -28,6 +28,9 @@
import glob
import imp
import time
import copy
from external import yaml
from external.yaml.error import MarkedYAMLError
@ -43,8 +46,18 @@
from spack.util.naming import mod_to_class, validate_module_name
def _autospec(function):
"""Decorator that automatically converts the argument of a single-arg
function to a Spec."""
def converter(self, spec_like, **kwargs):
if not isinstance(spec_like, spack.spec.Spec):
spec_like = spack.spec.Spec(spec_like)
return function(self, spec_like, **kwargs)
return converter
class Database(object):
def __init__(self,file_name="specDB.yaml"):
def __init__(self,root,file_name="specDB.yaml"):
"""
Create an empty Database
Location defaults to root/specDB.yaml
@ -53,13 +66,16 @@ def __init__(self,file_name="specDB.yaml"):
path: the path to the install of that package
dep_hash: a hash of the dependence DAG for that package
"""
self.root = root
self.file_name = file_name
self.file_path = join_path(self.root,self.file_name)
self.data = []
self.last_write_time = 0
def from_yaml(self,stream):
"""
Fill database from YAML
Fill database from YAML, do not maintain old data
Translate the spec portions from node-dict form to spec from
"""
try:
@ -70,6 +86,7 @@ def from_yaml(self,stream):
if file==None:
return
self.data = []
for sp in file['database']:
spec = Spec.from_node_dict(sp['spec'])
path = sp['path']
@ -78,19 +95,14 @@ def from_yaml(self,stream):
self.data.append(db_entry)
@staticmethod
def read_database(root):
"""Create a Database from the data in the standard location"""
database = Database()
full_path = join_path(root,database.file_name)
if os.path.isfile(full_path):
with open(full_path,'r') as f:
database.from_yaml(f)
def read_database(self):
"""Reread Database from the data in the set location"""
if os.path.isfile(self.file_path):
with open(self.file_path,'r') as f:
self.from_yaml(f)
else:
with open(full_path,'w+') as f:
database.from_yaml(f)
return database
#The file doesn't exist, construct empty data.
self.data = []
def write_database_to_yaml(self,stream):
@ -110,48 +122,104 @@ def write_database_to_yaml(self,stream):
stream=stream, default_flow_style=False)
def write(self,root):
def write(self):
"""Write the database to the standard location"""
full_path = join_path(root,self.file_name)
#test for file existence
with open(full_path,'w') as f:
#creates file if necessary
with open(self.file_path,'w') as f:
self.last_write_time = int(time.time())
self.write_database_to_yaml(f)
@staticmethod
def add(root, spec, path):
"""Read the database from the standard location
def is_dirty(self):
"""
Returns true iff the database file exists
and was most recently written to by another spack instance.
"""
return (os.path.isfile(self.file_path) and (os.path.getmtime(self.file_path) > self.last_write_time))
# @_autospec
def add(self, spec, path):
"""Re-read the database from the set location if data is dirty
Add the specified entry as a dict
Write the database back to memory
TODO: Caching databases
"""
database = Database.read_database(root)
if self.is_dirty():
self.read_database()
sph = {}
sph['spec']=spec
sph['path']=path
sph['hash']=spec.dag_hash()
database.data.append(sph)
self.data.append(sph)
database.write(root)
self.write()
@staticmethod
def remove(root, spec):
@_autospec
def remove(self, spec):
"""
Reads the database from the standard location
Re-reads the database from the set location if data is dirty
Searches for and removes the specified spec
Writes the database back to memory
TODO: Caching databases
"""
database = Database.read_database(root)
if self.is_dirty():
self.read_database()
for sp in database.data:
#This requires specs w/o dependencies, is that sustainable?
if sp['spec'] == spec:
database.data.remove(sp)
for sp in self.data:
if sp['hash'] == spec.dag_hash() and sp['spec'] == Spec.from_node_dict(spec.to_node_dict()):
self.data.remove(sp)
self.write()
@_autospec
def get_installed(self, spec):
"""
Get all the installed specs that satisfy the provided spec constraint
"""
return [s for s in self.installed_package_specs() if s.satisfies(spec)]
@_autospec
def installed_extensions_for(self, extendee_spec):
"""
Return the specs of all packages that extend
the given spec
"""
for s in self.installed_package_specs():
try:
if s.package.extends(extendee_spec):
yield s.package
except UnknownPackageError, e:
continue
#skips unknown packages
#TODO: conditional way to do this instead of catching exceptions
def installed_package_specs(self):
"""
Read installed package names from the database
and return their specs
"""
if self.is_dirty():
self.read_database()
installed = []
for sph in self.data:
sph['spec'].normalize()
sph['spec'].concretize()
installed.append(sph['spec'])
return installed
def installed_known_package_specs(self):
"""
Read installed package names from the database.
Return only the specs for which the package is known
to this version of spack
"""
return [s for s in self.installed_package_specs() if spack.db.exists(s.name)]
database.write(root)

6
lib/spack/spack/directory_layout.py
View file

@ -153,7 +153,6 @@ def remove_install_directory(self, spec):
os.rmdir(path)
path = os.path.dirname(path)
Database.remove(self.root,spec)
class YamlDirectoryLayout(DirectoryLayout):
@ -266,11 +265,6 @@ def create_install_directory(self, spec):
self.write_spec(spec, spec_file_path)
def add_to_database(self, spec):
"""Simply adds a spec to the database"""
Database.add(self.root, spec, self.path_for_spec(spec))
@memoized
def all_specs(self):
if not os.path.isdir(self.root):

6
lib/spack/spack/package.py
View file

@ -565,7 +565,7 @@ def installed_dependents(self):
"""Return a list of the specs of all installed packages that depend
on this one."""
dependents = []
for spec in spack.db.installed_package_specs():
for spec in spack.installed_db.installed_package_specs():
if self.name == spec.name:
continue
for dep in spec.traverse():
@ -601,7 +601,7 @@ def url_version(self, version):
def remove_prefix(self):
"""Removes the prefix for a package along with any empty parent directories."""
spack.install_layout.remove_install_directory(self.spec)
spack.installed_db.remove(self.spec)
def do_fetch(self):
"""Creates a stage directory and downloads the taball for this package.
@ -818,7 +818,7 @@ def real_work():
install(log_path, log_install_path)
#Update the database once we know install successful
spack.install_layout.add_to_database(self.spec)
spack.installed_db.add(self.spec, spack.install_layout.path_for_spec(self.spec))
# On successful install, remove the stage.
if not keep_stage:

42
lib/spack/spack/packages.py
View file

@ -95,12 +95,6 @@ def purge(self):
self.instances.clear()
@_autospec
def get_installed(self, spec):
"""Get all the installed specs that satisfy the provided spec constraint."""
return [s for s in self.installed_package_specs() if s.satisfies(spec)]
@_autospec
def providers_for(self, vpkg_spec):
if self.provider_index is None:
@ -117,19 +111,6 @@ def extensions_for(self, extendee_spec):
return [p for p in self.all_packages() if p.extends(extendee_spec)]
@_autospec
def installed_extensions_for(self, extendee_spec):
for s in self.installed_package_specs():
try:
if s.package.extends(extendee_spec):
yield s.package
except UnknownPackageError, e:
# Skip packages we know nothing about
continue
# TODO: add some conditional way to do this instead of
# catching exceptions.
def dirname_for_package_name(self, pkg_name):
"""Get the directory name for a particular package. This is the
directory that contains its package.py file."""
@ -150,29 +131,6 @@ def filename_for_package_name(self, pkg_name):
return join_path(pkg_dir, _package_file_name)
def installed_package_specs(self):
"""Read installed package names straight from the install directory
layout.
"""
# Get specs from the directory layout but ensure that they're
# all normalized properly.
installed = []
for spec in spack.install_layout.all_specs():
spec.normalize()
installed.append(spec)
return installed
def installed_known_package_specs(self):
"""Read installed package names straight from the install
directory layout, but return only specs for which the
package is known to this version of spack.
"""
for spec in spack.install_layout.all_specs():
if self.exists(spec.name):
yield spec
@memoized
def all_package_names(self):
"""Generator function for all packages. This looks for