First cut concretization works, with tests.

Mock packages now all have their own version lists.
This commit is contained in:
Todd Gamblin 2013-10-26 14:04:09 -07:00
parent 7bdf93234a
commit b5c565891f
18 changed files with 265 additions and 116 deletions

View file

@ -14,3 +14,6 @@ def spec(parser, args):
for spec in specs:
spec.normalize()
print spec.tree()
spec.concretize()
print spec.tree()

View file

@ -17,13 +17,21 @@ def setup_parser(subparser):
help="verbose output")
def find_test_modules():
"""Include all the modules under test, unless they set skip_test=True"""
for name in list_modules(spack.test_path):
module = __import__('spack.test.' + name, fromlist='skip_test')
if not getattr(module, 'skip_test', False):
yield name
def test(parser, args):
if args.list:
print "Available tests:"
colify(list_modules(spack.test_path, directories=False))
colify(find_test_modules())
elif not args.names:
for name in list_modules(spack.test_path, directories=False):
for name in find_test_modules():
print "Running Tests: %s" % name
spack.test.run(name, verbose=args.verbose)

View file

@ -24,12 +24,12 @@ def concretize_version(spec):
if spec.versions.concrete:
return
pkg = speck.package
available = pkg.available_versions
pkg = spec.package
# If there are known avaialble versions, return the most recent
if versions:
spec.versions = ver([avaialble[-1]])
available_versions = pkg.available_versions
if available_versions:
spec.versions = ver([available_versions[-1]])
else:
spec.versions = ver([pkg.version])
@ -73,7 +73,7 @@ def concretize_compiler(spec):
build with the compiler that will be used by libraries that
link to this one, to maximize compatibility.
"""
if spec.compiler.concrete:
if spec.compiler and spec.compiler.concrete:
if spec.compiler != spack.compilers.default_compiler():
raise spack.spec.UnknownCompilerError(str(spec.compiler))
else:

View file

@ -298,6 +298,11 @@ def __init__(self, spec):
# This is set by scraping a web page.
self._available_versions = None
# This list overrides available_versions if set by the user.
attr.setdefault(self, 'versions', None)
if self.versions and type(self.versions) != VersionList:
self.versions = VersionList(self.versions)
# stage used to build this package.
self.stage = Stage("%s-%s" % (self.name, self.version), self.url)
@ -637,6 +642,11 @@ def do_clean_dist(self):
@property
def available_versions(self):
# If the package overrode available_versions, then use that.
if self.versions is not None:
return self.versions
# If not, then try to fetch using list_url
if not self._available_versions:
self._available_versions = ver([self.version])
try:
@ -656,7 +666,7 @@ def available_versions(self):
"to the package to tell Spack where to look for versions.")
except subprocess.CalledProcessError:
tty.warn("Fetching %s failed." % self.list_url,
tty.warn("Could not connect to %s" % self.list_url,
"Package.available_versions requires an internet connection.",
"Version list may be incomplete.")

View file

@ -170,7 +170,7 @@ def _cmp_key(self):
def __str__(self):
out = self.name
if self.versions:
vlist = ",".join(str(v) for v in sorted(self.versions))
vlist = ",".join(str(v) for v in self.versions)
out += "@%s" % vlist
return out
@ -190,6 +190,10 @@ def _cmp_key(self):
return (self.name, self.enabled)
def copy(self):
return Variant(self.name, self.enabled)
def __str__(self):
out = '+' if self.enabled else '~'
return out + self.name
@ -350,43 +354,58 @@ def preorder_traversal(self, visited=None, d=0, **kwargs):
yield spec
def _concretize(self):
def _concretize_helper(self, presets):
for name in sorted(self.dependencies.keys()):
self.dependencies[name]._concretize_helper(presets)
if self.name in presets:
self.constrain(presets[self.name])
else:
spack.concretize.concretize_architecture(self)
spack.concretize.concretize_compiler(self)
spack.concretize.concretize_version(self)
presets[self.name] = self
def concretize(self, *presets):
"""A spec is concrete if it describes one build of a package uniquely.
This will ensure that this spec is concrete.
If this spec could describe more than one version, variant, or build
of a package, this will resolve it to be concrete.
of a package, this will add constraints to make it concrete.
Ensures that the spec is in canonical form.
This means:
1. All dependencies of this package and of its dependencies are
in the dependencies list (transitive closure of deps).
2. All dependencies in the dependencies list are canonicalized.
This function also serves to validate the spec, in that it makes sure
that each package exists an that spec criteria don't violate package
criteria.
Some rigorous validation and checks are also performed on the spec.
Concretizing ensures that it is self-consistent and that it's consistent
with requirements of its pacakges. See flatten() and normalize() for
more details on this.
"""
# TODO: modularize the process of selecting concrete versions.
# There should be a set of user-configurable policies for these decisions.
self.validate()
# Build specs out of user-provided presets
specs = [Spec(p) for p in presets]
# take the system's architecture for starters
if not self.architecture:
self.architecture = arch.sys_type()
# Concretize the presets first. They could be partial specs, like just
# a particular version that the caller wants.
for spec in specs:
if not spec.concrete:
try:
spec.concretize()
except UnsatisfiableSpecError, e:
e.message = ("Unsatisfiable preset in concretize: %s."
% e.message)
raise e
if self.compiler:
self.compiler._concretize()
# build preset specs into a map
preset_dict = {spec.name : spec for spec in specs}
# TODO: handle variants.
# Concretize bottom up, passing in presets to force concretization
# for certain specs.
self.normalize()
self._concretize_helper(preset_dict)
# Take the highest version in a range
if not self.versions.concrete:
preferred = self.versions.highest() or self.package.version
self.versions = VersionList([preferred])
# Ensure dependencies have right versions
def concretized(self, *presets):
clone = self.copy()
clone.concretize(*presets)
return clone
def flat_dependencies(self):
@ -406,7 +425,9 @@ def flat_dependencies(self):
try:
for spec in self.preorder_traversal():
if spec.name not in flat_deps:
flat_deps[spec.name] = spec
new_spec = spec.copy(dependencies=False)
flat_deps[spec.name] = new_spec
else:
flat_deps[spec.name].constrain(spec)
@ -466,7 +487,7 @@ def normalize(self):
# provided spec is sane, and that all dependency specs are in the
# root node of the spec. flat_dependencies will do this for us.
spec_deps = self.flat_dependencies()
self.dependencies = DependencyMap()
self.dependencies.clear()
visited = set()
self._normalize_helper(visited, spec_deps)
@ -523,30 +544,37 @@ def sat(attribute):
self.dependencies.satisfies(other.dependencies))
def concretized(self):
clone = self.copy()
clone._concretize()
return clone
def _dup(self, other):
def _dup(self, other, **kwargs):
"""Copy the spec other into self. This is a
first-party, overwriting copy."""
first-party, overwriting copy. This does not copy
parent; if the other spec has a parent, this one will not.
To duplicate an entire DAG, Duplicate the root of the DAG.
"""
# TODO: this needs to handle DAGs.
self.name = other.name
self.parent = None
self.versions = other.versions.copy()
self.variants = other.variants.copy()
self.architecture = other.architecture
self.compiler = None
if other.compiler:
self.compiler = other.compiler.copy()
copy_deps = kwargs.get('dependencies', True)
if copy_deps:
self.dependencies = other.dependencies.copy()
else:
self.dependencies = DependencyMap()
def copy(self):
"""Return a deep copy of this spec."""
return Spec(self)
def copy(self, **kwargs):
"""Return a copy of this spec.
By default, returns a deep copy. Supply dependencies=False
to get a shallow copy.
"""
clone = Spec.__new__(Spec)
clone._dup(self, **kwargs)
return clone
@property
def version(self):
@ -564,7 +592,7 @@ def colorized(self):
return colorize_spec(self)
def str_no_deps(self):
def str_no_deps(self, **kwargs):
out = self.name
# If the version range is entirely open, omit it
@ -579,12 +607,17 @@ def str_no_deps(self):
if self.architecture:
out += "=%s" % self.architecture
if kwargs.get('color', False):
return colorize_spec(out)
else:
return out
def tree(self):
def tree(self, **kwargs):
"""Prints out this spec and its dependencies, tree-formatted
with indentation."""
color = kwargs.get('color', False)
out = ""
cur_id = 0
ids = {}
@ -594,7 +627,7 @@ def tree(self):
ids[id(node)] = cur_id
out += str(ids[id(node)])
out += " "+ (" " * d)
out += node.str_no_deps() + "\n"
out += node.str_no_deps(color=color) + "\n"
return out

View file

@ -1,17 +1,60 @@
import unittest
from spack.spec import Spec
from spack.test.mock_packages_test import *
class ConcretizeTest(MockPackagesTest):
def check_spec(self, abstract, concrete):
if abstract.versions.concrete:
self.assertEqual(abstract.versions, concrete.versions)
if abstract.variants:
self.assertEqual(abstract.versions, concrete.versions)
if abstract.compiler and abstract.compiler.concrete:
self.assertEqual(abstract.compiler, concrete.compiler)
if abstract.architecture and abstract.architecture.concrete:
self.assertEqual(abstract.architecture, concrete.architecture)
class ConcretizeTest(unittest.TestCase):
def check_concretize(self, abstract_spec):
def check_concretize(self, abstract_spec, *presets):
abstract = Spec(abstract_spec)
print abstract
print abstract.concretized()
print abstract.concretized().concrete
self.assertTrue(abstract.concretized().concrete)
concrete = abstract.concretized(*presets)
self.assertFalse(abstract.concrete)
self.assertTrue(concrete.concrete)
self.check_spec(abstract, concrete)
return concrete
def test_packages(self):
pass
#self.check_concretize("libelf")
def check_presets(self, abstract, *presets):
abstract = Spec(abstract)
concrete = self.check_concretize(abstract, *presets)
flat_deps = concrete.flat_dependencies()
for preset in presets:
preset_spec = Spec(preset)
name = preset_spec.name
self.assertTrue(name in flat_deps)
self.check_spec(preset_spec, flat_deps[name])
return concrete
def test_concretize_no_deps(self):
self.check_concretize('libelf')
self.check_concretize('libelf@0.8.13')
def test_concretize_dag(self):
self.check_concretize('mpileaks')
self.check_concretize('callpath')
def test_concretize_with_presets(self):
self.check_presets('mpileaks', 'callpath@0.8')
self.check_presets('mpileaks', 'callpath@0.9', 'dyninst@8.0+debug')
self.check_concretize('callpath', 'libelf@0.8.13+debug~foo', 'mpich@1.0')

View file

@ -0,0 +1 @@
skip_test = True

View file

@ -5,6 +5,8 @@ class Callpath(Package):
url = "http://github.com/tgamblin/callpath-0.2.tar.gz"
md5 = "foobarbaz"
versions = [0.8, 0.9, 1.0]
depends_on("dyninst")
depends_on("mpich")

View file

@ -5,6 +5,8 @@ class Dyninst(Package):
url = "http://www.dyninst.org/sites/default/files/downloads/dyninst/8.1.2/DyninstAPI-8.1.2.tgz"
md5 = "bf03b33375afa66fe0efa46ce3f4b17a"
versions = '7.0, 7.0.1, 8.0, 8.1.1, 8.1.2'
depends_on("libelf")
depends_on("libdwarf")

View file

@ -10,6 +10,7 @@ class Libdwarf(Package):
md5 = "64b42692e947d5180e162e46c689dfbf"
list_url = "http://reality.sgiweb.org/davea/dwarf.html"
versions = '20111030, 20120410, 20130207'
depends_on("libelf")

View file

@ -5,6 +5,8 @@ class Libelf(Package):
url = "http://www.mr511.de/software/libelf-0.8.13.tar.gz"
md5 = "4136d7b4c04df68b686570afa26988ac"
versions = '0.8.10, 0.8.12, 0.8.13'
def install(self, prefix):
configure("--prefix=%s" % prefix,
"--enable-shared",

View file

@ -5,6 +5,8 @@ class Mpich(Package):
url = "http://www.mpich.org/static/downloads/3.0.4/mpich-3.0.4.tar.gz"
md5 = "9c5d5d4fe1e17dd12153f40bc5b6dbc0"
versions = '1.0.3, 1.3.2p1, 1.4.1p1, 3.0.4, 3.1b1'
def install(self, prefix):
configure("--prefix=%s" % prefix)
make()

View file

@ -5,6 +5,8 @@ class Mpileaks(Package):
url = "http://www.llnl.gov/mpileaks-1.0.tar.gz"
md5 = "foobarbaz"
versions = [1.0, 2.1, 2.2, 2.3]
depends_on("mpich")
depends_on("callpath")

View file

@ -0,0 +1,55 @@
import unittest
import spack
import spack.packages as packages
from spack.spec import Spec
from spack.util.lang import new_path, list_modules
mock_packages_path = new_path(spack.module_path, 'test', 'mock_packages')
original_deps = None
def set_pkg_dep(pkg, spec):
"""Alters dependence information for a pacakge.
Use this to mock up constraints.
"""
spec = Spec(spec)
packages.get(pkg).dependencies[spec.name] = spec
def restore_dependencies():
# each time through restore original dependencies & constraints
global original_deps
for pkg_name, deps in original_deps.iteritems():
packages.get(pkg_name).dependencies.clear()
for dep in deps:
set_pkg_dep(pkg_name, dep)
class MockPackagesTest(unittest.TestCase):
@classmethod
def setUpClass(cls):
# Use a different packages directory for these tests. We want to use
# mocked up packages that don't interfere with the real ones.
cls.real_packages_path = spack.packages_path
spack.packages_path = mock_packages_path
# First time through, record original relationships bt/w packages
global original_deps
original_deps = {}
for name in list_modules(mock_packages_path):
pkg = packages.get(name)
original_deps[name] = [
spec for spec in pkg.dependencies.values()]
@classmethod
def tearDownClass(cls):
"""Restore the real packages path after any test."""
restore_dependencies()
spack.packages_path = cls.real_packages_path
def setUp(self):
"""Before each test, restore deps between packages to original state."""
restore_dependencies()

View file

@ -6,61 +6,18 @@
Each test validates conditions with the packages in those directories.
"""
import unittest
import spack
import spack.package
import spack.packages as packages
from spack.util.lang import new_path, list_modules
from spack.spec import Spec
from spack.test.mock_packages_test import *
mock_packages_path = new_path(spack.module_path, 'test', 'mock_packages')
def set_pkg_dep(pkg, spec):
"""Alters dependence information for a pacakge.
Use this to mock up constraints.
"""
spec = Spec(spec)
packages.get(pkg).dependencies[spec.name] = spec
class ValidationTest(unittest.TestCase):
@classmethod
def setUpClass(cls):
# Use a different packages directory for these tests. We want to use
# mocked up packages that don't interfere with the real ones.
cls.real_packages_path = spack.packages_path
spack.packages_path = mock_packages_path
# First time through, record original relationships bt/w packages
cls.original_deps = {}
for name in list_modules(mock_packages_path):
pkg = packages.get(name)
cls.original_deps[name] = [
spec for spec in pkg.dependencies.values()]
@classmethod
def restore(cls):
# each time through restore original dependencies & constraints
for pkg_name, deps in cls.original_deps.iteritems():
packages.get(pkg_name).dependencies.clear()
for dep in deps:
set_pkg_dep(pkg_name, dep)
@classmethod
def tearDownClass(cls):
"""Restore the real packages path after any test."""
cls.restore()
spack.packages_path = cls.real_packages_path
def setUp(self):
"""Before each test, restore deps between packages to original state."""
ValidationTest.restore()
class ValidationTest(MockPackagesTest):
def test_conflicting_package_constraints(self):
set_pkg_dep('mpileaks', 'mpich@1.0')
@ -87,6 +44,25 @@ def test_conflicting_spec_constraints(self):
self.assertRaises(spack.spec.InconsistentSpecError, mpileaks.flatten)
def test_normalize_twice(self):
"""Make sure normalize can be run twice on the same spec,
and that it is idempotent."""
spec = Spec('mpileaks')
spec.normalize()
n1 = spec.copy()
spec.normalize()
self.assertEqual(n1, spec)
def test_normalize_a_lot(self):
spec = Spec('mpileaks')
spec.normalize()
spec.normalize()
spec.normalize()
spec.normalize()
def test_unsatisfiable_version(self):
set_pkg_dep('mpileaks', 'mpich@1.0')
spec = Spec('mpileaks ^mpich@2.0 ^callpath ^dyninst ^libelf ^libdwarf')

View file

@ -24,15 +24,15 @@ def verbose(message, *args):
def debug(*args):
if spack.debug:
info(message, *args, format='*c')
info("Debug: " + message, *args, format='*c')
def error(message, *args):
info(message, *args, format='*r')
info("Error: " + message, *args, format='*r')
def warn(message, *args):
info(message, *args, format='*Y')
info("Warning: " + message, *args, format='*Y')
def die(message, *args):

View file

@ -97,5 +97,5 @@ def copy(self):
# Copy everything from this dict into it.
for key in self:
clone[key] = self[key]
clone[key] = self[key].copy()
return clone

View file

@ -347,6 +347,13 @@ class VersionList(object):
def __init__(self, vlist=None):
self.versions = []
if vlist != None:
if type(vlist) == str:
vlist = _string_to_version(vlist)
if type(vlist) == VersionList:
self.versions = vlist.versions
else:
self.versions = [vlist]
else:
vlist = list(vlist)
for v in vlist:
self.add(ver(v))
@ -540,6 +547,8 @@ def ver(obj):
return VersionList(obj)
elif t == str:
return _string_to_version(obj)
elif t in (int, float):
return _string_to_version(str(obj))
elif t in (Version, VersionRange, VersionList):
return obj
else: