hlrs-software-stack/spack
7
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Merge pull request #847 from epfl-scitas/features/test_install_with_time

Browse source 
test-install command : added elapsed time + xml is prettyprinted
This commit is contained in:
Todd Gamblin 2016-04-27 17:32:21 -07:00
commit 8773a0b747
4 changed files with 341 additions and 263 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

284
lib/spack/spack/cmd/test-install.py
View file

@ -23,87 +23,106 @@
# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
##############################################################################
import argparse
import xml.etree.ElementTree as ET
import itertools
import re
import os
import codecs
import os
import time
import xml.dom.minidom
import xml.etree.ElementTree as ET
import llnl.util.tty as tty
from llnl.util.filesystem import *
import spack
import spack.cmd
from llnl.util.filesystem import *
from spack.build_environment import InstallError
from spack.fetch_strategy import FetchError
import spack.cmd
description = "Run package installation as a unit test, output formatted results."
def setup_parser(subparser):
subparser.add_argument(
'-j', '--jobs', action='store', type=int,
help="Explicitly set number of make jobs. Default is #cpus.")
subparser.add_argument('-j',
'--jobs',
action='store',
type=int,
help="Explicitly set number of make jobs. Default is #cpus.")
subparser.add_argument(
'-n', '--no-checksum', action='store_true', dest='no_checksum',
help="Do not check packages against checksum")
subparser.add_argument('-n',
'--no-checksum',
action='store_true',
dest='no_checksum',
help="Do not check packages against checksum")
subparser.add_argument(
'-o', '--output', action='store', help="test output goes in this file")
subparser.add_argument('-o', '--output', action='store', help="test output goes in this file")
subparser.add_argument(
'package', nargs=argparse.REMAINDER, help="spec of package to install")
class JunitResultFormat(object):
def __init__(self):
self.root = ET.Element('testsuite')
self.tests = []
def add_test(self, buildId, testResult, buildInfo=None):
self.tests.append((buildId, testResult, buildInfo))
def write_to(self, stream):
self.root.set('tests', '{0}'.format(len(self.tests)))
for buildId, testResult, buildInfo in self.tests:
testcase = ET.SubElement(self.root, 'testcase')
testcase.set('classname', buildId.name)
testcase.set('name', buildId.stringId())
if testResult == TestResult.FAILED:
failure = ET.SubElement(testcase, 'failure')
failure.set('type', "Build Error")
failure.text = buildInfo
elif testResult == TestResult.SKIPPED:
skipped = ET.SubElement(testcase, 'skipped')
skipped.set('type', "Skipped Build")
skipped.text = buildInfo
ET.ElementTree(self.root).write(stream)
subparser.add_argument('package', nargs=argparse.REMAINDER, help="spec of package to install")
class TestResult(object):
PASSED = 0
FAILED = 1
SKIPPED = 2
ERRORED = 3
class BuildId(object):
def __init__(self, spec):
self.name = spec.name
self.version = spec.version
self.hashId = spec.dag_hash()
class TestSuite(object):
def __init__(self, filename):
self.filename = filename
self.root = ET.Element('testsuite')
self.tests = []
def stringId(self):
return "-".join(str(x) for x in (self.name, self.version, self.hashId))
def __enter__(self):
return self
def __hash__(self):
return hash((self.name, self.version, self.hashId))
def append(self, item):
if not isinstance(item, TestCase):
raise TypeError('only TestCase instances may be appended to a TestSuite instance')
self.tests.append(item) # Append the item to the list of tests
def __eq__(self, other):
if not isinstance(other, BuildId):
return False
def __exit__(self, exc_type, exc_val, exc_tb):
# Prepare the header for the entire test suite
number_of_errors = sum(x.result_type == TestResult.ERRORED for x in self.tests)
self.root.set('errors', str(number_of_errors))
number_of_failures = sum(x.result_type == TestResult.FAILED for x in self.tests)
self.root.set('failures', str(number_of_failures))
self.root.set('tests', str(len(self.tests)))
return ((self.name, self.version, self.hashId) ==
(other.name, other.version, other.hashId))
for item in self.tests:
self.root.append(item.element)
with open(self.filename, 'wb') as file:
xml_string = ET.tostring(self.root)
xml_string = xml.dom.minidom.parseString(xml_string).toprettyxml()
file.write(xml_string)
class TestCase(object):
results = {
TestResult.PASSED: None,
TestResult.SKIPPED: 'skipped',
TestResult.FAILED: 'failure',
TestResult.ERRORED: 'error',
}
def __init__(self, classname, name, time=None):
self.element = ET.Element('testcase')
self.element.set('classname', str(classname))
self.element.set('name', str(name))
if time is not None:
self.element.set('time', str(time))
self.result_type = None
def set_result(self, result_type, message=None, error_type=None, text=None):
self.result_type = result_type
result = TestCase.results[self.result_type]
if result is not None and result is not TestResult.PASSED:
subelement = ET.SubElement(self.element, result)
if error_type is not None:
subelement.set('type', error_type)
if message is not None:
subelement.set('message', str(message))
if text is not None:
subelement.text = text
def fetch_log(path):
@ -114,46 +133,76 @@ def fetch_log(path):
def failed_dependencies(spec):
return set(childSpec for childSpec in spec.dependencies.itervalues() if not
spack.repo.get(childSpec).installed)
return set(item for item in spec.dependencies.itervalues() if not spack.repo.get(item).installed)
def create_test_output(topSpec, newInstalls, output, getLogFunc=fetch_log):
# Post-order traversal is not strictly required but it makes sense to output
# tests for dependencies first.
for spec in topSpec.traverse(order='post'):
if spec not in newInstalls:
continue
def get_top_spec_or_die(args):
specs = spack.cmd.parse_specs(args.package, concretize=True)
if len(specs) > 1:
tty.die("Only 1 top-level package can be specified")
top_spec = iter(specs).next()
return top_spec
failedDeps = failed_dependencies(spec)
package = spack.repo.get(spec)
if failedDeps:
result = TestResult.SKIPPED
dep = iter(failedDeps).next()
depBID = BuildId(dep)
errOutput = "Skipped due to failed dependency: {0}".format(
depBID.stringId())
elif (not package.installed) and (not package.stage.source_path):
result = TestResult.FAILED
errOutput = "Failure to fetch package resources."
elif not package.installed:
result = TestResult.FAILED
lines = getLogFunc(package.build_log_path)
errMessages = list(line for line in lines if
re.search('error:', line, re.IGNORECASE))
errOutput = errMessages if errMessages else lines[-10:]
errOutput = '\n'.join(itertools.chain(
[spec.to_yaml(), "Errors:"], errOutput,
["Build Log:", package.build_log_path]))
else:
result = TestResult.PASSED
errOutput = None
bId = BuildId(spec)
output.add_test(bId, result, errOutput)
def install_single_spec(spec, number_of_jobs):
package = spack.repo.get(spec)
# If it is already installed, skip the test
if spack.repo.get(spec).installed:
testcase = TestCase(package.name, package.spec.short_spec, time=0.0)
testcase.set_result(TestResult.SKIPPED, message='Skipped [already installed]', error_type='already_installed')
return testcase
# If it relies on dependencies that did not install, skip
if failed_dependencies(spec):
testcase = TestCase(package.name, package.spec.short_spec, time=0.0)
testcase.set_result(TestResult.SKIPPED, message='Skipped [failed dependencies]', error_type='dep_failed')
return testcase
# Otherwise try to install the spec
try:
start_time = time.time()
package.do_install(keep_prefix=False,
keep_stage=True,
ignore_deps=False,
make_jobs=number_of_jobs,
verbose=True,
fake=False)
duration = time.time() - start_time
testcase = TestCase(package.name, package.spec.short_spec, duration)
testcase.set_result(TestResult.PASSED)
except InstallError:
# An InstallError is considered a failure (the recipe didn't work correctly)
duration = time.time() - start_time
# Try to get the log
lines = fetch_log(package.build_log_path)
text = '\n'.join(lines)
testcase = TestCase(package.name, package.spec.short_spec, duration)
testcase.set_result(TestResult.FAILED, message='Installation failure', text=text)
except FetchError:
# A FetchError is considered an error (we didn't even start building)
duration = time.time() - start_time
testcase = TestCase(package.name, package.spec.short_spec, duration)
testcase.set_result(TestResult.ERRORED, message='Unable to fetch package')
return testcase
def get_filename(args, top_spec):
if not args.output:
fname = 'test-{x.name}-{x.version}-{hash}.xml'.format(x=top_spec, hash=top_spec.dag_hash())
output_directory = join_path(os.getcwd(), 'test-output')
if not os.path.exists(output_directory):
os.mkdir(output_directory)
output_filename = join_path(output_directory, fname)
else:
output_filename = args.output
return output_filename
def test_install(parser, args):
# Check the input
if not args.package:
tty.die("install requires a package argument")
@ -162,50 +211,15 @@ def test_install(parser, args):
tty.die("The -j option must be a positive integer!")
if args.no_checksum:
spack.do_checksum = False # TODO: remove this global.
spack.do_checksum = False # TODO: remove this global.
specs = spack.cmd.parse_specs(args.package, concretize=True)
if len(specs) > 1:
tty.die("Only 1 top-level package can be specified")
topSpec = iter(specs).next()
newInstalls = set()
for spec in topSpec.traverse():
package = spack.repo.get(spec)
if not package.installed:
newInstalls.add(spec)
if not args.output:
bId = BuildId(topSpec)
outputDir = join_path(os.getcwd(), "test-output")
if not os.path.exists(outputDir):
os.mkdir(outputDir)
outputFpath = join_path(outputDir, "test-{0}.xml".format(bId.stringId()))
else:
outputFpath = args.output
for spec in topSpec.traverse(order='post'):
# Calling do_install for the top-level package would be sufficient but
# this attempts to keep going if any package fails (other packages which
# are not dependents may succeed)
package = spack.repo.get(spec)
if (not failed_dependencies(spec)) and (not package.installed):
try:
package.do_install(
keep_prefix=False,
keep_stage=True,
ignore_deps=False,
make_jobs=args.jobs,
verbose=True,
fake=False)
except InstallError:
pass
except FetchError:
pass
jrf = JunitResultFormat()
handled = {}
create_test_output(topSpec, newInstalls, jrf)
with open(outputFpath, 'wb') as F:
jrf.write_to(F)
# Get the one and only top spec
top_spec = get_top_spec_or_die(args)
# Get the filename of the test
output_filename = get_filename(args, top_spec)
# TEST SUITE
with TestSuite(output_filename) as test_suite:
# Traverse in post order : each spec is a test case
for spec in top_spec.traverse(order='post'):
test_case = install_single_spec(spec, args.jobs)
test_suite.append(test_case)

4
lib/spack/spack/test/__init__.py
View file

@ -61,14 +61,14 @@
'optional_deps',
'make_executable',
'configure_guess',
'unit_install',
'lock',
'database',
'namespace_trie',
'yaml',
'sbang',
'environment',
'cmd.uninstall']
'cmd.uninstall',
'cmd.test_install']
def list_tests():

190
lib/spack/spack/test/cmd/test_install.py Normal file
View file

@ -0,0 +1,190 @@
##############################################################################
# Copyright (c) 2013, Lawrence Livermore National Security, LLC.
# Produced at the Lawrence Livermore National Laboratory.
#
# This file is part of Spack.
# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
# LLNL-CODE-647188
#
# For details, see https://github.com/llnl/spack
# Please also see the LICENSE file for our notice and the LGPL.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License (as published by
# the Free Software Foundation) version 2.1 dated February 1999.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and
# conditions of the GNU General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with this program; if not, write to the Free Software Foundation,
# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
##############################################################################
import collections
from contextlib import contextmanager
import StringIO
FILE_REGISTRY = collections.defaultdict(StringIO.StringIO)
# Monkey-patch open to write module files to a StringIO instance
@contextmanager
def mock_open(filename, mode):
if not mode == 'wb':
raise RuntimeError('test.test_install : unexpected opening mode for monkey-patched open')
FILE_REGISTRY[filename] = StringIO.StringIO()
try:
yield FILE_REGISTRY[filename]
finally:
handle = FILE_REGISTRY[filename]
FILE_REGISTRY[filename] = handle.getvalue()
handle.close()
import os
import itertools
import unittest
import spack
import spack.cmd
# The use of __import__ is necessary to maintain a name with hyphen (which cannot be an identifier in python)
test_install = __import__("spack.cmd.test-install", fromlist=['test_install'])
class MockSpec(object):
def __init__(self, name, version, hashStr=None):
self.dependencies = {}
self.name = name
self.version = version
self.hash = hashStr if hashStr else hash((name, version))
def traverse(self, order=None):
for _, spec in self.dependencies.items():
yield spec
yield self
#allDeps = itertools.chain.from_iterable(i.traverse() for i in self.dependencies.itervalues())
#return set(itertools.chain([self], allDeps))
def dag_hash(self):
return self.hash
@property
def short_spec(self):
return '-'.join([self.name, str(self.version), str(self.hash)])
class MockPackage(object):
def __init__(self, spec, buildLogPath):
self.name = spec.name
self.spec = spec
self.installed = False
self.build_log_path = buildLogPath
def do_install(self, *args, **kwargs):
self.installed = True
class MockPackageDb(object):
def __init__(self, init=None):
self.specToPkg = {}
if init:
self.specToPkg.update(init)
def get(self, spec):
return self.specToPkg[spec]
def mock_fetch_log(path):
return []
specX = MockSpec('X', "1.2.0")
specY = MockSpec('Y', "2.3.8")
specX.dependencies['Y'] = specY
pkgX = MockPackage(specX, 'logX')
pkgY = MockPackage(specY, 'logY')
class MockArgs(object):
def __init__(self, package):
self.package = package
self.jobs = None
self.no_checksum = False
self.output = None
# TODO: add test(s) where Y fails to install
class TestInstallTest(unittest.TestCase):
"""
Tests test-install where X->Y
"""
def setUp(self):
super(TestInstallTest, self).setUp()
# Monkey patch parse specs
def monkey_parse_specs(x, concretize):
if x == 'X':
return [specX]
elif x == 'Y':
return [specY]
return []
self.parse_specs = spack.cmd.parse_specs
spack.cmd.parse_specs = monkey_parse_specs
# Monkey patch os.mkdirp
self.os_mkdir = os.mkdir
os.mkdir = lambda x: True
# Monkey patch open
test_install.open = mock_open
# Clean FILE_REGISTRY
FILE_REGISTRY = collections.defaultdict(StringIO.StringIO)
pkgX.installed = False
pkgY.installed = False
# Monkey patch pkgDb
self.saved_db = spack.repo
pkgDb = MockPackageDb({specX: pkgX, specY: pkgY})
spack.repo = pkgDb
def tearDown(self):
# Remove the monkey patched test_install.open
test_install.open = open
# Remove the monkey patched os.mkdir
os.mkdir = self.os_mkdir
del self.os_mkdir
# Remove the monkey patched parse_specs
spack.cmd.parse_specs = self.parse_specs
del self.parse_specs
super(TestInstallTest, self).tearDown()
spack.repo = self.saved_db
def test_installing_both(self):
test_install.test_install(None, MockArgs('X') )
self.assertEqual(len(FILE_REGISTRY), 1)
for _, content in FILE_REGISTRY.items():
self.assertTrue('tests="2"' in content)
self.assertTrue('failures="0"' in content)
self.assertTrue('errors="0"' in content)
def test_dependency_already_installed(self):
pkgX.installed = True
pkgY.installed = True
test_install.test_install(None, MockArgs('X'))
self.assertEqual(len(FILE_REGISTRY), 1)
for _, content in FILE_REGISTRY.items():
self.assertTrue('tests="2"' in content)
self.assertTrue('failures="0"' in content)
self.assertTrue('errors="0"' in content)
self.assertEqual(sum('skipped' in line for line in content.split('\n')), 2)

126
lib/spack/spack/test/unit_install.py
View file

@ -1,126 +0,0 @@
##############################################################################
# Copyright (c) 2013, Lawrence Livermore National Security, LLC.
# Produced at the Lawrence Livermore National Laboratory.
#
# This file is part of Spack.
# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
# LLNL-CODE-647188
#
# For details, see https://github.com/llnl/spack
# Please also see the LICENSE file for our notice and the LGPL.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License (as published by
# the Free Software Foundation) version 2.1 dated February 1999.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and
# conditions of the GNU General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with this program; if not, write to the Free Software Foundation,
# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
##############################################################################
import itertools
import unittest
import spack
test_install = __import__("spack.cmd.test-install",
fromlist=["BuildId", "create_test_output", "TestResult"])
class MockOutput(object):
def __init__(self):
self.results = {}
def add_test(self, buildId, passed=True, buildInfo=None):
self.results[buildId] = passed
def write_to(self, stream):
pass
class MockSpec(object):
def __init__(self, name, version, hashStr=None):
self.dependencies = {}
self.name = name
self.version = version
self.hash = hashStr if hashStr else hash((name, version))
def traverse(self, order=None):
allDeps = itertools.chain.from_iterable(i.traverse() for i in
self.dependencies.itervalues())
return set(itertools.chain([self], allDeps))
def dag_hash(self):
return self.hash
def to_yaml(self):
return "<<<MOCK YAML {0}>>>".format(test_install.BuildId(self).stringId())
class MockPackage(object):
def __init__(self, buildLogPath):
self.installed = False
self.build_log_path = buildLogPath
specX = MockSpec("X", "1.2.0")
specY = MockSpec("Y", "2.3.8")
specX.dependencies['Y'] = specY
pkgX = MockPackage('logX')
pkgY = MockPackage('logY')
bIdX = test_install.BuildId(specX)
bIdY = test_install.BuildId(specY)
class UnitInstallTest(unittest.TestCase):
"""Tests test-install where X->Y"""
def setUp(self):
super(UnitInstallTest, self).setUp()
pkgX.installed = False
pkgY.installed = False
self.saved_db = spack.repo
pkgDb = MockPackageDb({specX:pkgX, specY:pkgY})
spack.repo = pkgDb
def tearDown(self):
super(UnitInstallTest, self).tearDown()
spack.repo = self.saved_db
def test_installing_both(self):
mo = MockOutput()
pkgX.installed = True
pkgY.installed = True
test_install.create_test_output(specX, [specX, specY], mo, getLogFunc=mock_fetch_log)
self.assertEqual(mo.results,
{bIdX:test_install.TestResult.PASSED,
bIdY:test_install.TestResult.PASSED})
def test_dependency_already_installed(self):
mo = MockOutput()
pkgX.installed = True
pkgY.installed = True
test_install.create_test_output(specX, [specX], mo, getLogFunc=mock_fetch_log)
self.assertEqual(mo.results, {bIdX:test_install.TestResult.PASSED})
#TODO: add test(s) where Y fails to install
class MockPackageDb(object):
def __init__(self, init=None):
self.specToPkg = {}
if init:
self.specToPkg.update(init)
def get(self, spec):
return self.specToPkg[spec]
def mock_fetch_log(path):
return []