Merge pull request #847 from epfl-scitas/features/test_install_with_time

test-install command : added elapsed time + xml is prettyprinted

lib/spack/spack/cmd/test-install.py

@@ -23,87 +23,106 @@
 # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
 ##############################################################################
 import argparse
-import xml.etree.ElementTree as ET
-import itertools
-import re
-import os
 import codecs
+import os
+import time
+import xml.dom.minidom
+import xml.etree.ElementTree as ET
 
 import llnl.util.tty as tty
-from llnl.util.filesystem import *
-
 import spack
+import spack.cmd
+from llnl.util.filesystem import *
 from spack.build_environment import InstallError
 from spack.fetch_strategy import FetchError
-import spack.cmd
 
 description = "Run package installation as a unit test, output formatted results."
 
+
 def setup_parser(subparser):
-    subparser.add_argument(
+    subparser.add_argument('-j',
-        '-j', '--jobs', action='store', type=int,
+                           '--jobs',
-        help="Explicitly set number of make jobs.  Default is #cpus.")
+                           action='store',
+                           type=int,
+                           help="Explicitly set number of make jobs.  Default is #cpus.")
 
-    subparser.add_argument(
+    subparser.add_argument('-n',
-        '-n', '--no-checksum', action='store_true', dest='no_checksum',
+                           '--no-checksum',
-        help="Do not check packages against checksum")
+                           action='store_true',
+                           dest='no_checksum',
+                           help="Do not check packages against checksum")
 
-    subparser.add_argument(
+    subparser.add_argument('-o', '--output', action='store', help="test output goes in this file")
-        '-o', '--output', action='store', help="test output goes in this file")
 
-    subparser.add_argument(
+    subparser.add_argument('package', nargs=argparse.REMAINDER, help="spec of package to install")
-        'package', nargs=argparse.REMAINDER, help="spec of package to install")
-
-
-class JunitResultFormat(object):
-    def __init__(self):
-        self.root = ET.Element('testsuite')
-        self.tests = []
-
-    def add_test(self, buildId, testResult, buildInfo=None):
-        self.tests.append((buildId, testResult, buildInfo))
-
-    def write_to(self, stream):
-        self.root.set('tests', '{0}'.format(len(self.tests)))
-        for buildId, testResult, buildInfo in self.tests:
-            testcase = ET.SubElement(self.root, 'testcase')
-            testcase.set('classname', buildId.name)
-            testcase.set('name', buildId.stringId())
-            if testResult == TestResult.FAILED:
-                failure = ET.SubElement(testcase, 'failure')
-                failure.set('type', "Build Error")
-                failure.text = buildInfo
-            elif testResult == TestResult.SKIPPED:
-                skipped = ET.SubElement(testcase, 'skipped')
-                skipped.set('type', "Skipped Build")
-                skipped.text = buildInfo
-        ET.ElementTree(self.root).write(stream)
 
 
 class TestResult(object):
     PASSED = 0
     FAILED = 1
     SKIPPED = 2
+    ERRORED = 3
 
 
-class BuildId(object):
+class TestSuite(object):
-    def __init__(self, spec):
+    def __init__(self, filename):
-        self.name = spec.name
+        self.filename = filename
-        self.version = spec.version
+        self.root = ET.Element('testsuite')
-        self.hashId = spec.dag_hash()
+        self.tests = []
 
-    def stringId(self):
+    def __enter__(self):
-        return "-".join(str(x) for x in (self.name, self.version, self.hashId))
+        return self
 
-    def __hash__(self):
+    def append(self, item):
-        return hash((self.name, self.version, self.hashId))
+        if not isinstance(item, TestCase):
+            raise TypeError('only TestCase instances may be appended to a TestSuite instance')
+        self.tests.append(item)  # Append the item to the list of tests
 
-    def __eq__(self, other):
+    def __exit__(self, exc_type, exc_val, exc_tb):
-        if not isinstance(other, BuildId):
+        # Prepare the header for the entire test suite
-            return False
+        number_of_errors = sum(x.result_type == TestResult.ERRORED for x in self.tests)
+        self.root.set('errors', str(number_of_errors))
+        number_of_failures = sum(x.result_type == TestResult.FAILED for x in self.tests)
+        self.root.set('failures', str(number_of_failures))
+        self.root.set('tests', str(len(self.tests)))
 
-        return ((self.name, self.version, self.hashId) ==
+        for item in self.tests:
-            (other.name, other.version, other.hashId))
+            self.root.append(item.element)
+
+        with open(self.filename, 'wb') as file:
+            xml_string = ET.tostring(self.root)
+            xml_string = xml.dom.minidom.parseString(xml_string).toprettyxml()
+            file.write(xml_string)
+
+
+class TestCase(object):
+
+    results = {
+        TestResult.PASSED: None,
+        TestResult.SKIPPED: 'skipped',
+        TestResult.FAILED: 'failure',
+        TestResult.ERRORED: 'error',
+    }
+
+    def __init__(self, classname, name, time=None):
+        self.element = ET.Element('testcase')
+        self.element.set('classname', str(classname))
+        self.element.set('name', str(name))
+        if time is not None:
+            self.element.set('time', str(time))
+        self.result_type = None
+
+    def set_result(self, result_type, message=None, error_type=None, text=None):
+        self.result_type = result_type
+        result = TestCase.results[self.result_type]
+        if result is not None and result is not TestResult.PASSED:
+            subelement = ET.SubElement(self.element, result)
+            if error_type is not None:
+                subelement.set('type', error_type)
+            if message is not None:
+                subelement.set('message', str(message))
+            if text is not None:
+                subelement.text = text
 
 
 def fetch_log(path):
@@ -114,46 +133,76 @@ def fetch_log(path):
 
 
 def failed_dependencies(spec):
-    return set(childSpec for childSpec in spec.dependencies.itervalues() if not
+    return set(item for item in spec.dependencies.itervalues() if not spack.repo.get(item).installed)
-        spack.repo.get(childSpec).installed)
 
 
-def create_test_output(topSpec, newInstalls, output, getLogFunc=fetch_log):
+def get_top_spec_or_die(args):
-    # Post-order traversal is not strictly required but it makes sense to output
+    specs = spack.cmd.parse_specs(args.package, concretize=True)
-    # tests for dependencies first.
+    if len(specs) > 1:
-    for spec in topSpec.traverse(order='post'):
+        tty.die("Only 1 top-level package can be specified")
-        if spec not in newInstalls:
+    top_spec = iter(specs).next()
-            continue
+    return top_spec
 
-        failedDeps = failed_dependencies(spec)
-        package = spack.repo.get(spec)
-        if failedDeps:
-            result = TestResult.SKIPPED
-            dep = iter(failedDeps).next()
-            depBID = BuildId(dep)
-            errOutput = "Skipped due to failed dependency: {0}".format(
-                depBID.stringId())
-        elif (not package.installed) and (not package.stage.source_path):
-            result = TestResult.FAILED
-            errOutput = "Failure to fetch package resources."
-        elif not package.installed:
-            result = TestResult.FAILED
-            lines = getLogFunc(package.build_log_path)
-            errMessages = list(line for line in lines if
-                re.search('error:', line, re.IGNORECASE))
-            errOutput = errMessages if errMessages else lines[-10:]
-            errOutput = '\n'.join(itertools.chain(
-                    [spec.to_yaml(), "Errors:"], errOutput,
-                    ["Build Log:", package.build_log_path]))
-        else:
-            result = TestResult.PASSED
-            errOutput = None
 
-        bId = BuildId(spec)
+def install_single_spec(spec, number_of_jobs):
-        output.add_test(bId, result, errOutput)
+    package = spack.repo.get(spec)
+
+    # If it is already installed, skip the test
+    if spack.repo.get(spec).installed:
+        testcase = TestCase(package.name, package.spec.short_spec, time=0.0)
+        testcase.set_result(TestResult.SKIPPED, message='Skipped [already installed]', error_type='already_installed')
+        return testcase
+
+    # If it relies on dependencies that did not install, skip
+    if failed_dependencies(spec):
+        testcase = TestCase(package.name, package.spec.short_spec, time=0.0)
+        testcase.set_result(TestResult.SKIPPED, message='Skipped [failed dependencies]', error_type='dep_failed')
+        return testcase
+
+    # Otherwise try to install the spec
+    try:
+        start_time = time.time()
+        package.do_install(keep_prefix=False,
+                           keep_stage=True,
+                           ignore_deps=False,
+                           make_jobs=number_of_jobs,
+                           verbose=True,
+                           fake=False)
+        duration = time.time() - start_time
+        testcase = TestCase(package.name, package.spec.short_spec, duration)
+        testcase.set_result(TestResult.PASSED)
+    except InstallError:
+        # An InstallError is considered a failure (the recipe didn't work correctly)
+        duration = time.time() - start_time
+        # Try to get the log
+        lines = fetch_log(package.build_log_path)
+        text = '\n'.join(lines)
+        testcase = TestCase(package.name, package.spec.short_spec, duration)
+        testcase.set_result(TestResult.FAILED, message='Installation failure', text=text)
+
+    except FetchError:
+        # A FetchError is considered an error (we didn't even start building)
+        duration = time.time() - start_time
+        testcase = TestCase(package.name, package.spec.short_spec, duration)
+        testcase.set_result(TestResult.ERRORED, message='Unable to fetch package')
+
+    return testcase
+
+
+def get_filename(args, top_spec):
+    if not args.output:
+        fname = 'test-{x.name}-{x.version}-{hash}.xml'.format(x=top_spec, hash=top_spec.dag_hash())
+        output_directory = join_path(os.getcwd(), 'test-output')
+        if not os.path.exists(output_directory):
+            os.mkdir(output_directory)
+        output_filename = join_path(output_directory, fname)
+    else:
+        output_filename = args.output
+    return output_filename
 
 
 def test_install(parser, args):
+    # Check the input
     if not args.package:
         tty.die("install requires a package argument")
 
@@ -162,50 +211,15 @@ def test_install(parser, args):
             tty.die("The -j option must be a positive integer!")
 
     if args.no_checksum:
-        spack.do_checksum = False        # TODO: remove this global.
+        spack.do_checksum = False  # TODO: remove this global.
 
-    specs = spack.cmd.parse_specs(args.package, concretize=True)
+    # Get the one and only top spec
-    if len(specs) > 1:
+    top_spec = get_top_spec_or_die(args)
-        tty.die("Only 1 top-level package can be specified")
+    # Get the filename of the test
-    topSpec = iter(specs).next()
+    output_filename = get_filename(args, top_spec)
-
+    # TEST SUITE
-    newInstalls = set()
+    with TestSuite(output_filename) as test_suite:
-    for spec in topSpec.traverse():
+        # Traverse in post order : each spec is a test case
-        package = spack.repo.get(spec)
+        for spec in top_spec.traverse(order='post'):
-        if not package.installed:
+            test_case = install_single_spec(spec, args.jobs)
-            newInstalls.add(spec)
+            test_suite.append(test_case)
-
-    if not args.output:
-        bId = BuildId(topSpec)
-        outputDir = join_path(os.getcwd(), "test-output")
-        if not os.path.exists(outputDir):
-            os.mkdir(outputDir)
-        outputFpath = join_path(outputDir, "test-{0}.xml".format(bId.stringId()))
-    else:
-        outputFpath = args.output
-
-    for spec in topSpec.traverse(order='post'):
-        # Calling do_install for the top-level package would be sufficient but
-        # this attempts to keep going if any package fails (other packages which
-        # are not dependents may succeed)
-        package = spack.repo.get(spec)
-        if (not failed_dependencies(spec)) and (not package.installed):
-            try:
-                package.do_install(
-                    keep_prefix=False,
-                    keep_stage=True,
-                    ignore_deps=False,
-                    make_jobs=args.jobs,
-                    verbose=True,
-                    fake=False)
-            except InstallError:
-                pass
-            except FetchError:
-                pass
-
-    jrf = JunitResultFormat()
-    handled = {}
-    create_test_output(topSpec, newInstalls, jrf)
-
-    with open(outputFpath, 'wb') as F:
-            jrf.write_to(F)

lib/spack/spack/test/__init__.py

@@ -61,14 +61,14 @@
               'optional_deps',
               'make_executable',
               'configure_guess',
-              'unit_install',
               'lock',
               'database',
               'namespace_trie',
               'yaml',
               'sbang',
               'environment',
-              'cmd.uninstall']
+              'cmd.uninstall',
+              'cmd.test_install']
 
 
 def list_tests():

lib/spack/spack/test/cmd/test_install.py

@@ -0,0 +1,190 @@
+##############################################################################
+# Copyright (c) 2013, Lawrence Livermore National Security, LLC.
+# Produced at the Lawrence Livermore National Laboratory.
+#
+# This file is part of Spack.
+# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
+# LLNL-CODE-647188
+#
+# For details, see https://github.com/llnl/spack
+# Please also see the LICENSE file for our notice and the LGPL.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License (as published by
+# the Free Software Foundation) version 2.1 dated February 1999.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and
+# conditions of the GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with this program; if not, write to the Free Software Foundation,
+# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+##############################################################################
+import collections
+from contextlib import contextmanager
+
+import StringIO
+
+FILE_REGISTRY = collections.defaultdict(StringIO.StringIO)
+
+# Monkey-patch open to write module files to a StringIO instance
+@contextmanager
+def mock_open(filename, mode):
+    if not mode == 'wb':
+        raise RuntimeError('test.test_install : unexpected opening mode for monkey-patched open')
+
+    FILE_REGISTRY[filename] = StringIO.StringIO()
+
+    try:
+        yield FILE_REGISTRY[filename]
+    finally:
+        handle = FILE_REGISTRY[filename]
+        FILE_REGISTRY[filename] = handle.getvalue()
+        handle.close()
+
+import os
+import itertools
+import unittest
+
+import spack
+import spack.cmd
+
+
+# The use of __import__ is necessary to maintain a name with hyphen (which cannot be an identifier in python)
+test_install = __import__("spack.cmd.test-install", fromlist=['test_install'])
+
+
+class MockSpec(object):
+    def __init__(self, name, version, hashStr=None):
+        self.dependencies = {}
+        self.name = name
+        self.version = version
+        self.hash = hashStr if hashStr else hash((name, version))
+
+    def traverse(self, order=None):
+        for _, spec in self.dependencies.items():
+            yield spec
+        yield self
+        #allDeps = itertools.chain.from_iterable(i.traverse() for i in self.dependencies.itervalues())
+        #return set(itertools.chain([self], allDeps))
+
+    def dag_hash(self):
+        return self.hash
+
+    @property
+    def short_spec(self):
+        return '-'.join([self.name, str(self.version), str(self.hash)])
+
+
+class MockPackage(object):
+    def __init__(self, spec, buildLogPath):
+        self.name = spec.name
+        self.spec = spec
+        self.installed = False
+        self.build_log_path = buildLogPath
+
+    def do_install(self, *args, **kwargs):
+        self.installed = True
+
+
+class MockPackageDb(object):
+    def __init__(self, init=None):
+        self.specToPkg = {}
+        if init:
+            self.specToPkg.update(init)
+
+    def get(self, spec):
+        return self.specToPkg[spec]
+
+
+def mock_fetch_log(path):
+    return []
+
+specX = MockSpec('X', "1.2.0")
+specY = MockSpec('Y', "2.3.8")
+specX.dependencies['Y'] = specY
+pkgX = MockPackage(specX, 'logX')
+pkgY = MockPackage(specY, 'logY')
+
+
+class MockArgs(object):
+    def __init__(self, package):
+        self.package = package
+        self.jobs = None
+        self.no_checksum = False
+        self.output = None
+
+
+# TODO: add test(s) where Y fails to install
+class TestInstallTest(unittest.TestCase):
+    """
+    Tests test-install where X->Y
+    """
+
+    def setUp(self):
+        super(TestInstallTest, self).setUp()
+
+        # Monkey patch parse specs
+        def monkey_parse_specs(x, concretize):
+            if x == 'X':
+                return [specX]
+            elif x == 'Y':
+                return [specY]
+            return []
+
+        self.parse_specs = spack.cmd.parse_specs
+        spack.cmd.parse_specs = monkey_parse_specs
+
+        # Monkey patch os.mkdirp
+        self.os_mkdir = os.mkdir
+        os.mkdir = lambda x: True
+
+        # Monkey patch open
+        test_install.open = mock_open
+
+        # Clean FILE_REGISTRY
+        FILE_REGISTRY = collections.defaultdict(StringIO.StringIO)
+
+        pkgX.installed = False
+        pkgY.installed = False
+
+        # Monkey patch pkgDb
+        self.saved_db = spack.repo
+        pkgDb = MockPackageDb({specX: pkgX, specY: pkgY})
+        spack.repo = pkgDb
+
+    def tearDown(self):
+        # Remove the monkey patched test_install.open
+        test_install.open = open
+
+        # Remove the monkey patched os.mkdir
+        os.mkdir = self.os_mkdir
+        del self.os_mkdir
+
+        # Remove the monkey patched parse_specs
+        spack.cmd.parse_specs = self.parse_specs
+        del self.parse_specs
+        super(TestInstallTest, self).tearDown()
+
+        spack.repo = self.saved_db
+
+    def test_installing_both(self):
+        test_install.test_install(None, MockArgs('X') )
+        self.assertEqual(len(FILE_REGISTRY), 1)
+        for _, content in FILE_REGISTRY.items():
+            self.assertTrue('tests="2"' in content)
+            self.assertTrue('failures="0"' in content)
+            self.assertTrue('errors="0"' in content)
+
+    def test_dependency_already_installed(self):
+        pkgX.installed = True
+        pkgY.installed = True
+        test_install.test_install(None, MockArgs('X'))
+        self.assertEqual(len(FILE_REGISTRY), 1)
+        for _, content in FILE_REGISTRY.items():
+            self.assertTrue('tests="2"' in content)
+            self.assertTrue('failures="0"' in content)
+            self.assertTrue('errors="0"' in content)
+            self.assertEqual(sum('skipped' in line for line in content.split('\n')), 2)

lib/spack/spack/test/unit_install.py

@@ -1,126 +0,0 @@
-##############################################################################
-# Copyright (c) 2013, Lawrence Livermore National Security, LLC.
-# Produced at the Lawrence Livermore National Laboratory.
-#
-# This file is part of Spack.
-# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
-# LLNL-CODE-647188
-#
-# For details, see https://github.com/llnl/spack
-# Please also see the LICENSE file for our notice and the LGPL.
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License (as published by
-# the Free Software Foundation) version 2.1 dated February 1999.
-#
-# This program is distributed in the hope that it will be useful, but
-# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and
-# conditions of the GNU General Public License for more details.
-#
-# You should have received a copy of the GNU Lesser General Public License
-# along with this program; if not, write to the Free Software Foundation,
-# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
-##############################################################################
-import itertools
-import unittest
-
-import spack
-
-test_install = __import__("spack.cmd.test-install",
-    fromlist=["BuildId", "create_test_output", "TestResult"])
-
-class MockOutput(object):
-    def __init__(self):
-        self.results = {}
-
-    def add_test(self, buildId, passed=True, buildInfo=None):
-        self.results[buildId] = passed
-
-    def write_to(self, stream):
-        pass
-
-class MockSpec(object):
-    def __init__(self, name, version, hashStr=None):
-        self.dependencies = {}
-        self.name = name
-        self.version = version
-        self.hash = hashStr if hashStr else hash((name, version))
-
-    def traverse(self, order=None):
-        allDeps = itertools.chain.from_iterable(i.traverse() for i in
-            self.dependencies.itervalues())
-        return set(itertools.chain([self], allDeps))
-
-    def dag_hash(self):
-        return self.hash
-
-    def to_yaml(self):
-        return "<<<MOCK YAML {0}>>>".format(test_install.BuildId(self).stringId())
-
-class MockPackage(object):
-    def __init__(self, buildLogPath):
-        self.installed = False
-        self.build_log_path = buildLogPath
-
-specX = MockSpec("X", "1.2.0")
-specY = MockSpec("Y", "2.3.8")
-specX.dependencies['Y'] = specY
-pkgX = MockPackage('logX')
-pkgY = MockPackage('logY')
-bIdX = test_install.BuildId(specX)
-bIdY = test_install.BuildId(specY)
-
-class UnitInstallTest(unittest.TestCase):
-    """Tests test-install where X->Y"""
-
-    def setUp(self):
-        super(UnitInstallTest, self).setUp()
-
-        pkgX.installed = False
-        pkgY.installed = False
-
-        self.saved_db = spack.repo
-        pkgDb = MockPackageDb({specX:pkgX, specY:pkgY})
-        spack.repo = pkgDb
-
-
-    def tearDown(self):
-        super(UnitInstallTest, self).tearDown()
-
-        spack.repo = self.saved_db
-
-    def test_installing_both(self):
-        mo = MockOutput()
-
-        pkgX.installed = True
-        pkgY.installed = True
-        test_install.create_test_output(specX, [specX, specY], mo, getLogFunc=mock_fetch_log)
-
-        self.assertEqual(mo.results,
-            {bIdX:test_install.TestResult.PASSED,
-             bIdY:test_install.TestResult.PASSED})
-
-
-    def test_dependency_already_installed(self):
-        mo = MockOutput()
-
-        pkgX.installed = True
-        pkgY.installed = True
-        test_install.create_test_output(specX, [specX], mo, getLogFunc=mock_fetch_log)
-        self.assertEqual(mo.results, {bIdX:test_install.TestResult.PASSED})
-
-    #TODO: add test(s) where Y fails to install
-
-
-class MockPackageDb(object):
-    def __init__(self, init=None):
-        self.specToPkg = {}
-        if init:
-            self.specToPkg.update(init)
-
-    def get(self, spec):
-        return self.specToPkg[spec]
-
-def mock_fetch_log(path):
-    return []