Rework Spack's Mercurial support (#3834)

* Add tests to mercurial package * Add support for --insecure with mercurial fetching * Install man pages and tab-completion scripts * Add tests and latest version for all deps * Flake8 fix * Use certifi module to find CA certificate * Flake8 fix * Unset PYTHONPATH when running hg * svn_fetch should use to svn-test, not hg-test * Drop Python 3 support in Mercurial Python 3 support is a work in progress and isn't currently recommended: https://www.mercurial-scm.org/wiki/SupportedPythonVersions * Test both secure and insecure hg fetching * Test both secure and insecure git and svn fetching
2017-04-19 20:59:04 -05:00 · 2017-04-19 20:59:04 -05:00 · f4858cb7a7
commit f4858cb7a7
parent 53763f7698
15 changed files with 176 additions and 52 deletions
--- a/lib/spack/spack/fetch_strategy.py
+++ b/lib/spack/spack/fetch_strategy.py
@ -808,8 +808,17 @@ def __init__(self, **kwargs):

    @property
    def hg(self):
+        """:returns: The hg executable
+        :rtype: Executable
+        """
        if not self._hg:
            self._hg = which('hg', required=True)
+
+            # When building PythonPackages, Spack automatically sets
+            # PYTHONPATH. This can interfere with hg, which is a Python
+            # script. Unset PYTHONPATH while running hg.
+            self._hg.add_default_env('PYTHONPATH', '')
+
        return self._hg

    @property
@ -829,9 +838,15 @@ def fetch(self):
            args.append('at revision %s' % self.revision)
        tty.msg("Trying to clone Mercurial repository:", self.url, *args)

-        args = ['clone', self.url]
+        args = ['clone']
+
+        if spack.insecure:
+            args.append('--insecure')
+
+        args.append(self.url)
+
        if self.revision:
-            args += ['-r', self.revision]
+            args.extend(['-r', self.revision])

        self.hg(*args)

--- a/lib/spack/spack/test/git_fetch.py
+++ b/lib/spack/spack/test/git_fetch.py
@ -37,8 +37,15 @@ def type_of_test(request):
    return request.param


+@pytest.fixture(params=[True, False])
+def secure(request):
+    """Attempt both secure and insecure fetching"""
+    return request.param
+
+
 def test_fetch(
        type_of_test,
+        secure,
        mock_git_repository,
        config,
        refresh_builtin_mock
@ -62,7 +69,12 @@ def test_fetch(
    pkg.versions[ver('git')] = t.args
    # Enter the stage directory and check some properties
    with pkg.stage:
+        try:
+            spack.insecure = secure
            pkg.do_stage()
+        finally:
+            spack.insecure = False
+
        assert h('HEAD') == h(t.revision)

        file_path = join_path(pkg.stage.source_path, t.file)
--- a/lib/spack/spack/test/hg_fetch.py
+++ b/lib/spack/spack/test/hg_fetch.py
@ -37,8 +37,15 @@ def type_of_test(request):
    return request.param


+@pytest.fixture(params=[True, False])
+def secure(request):
+    """Attempt both secure and insecure fetching"""
+    return request.param
+
+
 def test_fetch(
        type_of_test,
+        secure,
        mock_hg_repository,
        config,
        refresh_builtin_mock
@ -62,7 +69,12 @@ def test_fetch(
    pkg.versions[ver('hg')] = t.args
    # Enter the stage directory and check some properties
    with pkg.stage:
+        try:
+            spack.insecure = secure
            pkg.do_stage()
+        finally:
+            spack.insecure = False
+
        assert h() == t.revision

        file_path = join_path(pkg.stage.source_path, t.file)
--- a/lib/spack/spack/test/svn_fetch.py
+++ b/lib/spack/spack/test/svn_fetch.py
@ -33,12 +33,19 @@

@pytest.fixture(params=['default', 'rev0'])
 def type_of_test(request):
-    """Returns one of the test type available for the mock_hg_repository"""
+    """Returns one of the test type available for the mock_svn_repository"""
+    return request.param
+
+
+@pytest.fixture(params=[True, False])
+def secure(request):
+    """Attempt both secure and insecure fetching"""
    return request.param


 def test_fetch(
        type_of_test,
+        secure,
        mock_svn_repository,
        config,
        refresh_builtin_mock
@ -56,13 +63,18 @@ def test_fetch(
    t = mock_svn_repository.checks[type_of_test]
    h = mock_svn_repository.hash
    # Construct the package under test
-    spec = Spec('hg-test')
+    spec = Spec('svn-test')
    spec.concretize()
    pkg = spack.repo.get(spec, new=True)
-    pkg.versions[ver('hg')] = t.args
+    pkg.versions[ver('svn')] = t.args
    # Enter the stage directory and check some properties
    with pkg.stage:
+        try:
+            spack.insecure = secure
            pkg.do_stage()
+        finally:
+            spack.insecure = False
+
        assert h() == t.revision

        file_path = join_path(pkg.stage.source_path, t.file)
--- a/var/spack/repos/builtin/packages/mercurial/package.py
+++ b/var/spack/repos/builtin/packages/mercurial/package.py
@ -23,16 +23,23 @@
 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
 ##############################################################################
 from spack import *
-import llnl.util.tty as tty
-import os
+from llnl.util import tty


-class Mercurial(Package):
+class Mercurial(PythonPackage):
    """Mercurial is a free, distributed source control management tool."""

    homepage = "https://www.mercurial-scm.org"
-    url      = "https://www.mercurial-scm.org/release/mercurial-3.9.tar.gz"
+    url      = "https://www.mercurial-scm.org/release/mercurial-4.1.2.tar.gz"

+    import_modules = [
+        'hgext', 'hgext3rd', 'mercurial', 'hgext.convert', 'hgext.fsmonitor',
+        'hgext.highlight', 'hgext.largefiles', 'hgext.zeroconf',
+        'hgext.fsmonitor.pywatchman', 'mercurial.hgweb',
+        'mercurial.httpclient', 'mercurial.pure'
+    ]
+
+    version('4.1.2', '934c99808bdc8385e074b902d59b0d93')
    version('3.9.1', '3759dd10edb8c1a6dfb8ff0ce82658ce')
    version('3.9',   'e2b355da744e94747daae3a5339d28a0')
    version('3.8.4', 'cec2c3db688cb87142809089c6ae13e9')
@ -40,30 +47,62 @@ class Mercurial(Package):
    version('3.8.2', 'c38daa0cbe264fc621dc3bb05933b0b3')
    version('3.8.1', '172a8c588adca12308c2aca16608d7f4')

-    extends('python')
    depends_on('python@2.6:2.8')
    depends_on('py-docutils', type='build')
+    depends_on('py-pygments', type=('build', 'run'))
+    depends_on('py-certifi',  type=('build', 'run'))

-    def install(self, spec, prefix):
-        make('install', 'PREFIX={0}'.format(prefix))
+    @run_after('install')
+    def post_install(self):
+        prefix = self.prefix

-        # Configuration of HTTPS certificate authorities
-        # https://www.mercurial-scm.org/wiki/CACertificates
-        hgrc_filename = join_path(prefix.etc, 'mercurial', 'hgrc')
-        mkdirp(os.path.dirname(hgrc_filename))
+        # Install man pages
+        mkdirp(prefix.man1)
+        mkdirp(prefix.man5)
+        mkdirp(prefix.man8)
+        with working_dir('doc'):
+            install('hg.1', prefix.man1)
+            install('hgignore.5', prefix.man5)
+            install('hgrc.5', prefix.man5)
+            install('hg-ssh.8', prefix.man8)

-        with open(hgrc_filename, 'w') as hgrc:
-            if os.path.exists('/etc/ssl/certs/ca-certificates.crt'):
-                # Debian/Ubuntu/Gentoo/Arch Linux
-                hgrc.write('[web]\ncacerts = /etc/ssl/certs/ca-certificates.crt')  # noqa
-            elif os.path.exists('/etc/pki/tls/certs/ca-bundle.crt'):
-                # Fedora/RHEL/CentOS
-                hgrc.write('[web]\ncacerts = /etc/pki/tls/certs/ca-bundle.crt')
-            elif os.path.exists('/etc/ssl/ca-bundle.pem'):
-                # openSUSE/SLE
-                hgrc.write('[web]\ncacerts = /etc/ssl/ca-bundle.pem')
-            else:
+        # Install completion scripts
+        contrib = join_path(prefix, 'contrib')
+        mkdir(contrib)
+        with working_dir('contrib'):
+            install('bash_completion', join_path(contrib, 'bash_completion'))
+            install('zsh_completion',  join_path(contrib, 'zsh_completion'))
+
+    @run_after('install')
+    def configure_certificates(self):
+        """Configuration of HTTPS certificate authorities
+        https://www.mercurial-scm.org/wiki/CACertificates"""
+
+        etc_dir = join_path(self.prefix.etc, 'mercurial')
+        mkdirp(etc_dir)
+
+        hgrc_filename = join_path(etc_dir, 'hgrc')
+
+        # Use certifi to find the location of the CA certificate
+        certificate = python('-c', 'import certifi; print certifi.where()',
+                             output=str)
+
+        if not certificate:
            tty.warn('CA certificate not found. You may not be able to '
                     'connect to an HTTPS server. If your CA certificate '
                     'is in a non-standard location, you should add it to '
-                         '{0}'.format(hgrc_filename))
+                     '{0}.'.format(hgrc_filename))
+
+        # Write the global mercurial configuration file
+        with open(hgrc_filename, 'w') as hgrc:
+            hgrc.write('[web]\ncacerts = {0}'.format(certificate))
+
+    @run_after('install')
+    @on_package_attributes(run_tests=True)
+    def check_install(self):
+        """Sanity-check setup."""
+
+        hg = Executable(join_path(self.prefix.bin, 'hg'))
+
+        hg('debuginstall')
+        hg('version')
--- a/var/spack/repos/builtin/packages/py-appdirs/package.py
+++ b/var/spack/repos/builtin/packages/py-appdirs/package.py
@ -30,8 +30,11 @@ class PyAppdirs(PythonPackage):
    dirs, e.g. a "user data dir"."""

    homepage = "https://github.com/ActiveState/appdirs"
-    url      = "https://pypi.io/packages/source/a/appdirs/appdirs-1.4.0.tar.gz"
+    url      = "https://pypi.io/packages/source/a/appdirs/appdirs-1.4.3.tar.gz"

+    import_modules = ['appdirs']
+
+    version('1.4.3', '44c679904082a2133f5566c8a0d3ab42')
    version('1.4.0', '1d17b4c9694ab84794e228f28dc3275b')

    patch('setuptools-import.patch', when='@:1.4.0')
--- a/var/spack/repos/builtin/packages/py-certifi/package.py
+++ b/var/spack/repos/builtin/packages/py-certifi/package.py
@ -29,9 +29,12 @@ class PyCertifi(PythonPackage):
    """Certifi: A carefully curated collection of Root Certificates for validating
    the trustworthiness of SSL certificates while verifying the identity of TLS
    hosts."""
-    homepage = "https://github.com/certifi/python-certifi"
-    url      = "https://github.com/certifi/python-certifi/archive/2016.02.28.tar.gz"
+    homepage = "http://certifi.io/"
+    url      = "https://pypi.io/packages/source/c/certifi/certifi-2017.1.23.tar.gz"

+    import_modules = ['certifi']
+
+    version('2017.1.23',  'b72521a8badff5e89a8eabea586d79ab')
    version('2016.02.28', '5ccfc23bd5e931863f0b01ef3e9d2dbd3bef0e1b')

    depends_on('py-setuptools', type='build')
--- a/var/spack/repos/builtin/packages/py-cffi/package.py
+++ b/var/spack/repos/builtin/packages/py-cffi/package.py
@ -23,14 +23,17 @@
 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
 ##############################################################################
 from spack import *
+import sys


 class PyCffi(PythonPackage):
    """Foreign Function Interface for Python calling C code"""
    homepage = "http://cffi.readthedocs.org/en/latest/"
-    # base https://pypi.python.org/pypi/cffi
-    url      = "https://pypi.python.org/packages/source/c/cffi/cffi-1.1.2.tar.gz"
+    url      = "https://pypi.io/packages/source/c/cffi/cffi-1.10.0.tar.gz"

+    import_modules = ['cffi']
+
+    version('1.10.0', '2b5fa41182ed0edaf929a789e602a070')
    version('1.1.2',  'ca6e6c45b45caa87aee9adc7c796eaea')

    depends_on('py-setuptools', type='build')
@ -44,4 +47,5 @@ def setup_environment(self, spack_env, run_env):
        # other compilation.  We are setting the 'LDSHARED" to the
        # spack compiler wrapper plus a few extra flags necessary for
        # building the shared library.
+        if not sys.platform == 'darwin':
            spack_env.set('LDSHARED', "{0} -shared -pthread".format(spack_cc))
--- a/var/spack/repos/builtin/packages/py-docutils/package.py
+++ b/var/spack/repos/builtin/packages/py-docutils/package.py
@ -33,8 +33,18 @@ class PyDocutils(PythonPackage):
    markup language."""

    homepage = "http://docutils.sourceforge.net/"
-    url      = "https://pypi.python.org/packages/source/d/docutils/docutils-0.12.tar.gz"
+    url      = "https://pypi.io/packages/source/d/docutils/docutils-0.13.1.tar.gz"

-    version('0.13.1', 'ea4a893c633c788be9b8078b6b305d53',
-            url="https://pypi.python.org/packages/05/25/7b5484aca5d46915493f1fd4ecb63c38c333bd32aa9ad6e19da8d08895ae/docutils-0.13.1.tar.gz")
+    import_modules = [
+        'docutils', 'docutils.languages', 'docutils.parsers',
+        'docutils.readers', 'docutils.transforms', 'docutils.utils',
+        'docutils.writers', 'docutils.parsers.rst',
+        'docutils.parsers.rst.directives', 'docutils.parsers.rst.languages',
+        'docutils.utils.math', 'docutils.writers.html4css1',
+        'docutils.writers.html5_polyglot', 'docutils.writers.latex2e',
+        'docutils.writers.odf_odt', 'docutils.writers.pep_html',
+        'docutils.writers.s5_html', 'docutils.writers.xetex'
+    ]
+
+    version('0.13.1', 'ea4a893c633c788be9b8078b6b305d53')
    version('0.12',   '4622263b62c5c771c03502afa3157768')
--- a/var/spack/repos/builtin/packages/py-packaging/package.py
+++ b/var/spack/repos/builtin/packages/py-packaging/package.py
@ -31,6 +31,8 @@ class PyPackaging(PythonPackage):
    homepage = "https://github.com/pypa/packaging"
    url      = "https://pypi.io/packages/source/p/packaging/packaging-16.8.tar.gz"

+    import_modules = ['packaging']
+
    version('16.8', '53895cdca04ecff80b54128e475b5d3b')

    # Not needed for the installation, but used at runtime
--- a/var/spack/repos/builtin/packages/py-pycparser/package.py
+++ b/var/spack/repos/builtin/packages/py-pycparser/package.py
@ -28,8 +28,11 @@
 class PyPycparser(PythonPackage):
    """A complete parser of the C language, written in pure python."""
    homepage = "https://github.com/eliben/pycparser"
-    url      = "https://pypi.python.org/packages/source/p/pycparser/pycparser-2.13.tar.gz"
+    url      = "https://pypi.io/packages/source/p/pycparser/pycparser-2.17.tar.gz"

+    import_modules = ['pycparser', 'pycparser.ply']
+
+    version('2.17', 'ca98dcb50bc1276f230118f6af5a40c7')
    version('2.13', 'e4fe1a2d341b22e25da0d22f034ef32f')

    depends_on('py-setuptools', type='build')
--- a/var/spack/repos/builtin/packages/py-pygments/package.py
+++ b/var/spack/repos/builtin/packages/py-pygments/package.py
@ -29,8 +29,14 @@ class PyPygments(PythonPackage):
    """Pygments is a syntax highlighting package written in Python."""

    homepage = "https://pypi.python.org/pypi/pygments"
-    url      = "https://pypi.python.org/packages/source/P/Pygments/Pygments-2.0.1.tar.gz"
+    url      = "https://pypi.io/packages/source/P/Pygments/Pygments-2.2.0.tar.gz"

+    import_modules = [
+        'pygments', 'pygments.filters', 'pygments.formatters',
+        'pygments.lexers', 'pygments.styles'
+    ]
+
+    version('2.2.0', '13037baca42f16917cbd5ad2fab50844')
    version('2.1.3', 'ed3fba2467c8afcda4d317e4ef2c6150')
    version('2.0.1', 'e0daf4c14a4fe5b630da765904de4d6c')
    version('2.0.2', '238587a1370d62405edabd0794b3ec4a')
--- a/var/spack/repos/builtin/packages/py-pyparsing/package.py
+++ b/var/spack/repos/builtin/packages/py-pyparsing/package.py
@ -28,8 +28,11 @@
 class PyPyparsing(PythonPackage):
    """A Python Parsing Module."""
    homepage = "https://pypi.python.org/pypi/pyparsing"
-    url      = "https://pypi.io/packages/source/p/pyparsing/pyparsing-2.0.3.tar.gz"
+    url      = "https://pypi.io/packages/source/p/pyparsing/pyparsing-2.2.0.tar.gz"

+    import_modules = ['pyparsing']
+
+    version('2.2.0',  '0214e42d63af850256962b6744c948d9')
    version('2.1.10', '065908b92904e0d3634eb156f44cc80e')
    version('2.0.3',  '0fe479be09fc2cf005f753d3acc35939')

--- a/var/spack/repos/builtin/packages/py-setuptools/package.py
+++ b/var/spack/repos/builtin/packages/py-setuptools/package.py
@ -32,12 +32,10 @@ class PySetuptools(PythonPackage):
    homepage = "https://pypi.python.org/pypi/setuptools"
    url      = "https://pypi.io/packages/source/s/setuptools/setuptools-25.2.0.tar.gz"

-    import_modules = [
-        'pkg_resources', 'setuptools', 'pkg_resources.extern',
-        'pkg_resources._vendor', 'pkg_resources._vendor.packaging',
-        'setuptools.extern', 'setuptools.command'
-    ]
+    import_modules = ['pkg_resources', 'setuptools', 'setuptools.command']

+    version('34.4.1', '5f9b07aeaafd29eac2548fc0b89a4934',
+            url="https://pypi.io/packages/source/s/setuptools/setuptools-34.4.1.zip")
    version('34.2.0', '41b630da4ea6cfa5894d9eb3142922be',
            url="https://pypi.io/packages/source/s/setuptools/setuptools-34.2.0.zip")
    version('25.2.0', 'a0dbb65889c46214c691f6c516cf959c')
--- a/var/spack/repos/builtin/packages/py-six/package.py
+++ b/var/spack/repos/builtin/packages/py-six/package.py
@ -29,7 +29,9 @@ class PySix(PythonPackage):
    """Python 2 and 3 compatibility utilities."""

    homepage = "https://pypi.python.org/pypi/six"
-    url      = "https://pypi.io/packages/source/s/six/six-1.9.0.tar.gz"
+    url      = "https://pypi.io/packages/source/s/six/six-1.10.0.tar.gz"
+
+    import_modules = ['six']

    version('1.10.0', '34eed507548117b2ab523ab14b2f8b55')
    version('1.9.0',  '476881ef4012262dfc8adc645ee786c4')